Merge pull request #2747 from intelowlproject/develop_old

v6.3.1

Author: mlodic

.github/CHANGELOG.md

@@ -2,6 +2,9 @@
 
 [**Upgrade Guide**](https://intelowlproject.github.io/docs/IntelOwl/installation/#update-to-the-most-recent-version)
 
+## [v6.3.1](https://github.com/intelowlproject/IntelOwl/releases/tag/v6.3.1)
+This release provides fixes to the recent added ARM support. (ARM build for v6.3.0 was broken due to some dependencies)
+
 ## [v6.3.0](https://github.com/intelowlproject/IntelOwl/releases/tag/v6.3.0)
 
 This release brings official support for ARM architecture. From now on, our Docker builds are multi-platform. You can now run IntelOwl in your favourite ARM machine smoothly, e.g. Apple Silicon Mac and Raspberry PI.

.github/release_template.md

@@ -11,7 +11,7 @@
 ```commandline
 please refer to the [Changelog](https://github.com/intelowlproject/IntelOwl/blob/develop/.github/CHANGELOG.md#v331)
 
-WARNING: The release will be live within an hour!
+WARNING: We are building the new version of the project! The release will be officially available within 2 hours!
 ```
 
 - [ ] Wait for [dockerHub](https://hub.docker.com/repository/docker/intelowlproject/intelowl) to finish the builds

README.md

@@ -112,6 +112,8 @@ In 2021 IntelOwl joined the official [Docker Open Source Program](https://www.do
 
 #### DigitalOcean
 
+[![DigitalOcean Referral Badge](https://web-platforms.sfo2.cdn.digitaloceanspaces.com/WWW/Badge%201.svg)](https://www.digitalocean.com/?refcode=128f2c68f93b&utm_campaign=Referral_Invite&utm_medium=Referral_Program&utm_source=badge)
+
 In 2022 IntelOwl joined the official [DigitalOcean Open Source Program](https://www.digitalocean.com/open-source?utm_medium=opensource&utm_source=IntelOwl).
 
 

api_app/analyzers_manager/file_analyzers/detectiteasy.py

@@ -1,7 +1,12 @@
 import json
 import logging
 
-import die
+from api_app.analyzers_manager.exceptions import AnalyzerRunException
+
+try:
+    import die
+except ImportError:
+    die = None
 
 from api_app.analyzers_manager.classes import FileAnalyzer
 from tests.mock_utils import MockUpResponse
@@ -17,11 +22,20 @@ def update(self):
     def run(self):
         logger.info(f"Running DIE on {self.filepath} for {self.md5}")
 
-        json_report = die.scan_file(
-            self.filepath, die.ScanFlags.RESULT_AS_JSON, str(die.database_path / "db")
-        )
+        if die:
+            json_report = die.scan_file(
+                self.filepath,
+                die.ScanFlags.RESULT_AS_JSON,
+                str(die.database_path / "db"),
+            )
+            result = json.loads(json_report)
+        else:
+            message = "DIE package not imported because incompatible in ARM"
+            self.report.errors.append(message)
+            result = {"errors": message}
+            raise AnalyzerRunException(message)
 
-        return json.loads(json_report)
+        return result
 
     @staticmethod
     def mocked_docker_analyzer_get(*args, **kwargs):

api_app/analyzers_manager/file_analyzers/goresym.py

@@ -49,12 +49,18 @@ def run(self):
         )
         result = self._docker_run(req_data, req_files, analyzer_name=self.analyzer_name)
         if "error" in result:
-            er = (
-                "Failed to parse file: failed to read pclntab: failed to locate pclntab"
-            )
-            if result["error"] == er:
-                logger.warning(f"Not a GO-compiled file: {result['error']}")
-                return f"Not a Go-compiled file: {result['error']}"
+            # the error message may change based on the version of the program
+            partial_error_keywords = ["failed", "no"]
+            found_negative_clause = False
+            if "pclntab" in result["error"]:
+                for partial_error_keyword in partial_error_keywords:
+                    if partial_error_keyword in result["error"]:
+                        found_negative_clause = True
+                        break
+            if found_negative_clause:
+                message = f"Not a GO-compiled file: {result['error']}"
+                logger.warning(message)
+                raise AnalyzerRunException(message)
             raise AnalyzerRunException(result["error"])
         return result
 

api_app/analyzers_manager/file_analyzers/phishing/phishing_form_compiler.py

@@ -1,13 +1,13 @@
 import logging
 from datetime import date, timedelta
 from typing import Dict
+from urllib.parse import urljoin
 
 import requests
 from faker import Faker  # skipcq: BAN-B410
 from lxml.etree import HTMLParser  # skipcq: BAN-B410
 from lxml.html import document_fromstring
 from requests import HTTPError, Response
-from requests.exceptions import MissingSchema
 
 from api_app.analyzers_manager.classes import FileAnalyzer
 from api_app.models import PythonConfig
@@ -137,32 +137,25 @@ def identify_text_input(self, input_name: str) -> str:
             if input_name in names:
                 return fake_value
 
-    def extract_action_attribute(self, form) -> str:
+    #         guarda anche i log di errore
+
+    @staticmethod
+    def extract_action_attribute(base_site: str, form) -> str:
+        # we always return an URL to prevent MissingSchema error in request
         form_action: str = form.get("action", None)
         if not form_action:
             logger.info(
-                f"'action' attribute not found in form. Defaulting to {self.target_site=}"
+                f"'action' attribute not found in form. Defaulting to {base_site=}"
             )
-            form_action = self.target_site
-        elif form_action.startswith("/"):  # pure relative url
-            logger.info(f"Found relative url in {form_action=}")
-            form_action = form_action.replace("/", "", 1)
-            base_site = self.target_site
-
-            if base_site.endswith("/"):
-                base_site = base_site[:-1]
-            form_action = base_site + "/" + form_action
-        elif (
-            "." in form_action and "://" not in form_action
-        ):  # found a domain (relative file names such as "login.php" should start with /)
-            logger.info(f"Found a domain in form action {form_action=}")
-        else:
-            base_site = self.target_site
-
-            if base_site.endswith("/"):
-                base_site = base_site[:-1]
-            form_action = base_site + "/" + form_action
-
+            return base_site
+        if "://" not in base_site:
+            # if target site is a domain add a temporary default
+            # schema so we can use urljoin as if it was an url
+            base_site = "https://" + base_site
+
+        form_action = urljoin(base_site, form_action)
+        if "://" not in form_action:
+            form_action = "https://" + form_action
         logger.info(f"Extracted action to post data to: {form_action}")
 
         return form_action
@@ -203,34 +196,21 @@ def compile_form_field(self, form) -> dict:
 
     def perform_request_to_form(self, form) -> Response:
         params = self.compile_form_field(form)
-        dest_url = self.extract_action_attribute(form)
+        dest_url = self.extract_action_attribute(self.target_site, form)
         logger.info(f"Job #{self.job_id}: Sending {params=} to submit url {dest_url}")
         headers = {
             "User-Agent": self.user_agent,
         }
-        try:
-            response = requests.post(
-                url=dest_url,
-                data=params,
-                headers=headers,
-                proxies=(
-                    {"http": self.proxy_address, "https": self.proxy_address}
-                    if self.proxy_address
-                    else None
-                ),
-            )
-        except MissingSchema:
-            logger.info(f"Adding default 'https://' schema to {dest_url}")
-            response = requests.post(
-                url="https://" + dest_url,
-                data=params,
-                headers=headers,
-                proxies=(
-                    {"http": self.proxy_address, "https": self.proxy_address}
-                    if self.proxy_address
-                    else None
-                ),
-            )
+        response = requests.post(
+            url=dest_url,
+            data=params,
+            headers=headers,
+            proxies=(
+                {"http": self.proxy_address, "https": self.proxy_address}
+                if self.proxy_address
+                else None
+            ),
+        )
         logger.info(f"Request headers: {response.request.headers}")
         return response
 

api_app/analyzers_manager/file_analyzers/qiling.py

@@ -21,6 +21,9 @@ class Qiling(FileAnalyzer, DockerBasedAnalyzer):
     shellcode: bool
     profile: str
 
+    def update(self):
+        pass
+
     def config(self, runtime_configuration: Dict):
         super().config(runtime_configuration)
         self.args = [self.os, self.arch]
@@ -41,4 +44,6 @@ def run(self):
             raise AnalyzerRunException(report["setup_error"])
         if report.get("execution_error"):
             raise AnalyzerRunException(report["execution_error"])
+        if report.get("qiling_not_available_error"):
+            raise AnalyzerRunException(report["qiling_not_available_error"])
         return report

api_app/analyzers_manager/observable_analyzers/thug_url.py

@@ -23,19 +23,16 @@ class ThugUrl(ObservableAnalyzer, DockerBasedAnalyzer):
 
     def _thug_args_builder(self):
         user_agent = self.user_agent
-        if not user_agent:
-            user_agent = (
-                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
-                "(KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36 Edg/131.0.2903.86"
-            )
         dom_events = self.dom_events
         use_proxy = self.use_proxy
         proxy = self.proxy
         enable_awis = self.enable_awis
         enable_img_proc = self.enable_image_processing_analysis
         # make request arguments
         # analysis timeout is set to 5 minutes
-        args = ["-T", "300", "-u", str(user_agent)]
+        args = ["-T", "300"]
+        if user_agent:
+            args.extend(["-u", str(user_agent)])
         if dom_events:
             args.extend(["-e", str(dom_events)])
         if use_proxy and proxy:
@@ -53,7 +50,6 @@ def run(self):
         tmp_dir = secrets.token_hex(4)
         tmp_dir_full_path = "/opt/deploy/thug" + tmp_dir
         # make request data
-        # the option -n is bugged and does not work https://github.com/intelowlproject/IntelOwl/issues/2656
         args.extend(["-n", tmp_dir_full_path, self.observable_name])
 
         req_data = {

api_app/visualizers_manager/visualizers/domain_reputation_services.py

@@ -238,7 +238,7 @@ def run(self) -> List[Dict]:
             third_level_elements.append(
                 self.Bool(
                     value=printable_analyzer_name,
-                    disable=not analyzer_report.report["malicious"],
+                    disable=not analyzer_report.report.get("malicious"),
                 )
             )
 

docker/.env

@@ -1,6 +1,6 @@
 ### DO NOT CHANGE THIS VALUE !!
 ### It should be updated only when you pull latest changes off from the 'master' branch of IntelOwl.
 # this variable must start with "REACT_APP_" to be used in the frontend too
-REACT_APP_INTELOWL_VERSION=v6.3.0
+REACT_APP_INTELOWL_VERSION=v6.3.1
 # if you want to use a nfs volume for shared files
 # NFS_ADDRESS=