diff --git a/api_app/admin.py b/api_app/admin.py index fb81ed5143..b11d62393a 100644 --- a/api_app/admin.py +++ b/api_app/admin.py @@ -49,10 +49,8 @@ class JobAdminView(CustomAdminView): "id", "status", "user", - "observable_name", - "observable_classification", - "file_name", - "file_mimetype", + "get_analyzable_name", + "get_analyzable_classification", "received_request_time", "analyzers_executed", "connectors_executed", @@ -64,13 +62,16 @@ class JobAdminView(CustomAdminView): "user", "status", ) - search_fields = ( - "md5", - "observable_name", - "file_name", - ) list_filter = ("status", "user", "tags") + @admin.display(description="Name") + def get_analyzable_name(self, instance): + return instance.analyzable.name + + @admin.display(description="Classification") + def get_analyzable_classification(self, instance): + return instance.analyzable.classification + @staticmethod def has_add_permission(request: HttpRequest) -> bool: return False diff --git a/api_app/analyzables_manager/__init__.py b/api_app/analyzables_manager/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api_app/analyzables_manager/admin.py b/api_app/analyzables_manager/admin.py new file mode 100644 index 0000000000..6d9ea8d6f2 --- /dev/null +++ b/api_app/analyzables_manager/admin.py @@ -0,0 +1,11 @@ +from django.contrib import admin + +from api_app.analyzables_manager.models import Analyzable + + +@admin.register(Analyzable) +class AnalyzableAdmin(admin.ModelAdmin): + list_display = ["pk", "name", "sha1", "sha256", "md5"] + search_fields = ["name", "sha1", "sha256", "md5"] + ordering = ["name"] + list_filter = ["discovery_date"] diff --git a/api_app/analyzables_manager/apps.py b/api_app/analyzables_manager/apps.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api_app/analyzables_manager/migrations/0001_initial.py b/api_app/analyzables_manager/migrations/0001_initial.py new file mode 100644 index 0000000000..de29326e6b --- /dev/null +++ b/api_app/analyzables_manager/migrations/0001_initial.py @@ -0,0 +1,70 @@ +# Generated by Django 4.2.17 on 2025-01-22 08:59 + +from django.db import migrations, models +from django.utils.timezone import now + +import api_app.defaults + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="Analyzable", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("md5", models.CharField(max_length=255, unique=True, editable=False)), + ( + "sha256", + models.CharField(max_length=255, unique=True, editable=False), + ), + ("sha1", models.CharField(max_length=255, unique=True, editable=False)), + ("name", models.CharField(max_length=255)), + ( + "mimetype", + models.CharField( + blank=True, max_length=80, null=True, default=None + ), + ), + ( + "file", + models.FileField( + null=True, + default=None, + blank=True, + upload_to=api_app.defaults.file_directory_path, + ), + ), + ( + "classification", + models.CharField( + max_length=100, + choices=[ + ("ip", "Ip"), + ("url", "Url"), + ("domain", "Domain"), + ("hash", "Hash"), + ("generic", "Generic"), + ("file", "File"), + ], + ), + ), + ("discovery_date", models.DateTimeField(default=now)), + ], + options={ + "abstract": False, + }, + ), + ] diff --git a/api_app/analyzables_manager/migrations/0002_migrate_data.py b/api_app/analyzables_manager/migrations/0002_migrate_data.py new file mode 100644 index 0000000000..2e6c1a76f7 --- /dev/null +++ b/api_app/analyzables_manager/migrations/0002_migrate_data.py @@ -0,0 +1,78 @@ +import hashlib + +from django.db import migrations +from django.db.models import F, OuterRef, Subquery, Window +from django.db.models.functions import RowNumber + + +def calculate_sha1(value: bytes) -> str: + return hashlib.sha1(value).hexdigest() # skipcq BAN-B324 + + +def calculate_sha256(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() # skipcq BAN-B324 + + +def migrate(apps, schema_editor): + Job = apps.get_model("api_app", "Job") + Analyzable = apps.get_model("analyzables_manager", "Analyzable") + # get only on job for md5 + jobs = Job.objects.alias( + row_number=Window( + RowNumber(), partition_by=(F("md5"),), order_by="received_request_time" + ) + ).filter(row_number=1) + for job in jobs: + if job.is_sample: + an = Analyzable.objects.create( + md5=job.md5, + sha256=job.sha256, + sha1=job.sha1, + file=job.file, + mimetype=job.file_mimetype, + name=job.file_name, + classification="sample", + discovery_date=job.received_request_time, + ) + + p = job.file.path + try: + p.rename(p.parent / job.md5) + except Exception: + print(f"Error: unable to rename {job}") + else: + job.file.name = job.md5 + with open(p, "rb") as f: + content = f.read() + f.seek(0) + an.sha1 = calculate_sha1(content) + an.sha256 = calculate_sha256(content) + else: + an = Analyzable.objects.create( + md5=job.md5, + name=job.observable_name, + classification=job.observable_classification, + discovery_date=job.received_request_time, + ) + an.sha1 = calculate_sha1(an.name.encode("utf-8")) + an.sha256 = calculate_sha256(an.name.encode("utf-8")) + an.save() + Job.objects.update( + analyzable=Subquery( + Analyzable.objects.filter(md5=OuterRef("md5")).values("pk")[:1] + ) + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("contenttypes", "0002_remove_content_type_name"), + ("api_app", "0067_add_analyzable"), + ("analyzables_manager", "0001_initial"), + ("visualizers_manager", "0040_visualizer_config_data_model"), + ] + + operations = [ + migrations.RunPython(migrate, migrations.RunPython.noop), + ] diff --git a/api_app/analyzables_manager/migrations/0003_analyzable_analyzables_classif_adf7ca_idx_and_more.py b/api_app/analyzables_manager/migrations/0003_analyzable_analyzables_classif_adf7ca_idx_and_more.py new file mode 100644 index 0000000000..2b4ff4b086 --- /dev/null +++ b/api_app/analyzables_manager/migrations/0003_analyzable_analyzables_classif_adf7ca_idx_and_more.py @@ -0,0 +1,25 @@ +# Generated by Django 4.2.17 on 2025-01-23 14:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("analyzables_manager", "0002_migrate_data"), + ] + + operations = [ + migrations.AddIndex( + model_name="analyzable", + index=models.Index( + fields=["classification"], name="analyzables_classif_adf7ca_idx" + ), + ), + migrations.AddIndex( + model_name="analyzable", + index=models.Index( + fields=["mimetype"], name="analyzables_mimetyp_321d7d_idx" + ), + ), + ] diff --git a/api_app/analyzables_manager/migrations/__init__.py b/api_app/analyzables_manager/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api_app/analyzables_manager/models.py b/api_app/analyzables_manager/models.py new file mode 100644 index 0000000000..6381ec54ba --- /dev/null +++ b/api_app/analyzables_manager/models.py @@ -0,0 +1,96 @@ +from typing import Type, Union + +from django.core.exceptions import ValidationError +from django.db import models +from django.utils.timezone import now + +from api_app.analyzables_manager.queryset import AnalyzableQuerySet +from api_app.choices import Classification +from api_app.data_model_manager.models import ( + BaseDataModel, + DomainDataModel, + FileDataModel, + IPDataModel, +) +from api_app.defaults import file_directory_path +from api_app.helpers import calculate_md5, calculate_sha1, calculate_sha256 + + +class Analyzable(models.Model): + name = models.CharField(max_length=255) + discovery_date = models.DateTimeField(default=now) + md5 = models.CharField(max_length=255, unique=True, editable=False) + sha256 = models.CharField(max_length=255, unique=True, editable=False) + sha1 = models.CharField(max_length=255, unique=True, editable=False) + classification = models.CharField(max_length=100, choices=Classification.choices) + mimetype = models.CharField(max_length=80, blank=True, null=True, default=None) + file = models.FileField( + upload_to=file_directory_path, null=True, blank=True, default=None + ) + + objects = AnalyzableQuerySet.as_manager() + + class Meta: + indexes = [ + models.Index(fields=["classification"]), + models.Index(fields=["mimetype"]), + ] + + def __str__(self): + return self.name + + @property + def analyzed_object(self): + return self.file if self.is_sample else self.name + + @property + def is_sample(self) -> bool: + return self.classification == Classification.FILE.value + + def get_data_model_class(self) -> Type[BaseDataModel]: + if self.classification == Classification.IP.value: + return IPDataModel + elif self.classification in [ + Classification.URL.value, + Classification.DOMAIN.value, + ]: + return DomainDataModel + elif self.classification in [ + Classification.HASH.value, + Classification.FILE.value, + ]: + return FileDataModel + else: + raise NotImplementedError() + + def _set_hashes(self, value: Union[str, bytes]): + if isinstance(value, str): + value = value.encode("utf-8") + if not self.md5: + self.md5 = calculate_md5(value) + if not self.sha256: + self.sha256 = calculate_sha256(value) + if not self.sha1: + self.sha1 = calculate_sha1(value) + + def clean(self): + if self.classification == Classification.FILE.value: + from api_app.analyzers_manager.models import MimeTypes + + if not self.file: + raise ValidationError("File must be set for samples") + content = self.read() + if not self.mimetype: + self.mimetype = MimeTypes.calculate(content, self.name) + else: + if self.mimetype or self.file: + raise ValidationError( + "Mimetype and file must not be set for observables" + ) + content = self.name + self._set_hashes(content) + + def read(self) -> bytes: + if self.classification == Classification.FILE.value: + self.file.seek(0) + return self.file.read() diff --git a/api_app/analyzables_manager/queryset.py b/api_app/analyzables_manager/queryset.py new file mode 100644 index 0000000000..90d60a02f6 --- /dev/null +++ b/api_app/analyzables_manager/queryset.py @@ -0,0 +1,31 @@ +import logging + +from django.db.models import QuerySet + +logger = logging.getLogger(__name__) + + +class AnalyzableQuerySet(QuerySet): + + def visible_for_user(self, user): + + from api_app.models import Job + + analyzables = ( + Job.objects.visible_for_user(user) + .values("analyzable") + .distinct() + .values_list("analyzable__pk", flat=True) + ) + return self.filter(pk__in=analyzables) + + def create(self, *args, **kwargs): + obj = self.model(**kwargs) + self._for_write = True + try: + obj.full_clean() + except Exception as e: + logger.error(f"Already exists obj {obj.md5}") + raise e + obj.save(force_insert=True, using=self.db) + return obj diff --git a/api_app/analyzables_manager/serializers.py b/api_app/analyzables_manager/serializers.py new file mode 100644 index 0000000000..5790b700bd --- /dev/null +++ b/api_app/analyzables_manager/serializers.py @@ -0,0 +1,12 @@ +from rest_framework.serializers import ModelSerializer + +from api_app.analyzables_manager.models import Analyzable +from api_app.serializers.job import JobRelatedField + + +class AnalyzableSerializer(ModelSerializer): + jobs = JobRelatedField(many=True, read_only=True) + + class Meta: + model = Analyzable + fields = "__all__" diff --git a/api_app/analyzables_manager/signals.py b/api_app/analyzables_manager/signals.py new file mode 100644 index 0000000000..cfd708a2d5 --- /dev/null +++ b/api_app/analyzables_manager/signals.py @@ -0,0 +1,19 @@ +from django.db import models +from django.dispatch import receiver + +from api_app.analyzables_manager.models import Analyzable + + +@receiver(models.signals.pre_delete, sender=Analyzable) +def pre_delete_analyzable(sender, instance: Analyzable, **kwargs): + """ + Signal receiver for the pre_delete signal of the Analyzable model. + Deletes the associated file if it exists. + + Args: + sender (Model): The model class sending the signal. + instance (Analyzable): The instance of the model being deleted. + **kwargs: Additional keyword arguments. + """ + if instance.file: + instance.file.delete() diff --git a/api_app/analyzables_manager/urls.py b/api_app/analyzables_manager/urls.py new file mode 100644 index 0000000000..58ee5be224 --- /dev/null +++ b/api_app/analyzables_manager/urls.py @@ -0,0 +1,18 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from django.urls import include, path +from rest_framework import routers + +from api_app.analyzables_manager.views import AnalyzableViewSet + +# Routers provide an easy way of automatically determining the URL conf. + + +router = routers.DefaultRouter(trailing_slash=False) +router.register(r"analyzable", AnalyzableViewSet, basename="analyzable") + +urlpatterns = [ + # Viewsets + path(r"", include(router.urls)), +] diff --git a/api_app/analyzables_manager/views.py b/api_app/analyzables_manager/views.py new file mode 100644 index 0000000000..bd978ff199 --- /dev/null +++ b/api_app/analyzables_manager/views.py @@ -0,0 +1,14 @@ +from rest_framework import viewsets +from rest_framework.permissions import IsAuthenticated + +from api_app.analyzables_manager.serializers import AnalyzableSerializer + + +class AnalyzableViewSet(viewsets.ReadOnlyModelViewSet): + + serializer_class = AnalyzableSerializer + permission_classes = [IsAuthenticated] + + def get_queryset(self): + user = self.request.user + return super().get_queryset().visible_for_user(user) diff --git a/api_app/analyzers_manager/classes.py b/api_app/analyzers_manager/classes.py index 7ac5a26bd1..9e4f550597 100644 --- a/api_app/analyzers_manager/classes.py +++ b/api_app/analyzers_manager/classes.py @@ -14,10 +14,10 @@ from certego_saas.apps.user.models import User from tests.mock_utils import MockUpResponse, if_mock_connections, patch -from ..choices import PythonModuleBasePaths +from ..choices import Classification, PythonModuleBasePaths from ..classes import Plugin from ..models import PythonConfig -from .constants import HashChoices, ObservableTypes, TypeChoices +from .constants import HashChoices, TypeChoices from .exceptions import AnalyzerConfigurationException, AnalyzerRunException from .models import AnalyzerConfig, AnalyzerReport @@ -32,7 +32,6 @@ class BaseAnalyzerMixin(Plugin, metaclass=ABCMeta): """ HashChoices = HashChoices - ObservableTypes = ObservableTypes TypeChoices = TypeChoices MALICIOUS_EVALUATION = 75 @@ -54,7 +53,7 @@ def threat_to_evaluation(self, threat_level): return evaluation def _do_create_data_model(self) -> bool: - if self.report.job.observable_classification == ObservableTypes.GENERIC: + if self.report.job.analyzable.classification == Classification.GENERIC.value: return False if ( not self._config.mapping_data_model @@ -190,16 +189,16 @@ def config(self, runtime_configuration: Dict): super().config(runtime_configuration) self._config: AnalyzerConfig if self._job.is_sample and self._config.run_hash: - self.observable_classification = ObservableTypes.HASH + self.observable_classification = Classification.HASH # check which kind of hash the analyzer needs run_hash_type = self._config.run_hash_type if run_hash_type == HashChoices.SHA256: - self.observable_name = self._job.sha256 + self.observable_name = self._job.analyzable.sha256 else: - self.observable_name = self._job.md5 + self.observable_name = self._job.analyzable.md5 else: - self.observable_name = self._job.observable_name - self.observable_classification = self._job.observable_classification + self.observable_name = self._job.analyzable.name + self.observable_classification = self._job.analyzable.classification @classmethod @property @@ -242,13 +241,13 @@ def __init__( def config(self, runtime_configuration: Dict): super().config(runtime_configuration) - self.md5 = self._job.md5 - self.filename = self._job.file_name + self.md5 = self._job.analyzable.md5 + self.filename = self._job.analyzable.name # this is updated in the filepath property, like a cache decorator. # if the filepath is requested, it means that the analyzer downloads... # ...the file from AWS because it requires a path and it needs to be deleted self.__filepath = None - self.file_mimetype = self._job.file_mimetype + self.file_mimetype = self._job.analyzable.mimetype @classmethod @property @@ -256,8 +255,7 @@ def python_base_path(cls) -> PosixPath: return PythonModuleBasePaths[FileAnalyzer.__name__].value def read_file_bytes(self) -> bytes: - self._job.file.seek(0) - return self._job.file.read() + return self._job.analyzable.read() @property def filepath(self) -> str: @@ -267,9 +265,7 @@ def filepath(self) -> str: str: The file path. """ if not self.__filepath: - self.__filepath = self._job.file.storage.retrieve( - file=self._job.file, analyzer=self.analyzer_name - ) + self.__filepath = self._job.analyzable.file.path return self.__filepath def before_run(self): diff --git a/api_app/analyzers_manager/constants.py b/api_app/analyzers_manager/constants.py index 70807cd445..afb9c34a2a 100644 --- a/api_app/analyzers_manager/constants.py +++ b/api_app/analyzers_manager/constants.py @@ -1,7 +1,5 @@ # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl # See the file 'LICENSE' for copying permission. -import ipaddress -import re from logging import getLogger from django.db import models @@ -19,72 +17,6 @@ class HashChoices(models.TextChoices): SHA256 = "sha256" -class ObservableTypes(models.TextChoices): - IP = "ip" - URL = "url" - DOMAIN = "domain" - HASH = "hash" - GENERIC = "generic" - - @classmethod - def calculate(cls, value: str) -> str: - """Returns observable classification for the given value.\n - Only following types are supported: - ip, domain, url, hash (md5, sha1, sha256), generic (if no match) - - Args: - value (str): - observable value - Returns: - str: one of `ip`, `url`, `domain`, `hash` or 'generic'. - """ - try: - ipaddress.ip_address(value) - except ValueError: - if re.match( - r"^.+://[a-z\d-]{1,200}" - r"(?:\.[a-zA-Z\d\u2044\u2215!#$&(-;=?-\[\]_~]{1,200})+" - r"(?::\d{2,6})?" - r"(?:/[a-zA-Z\d\u2044\u2215!#$&(-;=?-\[\]_~]{1,200})*" - r"(?:\.\w+)?", - value, - ): - classification = cls.URL - elif re.match( - r"^([\[\\]?\.[\]\\]?)?[a-z\d\-_]{1,63}" - r"(([\[\\]?\.[\]\\]?)[a-z\d\-_]{1,63})+$", - value, - re.IGNORECASE, - ): - classification = cls.DOMAIN - elif ( - re.match(r"^[a-f\d]{32}$", value, re.IGNORECASE) - or re.match(r"^[a-f\d]{40}$", value, re.IGNORECASE) - or re.match(r"^[a-f\d]{64}$", value, re.IGNORECASE) - ): - classification = cls.HASH - else: - classification = cls.GENERIC - logger.info( - "Couldn't detect observable classification" - f" for {value}, setting as 'generic'" - ) - else: - # it's a simple IP - classification = cls.IP - - return classification - - -class AllTypes(models.TextChoices): - IP = "ip" - URL = "url" - DOMAIN = "domain" - HASH = "hash" - GENERIC = "generic" - FILE = "file" - - class HTTPMethods(models.TextChoices): GET = "get" POST = "post" diff --git a/api_app/analyzers_manager/file_analyzers/androguard.py b/api_app/analyzers_manager/file_analyzers/androguard.py index aeff03bb1a..21622c37be 100644 --- a/api_app/analyzers_manager/file_analyzers/androguard.py +++ b/api_app/analyzers_manager/file_analyzers/androguard.py @@ -1,9 +1,7 @@ -import androguard -import androguard.core -import androguard.core.bytecodes -import androguard.core.bytecodes.apk +from androguard.misc import get_default_session from api_app.analyzers_manager.classes import FileAnalyzer +from api_app.analyzers_manager.models import MimeTypes class AndroguardAnalyzer(FileAnalyzer): @@ -13,23 +11,29 @@ def update(self) -> bool: def run(self): - binary = self.read_file_bytes() - apk = androguard.core.bytecodes.apk.APK(binary, raw=True) - results = { - "app_name": apk.get_app_name(), - "permissions": apk.get_permissions(), - "activities": apk.get_activities(), - "requested_third_party_permissions": apk.get_requested_third_party_permissions(), - "providers": apk.get_providers(), - "features": apk.get_features(), - "receivers": apk.get_receivers(), - "services": apk.get_services(), - "is_valid_apk": apk.is_valid_APK(), - "min_sdk_version": apk.get_min_sdk_version(), - "max_sdk_version": apk.get_max_sdk_version(), - "target_sdk_version": apk.get_target_sdk_version(), - "android_version_code": apk.get_androidversion_code(), - "android_version_name": apk.get_androidversion_name(), - } + self.read_file_bytes() + session = get_default_session() + + if self._job.analyzable.mimetype == MimeTypes.DEX: + _, _, dx = session.addDEX(self._job.analyzable.name, self.read_file_bytes()) + results = {} + else: + _, apk = session.addAPK(self._job.analyzable.name, self.read_file_bytes()) + results = { + "app_name": apk.get_app_name(), + "permissions": apk.get_permissions(), + "activities": apk.get_activities(), + "requested_third_party_permissions": apk.get_requested_third_party_permissions(), + "providers": apk.get_providers(), + "features": apk.get_features(), + "receivers": apk.get_receivers(), + "services": apk.get_services(), + "is_valid_apk": apk.is_valid_APK(), + "min_sdk_version": apk.get_min_sdk_version(), + "max_sdk_version": apk.get_max_sdk_version(), + "target_sdk_version": apk.get_target_sdk_version(), + "android_version_code": apk.get_androidversion_code(), + "android_version_name": apk.get_androidversion_name(), + } return results diff --git a/api_app/analyzers_manager/file_analyzers/lnk_info.py b/api_app/analyzers_manager/file_analyzers/lnk_info.py index 77a7bbaf56..77ca3cc66e 100644 --- a/api_app/analyzers_manager/file_analyzers/lnk_info.py +++ b/api_app/analyzers_manager/file_analyzers/lnk_info.py @@ -7,7 +7,7 @@ import pylnk3 from api_app.analyzers_manager.classes import FileAnalyzer -from api_app.analyzers_manager.constants import ObservableTypes +from api_app.choices import Classification logger = logging.getLogger(__name__) @@ -28,7 +28,7 @@ def run(self): if arguments := getattr(parsed, "arguments", None): args = arguments.split() for a in args: - if ObservableTypes.calculate(a) == ObservableTypes.URL: + if Classification.calculate_observable(a) == Classification.URL: # remove strings delimiters used in commands a = re.sub(r"[\"\']", "", a) result["uris"].append(a) diff --git a/api_app/analyzers_manager/file_analyzers/mwdb_scan.py b/api_app/analyzers_manager/file_analyzers/mwdb_scan.py index 29e29f4d77..47d26c454d 100644 --- a/api_app/analyzers_manager/file_analyzers/mwdb_scan.py +++ b/api_app/analyzers_manager/file_analyzers/mwdb_scan.py @@ -90,7 +90,7 @@ def adjust_relations(self, base, key, recursive=True): def run(self): result = {} binary = self.read_file_bytes() - query = self._job.sha256 + query = self._job.analyzable.sha256 self.mwdb = mwdblib.MWDB(api_key=self._api_key_name) if self.upload_file: diff --git a/api_app/analyzers_manager/file_analyzers/phishing/phishing_form_compiler.py b/api_app/analyzers_manager/file_analyzers/phishing/phishing_form_compiler.py index a552fc533e..1d939fb683 100644 --- a/api_app/analyzers_manager/file_analyzers/phishing/phishing_form_compiler.py +++ b/api_app/analyzers_manager/file_analyzers/phishing/phishing_form_compiler.py @@ -53,7 +53,7 @@ def config(self, runtime_configuration: Dict): super().config(runtime_configuration) if hasattr(self._job, "pivot_parent"): # extract target site from parent job - self.target_site = self._job.pivot_parent.starting_job.observable_name + self.target_site = self._job.pivot_parent.starting_job.analyzable.name else: logger.warning( f"Job #{self.job_id}: Analyzer {self.analyzer_name} should be ran from PhishingAnalysis playbook." diff --git a/api_app/analyzers_manager/file_analyzers/strings_info.py b/api_app/analyzers_manager/file_analyzers/strings_info.py index 2230f34ea0..77292f8735 100644 --- a/api_app/analyzers_manager/file_analyzers/strings_info.py +++ b/api_app/analyzers_manager/file_analyzers/strings_info.py @@ -4,8 +4,8 @@ from json import dumps as json_dumps from api_app.analyzers_manager.classes import DockerBasedAnalyzer, FileAnalyzer -from api_app.analyzers_manager.constants import ObservableTypes from api_app.analyzers_manager.models import MimeTypes +from api_app.choices import Classification class StringsInfo(FileAnalyzer, DockerBasedAnalyzer): @@ -84,7 +84,7 @@ def run(self): import re for d in result["data"]: - if ObservableTypes.calculate(d) == ObservableTypes.URL: + if Classification.calculate_observable(d) == Classification.URL: extracted_urls = re.findall( r"[a-z]{1,5}://[a-z\d-]{1,200}" r"(?:\.[a-zA-Z\d\u2044\u2215!#$&(-;=?-\[\]_~]{1,200})+" diff --git a/api_app/analyzers_manager/file_analyzers/sublime.py b/api_app/analyzers_manager/file_analyzers/sublime.py index aad0d74a03..0cb9e5e357 100644 --- a/api_app/analyzers_manager/file_analyzers/sublime.py +++ b/api_app/analyzers_manager/file_analyzers/sublime.py @@ -87,7 +87,7 @@ def raw_message(self) -> str: file.seek(0) proc = subprocess.run(command, check=True, stdout=subprocess.PIPE) return base64.b64encode(proc.stdout.strip()).decode("utf-8") - return self._job.b64 + return str(base64.b64encode(self._job.analyzable.read()), "utf-8") def _analysis(self, session: requests.Session, content: str): result = session.post( diff --git a/api_app/analyzers_manager/migrations/0059_alter_analyzer_config_dns0_rrsets_data.py b/api_app/analyzers_manager/migrations/0059_alter_analyzer_config_dns0_rrsets_data.py index ddcf337bec..52f96a2dc3 100644 --- a/api_app/analyzers_manager/migrations/0059_alter_analyzer_config_dns0_rrsets_data.py +++ b/api_app/analyzers_manager/migrations/0059_alter_analyzer_config_dns0_rrsets_data.py @@ -1,15 +1,13 @@ from django.db import migrations -from api_app.analyzers_manager.constants import ObservableTypes - def migrate(apps, schema_editor): AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig") config = AnalyzerConfig.objects.get(name="DNS0_rrsets_data") config.observable_supported = [ - ObservableTypes.DOMAIN, - ObservableTypes.GENERIC, - ObservableTypes.IP, + "domain", + "generic", + "ip", ] config.full_clean() config.save() @@ -19,10 +17,10 @@ def reverse_migrate(apps, schema_editor): AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig") config = AnalyzerConfig.objects.get(name="DNS0_rrsets_data") config.observable_supported = [ - ObservableTypes.DOMAIN, - ObservableTypes.URL, - ObservableTypes.GENERIC, - ObservableTypes.IP, + "domain", + "url", + "generic", + "ip", ] config.full_clean() config.save() diff --git a/api_app/analyzers_manager/migrations/0131_analyzer_config_vt_sample_download.py b/api_app/analyzers_manager/migrations/0131_analyzer_config_vt_sample_download.py index 751f632807..c7b91d2620 100644 --- a/api_app/analyzers_manager/migrations/0131_analyzer_config_vt_sample_download.py +++ b/api_app/analyzers_manager/migrations/0131_analyzer_config_vt_sample_download.py @@ -1,6 +1,6 @@ from django.db import migrations -from api_app.analyzers_manager.constants import ObservableTypes, TypeChoices +from api_app.analyzers_manager.constants import TypeChoices from api_app.choices import TLP @@ -15,7 +15,7 @@ def migrate(apps, schema_editor): python_module=PythonModule.objects.get( module="vt.vt3_sample_download.VirusTotalv3SampleDownload" ), - observable_supported=[ObservableTypes.HASH.value], + observable_supported=["hash"], ) diff --git a/api_app/analyzers_manager/migrations/0139_alter_analyzerconfig_mapping_data_model.py b/api_app/analyzers_manager/migrations/0139_alter_analyzerconfig_mapping_data_model.py index c7077710eb..3af46871dd 100644 --- a/api_app/analyzers_manager/migrations/0139_alter_analyzerconfig_mapping_data_model.py +++ b/api_app/analyzers_manager/migrations/0139_alter_analyzerconfig_mapping_data_model.py @@ -6,13 +6,17 @@ class Migration(migrations.Migration): dependencies = [ - ('analyzers_manager', '0138_alter_analyzerreport_data_model_content_type'), + ("analyzers_manager", "0138_alter_analyzerreport_data_model_content_type"), ] operations = [ migrations.AlterField( - model_name='analyzerconfig', - name='mapping_data_model', - field=models.JSONField(blank=True, default=dict, help_text='Mapping analyzer_report_key: data_model_key. Keys preceded by the symbol $ will be considered as constants.'), + model_name="analyzerconfig", + name="mapping_data_model", + field=models.JSONField( + blank=True, + default=dict, + help_text="Mapping analyzer_report_key: data_model_key. Keys preceded by the symbol $ will be considered as constants.", + ), ), ] diff --git a/api_app/analyzers_manager/models.py b/api_app/analyzers_manager/models.py index cd1458113a..803baa3a24 100644 --- a/api_app/analyzers_manager/models.py +++ b/api_app/analyzers_manager/models.py @@ -6,25 +6,14 @@ from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation from django.contrib.contenttypes.models import ContentType -from django.contrib.postgres.fields import ArrayField from django.core.exceptions import ValidationError from django.db import models -from django.db.models import ForeignKey -from api_app.analyzers_manager.constants import ( - HashChoices, - ObservableTypes, - TypeChoices, -) +from api_app.analyzers_manager.constants import HashChoices, TypeChoices from api_app.analyzers_manager.exceptions import AnalyzerConfigurationException from api_app.analyzers_manager.queryset import AnalyzerReportQuerySet -from api_app.choices import TLP, PythonModuleBasePaths -from api_app.data_model_manager.models import ( - BaseDataModel, - DomainDataModel, - FileDataModel, - IPDataModel, -) +from api_app.choices import TLP, Classification, PythonModuleBasePaths +from api_app.data_model_manager.models import BaseDataModel from api_app.fields import ChoiceArrayField from api_app.models import AbstractReport, PythonConfig, PythonModule @@ -63,24 +52,9 @@ def clean(self): ): raise ValidationError("Wrong data model for this report") - @classmethod - def get_data_model_class(cls, job) -> Type[BaseDataModel]: - if job.is_sample or job.observable_classification == ObservableTypes.HASH.value: - return FileDataModel - if job.observable_classification == ObservableTypes.IP.value: - return IPDataModel - if job.observable_classification in [ - ObservableTypes.DOMAIN.value, - ObservableTypes.URL.value, - ]: - return DomainDataModel - raise NotImplementedError( - f"Unable to find data model for {job.observable_classification}" - ) - @property def data_model_class(self) -> Type[BaseDataModel]: - return self.get_data_model_class(self.job) + return self.job.analyzable.get_data_model_class() def _validation_before_data_model(self) -> bool: if not self.status == self.STATUSES.SUCCESS.value: @@ -115,7 +89,6 @@ def _create_data_model_dictionary(self) -> Dict: result = {"malware_family": "MalwareFamily"}. """ result = {} - data_model_fields = self.data_model_class.get_fields() logger.debug(f"Mapping is {json.dumps(self.config.mapping_data_model)}") for report_key, data_model_key in self.config.mapping_data_model.items(): # this is a constant @@ -130,40 +103,20 @@ def _create_data_model_dictionary(self) -> Dict: # validation self.errors.append(f"Field {report_key} not available in report") continue - - # create the related object if necessary - if isinstance(data_model_fields[data_model_key], ForeignKey): - # to create an object we need at least a dictionary - if not isinstance(value, dict): - self.errors.append( - f"Field {report_key} has type {type(report_key)} while a dictionary is expected" - ) - continue - value, _ = data_model_fields[ - data_model_key - ].related_model.objects.get_or_create(**value) - result[data_model_key] = value - elif isinstance(data_model_fields[data_model_key], ArrayField): - if data_model_key not in result: - result[data_model_key] = [] - if isinstance(value, list): - result[data_model_key].extend(value) - elif isinstance(value, dict): - result[data_model_key].extend(list(value.keys())) - else: - result[data_model_key].append(value) - else: - result[data_model_key] = value + result[data_model_key] = value return result def create_data_model(self) -> Optional[BaseDataModel]: + # TODO we don't need to actually crate a new object every time. + # if the report is the same of the previous one, we can just link it if not self._validation_before_data_model(): return None dictionary = self._create_data_model_dictionary() - data_model = self.data_model_class.objects.create(**dictionary) - self.data_model = data_model + + self.data_model: BaseDataModel = self.data_model_class.objects.create() + self.data_model.merge(dictionary) self.save() - return data_model + return self.data_model class MimeTypes(models.TextChoices): @@ -299,7 +252,9 @@ class AnalyzerConfig(PythonConfig): ) # obs observable_supported = ChoiceArrayField( - models.CharField(null=False, choices=ObservableTypes.choices, max_length=30), + models.CharField( + null=False, choices=Classification.choices[:-1], max_length=30 + ), default=list, blank=True, ) diff --git a/api_app/analyzers_manager/observable_analyzers/apivoid.py b/api_app/analyzers_manager/observable_analyzers/apivoid.py index f7548844b1..85117d65d5 100644 --- a/api_app/analyzers_manager/observable_analyzers/apivoid.py +++ b/api_app/analyzers_manager/observable_analyzers/apivoid.py @@ -7,6 +7,7 @@ from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerConfigurationException +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch @@ -18,13 +19,13 @@ def update(self): pass def run(self): - if self.observable_classification == self.ObservableTypes.DOMAIN.value: + if self.observable_classification == Classification.DOMAIN.value: path = "domainbl" parameter = "host" - elif self.observable_classification == self.ObservableTypes.IP.value: + elif self.observable_classification == Classification.IP.value: path = "iprep" parameter = "ip" - elif self.observable_classification == self.ObservableTypes.URL.value: + elif self.observable_classification == Classification.URL.value: path = "urlrep" parameter = "url" else: diff --git a/api_app/analyzers_manager/observable_analyzers/binaryedge.py b/api_app/analyzers_manager/observable_analyzers/binaryedge.py index 5db03d71e3..f97b58208e 100644 --- a/api_app/analyzers_manager/observable_analyzers/binaryedge.py +++ b/api_app/analyzers_manager/observable_analyzers/binaryedge.py @@ -6,6 +6,7 @@ from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch @@ -24,7 +25,7 @@ def config(self, runtime_configuration: Dict): def run(self): results = {} - if self.observable_classification == self.ObservableTypes.IP: + if self.observable_classification == Classification.IP: try: response_recent_ip_info = requests.get( self.url + "ip/" + self.observable_name, headers=self.headers @@ -44,7 +45,7 @@ def run(self): "ip_recent_report": response_recent_ip_info.json(), "ip_query_report": response_query_ip.json(), } - elif self.observable_classification == self.ObservableTypes.DOMAIN: + elif self.observable_classification == Classification.DOMAIN: try: response_domain_report = requests.get( self.url + "domains/subdomain/" + self.observable_name, diff --git a/api_app/analyzers_manager/observable_analyzers/circl_pdns.py b/api_app/analyzers_manager/observable_analyzers/circl_pdns.py index 789200b4d4..e65e82929e 100644 --- a/api_app/analyzers_manager/observable_analyzers/circl_pdns.py +++ b/api_app/analyzers_manager/observable_analyzers/circl_pdns.py @@ -10,6 +10,7 @@ from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from certego_saas.apps.user.models import User from tests.mock_utils import MockResponseNoOp, if_mock_connections, patch @@ -21,7 +22,7 @@ class CIRCL_PDNS(classes.ObservableAnalyzer): def config(self, runtime_configuration: Dict): super().config(runtime_configuration) self.domain = self.observable_name - if self.observable_classification == self.ObservableTypes.URL: + if self.observable_classification == Classification.URL: self.domain = urlparse(self.observable_name).hostname # You should save CIRCL credentials with this template: "|" self.split_credentials = self._pdns_credentials.split("|") diff --git a/api_app/analyzers_manager/observable_analyzers/criminalip/criminalip.py b/api_app/analyzers_manager/observable_analyzers/criminalip/criminalip.py index 0e525342af..e11162eabc 100644 --- a/api_app/analyzers_manager/observable_analyzers/criminalip/criminalip.py +++ b/api_app/analyzers_manager/observable_analyzers/criminalip/criminalip.py @@ -4,6 +4,7 @@ import requests from api_app.analyzers_manager import classes +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch from .criminalip_base import CriminalIpBase @@ -29,7 +30,7 @@ def make_request(self, url: str, params: Dict[str, str] = None) -> Dict: def run(self): URLs = { - self.ObservableTypes.IP.value: { + Classification.IP.value: { "endpoints": { "malicious_info": "/v1/feature/ip/malicious-info", "privacy_threat": "/v1/feature/ip/privacy-threat", @@ -38,13 +39,13 @@ def run(self): }, "params": {"ip": self.observable_name}, }, - self.ObservableTypes.DOMAIN.value: { + Classification.DOMAIN.value: { "endpoints": { "hash_view": "/v1/domain/quick/hash/view", }, "params": {"domain": self.observable_name}, }, - self.ObservableTypes.GENERIC.value: { + Classification.GENERIC.value: { "endpoints": { "banner_search": "/v1/banner/search", "banner_stats": "/v1/banner/stats", diff --git a/api_app/analyzers_manager/observable_analyzers/crowdsec.py b/api_app/analyzers_manager/observable_analyzers/crowdsec.py index 4d7676de3d..289075ba6d 100644 --- a/api_app/analyzers_manager/observable_analyzers/crowdsec.py +++ b/api_app/analyzers_manager/observable_analyzers/crowdsec.py @@ -61,14 +61,14 @@ def _update_data_model(self, data_model): self.report.data_model_class.EVALUATIONS.CLEAN.value ) elif "Proxy" in label or "VPN" in label: - data_model.tags = [DataModelTags.ANONYMIZER] + data_model.tags = [DataModelTags.ANONYMIZER.value] data_model.evaluation = ( self.report.data_model_class.EVALUATIONS.CLEAN.value ) elif label in ["TOR exit node"]: data_model.tags = [ - DataModelTags.ANONYMIZER, - DataModelTags.TOR_EXIT_NODE, + DataModelTags.ANONYMIZER.value, + DataModelTags.TOR_EXIT_NODE.value, ] data_model.evaluation = ( self.report.data_model_class.EVALUATIONS.CLEAN.value diff --git a/api_app/analyzers_manager/observable_analyzers/cymru.py b/api_app/analyzers_manager/observable_analyzers/cymru.py index 5d7fcd8817..17b359e56f 100644 --- a/api_app/analyzers_manager/observable_analyzers/cymru.py +++ b/api_app/analyzers_manager/observable_analyzers/cymru.py @@ -6,6 +6,7 @@ from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification logger = logging.getLogger(__name__) @@ -13,7 +14,7 @@ class Cymru(ObservableAnalyzer): def run(self): results = {} - if self.observable_classification != self.ObservableTypes.HASH: + if self.observable_classification != Classification.HASH: raise AnalyzerRunException( f"observable type {self.observable_classification} not supported" ) diff --git a/api_app/analyzers_manager/observable_analyzers/dehashed.py b/api_app/analyzers_manager/observable_analyzers/dehashed.py index d26ff966f2..4a8bfe5d5f 100644 --- a/api_app/analyzers_manager/observable_analyzers/dehashed.py +++ b/api_app/analyzers_manager/observable_analyzers/dehashed.py @@ -9,7 +9,7 @@ from requests.structures import CaseInsensitiveDict from api_app.analyzers_manager.classes import ObservableAnalyzer -from api_app.analyzers_manager.constants import ObservableTypes +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch logger = logging.getLogger(__name__) @@ -56,13 +56,14 @@ def run(self): } def __identify_search_operator(self): - if self.observable_classification == ObservableTypes.IP: + if self.observable_classification == Classification.IP: self.operator = "ip_address" - elif self.observable_classification == ObservableTypes.DOMAIN: + elif self.observable_classification in [ + Classification.DOMAIN, + Classification.URL, + ]: self.operator = "domain" - elif self.observable_classification == ObservableTypes.URL: - self.operator = "domain" - elif self.observable_classification == ObservableTypes.GENERIC: + elif self.observable_classification == Classification.GENERIC: if re.match(r"^[\w\.\+\-]+\@[\w]+\.[a-z]{2,3}$", self.observable_name): self.operator = "email" # order matters! it's important "address" is placed before "phone" diff --git a/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/adguard.py b/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/adguard.py index 81c1676f69..d75d1d5390 100644 --- a/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/adguard.py +++ b/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/adguard.py @@ -8,6 +8,7 @@ from dns.rrset import RRset from api_app.analyzers_manager import classes +from api_app.choices import Classification from ..dns_responses import malicious_detector_response @@ -72,7 +73,7 @@ def run(self): logger.info(f"Running AdGuard DNS analyzer for {self.observable_name}") observable = self.observable_name # for URLs we are checking the relative domain - if self.observable_classification == self.ObservableTypes.URL: + if self.observable_classification == Classification.URL: logger.info(f"Extracting domain from URL {observable}") observable = urlparse(self.observable_name).hostname encoded_query = self.encode_query(observable) diff --git a/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/cloudflare_malicious_detector.py b/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/cloudflare_malicious_detector.py index aad873b7fe..2685fddedd 100644 --- a/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/cloudflare_malicious_detector.py +++ b/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/cloudflare_malicious_detector.py @@ -9,6 +9,7 @@ from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch from ..dns_responses import malicious_detector_response @@ -24,7 +25,7 @@ def run(self): is_malicious = False observable = self.observable_name # for URLs we are checking the relative domain - if self.observable_classification == self.ObservableTypes.URL: + if self.observable_classification == Classification.URL: observable = urlparse(self.observable_name).hostname params = { diff --git a/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/dns0_eu_malicious_detector.py b/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/dns0_eu_malicious_detector.py index a9e362749d..d6bbe1a162 100644 --- a/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/dns0_eu_malicious_detector.py +++ b/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/dns0_eu_malicious_detector.py @@ -11,6 +11,7 @@ from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch from ..dns_responses import malicious_detector_response @@ -27,7 +28,7 @@ def run(self): is_malicious = False try: # for URLs we are checking the relative domain - if self.observable_classification == self.ObservableTypes.URL: + if self.observable_classification == Classification.URL: observable = urlparse(self.observable_name).hostname try: IPv4Address(observable) diff --git a/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/google_webrisk.py b/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/google_webrisk.py index d5e4bf2d48..f2ea00f44d 100644 --- a/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/google_webrisk.py +++ b/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/google_webrisk.py @@ -13,6 +13,7 @@ from api_app.analyzers_manager.observable_analyzers.dns.dns_responses import ( malicious_detector_response, ) +from api_app.choices import Classification from tests.mock_utils import if_mock_connections, patch logger = logging.getLogger(__name__) @@ -43,7 +44,7 @@ class WebRisk(classes.ObservableAnalyzer): def run(self): if ( - self.observable_classification == self.ObservableTypes.URL + self.observable_classification == Classification.URL and not self.observable_name.startswith("http") ): raise AnalyzerRunException( diff --git a/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/quad9_malicious_detector.py b/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/quad9_malicious_detector.py index 0fa5803ef8..9f786c5f83 100644 --- a/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/quad9_malicious_detector.py +++ b/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/quad9_malicious_detector.py @@ -8,6 +8,7 @@ import requests from api_app.analyzers_manager import classes +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch from ..dns_responses import malicious_detector_response @@ -33,7 +34,7 @@ def update(self) -> bool: def run(self): observable = self.observable_name # for URLs we are checking the relative domain - if self.observable_classification == self.ObservableTypes.URL: + if self.observable_classification == Classification.URL: observable = urlparse(self.observable_name).hostname quad9_answer = self._quad9_dns_query(observable) diff --git a/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/spamhaus_wqs.py b/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/spamhaus_wqs.py index c85c1aec72..d76a3bc865 100644 --- a/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/spamhaus_wqs.py +++ b/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/spamhaus_wqs.py @@ -4,6 +4,7 @@ from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch from ..dns_responses import malicious_detector_response @@ -24,7 +25,7 @@ def run(self): url=f"""{self.url}/ { "DBL" - if self.observable_classification == self.ObservableTypes.DOMAIN.value + if self.observable_classification == Classification.DOMAIN.value else "AUTHBL" } /{self.observable_name}""", diff --git a/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/ultradns_malicious_detector.py b/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/ultradns_malicious_detector.py index e1c3dc9155..478ab79e38 100644 --- a/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/ultradns_malicious_detector.py +++ b/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/ultradns_malicious_detector.py @@ -5,6 +5,7 @@ from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from ..dns_responses import malicious_detector_response @@ -22,7 +23,7 @@ def run(self): observable = self.observable_name # for URLs we are checking the relative domain - if self.observable_classification == self.ObservableTypes.URL: + if self.observable_classification == Classification.URL: observable = urlparse(self.observable_name).hostname # Configure resolver with both nameservers diff --git a/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/classic_dns_resolver.py b/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/classic_dns_resolver.py index cd817beda8..cf38d95dd4 100644 --- a/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/classic_dns_resolver.py +++ b/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/classic_dns_resolver.py @@ -11,6 +11,7 @@ import dns.resolver from api_app.analyzers_manager import classes +from api_app.choices import Classification from ..dns_responses import dns_resolver_response @@ -25,7 +26,7 @@ class ClassicDNSResolver(classes.ObservableAnalyzer): def run(self): resolutions = [] timeout = False - if self.observable_classification == self.ObservableTypes.IP: + if self.observable_classification == Classification.IP: try: ipaddress.ip_address(self.observable_name) hostname, alias, _ = socket.gethostbyaddr(self.observable_name) @@ -49,12 +50,12 @@ def run(self): timeout = True elif self.observable_classification in [ - self.ObservableTypes.DOMAIN, - self.ObservableTypes.URL, + Classification.DOMAIN, + Classification.URL, ]: observable = self.observable_name # for URLs we are checking the relative domain - if self.observable_classification == self.ObservableTypes.URL: + if self.observable_classification == Classification.URL: observable = urlparse(self.observable_name).hostname try: diff --git a/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/cloudflare_dns_resolver.py b/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/cloudflare_dns_resolver.py index 63cf02873a..ea3efcd8c5 100644 --- a/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/cloudflare_dns_resolver.py +++ b/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/cloudflare_dns_resolver.py @@ -10,6 +10,7 @@ from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch from ..dns_responses import dns_resolver_response @@ -26,7 +27,7 @@ def run(self): try: observable = self.observable_name # for URLs we are checking the relative domain - if self.observable_classification == self.ObservableTypes.URL: + if self.observable_classification == Classification.URL: observable = urlparse(self.observable_name).hostname params = { diff --git a/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/dns0_eu_resolver.py b/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/dns0_eu_resolver.py index e9514e90a9..2a18c0373d 100644 --- a/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/dns0_eu_resolver.py +++ b/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/dns0_eu_resolver.py @@ -9,6 +9,7 @@ from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch from ..dns_responses import dns_resolver_response @@ -32,7 +33,7 @@ def run(self): resolutions = None try: # for URLs we are checking the relative domain - if self.observable_classification == self.ObservableTypes.URL: + if self.observable_classification == Classification.URL: observable = urlparse(self.observable_name).hostname try: IPv4Address(observable) diff --git a/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/google_dns_resolver.py b/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/google_dns_resolver.py index 0987088018..f8abc26fa2 100644 --- a/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/google_dns_resolver.py +++ b/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/google_dns_resolver.py @@ -10,6 +10,7 @@ from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch from ..dns_responses import dns_resolver_response @@ -26,7 +27,7 @@ def run(self): try: observable = self.observable_name # for URLs we are checking the relative domain - if self.observable_classification == self.ObservableTypes.URL: + if self.observable_classification == Classification.URL: observable = urlparse(self.observable_name).hostname params = { diff --git a/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/quad9_dns_resolver.py b/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/quad9_dns_resolver.py index 57452e8ee0..689631fee3 100644 --- a/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/quad9_dns_resolver.py +++ b/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/quad9_dns_resolver.py @@ -7,6 +7,7 @@ import requests from api_app.analyzers_manager import classes +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch from ..dns_responses import dns_resolver_response @@ -22,7 +23,7 @@ class Quad9DNSResolver(classes.ObservableAnalyzer): def run(self): observable = self.observable_name # for URLs we are checking the relative domain - if self.observable_classification == self.ObservableTypes.URL: + if self.observable_classification == Classification.URL: observable = urlparse(self.observable_name).hostname params = {"name": observable, "type": self.query_type} diff --git a/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/ultradns_dns_resolver.py b/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/ultradns_dns_resolver.py index 40dccb79cd..3e10f34173 100644 --- a/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/ultradns_dns_resolver.py +++ b/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/ultradns_dns_resolver.py @@ -9,6 +9,7 @@ import dns.resolver from api_app.analyzers_manager import classes +from api_app.choices import Classification from ..dns_responses import dns_resolver_response @@ -27,7 +28,7 @@ def run(self): resolutions = [] observable = self.observable_name - if self.observable_classification == self.ObservableTypes.URL: + if self.observable_classification == Classification.URL: observable = urlparse(self.observable_name).hostname resolver = dns.resolver.Resolver() diff --git a/api_app/analyzers_manager/observable_analyzers/dnsdb.py b/api_app/analyzers_manager/observable_analyzers/dnsdb.py index 489947b5ce..8c26cfaecf 100644 --- a/api_app/analyzers_manager/observable_analyzers/dnsdb.py +++ b/api_app/analyzers_manager/observable_analyzers/dnsdb.py @@ -10,6 +10,7 @@ from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from certego_saas.apps.user.models import User from tests.mock_utils import MockUpResponse, if_mock_connections, patch @@ -174,14 +175,14 @@ def _create_url(self): api_version = self._get_version_endpoint(self.api_version) observable_to_check = self.observable_name # for URLs we are checking the relative domain - if self.observable_classification == self.ObservableTypes.URL: + if self.observable_classification == Classification.URL: observable_to_check = urlparse(self.observable_name).hostname - if self.observable_classification == self.ObservableTypes.IP: + if self.observable_classification == Classification.IP: endpoint = "rdata/ip" elif self.observable_classification in [ - self.ObservableTypes.DOMAIN, - self.ObservableTypes.URL, + Classification.DOMAIN, + Classification.URL, ]: if self.query_type == "domain": endpoint = "rrset/name" diff --git a/api_app/analyzers_manager/observable_analyzers/dnstwist.py b/api_app/analyzers_manager/observable_analyzers/dnstwist.py index 23cc0c7c8f..0dd20187f3 100644 --- a/api_app/analyzers_manager/observable_analyzers/dnstwist.py +++ b/api_app/analyzers_manager/observable_analyzers/dnstwist.py @@ -12,6 +12,7 @@ from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from tests.mock_utils import if_mock_connections logger = logging.getLogger(__name__) @@ -30,7 +31,7 @@ class DNStwist(classes.ObservableAnalyzer): def run(self): domain = self.observable_name - if self.observable_classification == self.ObservableTypes.URL: + if self.observable_classification == Classification.URL: domain = urlparse(self.observable_name).hostname try: IPv4Address(domain) diff --git a/api_app/analyzers_manager/observable_analyzers/docguard_get.py b/api_app/analyzers_manager/observable_analyzers/docguard_get.py index b938bc58ac..e4978e7a3c 100644 --- a/api_app/analyzers_manager/observable_analyzers/docguard_get.py +++ b/api_app/analyzers_manager/observable_analyzers/docguard_get.py @@ -7,6 +7,7 @@ from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch logger = logging.getLogger(__name__) @@ -45,7 +46,7 @@ def run(self): self.report.errors.append(warning) uri = f"{self.observable_name}" - if self.observable_classification == self.ObservableTypes.HASH: + if self.observable_classification == Classification.HASH: try: response = requests.get(self.url + uri, headers=headers) response.raise_for_status() diff --git a/api_app/analyzers_manager/observable_analyzers/emailrep.py b/api_app/analyzers_manager/observable_analyzers/emailrep.py index 4d7d46cae3..4d88762a07 100644 --- a/api_app/analyzers_manager/observable_analyzers/emailrep.py +++ b/api_app/analyzers_manager/observable_analyzers/emailrep.py @@ -5,6 +5,7 @@ from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch @@ -30,7 +31,7 @@ def run(self): "Accept": "application/json", } - if self.observable_classification not in [self.ObservableTypes.GENERIC]: + if self.observable_classification not in [Classification.GENERIC]: raise AnalyzerRunException( f"not supported observable type {self.observable_classification}." f" Supported: generic" diff --git a/api_app/analyzers_manager/observable_analyzers/ha_get.py b/api_app/analyzers_manager/observable_analyzers/ha_get.py index 4146675f5a..f1731efdbb 100644 --- a/api_app/analyzers_manager/observable_analyzers/ha_get.py +++ b/api_app/analyzers_manager/observable_analyzers/ha_get.py @@ -5,6 +5,7 @@ from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch @@ -27,16 +28,16 @@ def run(self): } obs_clsfn = self.observable_classification - if obs_clsfn == self.ObservableTypes.DOMAIN: + if obs_clsfn == Classification.DOMAIN: data = {"domain": self.observable_name} uri = "search/terms" - elif obs_clsfn == self.ObservableTypes.IP: + elif obs_clsfn == Classification.IP: data = {"host": self.observable_name} uri = "search/terms" - elif obs_clsfn == self.ObservableTypes.URL: + elif obs_clsfn == Classification.URL: data = {"url": self.observable_name} uri = "search/terms" - elif obs_clsfn == self.ObservableTypes.HASH: + elif obs_clsfn == Classification.HASH: data = {"hash": self.observable_name} uri = "search/hash" else: diff --git a/api_app/analyzers_manager/observable_analyzers/hudsonrock.py b/api_app/analyzers_manager/observable_analyzers/hudsonrock.py index cd46a3e021..c08cb09904 100644 --- a/api_app/analyzers_manager/observable_analyzers/hudsonrock.py +++ b/api_app/analyzers_manager/observable_analyzers/hudsonrock.py @@ -5,6 +5,7 @@ from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerConfigurationException +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch logger = logging.getLogger(__name__) @@ -56,7 +57,7 @@ def run(self): "api-key": self._api_key_name, "Content-Type": "application/json", } - if self.observable_classification == self.ObservableTypes.IP: + if self.observable_classification == Classification.IP: url = ( self.url + "/search-by-ip" @@ -75,7 +76,7 @@ def run(self): url, headers=headers, json={"ip": self.observable_name} ) - elif self.observable_classification == self.ObservableTypes.DOMAIN: + elif self.observable_classification == Classification.DOMAIN: url = ( self.url + "/search-by-domain" @@ -97,7 +98,7 @@ def run(self): url, headers=headers, json={"domains": [self.observable_name]} ) - elif self.observable_classification == self.ObservableTypes.GENERIC: + elif self.observable_classification == Classification.GENERIC: # checking for email regex = r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,7}\b" if re.fullmatch(regex, self.observable_name): diff --git a/api_app/analyzers_manager/observable_analyzers/hunter_how.py b/api_app/analyzers_manager/observable_analyzers/hunter_how.py index 26dc48d323..ed595db2f9 100644 --- a/api_app/analyzers_manager/observable_analyzers/hunter_how.py +++ b/api_app/analyzers_manager/observable_analyzers/hunter_how.py @@ -8,6 +8,7 @@ from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch @@ -21,9 +22,9 @@ class Hunter_How(classes.ObservableAnalyzer): def config(self, runtime_configuration: Dict): super().config(runtime_configuration) - if self.observable_classification == self.ObservableTypes.IP: + if self.observable_classification == Classification.IP: self.query = f'ip="{self.observable_name}"' - elif self.observable_classification == self.ObservableTypes.DOMAIN: + elif self.observable_classification == Classification.DOMAIN: self.query = f'domain="{self.observable_name}"' self.encoded_query = base64.urlsafe_b64encode( diff --git a/api_app/analyzers_manager/observable_analyzers/inquest.py b/api_app/analyzers_manager/observable_analyzers/inquest.py index ad2e95ef7b..feb4ca8bed 100644 --- a/api_app/analyzers_manager/observable_analyzers/inquest.py +++ b/api_app/analyzers_manager/observable_analyzers/inquest.py @@ -12,6 +12,7 @@ AnalyzerConfigurationException, AnalyzerRunException, ) +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch logger = logging.getLogger(__name__) @@ -64,22 +65,22 @@ def run(self): if self.inquest_analysis == "dfi_search": link = "dfi" - if self.observable_classification == self.ObservableTypes.HASH: + if self.observable_classification == Classification.HASH: uri = ( f"/api/dfi/search/hash/{self.hash_type}?hash={self.observable_name}" ) elif self.observable_classification in [ - self.ObservableTypes.IP, - self.ObservableTypes.URL, - self.ObservableTypes.DOMAIN, + Classification.IP, + Classification.URL, + Classification.DOMAIN, ]: uri = ( f"/api/dfi/search/ioc/{self.observable_classification}" f"?keyword={self.observable_name}" ) - elif self.observable_classification == self.ObservableTypes.GENERIC: + elif self.observable_classification == Classification.GENERIC: try: type_, value = self.observable_name.split(":") except ValueError: @@ -113,7 +114,7 @@ def run(self): result = response.json() if ( self.inquest_analysis == "dfi_search" - and self.observable_classification == self.ObservableTypes.HASH + and self.observable_classification == Classification.HASH ): result["hash_type"] = self.hash_type diff --git a/api_app/analyzers_manager/observable_analyzers/misp.py b/api_app/analyzers_manager/observable_analyzers/misp.py index c757b08c9b..00628f908b 100644 --- a/api_app/analyzers_manager/observable_analyzers/misp.py +++ b/api_app/analyzers_manager/observable_analyzers/misp.py @@ -11,6 +11,7 @@ AnalyzerConfigurationException, AnalyzerRunException, ) +from api_app.choices import Classification from tests.mock_utils import MockResponseNoOp, if_mock_connections, patch @@ -73,9 +74,9 @@ def run(self): params["date_from"] = date_from.strftime("%Y-%m-%d %H:%M:%S") if self.filter_on_type: params["type_attribute"] = [self.observable_classification] - if self.observable_classification == self.ObservableTypes.HASH: + if self.observable_classification == Classification.HASH: params["type_attribute"] = ["md5", "sha1", "sha256"] - if self.observable_classification == self.ObservableTypes.IP: + if self.observable_classification == Classification.IP: params["type_attribute"] = [ "ip-dst", "ip-src", @@ -83,13 +84,13 @@ def run(self): "ip-dst|port", "domain|ip", ] - elif self.observable_classification == self.ObservableTypes.DOMAIN: + elif self.observable_classification == Classification.DOMAIN: params["type_attribute"] = [self.observable_classification, "domain|ip"] - elif self.observable_classification == self.ObservableTypes.HASH: + elif self.observable_classification == Classification.HASH: params["type_attribute"] = ["md5", "sha1", "sha256"] - elif self.observable_classification == self.ObservableTypes.URL: + elif self.observable_classification == Classification.URL: params["type_attribute"] = [self.observable_classification] - elif self.observable_classification == self.ObservableTypes.GENERIC: + elif self.observable_classification == Classification.GENERIC: pass else: raise AnalyzerConfigurationException( diff --git a/api_app/analyzers_manager/observable_analyzers/onyphe.py b/api_app/analyzers_manager/observable_analyzers/onyphe.py index d6f08d3806..8c64b6e34b 100644 --- a/api_app/analyzers_manager/observable_analyzers/onyphe.py +++ b/api_app/analyzers_manager/observable_analyzers/onyphe.py @@ -5,6 +5,7 @@ from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch @@ -24,11 +25,11 @@ def run(self): } obs_clsfn = self.observable_classification - if obs_clsfn == self.ObservableTypes.DOMAIN: + if obs_clsfn == Classification.DOMAIN: uri = f"domain/{self.observable_name}" - elif obs_clsfn == self.ObservableTypes.IP: + elif obs_clsfn == Classification.IP: uri = f"ip/{self.observable_name}" - elif obs_clsfn == self.ObservableTypes.URL: + elif obs_clsfn == Classification.URL: uri = f"hostname/{self.observable_name}" else: raise AnalyzerRunException( diff --git a/api_app/analyzers_manager/observable_analyzers/opencti.py b/api_app/analyzers_manager/observable_analyzers/opencti.py index fd8e7083cf..a330a4c5ca 100644 --- a/api_app/analyzers_manager/observable_analyzers/opencti.py +++ b/api_app/analyzers_manager/observable_analyzers/opencti.py @@ -46,7 +46,7 @@ def run(self): # search for observables observables = pycti.StixCyberObservable(opencti_instance, File).list( - search=self._job.observable_name + search=self._job.analyzable.name ) # Filter exact matches if exact_search is set @@ -54,7 +54,7 @@ def run(self): observables = [ obs for obs in observables - if obs["observable_value"] == self._job.observable_name + if obs["observable_value"] == self._job.analyzable.name ] for observable in observables: diff --git a/api_app/analyzers_manager/observable_analyzers/orkl_search.py b/api_app/analyzers_manager/observable_analyzers/orkl_search.py index 468036c212..8754aec291 100644 --- a/api_app/analyzers_manager/observable_analyzers/orkl_search.py +++ b/api_app/analyzers_manager/observable_analyzers/orkl_search.py @@ -3,6 +3,7 @@ import requests from api_app.analyzers_manager import classes +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch logger = logging.getLogger(__name__) @@ -20,7 +21,7 @@ def run(self): headers = { "accept": "application/json", } - if self.observable_classification == self.ObservableTypes.HASH.value: + if self.observable_classification == Classification.HASH.value: response = requests.get( url=f"{self.url}/library/entry/sha1/{self.observable_name}", headers=headers, diff --git a/api_app/analyzers_manager/observable_analyzers/otx.py b/api_app/analyzers_manager/observable_analyzers/otx.py index bd325b7028..2de23d04bb 100644 --- a/api_app/analyzers_manager/observable_analyzers/otx.py +++ b/api_app/analyzers_manager/observable_analyzers/otx.py @@ -10,6 +10,7 @@ from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from api_app.helpers import get_hash_type from tests.mock_utils import MockUpResponse, if_mock_connections, patch @@ -70,9 +71,9 @@ class OTX(classes.ObservableAnalyzer): def _extract_indicator_type(self) -> "OTXv2.IndicatorTypes": observable_classification = self.observable_classification - if observable_classification == self.ObservableTypes.IP: + if observable_classification == Classification.IP: otx_type = OTXv2.IndicatorTypes.IPv4 - elif observable_classification == self.ObservableTypes.URL: + elif observable_classification == Classification.URL: to_analyze_observable = urlparse(self.observable_name).hostname try: @@ -84,9 +85,9 @@ def _extract_indicator_type(self) -> "OTXv2.IndicatorTypes": if not to_analyze_observable: raise AnalyzerRunException("extracted observable is None") - elif observable_classification == self.ObservableTypes.DOMAIN: + elif observable_classification == Classification.DOMAIN: otx_type = OTXv2.IndicatorTypes.DOMAIN - elif observable_classification == self.ObservableTypes.HASH: + elif observable_classification == Classification.HASH: matched_type = get_hash_type(self.observable_name) if matched_type == "md5": otx_type = OTXv2.IndicatorTypes.FILE_HASH_MD5 diff --git a/api_app/analyzers_manager/observable_analyzers/phishing/phishing_extractor.py b/api_app/analyzers_manager/observable_analyzers/phishing/phishing_extractor.py index c3174dc179..0de1d9f85a 100644 --- a/api_app/analyzers_manager/observable_analyzers/phishing/phishing_extractor.py +++ b/api_app/analyzers_manager/observable_analyzers/phishing/phishing_extractor.py @@ -2,7 +2,7 @@ from typing import Dict from api_app.analyzers_manager.classes import DockerBasedAnalyzer, ObservableAnalyzer -from api_app.analyzers_manager.constants import ObservableTypes +from api_app.choices import Classification from api_app.models import PythonConfig logger = getLogger(__name__) @@ -32,7 +32,7 @@ def config(self, runtime_configuration: Dict): target = self.observable_name # handle domain names by appending default # protocol. selenium opens only URL types - if self.observable_classification == ObservableTypes.DOMAIN: + if self.observable_classification == Classification.DOMAIN: target = "http://" + target self.args.append(f"--target={target}") if self.proxy_address: diff --git a/api_app/analyzers_manager/observable_analyzers/phishing_army.py b/api_app/analyzers_manager/observable_analyzers/phishing_army.py index 05329a07f5..f454a7b0b2 100644 --- a/api_app/analyzers_manager/observable_analyzers/phishing_army.py +++ b/api_app/analyzers_manager/observable_analyzers/phishing_army.py @@ -10,6 +10,7 @@ from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch logger = logging.getLogger(__name__) @@ -37,7 +38,7 @@ def run(self): db_list = db.split("\n") to_analyze_observable = self.observable_name - if self.observable_classification == self.ObservableTypes.URL: + if self.observable_classification == Classification.URL: to_analyze_observable = urlparse(self.observable_name).hostname if to_analyze_observable in db_list: diff --git a/api_app/analyzers_manager/observable_analyzers/phishstats.py b/api_app/analyzers_manager/observable_analyzers/phishstats.py index 5ee7b77cff..9000145b38 100644 --- a/api_app/analyzers_manager/observable_analyzers/phishstats.py +++ b/api_app/analyzers_manager/observable_analyzers/phishstats.py @@ -8,6 +8,7 @@ from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch @@ -25,24 +26,24 @@ def update(cls) -> bool: def __build_phishstats_url(self) -> str: to_analyze_observable_classification = self.observable_classification to_analyze_observable_name = self.observable_name - if self.observable_classification == self.ObservableTypes.URL: + if self.observable_classification == Classification.URL: to_analyze_observable_name = urlparse(self.observable_name).hostname try: IPv4Address(to_analyze_observable_name) except AddressValueError: - to_analyze_observable_classification = self.ObservableTypes.DOMAIN + to_analyze_observable_classification = Classification.DOMAIN else: - to_analyze_observable_classification = self.ObservableTypes.IP + to_analyze_observable_classification = Classification.IP - if to_analyze_observable_classification == self.ObservableTypes.IP: + if to_analyze_observable_classification == Classification.IP: endpoint = ( f"phishing?_where=(ip,eq,{to_analyze_observable_name})&_sort=-date" ) - elif to_analyze_observable_classification == self.ObservableTypes.DOMAIN: + elif to_analyze_observable_classification == Classification.DOMAIN: endpoint = ( f"phishing?_where=(url,like,~{to_analyze_observable_name}~)&_sort=-date" ) - elif to_analyze_observable_classification == self.ObservableTypes.GENERIC: + elif to_analyze_observable_classification == Classification.GENERIC: endpoint = ( "phishing?_where=(title,like," f"~{to_analyze_observable_name}~)&_sort=-date" diff --git a/api_app/analyzers_manager/observable_analyzers/phishtank.py b/api_app/analyzers_manager/observable_analyzers/phishtank.py index abd88ef0c6..971b6fb9e6 100644 --- a/api_app/analyzers_manager/observable_analyzers/phishtank.py +++ b/api_app/analyzers_manager/observable_analyzers/phishtank.py @@ -9,6 +9,7 @@ from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch logger = logging.getLogger(__name__) @@ -20,7 +21,7 @@ class Phishtank(ObservableAnalyzer): def run(self): headers = {"User-Agent": "phishtank/IntelOwl"} observable_to_analyze = self.observable_name - if self.observable_classification == self.ObservableTypes.DOMAIN: + if self.observable_classification == Classification.DOMAIN: observable_to_analyze = "http://" + self.observable_name parsed = urlparse(observable_to_analyze) if not parsed.path: diff --git a/api_app/analyzers_manager/observable_analyzers/polyswarm_obs.py b/api_app/analyzers_manager/observable_analyzers/polyswarm_obs.py index e6ebadeaef..361cc19226 100644 --- a/api_app/analyzers_manager/observable_analyzers/polyswarm_obs.py +++ b/api_app/analyzers_manager/observable_analyzers/polyswarm_obs.py @@ -6,6 +6,8 @@ from api_app.analyzers_manager.exceptions import AnalyzerRunException from tests.mock_utils import if_mock_connections, patch +from ...choices import Classification + logger = logging.getLogger(__name__) from ..file_analyzers.polyswarm import PolyswarmBase @@ -14,15 +16,15 @@ class PolyswarmObs(ObservableAnalyzer, PolyswarmBase): def run(self): api = PolyswarmAPI(key=self._api_key, community=self.polyswarm_community) - if self.observable_classification == self.ObservableTypes.HASH.value: + if self.observable_classification == Classification.HASH.value: results = api.search(self.observable_name) result = self.get_results(results) return result - elif self.observable_classification == self.ObservableTypes.DOMAIN.value: + elif self.observable_classification == Classification.DOMAIN.value: # https://docs.polyswarm.io/consumers/polyswarm-customer-api-v3#ioc-searching return api.check_known_hosts(domains=[self.observable_name])[0].json() - elif self.observable_classification == self.ObservableTypes.IP.value: + elif self.observable_classification == Classification.IP.value: return api.check_known_hosts(ips=[self.observable_name])[0].json() def get_results(self, results): diff --git a/api_app/analyzers_manager/observable_analyzers/robtex.py b/api_app/analyzers_manager/observable_analyzers/robtex.py index 9c5458a142..902ca63f7d 100644 --- a/api_app/analyzers_manager/observable_analyzers/robtex.py +++ b/api_app/analyzers_manager/observable_analyzers/robtex.py @@ -8,6 +8,7 @@ from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch @@ -19,16 +20,16 @@ def update(cls) -> bool: pass def run(self): - if self.observable_classification == self.ObservableTypes.IP: + if self.observable_classification == Classification.IP: uris = [ f"ipquery/{self.observable_name}", f"pdns/reverse/{self.observable_name}", ] elif self.observable_classification in [ - self.ObservableTypes.URL, - self.ObservableTypes.DOMAIN, + Classification.URL, + Classification.DOMAIN, ]: - if self.observable_classification == self.ObservableTypes.URL: + if self.observable_classification == Classification.URL: domain = urlparse(self.observable_name).hostname else: domain = self.observable_name diff --git a/api_app/analyzers_manager/observable_analyzers/securitytrails.py b/api_app/analyzers_manager/observable_analyzers/securitytrails.py index fb135350fd..0f65facc6d 100644 --- a/api_app/analyzers_manager/observable_analyzers/securitytrails.py +++ b/api_app/analyzers_manager/observable_analyzers/securitytrails.py @@ -5,6 +5,7 @@ from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch @@ -22,9 +23,9 @@ def update(cls) -> bool: def run(self): headers = {"apikey": self._api_key_name, "Content-Type": "application/json"} - if self.observable_classification == self.ObservableTypes.IP: + if self.observable_classification == Classification.IP: uri = f"ips/nearby/{self.observable_name}" - elif self.observable_classification == self.ObservableTypes.DOMAIN: + elif self.observable_classification == Classification.DOMAIN: if self.securitytrails_analysis == "current": if self.securitytrails_current_type == "details": uri = f"domain/{self.observable_name}" diff --git a/api_app/analyzers_manager/observable_analyzers/spyse.py b/api_app/analyzers_manager/observable_analyzers/spyse.py index d5bf9e05fc..0697e08f62 100644 --- a/api_app/analyzers_manager/observable_analyzers/spyse.py +++ b/api_app/analyzers_manager/observable_analyzers/spyse.py @@ -7,6 +7,7 @@ from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from intel_owl.consts import REGEX_CVE, REGEX_EMAIL from tests.mock_utils import MockUpResponse, if_mock_connections, patch @@ -21,11 +22,11 @@ def update(cls) -> bool: pass def __build_spyse_api_uri(self) -> str: - if self.observable_classification == self.ObservableTypes.DOMAIN: + if self.observable_classification == Classification.DOMAIN: endpoint = "domain" - elif self.observable_classification == self.ObservableTypes.IP: + elif self.observable_classification == Classification.IP: endpoint = "ip" - elif self.observable_classification == self.ObservableTypes.GENERIC: + elif self.observable_classification == Classification.GENERIC: # it may be email if re.match(REGEX_EMAIL, self.observable_name): endpoint = "email" diff --git a/api_app/analyzers_manager/observable_analyzers/stalkphish.py b/api_app/analyzers_manager/observable_analyzers/stalkphish.py index 93b5505edf..69b6c2e7d6 100644 --- a/api_app/analyzers_manager/observable_analyzers/stalkphish.py +++ b/api_app/analyzers_manager/observable_analyzers/stalkphish.py @@ -5,6 +5,7 @@ from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch @@ -25,12 +26,12 @@ def run(self): obs_clsfn = self.observable_classification if obs_clsfn in [ - self.ObservableTypes.DOMAIN, - self.ObservableTypes.URL, - self.ObservableTypes.GENERIC, + Classification.DOMAIN, + Classification.URL, + Classification.GENERIC, ]: uri = f"search/url/{self.observable_name}" - elif obs_clsfn == self.ObservableTypes.IP: + elif obs_clsfn == Classification.IP: uri = f"search/ipv4/{self.observable_name}" else: raise AnalyzerRunException( diff --git a/api_app/analyzers_manager/observable_analyzers/talos.py b/api_app/analyzers_manager/observable_analyzers/talos.py index 3aa7e10b77..788b97252d 100644 --- a/api_app/analyzers_manager/observable_analyzers/talos.py +++ b/api_app/analyzers_manager/observable_analyzers/talos.py @@ -66,7 +66,7 @@ def _update_data_model(self, data_model): found = self.report.report.get("found", False) if found: data_model.external_references.append( - f"https://www.talosintelligence.com/reputation_center/lookup?search={self.report.job.observable_name}" + f"https://www.talosintelligence.com/reputation_center/lookup?search={self.report.job.analyzable.name}" ) data_model.evaluation = ( self.report.data_model_class.EVALUATIONS.MALICIOUS.value diff --git a/api_app/analyzers_manager/observable_analyzers/threatminer.py b/api_app/analyzers_manager/observable_analyzers/threatminer.py index 39b9db9b47..61788382bb 100644 --- a/api_app/analyzers_manager/observable_analyzers/threatminer.py +++ b/api_app/analyzers_manager/observable_analyzers/threatminer.py @@ -5,6 +5,7 @@ from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch @@ -21,11 +22,11 @@ def run(self): if self.rt_value: params["rt"] = self.rt_value - if self.observable_classification == self.ObservableTypes.DOMAIN: + if self.observable_classification == Classification.DOMAIN: uri = "domain.php" - elif self.observable_classification == self.ObservableTypes.IP: + elif self.observable_classification == Classification.IP: uri = "host.php" - elif self.observable_classification == self.ObservableTypes.HASH: + elif self.observable_classification == Classification.HASH: uri = "sample.php" else: raise AnalyzerRunException( diff --git a/api_app/analyzers_manager/observable_analyzers/threatstream.py b/api_app/analyzers_manager/observable_analyzers/threatstream.py index cfef6ce509..79a564afe8 100644 --- a/api_app/analyzers_manager/observable_analyzers/threatstream.py +++ b/api_app/analyzers_manager/observable_analyzers/threatstream.py @@ -8,6 +8,7 @@ AnalyzerConfigurationException, AnalyzerRunException, ) +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch @@ -46,9 +47,9 @@ def run(self): params = {"type": "confidence", "value": self.observable_name} uri = "v1/inteldetails/confidence_trend/" elif self.threatstream_analysis == "passive_dns": - if self.observable_classification == self.ObservableTypes.IP: + if self.observable_classification == Classification.IP: uri = f"v1/pdns/ip/{self.observable_name}" - elif self.observable_classification == self.ObservableTypes.DOMAIN: + elif self.observable_classification == Classification.DOMAIN: uri = f"v1/pdns/domain/{self.observable_name}" else: raise AnalyzerConfigurationException( diff --git a/api_app/analyzers_manager/observable_analyzers/tranco.py b/api_app/analyzers_manager/observable_analyzers/tranco.py index f5e3840789..25ea510e28 100644 --- a/api_app/analyzers_manager/observable_analyzers/tranco.py +++ b/api_app/analyzers_manager/observable_analyzers/tranco.py @@ -6,6 +6,7 @@ import requests from api_app.analyzers_manager import classes +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch @@ -18,7 +19,7 @@ def update(cls) -> bool: def run(self): observable_to_analyze = self.observable_name - if self.observable_classification == self.ObservableTypes.URL: + if self.observable_classification == Classification.URL: observable_to_analyze = urlparse(self.observable_name).hostname url = self.url + observable_to_analyze diff --git a/api_app/analyzers_manager/observable_analyzers/triage/triage_search.py b/api_app/analyzers_manager/observable_analyzers/triage/triage_search.py index 3476bd94ce..2cc935bf86 100644 --- a/api_app/analyzers_manager/observable_analyzers/triage/triage_search.py +++ b/api_app/analyzers_manager/observable_analyzers/triage/triage_search.py @@ -5,7 +5,6 @@ import time from api_app.analyzers_manager.classes import ObservableAnalyzer -from api_app.analyzers_manager.constants import ObservableTypes from api_app.analyzers_manager.exceptions import ( AnalyzerConfigurationException, AnalyzerRunException, @@ -13,6 +12,7 @@ from api_app.analyzers_manager.observable_analyzers.triage.triage_base import ( TriageMixin, ) +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch logger = logging.getLogger(__name__) @@ -35,7 +35,7 @@ def run(self): return self.final_report def __triage_search(self): - if self.observable_classification == self.ObservableTypes.HASH: + if self.observable_classification == Classification.HASH: query = self.observable_name else: query = f"{self.observable_classification}:{self.observable_name}" @@ -47,8 +47,8 @@ def __triage_search(self): def __triage_submit(self): data = { - "kind": ObservableTypes.URL, - ObservableTypes.URL: f"{self.observable_name}", + "kind": Classification.URL, + Classification.URL: f"{self.observable_name}", } logger.info(f"triage {self.observable_name} sending URL for analysis") diff --git a/api_app/analyzers_manager/observable_analyzers/urldna.py b/api_app/analyzers_manager/observable_analyzers/urldna.py index 0c1151bdd4..f5177d3247 100644 --- a/api_app/analyzers_manager/observable_analyzers/urldna.py +++ b/api_app/analyzers_manager/observable_analyzers/urldna.py @@ -8,6 +8,7 @@ from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch logger = logging.getLogger(__name__) @@ -95,11 +96,11 @@ def __poll_for_result(self, scan_id): def __urldna_search(self): uri = "/search" data = {"query": f"{self.observable_name}"} - if self.observable_classification == self.ObservableTypes.URL: + if self.observable_classification == Classification.URL: data["query"] = f"submitted_url = {self.observable_name}" - elif self.observable_classification == self.ObservableTypes.DOMAIN: + elif self.observable_classification == Classification.DOMAIN: data["query"] = f"domain = {self.observable_name}" - elif self.observable_classification == self.ObservableTypes.IP: + elif self.observable_classification == Classification.IP: data["query"] = f"ip = {self.observable_name}" else: data["query"] = f"{self.observable_name}" diff --git a/api_app/analyzers_manager/observable_analyzers/urlhaus.py b/api_app/analyzers_manager/observable_analyzers/urlhaus.py index 97fd172313..e4c55b392d 100644 --- a/api_app/analyzers_manager/observable_analyzers/urlhaus.py +++ b/api_app/analyzers_manager/observable_analyzers/urlhaus.py @@ -6,6 +6,7 @@ from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from api_app.mixins import AbuseCHMixin from tests.mock_utils import MockUpResponse, if_mock_connections, patch @@ -25,12 +26,12 @@ def run(self): headers = {"Accept": "application/json"} if self.observable_classification in [ - self.ObservableTypes.DOMAIN, - self.ObservableTypes.IP, + Classification.DOMAIN, + Classification.IP, ]: uri = "host/" post_data = {"host": self.observable_name} - elif self.observable_classification == self.ObservableTypes.URL: + elif self.observable_classification == Classification.URL: uri = "url/" post_data = {"url": self.observable_name} else: diff --git a/api_app/analyzers_manager/observable_analyzers/urlscan.py b/api_app/analyzers_manager/observable_analyzers/urlscan.py index acca95d4e3..858156ada2 100644 --- a/api_app/analyzers_manager/observable_analyzers/urlscan.py +++ b/api_app/analyzers_manager/observable_analyzers/urlscan.py @@ -8,6 +8,7 @@ from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch logger = logging.getLogger(__name__) @@ -80,7 +81,7 @@ def __urlscan_search(self): "q": f'{self.observable_classification}:"{self.observable_name}"', "size": self.search_size, } - if self.observable_classification == self.ObservableTypes.URL: + if self.observable_classification == Classification.URL: params["q"] = "page." + params["q"] resp = self.session.get(self.url + "/search/", params=params) resp.raise_for_status() diff --git a/api_app/analyzers_manager/observable_analyzers/xforce.py b/api_app/analyzers_manager/observable_analyzers/xforce.py index e3510865c9..c03b9b35ce 100644 --- a/api_app/analyzers_manager/observable_analyzers/xforce.py +++ b/api_app/analyzers_manager/observable_analyzers/xforce.py @@ -8,6 +8,7 @@ from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch @@ -31,7 +32,7 @@ def run(self): endpoints = self._get_endpoints() result = {} for endpoint in endpoints: - if self.observable_classification == self.ObservableTypes.URL: + if self.observable_classification == Classification.URL: observable_to_check = quote_plus(self.observable_name) else: observable_to_check = self.observable_name @@ -45,9 +46,9 @@ def run(self): response.raise_for_status() result[endpoint] = response.json() path = self.observable_classification - if self.observable_classification == self.ObservableTypes.DOMAIN: - path = self.ObservableTypes.URL - elif self.observable_classification == self.ObservableTypes.HASH: + if self.observable_classification == Classification.DOMAIN: + path = Classification.URL + elif self.observable_classification == Classification.HASH: path = "malware" result[endpoint]["link"] = f"{self.web_url}/{path}/{observable_to_check}" @@ -60,15 +61,15 @@ def _get_endpoints(self): :rtype: list """ endpoints = [] - if self.observable_classification == self.ObservableTypes.IP: + if self.observable_classification == Classification.IP: if not self.malware_only: endpoints.extend(["ipr", "ipr/history"]) endpoints.append("ipr/malware") - elif self.observable_classification == self.ObservableTypes.HASH: + elif self.observable_classification == Classification.HASH: endpoints.append("malware") elif self.observable_classification in [ - self.ObservableTypes.URL, - self.ObservableTypes.DOMAIN, + Classification.URL, + Classification.DOMAIN, ]: if not self.malware_only: endpoints.extend(["url", "url/history"]) diff --git a/api_app/analyzers_manager/observable_analyzers/yaraify.py b/api_app/analyzers_manager/observable_analyzers/yaraify.py index a1191d8ec7..c9b95c8978 100644 --- a/api_app/analyzers_manager/observable_analyzers/yaraify.py +++ b/api_app/analyzers_manager/observable_analyzers/yaraify.py @@ -5,6 +5,7 @@ import requests from api_app.analyzers_manager.classes import ObservableAnalyzer +from api_app.choices import Classification from api_app.mixins import AbuseCHMixin from tests.mock_utils import MockUpResponse, if_mock_connections, patch @@ -24,7 +25,7 @@ def update(self) -> bool: def run(self): data = {"search_term": self.observable_name, "query": self.query} - if self.observable_classification == self.ObservableTypes.GENERIC: + if self.observable_classification == Classification.GENERIC: data["result_max"] = self.result_max if getattr(self, "_api_key_name", None): diff --git a/api_app/analyzers_manager/observable_analyzers/yeti.py b/api_app/analyzers_manager/observable_analyzers/yeti.py index 7f213ac95e..b42df37ad6 100644 --- a/api_app/analyzers_manager/observable_analyzers/yeti.py +++ b/api_app/analyzers_manager/observable_analyzers/yeti.py @@ -17,7 +17,7 @@ class YETI(classes.ObservableAnalyzer): def run(self): # request payload payload = { - "filter": {"value": self._job.observable_name}, + "filter": {"value": self._job.analyzable.name}, "params": {"regex": self.regex, "range": self.results_count}, } headers = {"Accept": "application/json", "X-Api-Key": self._api_key_name} diff --git a/api_app/analyzers_manager/observable_analyzers/zoomeye.py b/api_app/analyzers_manager/observable_analyzers/zoomeye.py index bc3d0fed34..6db8d7eb2b 100644 --- a/api_app/analyzers_manager/observable_analyzers/zoomeye.py +++ b/api_app/analyzers_manager/observable_analyzers/zoomeye.py @@ -8,6 +8,7 @@ AnalyzerConfigurationException, AnalyzerRunException, ) +from api_app.choices import Classification from tests.mock_utils import MockUpResponse, if_mock_connections, patch @@ -26,7 +27,7 @@ def update(cls) -> bool: pass def __build_zoomeye_url(self): - if self.observable_classification == self.ObservableTypes.IP: + if self.observable_classification == Classification.IP: self.query += f" ip:{self.observable_name}" else: self.query += f" hostname:{self.observable_name}" diff --git a/api_app/analyzers_manager/queryset.py b/api_app/analyzers_manager/queryset.py index e1012c5a23..c0f6da2063 100644 --- a/api_app/analyzers_manager/queryset.py +++ b/api_app/analyzers_manager/queryset.py @@ -16,7 +16,7 @@ def _get_bi_serializer_class(cls) -> Type["AnalyzerReportBISerializer"]: return AnalyzerReportBISerializer def get_data_models(self, job) -> QuerySet: - DataModel = self.model.get_data_model_class(job) # noqa + DataModel = job.analyzable.get_data_model_class() # noqa return DataModel.objects.filter( pk__in=self.values_list("data_model_object_id", flat=True) ) diff --git a/api_app/choices.py b/api_app/choices.py index dee3bd94e1..800a3feea7 100644 --- a/api_app/choices.py +++ b/api_app/choices.py @@ -1,12 +1,17 @@ # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl # See the file 'LICENSE' for copying permission. import enum +import ipaddress +import logging +import re import typing from pathlib import PosixPath import _operator from django.db import models +logger = logging.getLogger(__name__) + class PythonModuleBasePaths(models.TextChoices): ObservableAnalyzer = ( @@ -99,13 +104,62 @@ def final_statuses(cls) -> typing.List["Status"]: ] -class ObservableClassification(models.TextChoices): +class Classification(models.TextChoices): IP = "ip" URL = "url" DOMAIN = "domain" HASH = "hash" GENERIC = "generic" - EMPTY = "" + FILE = "file" + + @classmethod + def calculate_observable(cls, value: str) -> str: + """Returns observable classification for the given value.\n + Only following types are supported: + ip, domain, url, hash (md5, sha1, sha256), generic (if no match) + + Args: + value (str): + observable value + Returns: + str: one of `ip`, `url`, `domain`, `hash` or 'generic'. + """ + try: + ipaddress.ip_address(value) + except ValueError: + if re.match( + r"^.+://[a-z\d-]{1,200}" + r"(?:\.[a-zA-Z\d\u2044\u2215!#$&(-;=?-\[\]_~]{1,200})+" + r"(?::\d{2,6})?" + r"(?:/[a-zA-Z\d\u2044\u2215!#$&(-;=?-\[\]_~]{1,200})*" + r"(?:\.\w+)?", + value, + ): + classification = cls.URL + elif re.match( + r"^([\[\\]?\.[\]\\]?)?[a-z\d\-_]{1,63}" + r"(([\[\\]?\.[\]\\]?)[a-z\d\-_]{1,63})+$", + value, + re.IGNORECASE, + ): + classification = cls.DOMAIN + elif ( + re.match(r"^[a-f\d]{32}$", value, re.IGNORECASE) + or re.match(r"^[a-f\d]{40}$", value, re.IGNORECASE) + or re.match(r"^[a-f\d]{64}$", value, re.IGNORECASE) + ): + classification = cls.HASH + else: + classification = cls.GENERIC + logger.info( + "Couldn't detect observable classification" + f" for {value}, setting as 'generic'" + ) + else: + # it's a simple IP + classification = cls.IP + + return classification class ScanMode(models.IntegerChoices): diff --git a/api_app/connectors_manager/connectors/abuse_submitter.py b/api_app/connectors_manager/connectors/abuse_submitter.py index c1c4cf6992..2715f4816b 100644 --- a/api_app/connectors_manager/connectors/abuse_submitter.py +++ b/api_app/connectors_manager/connectors/abuse_submitter.py @@ -7,7 +7,7 @@ class AbuseSubmitter(EmailSender): def subject(self) -> str: return ( "Takedown domain request for " - f"{self._job.parent_job.parent_job.observable_name}" + f"{self._job.parent_job.parent_job.analyzable.name}" ) @property @@ -18,7 +18,7 @@ def body(self) -> str: "This analyzer must be run only with the playbook Takedown_Request to work properly" ) return ( - f"Domain {self._job.parent_job.parent_job.observable_name} " + f"Domain {self._job.parent_job.parent_job.analyzable.name} " "has been detected as malicious by our team. We kindly request you to take " "it down as soon as possible." ) diff --git a/api_app/connectors_manager/connectors/email_sender.py b/api_app/connectors_manager/connectors/email_sender.py index d2c1ae52f6..df0316981d 100644 --- a/api_app/connectors_manager/connectors/email_sender.py +++ b/api_app/connectors_manager/connectors/email_sender.py @@ -28,7 +28,7 @@ def run(self) -> dict: base_eml = EmailMessage( subject=self.subject, from_email=sender, - to=[self._job.observable_name], + to=[self._job.analyzable.name], body=body, cc=self.CCs if self.CCs else [], ) diff --git a/api_app/connectors_manager/connectors/misp.py b/api_app/connectors_manager/connectors/misp.py index 377d1621b8..43a9da7a55 100644 --- a/api_app/connectors_manager/connectors/misp.py +++ b/api_app/connectors_manager/connectors/misp.py @@ -7,16 +7,16 @@ from django.conf import settings from api_app import helpers -from api_app.analyzers_manager.constants import ObservableTypes +from api_app.choices import Classification from api_app.connectors_manager.classes import Connector from tests.mock_utils import if_mock_connections, patch INTELOWL_MISP_TYPE_MAP = { - ObservableTypes.IP: "ip-src", - ObservableTypes.DOMAIN: "domain", - ObservableTypes.URL: "url", + Classification.IP: "ip-src", + Classification.DOMAIN: "domain", + Classification.URL: "url", # "hash" (checked from helpers.get_hash_type) - ObservableTypes.GENERIC: "text", # misc field, so keeping text + Classification.GENERIC: "text", # misc field, so keeping text "file": "filename|md5", } @@ -56,11 +56,11 @@ def _event_obj(self) -> pymisp.MISPEvent: def _base_attr_obj(self) -> pymisp.MISPAttribute: if self._job.is_sample: _type = INTELOWL_MISP_TYPE_MAP["file"] - value = f"{self._job.file_name}|{self._job.md5}" + value = f"{self._job.analyzable.name}|{self._job.analyzable.md5}" else: - _type = self._job.observable_classification - value = self._job.observable_name - if _type == ObservableTypes.HASH: + _type = self._job.analyzable.classification + value = self._job.analyzable.name + if _type == Classification.HASH: matched_type = helpers.get_hash_type(value) matched_type.replace("-", "") # convert sha-x to shax _type = matched_type if matched_type is not None else "text" @@ -79,7 +79,9 @@ def _secondary_attr_objs(self) -> List[pymisp.MISPAttribute]: obj_list = [] if self._job.is_sample: # mime-type - obj_list.append(create_misp_attribute("mime-type", self._job.file_mimetype)) + obj_list.append( + create_misp_attribute("mime-type", self._job.analyzable.file_mimetype) + ) return obj_list @property diff --git a/api_app/connectors_manager/connectors/opencti.py b/api_app/connectors_manager/connectors/opencti.py index 000db123cd..1ae4397259 100644 --- a/api_app/connectors_manager/connectors/opencti.py +++ b/api_app/connectors_manager/connectors/opencti.py @@ -8,20 +8,20 @@ from pycti.api.opencti_api_client import File from api_app import helpers -from api_app.analyzers_manager.constants import ObservableTypes +from api_app.choices import Classification from api_app.connectors_manager import classes from tests.mock_utils import if_mock_connections, patch INTELOWL_OPENCTI_TYPE_MAP = { - ObservableTypes.IP: { + Classification.IP: { "v4": "ipv4-addr", "v6": "ipv6-addr", }, - ObservableTypes.DOMAIN: "domain-name", - ObservableTypes.URL: "url", + Classification.DOMAIN: "domain-name", + Classification.URL: "url", # type hash is missing because it is combined with "file" # "generic" is misc field, so keeping text - ObservableTypes.GENERIC: "x-opencti-text", + Classification.GENERIC: "x-opencti-text", "file": "file", # hashes: md5, sha-1, sha-256 } @@ -36,8 +36,8 @@ class OpenCTI(classes.Connector): def get_observable_type(self) -> str: if self._job.is_sample: obs_type = INTELOWL_OPENCTI_TYPE_MAP["file"] - elif self._job.observable_classification == ObservableTypes.HASH: - matched_hash_type = helpers.get_hash_type(self._job.observable_name) + elif self._job.analyzable.classification == Classification.HASH: + matched_hash_type = helpers.get_hash_type(self._job.analyzable.name) if matched_hash_type in [ "md5", "sha-1", @@ -45,38 +45,38 @@ def get_observable_type(self) -> str: ]: # sha-512 not supported obs_type = INTELOWL_OPENCTI_TYPE_MAP["file"] else: - obs_type = INTELOWL_OPENCTI_TYPE_MAP[ObservableTypes.GENERIC] # text - elif self._job.observable_classification == ObservableTypes.IP: - ip_version = helpers.get_ip_version(self._job.observable_name) + obs_type = INTELOWL_OPENCTI_TYPE_MAP[Classification.GENERIC] # text + elif self._job.analyzable.classification == Classification.IP: + ip_version = helpers.get_ip_version(self._job.analyzable.name) if ip_version in [4, 6]: - obs_type = INTELOWL_OPENCTI_TYPE_MAP[ObservableTypes.IP][ + obs_type = INTELOWL_OPENCTI_TYPE_MAP[Classification.IP][ f"v{ip_version}" ] # v4/v6 else: - obs_type = INTELOWL_OPENCTI_TYPE_MAP[ObservableTypes.GENERIC] # text + obs_type = INTELOWL_OPENCTI_TYPE_MAP[Classification.GENERIC] # text else: - obs_type = INTELOWL_OPENCTI_TYPE_MAP[self._job.observable_classification] + obs_type = INTELOWL_OPENCTI_TYPE_MAP[self._job.analyzable.classification] return obs_type def generate_observable_data(self) -> dict: observable_data = {"type": self.get_observable_type()} if self._job.is_sample: - observable_data["name"] = self._job.file_name + observable_data["name"] = self._job.analyzable.name observable_data["hashes"] = { - "md5": self._job.md5, - "sha-1": self._job.sha1, - "sha-256": self._job.sha256, + "md5": self._job.analyzable.md5, + "sha-1": self._job.analyzable.sha1, + "sha-256": self._job.analyzable.sha256, } elif ( - self._job.observable_classification == ObservableTypes.HASH + self._job.analyzable.classification == Classification.HASH and observable_data["type"] == "file" ): # add hash instead of value - matched_type = helpers.get_hash_type(self._job.observable_name) - observable_data["hashes"] = {matched_type: self._job.observable_name} + matched_type = helpers.get_hash_type(self._job.analyzable.name) + observable_data["hashes"] = {matched_type: self._job.analyzable.name} else: - observable_data["value"] = self._job.observable_name + observable_data["value"] = self._job.analyzable.name return observable_data diff --git a/api_app/connectors_manager/connectors/slack.py b/api_app/connectors_manager/connectors/slack.py index 7bbf01d9a1..c50ba8c4cf 100644 --- a/api_app/connectors_manager/connectors/slack.py +++ b/api_app/connectors_manager/connectors/slack.py @@ -30,7 +30,7 @@ def body(self) -> str: return ( "Analysis executed " f"{f'by <@{self.slack_username}> ' if self.slack_username else ''}" - f"for <{self._job.url}/raw|{self._job.analyzed_object_name}>" + f"for <{self._job.url}/raw|{self._job.analyzable.name}>" ) def run(self) -> dict: diff --git a/api_app/connectors_manager/connectors/yeti.py b/api_app/connectors_manager/connectors/yeti.py index f9bdfbd9fb..747f6f9277 100644 --- a/api_app/connectors_manager/connectors/yeti.py +++ b/api_app/connectors_manager/connectors/yeti.py @@ -17,11 +17,11 @@ class YETI(classes.Connector): def run(self): # get observable value and type if self._job.is_sample: - obs_value = self._job.md5 + obs_value = self._job.analyzable.md5 obs_type = "file" else: - obs_value = self._job.observable_name - obs_type = self._job.observable_classification + obs_value = self._job.analyzable.name + obs_type = self._job.analyzable.classification # create context context = { diff --git a/api_app/data_model_manager/migrations/0005_alter_domaindatamodel_external_references_and_more.py b/api_app/data_model_manager/migrations/0005_alter_domaindatamodel_external_references_and_more.py index fe0d9d1912..f6e06d7a58 100644 --- a/api_app/data_model_manager/migrations/0005_alter_domaindatamodel_external_references_and_more.py +++ b/api_app/data_model_manager/migrations/0005_alter_domaindatamodel_external_references_and_more.py @@ -8,68 +8,134 @@ class Migration(migrations.Migration): dependencies = [ - ('data_model_manager', '0004_alter_domaindatamodel_evaluation_and_more'), + ("data_model_manager", "0004_alter_domaindatamodel_evaluation_and_more"), ] operations = [ migrations.AlterField( - model_name='domaindatamodel', - name='external_references', - field=api_app.data_model_manager.fields.SetField(base_field=models.URLField(), blank=True, default=list, size=None), + model_name="domaindatamodel", + name="external_references", + field=api_app.data_model_manager.fields.SetField( + base_field=models.URLField(), blank=True, default=list, size=None + ), ), migrations.AlterField( - model_name='domaindatamodel', - name='related_threats', - field=api_app.data_model_manager.fields.SetField(base_field=api_app.data_model_manager.fields.LowercaseCharField(max_length=100), blank=True, default=list, size=None), + model_name="domaindatamodel", + name="related_threats", + field=api_app.data_model_manager.fields.SetField( + base_field=api_app.data_model_manager.fields.LowercaseCharField( + max_length=100 + ), + blank=True, + default=list, + size=None, + ), ), migrations.AlterField( - model_name='domaindatamodel', - name='resolutions', - field=api_app.data_model_manager.fields.SetField(base_field=api_app.data_model_manager.fields.LowercaseCharField(max_length=100), default=list, size=None), + model_name="domaindatamodel", + name="resolutions", + field=api_app.data_model_manager.fields.SetField( + base_field=api_app.data_model_manager.fields.LowercaseCharField( + max_length=100 + ), + default=list, + size=None, + ), ), migrations.AlterField( - model_name='domaindatamodel', - name='tags', - field=api_app.data_model_manager.fields.SetField(base_field=api_app.data_model_manager.fields.LowercaseCharField(max_length=100), blank=True, default=None, null=True, size=None), + model_name="domaindatamodel", + name="tags", + field=api_app.data_model_manager.fields.SetField( + base_field=api_app.data_model_manager.fields.LowercaseCharField( + max_length=100 + ), + blank=True, + default=None, + null=True, + size=None, + ), ), migrations.AlterField( - model_name='filedatamodel', - name='comments', - field=api_app.data_model_manager.fields.SetField(base_field=api_app.data_model_manager.fields.LowercaseCharField(max_length=100), blank=True, default=list, size=None), + model_name="filedatamodel", + name="comments", + field=api_app.data_model_manager.fields.SetField( + base_field=api_app.data_model_manager.fields.LowercaseCharField( + max_length=100 + ), + blank=True, + default=list, + size=None, + ), ), migrations.AlterField( - model_name='filedatamodel', - name='external_references', - field=api_app.data_model_manager.fields.SetField(base_field=models.URLField(), blank=True, default=list, size=None), + model_name="filedatamodel", + name="external_references", + field=api_app.data_model_manager.fields.SetField( + base_field=models.URLField(), blank=True, default=list, size=None + ), ), migrations.AlterField( - model_name='filedatamodel', - name='related_threats', - field=api_app.data_model_manager.fields.SetField(base_field=api_app.data_model_manager.fields.LowercaseCharField(max_length=100), blank=True, default=list, size=None), + model_name="filedatamodel", + name="related_threats", + field=api_app.data_model_manager.fields.SetField( + base_field=api_app.data_model_manager.fields.LowercaseCharField( + max_length=100 + ), + blank=True, + default=list, + size=None, + ), ), migrations.AlterField( - model_name='filedatamodel', - name='tags', - field=api_app.data_model_manager.fields.SetField(base_field=api_app.data_model_manager.fields.LowercaseCharField(max_length=100), blank=True, default=None, null=True, size=None), + model_name="filedatamodel", + name="tags", + field=api_app.data_model_manager.fields.SetField( + base_field=api_app.data_model_manager.fields.LowercaseCharField( + max_length=100 + ), + blank=True, + default=None, + null=True, + size=None, + ), ), migrations.AlterField( - model_name='ipdatamodel', - name='external_references', - field=api_app.data_model_manager.fields.SetField(base_field=models.URLField(), blank=True, default=list, size=None), + model_name="ipdatamodel", + name="external_references", + field=api_app.data_model_manager.fields.SetField( + base_field=models.URLField(), blank=True, default=list, size=None + ), ), migrations.AlterField( - model_name='ipdatamodel', - name='related_threats', - field=api_app.data_model_manager.fields.SetField(base_field=api_app.data_model_manager.fields.LowercaseCharField(max_length=100), blank=True, default=list, size=None), + model_name="ipdatamodel", + name="related_threats", + field=api_app.data_model_manager.fields.SetField( + base_field=api_app.data_model_manager.fields.LowercaseCharField( + max_length=100 + ), + blank=True, + default=list, + size=None, + ), ), migrations.AlterField( - model_name='ipdatamodel', - name='resolutions', - field=api_app.data_model_manager.fields.SetField(base_field=models.URLField(), default=list, size=None), + model_name="ipdatamodel", + name="resolutions", + field=api_app.data_model_manager.fields.SetField( + base_field=models.URLField(), default=list, size=None + ), ), migrations.AlterField( - model_name='ipdatamodel', - name='tags', - field=api_app.data_model_manager.fields.SetField(base_field=api_app.data_model_manager.fields.LowercaseCharField(max_length=100), blank=True, default=None, null=True, size=None), + model_name="ipdatamodel", + name="tags", + field=api_app.data_model_manager.fields.SetField( + base_field=api_app.data_model_manager.fields.LowercaseCharField( + max_length=100 + ), + blank=True, + default=None, + null=True, + size=None, + ), ), ] diff --git a/api_app/data_model_manager/models.py b/api_app/data_model_manager/models.py index c0652f17e2..82edcd58a9 100644 --- a/api_app/data_model_manager/models.py +++ b/api_app/data_model_manager/models.py @@ -1,10 +1,14 @@ import json -from typing import Dict, Type +import logging +from typing import Dict, Type, Union from django.contrib.contenttypes.fields import GenericRelation from django.contrib.contenttypes.models import ContentType from django.contrib.postgres import fields as pg_fields +from django.contrib.postgres.fields import ArrayField from django.db import models +from django.db.models import ForeignKey, ManyToManyField +from django.forms import JSONField from django.utils.timezone import now from rest_framework.serializers import ModelSerializer @@ -17,6 +21,8 @@ from api_app.data_model_manager.queryset import BaseDataModelQuerySet from certego_saas.apps.user.models import User +logger = logging.getLogger(__name__) + class IETFReport(models.Model): rrname = LowercaseCharField(max_length=100) @@ -103,6 +109,11 @@ class BaseDataModel(models.Model): object_id_field="data_model_object_id", content_type_field="data_model_content_type", ) + jobs = GenericRelation( + to="api_app.Job", + object_id_field="data_model_object_id", + content_type_field="data_model_content_type", + ) TAGS = DataModelTags @@ -111,6 +122,77 @@ class BaseDataModel(models.Model): class Meta: abstract = True + @property + def owner(self) -> User: + return self.analyzers_report.first().user + + def merge( + self, other: Union["BaseDataModel", Dict], append: bool = True + ) -> "BaseDataModel": + if not self.pk: + raise ValueError("Unable to merge a model that was not saved.") + if not isinstance(other, (self.__class__, dict)): + raise TypeError(f"Different class between {self} and {type(other)}") + for field_name, field in self.get_fields().items(): + if field_name == "id": + continue + result_attr = getattr(self, field_name) + if isinstance(other, dict): + try: + other_attr = other[field_name] + except KeyError: + continue + else: + other_attr = getattr(other, field_name, None) + if not other_attr: + continue + if append: + if isinstance(field, ArrayField): + result_attr.extend(other_attr) + elif isinstance(field, (JSONField, SetField)): + result_attr |= other_attr + elif isinstance(field, ManyToManyField): + result_attr.add(*other_attr.values_list("pk", flat=True)) + continue + elif isinstance(field, ForeignKey): + if isinstance(other_attr, dict): + other_attr = field.related_model.objects.get_or_create( + **other_attr + ) + elif isinstance(other_attr, models.Model): + pass + else: + logger.error( + f"Field {field_name} has wrong type with value {other_attr}" + ) + continue + result_attr = other_attr + else: + result_attr = other_attr + else: + result_attr = other_attr + setattr(self, field_name, result_attr) + self.save() + return self + + def __sub__(self, other: "BaseDataModel") -> "BaseDataModel": + from deepdiff import DeepDiff + + if not isinstance(other, BaseDataModel): + raise TypeError(f"Different class between {self} and {type(other)}") + dict1 = self.serialize() + dict2 = other.serialize() + result = DeepDiff( + dict1, + dict2, + ignore_order=True, + verbose_level=1, + exclude_paths=["id", "analyzers_report", "jobs", "date"], + ) + + new = self.__class__.objects.create() + return new.merge(result) + @classmethod def get_content_type(cls) -> ContentType: return ContentType.objects.get_for_model(model=cls) @@ -121,14 +203,13 @@ def get_fields(cls) -> Dict: field.name: field for field in cls._meta.fields + cls._meta.many_to_many } - @property - def owner(self) -> User: - return self.analyzers_report.first().user - @classmethod def get_serializer(cls) -> Type[ModelSerializer]: raise NotImplementedError() + def serialize(self) -> Dict: + return self.get_serializer()(self, read_only=True).data + class DomainDataModel(BaseDataModel): ietf_report = models.ManyToManyField(IETFReport, related_name="domains") # pdns diff --git a/api_app/data_model_manager/queryset.py b/api_app/data_model_manager/queryset.py index 1cdfa1afe3..e343d5f038 100644 --- a/api_app/data_model_manager/queryset.py +++ b/api_app/data_model_manager/queryset.py @@ -1,8 +1,27 @@ +import typing from typing import Dict, List from django.db.models import QuerySet +if typing.TYPE_CHECKING: + from api_app.data_model_manager.models import BaseDataModel + class BaseDataModelQuerySet(QuerySet): + + def merge(self, append: bool = True) -> "BaseDataModel": + """ + Base method of merge of multiple data models. + :return: BaseDataModel + """ + result_obj: BaseDataModel = self.model.objects.create() + for obj in self: + result_obj.merge(obj, append=append) + return result_obj + def serialize(self) -> List[Dict]: - return self.model.get_serializer()(self, many=True, read_only=True).data + try: + serializer_class = self.model.get_serializer() + except NotImplementedError: + return [] + return serializer_class(self, many=True, read_only=True).data diff --git a/api_app/data_model_manager/serializers.py b/api_app/data_model_manager/serializers.py index a2c323a5a6..ed7bf5969f 100644 --- a/api_app/data_model_manager/serializers.py +++ b/api_app/data_model_manager/serializers.py @@ -13,13 +13,21 @@ class IETFReportSerializer(FlexFieldsModelSerializer): class Meta: model = IETFReport - fields = "__all__" + exclude = ["id"] + + def create(self, validated_data): + instance, _ = self.Meta.model.objects.get_or_create(**validated_data) + return instance class SignatureSerializer(FlexFieldsModelSerializer): class Meta: model = Signature - fields = "__all__" + exclude = ["id"] + + def create(self, validated_data): + instance, _ = self.Meta.model.objects.get_or_create(**validated_data) + return instance class DomainDataModelSerializer(FlexFieldsModelSerializer): diff --git a/api_app/defaults.py b/api_app/defaults.py index c26ea9c245..cc613b7f5c 100644 --- a/api_app/defaults.py +++ b/api_app/defaults.py @@ -1,5 +1,4 @@ from django.conf import settings -from django.utils import timezone def config_default(): @@ -16,5 +15,4 @@ def default_runtime(): def file_directory_path(instance, filename): - now = timezone.now().strftime("%Y_%m_%d_%H_%M_%S") - return f"job_{now}_{filename}" + return instance.md5 diff --git a/api_app/documents.py b/api_app/documents.py index 2306967045..651a1c4420 100644 --- a/api_app/documents.py +++ b/api_app/documents.py @@ -38,6 +38,7 @@ class JobDocument(Document): tlp = fields.KeywordField() observable_name = fields.KeywordField() observable_classification = fields.KeywordField() + is_sample = fields.BooleanField() file_name = fields.KeywordField() file_mimetype = fields.KeywordField() # Nested (ForeignKey) fields @@ -74,7 +75,6 @@ class Django: # The fields of the model you want to be indexed in Elasticsearch fields = [ - "is_sample", "received_request_time", "finished_analysis_time", "process_time", diff --git a/api_app/engines_manager/__init__.py b/api_app/engines_manager/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api_app/engines_manager/apps.py b/api_app/engines_manager/apps.py new file mode 100644 index 0000000000..a8f8937727 --- /dev/null +++ b/api_app/engines_manager/apps.py @@ -0,0 +1,12 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from django.apps import AppConfig + + +class EnginesManagerConfig(AppConfig): + name = "api_app.engines_manager" + + @staticmethod + def ready(**kwargs) -> None: + from . import signals # noqa diff --git a/api_app/engines_manager/classes.py b/api_app/engines_manager/classes.py new file mode 100644 index 0000000000..a31a7fc186 --- /dev/null +++ b/api_app/engines_manager/classes.py @@ -0,0 +1,11 @@ +from typing import Any, Dict + +from api_app.models import Job + + +class EngineModule: + def __init__(self, job: Job): + self.job = job + + def run(self) -> Dict[str, Any]: + raise NotImplementedError("Method run not implemented") diff --git a/api_app/engines_manager/engines/__init__.py b/api_app/engines_manager/engines/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api_app/engines_manager/engines/evaluation.py b/api_app/engines_manager/engines/evaluation.py new file mode 100644 index 0000000000..0d4ca0f8f5 --- /dev/null +++ b/api_app/engines_manager/engines/evaluation.py @@ -0,0 +1,25 @@ +from api_app.data_model_manager.enums import DataModelEvaluations +from api_app.engines_manager.classes import EngineModule + + +class EvaluationEngineModule(EngineModule): + evaluations_order = [ + DataModelEvaluations.TRUSTED.value, + DataModelEvaluations.MALICIOUS.value, + DataModelEvaluations.SUSPICIOUS.value, + DataModelEvaluations.CLEAN.value, + ] + + def run(self): + evaluations = self.job.get_analyzers_data_models().values_list( + "evaluation", flat=True + ) + + evaluation = DataModelEvaluations.CLEAN.value + for key in self.evaluations_order: + if key in evaluations: + evaluation = key + break + return { + "evaluation": evaluation, + } diff --git a/api_app/engines_manager/engines/malware_family.py b/api_app/engines_manager/engines/malware_family.py new file mode 100644 index 0000000000..5c81ba54d7 --- /dev/null +++ b/api_app/engines_manager/engines/malware_family.py @@ -0,0 +1,27 @@ +from django.db.models import Count + +from api_app.data_model_manager.enums import DataModelEvaluations +from api_app.engines_manager.engines.evaluation import EvaluationEngineModule + + +class MalwareFamilyEngineModule(EvaluationEngineModule): + def run(self): + result = super().run() + if result["evaluation"] in [ + DataModelEvaluations.TRUSTED.value, + DataModelEvaluations.CLEAN.value, + ]: + first_family = None + else: + first_family = ( + self.job.get_analyzers_data_models() + .values("malware_family") + .order_by("-malware_family") + .annotate(count=Count("malware_family", distinct=True)) + .first()["malware_family"] + ) + + return { + **result, + "malware_family": first_family, + } diff --git a/api_app/engines_manager/migrations/0001_initial.py b/api_app/engines_manager/migrations/0001_initial.py new file mode 100644 index 0000000000..000c5a681d --- /dev/null +++ b/api_app/engines_manager/migrations/0001_initial.py @@ -0,0 +1,41 @@ +# Generated by Django 4.2.17 on 2025-01-09 11:19 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="EngineConfig", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "modules", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=255), + blank=True, + default=list, + help_text="List of modules used by the engine. Each module has syntax `name_file.name_class`", + size=None, + ), + ), + ], + options={ + "abstract": False, + }, + ), + ] diff --git a/api_app/engines_manager/migrations/0002_alter_engineconfig_modules.py b/api_app/engines_manager/migrations/0002_alter_engineconfig_modules.py new file mode 100644 index 0000000000..b68b79fff4 --- /dev/null +++ b/api_app/engines_manager/migrations/0002_alter_engineconfig_modules.py @@ -0,0 +1,32 @@ +# Generated by Django 4.2.17 on 2025-01-15 09:34 + +import django.contrib.postgres.fields +from django.db import migrations, models + +import api_app.engines_manager.validators + + +class Migration(migrations.Migration): + + dependencies = [ + ("engines_manager", "0001_initial"), + ] + + operations = [ + migrations.AlterField( + model_name="engineconfig", + name="modules", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + max_length=255, + validators=[ + api_app.engines_manager.validators.validate_engine_module + ], + ), + blank=True, + default=list, + help_text="List of modules used by the engine. Each module has syntax `name_file.name_class`", + size=None, + ), + ), + ] diff --git a/api_app/engines_manager/migrations/0003_engine.py b/api_app/engines_manager/migrations/0003_engine.py new file mode 100644 index 0000000000..949f9b74b5 --- /dev/null +++ b/api_app/engines_manager/migrations/0003_engine.py @@ -0,0 +1,22 @@ +# Generated by Django 4.2.17 on 2025-01-15 09:34 + +from django.db import migrations + + +def migrate(apps, schema_editor): + EngineConfig = apps.get_model("engines_manager", "EngineConfig") + if not EngineConfig.objects.exists(): + EngineConfig.objects.create( + modules=["malware_family.MalwareFamilyEngineModule"] + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("engines_manager", "0002_alter_engineconfig_modules"), + ] + + operations = [ + migrations.RunPython(migrate, migrations.RunPython.noop), + ] diff --git a/api_app/engines_manager/migrations/__init__.py b/api_app/engines_manager/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api_app/engines_manager/models.py b/api_app/engines_manager/models.py new file mode 100644 index 0000000000..37880129f2 --- /dev/null +++ b/api_app/engines_manager/models.py @@ -0,0 +1,55 @@ +import uuid +from typing import Generator + +from celery import group +from celery.canvas import Signature +from django.conf import settings +from django.contrib.postgres.fields import ArrayField +from django.db import models +from solo.models import SingletonModel + +from api_app.engines_manager.validators import validate_engine_module +from api_app.models import Job +from intel_owl.celery import get_queue_name + + +class EngineConfig(SingletonModel): + modules = ArrayField( + models.CharField( + max_length=255, null=False, blank=False, validators=[validate_engine_module] + ), + blank=True, + default=list, + help_text="List of modules used by the engine. Each module has syntax `name_file.name_class`", + ) + + def get_modules_signatures(self, job) -> Generator[Signature, None, None]: + from api_app.engines_manager.tasks import execute_engine_module + + for path in self.modules: + yield execute_engine_module.signature( + args=[job.pk, f"{settings.BASE_ENGINE_MODULES_PYTHON_PATH}.{path}"], + queue=get_queue_name(settings.DEFAULT_QUEUE), + immutable=True, + MessageGroupId=str(uuid.uuid4()), + priority=job.priority, + ) + + def run(self, job: Job) -> None: + from api_app.data_model_manager.models import BaseDataModel + + data_model_result: BaseDataModel = job.get_analyzers_data_models().merge( + append=True + ) + if job.data_model: + job.data_model.delete() + job.data_model = data_model_result + job.save() + + runner = group(list(self.get_modules_signatures(job))) + runner.apply_async( + queue=get_queue_name(settings.DEFAULT_QUEUE), + immutable=True, + MessageGroupId=str(uuid.uuid4()), + priority=10, + ) diff --git a/api_app/engines_manager/queryset.py b/api_app/engines_manager/queryset.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api_app/engines_manager/signals.py b/api_app/engines_manager/signals.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api_app/engines_manager/tasks.py b/api_app/engines_manager/tasks.py new file mode 100644 index 0000000000..4462ed113c --- /dev/null +++ b/api_app/engines_manager/tasks.py @@ -0,0 +1,24 @@ +from celery import shared_task +from django.utils.module_loading import import_string + +from intel_owl.tasks import FailureLoggedTask + + +@shared_task(base=FailureLoggedTask, soft_time_limit=300) +def execute_engine(job_pk: int): + from api_app.engines_manager.models import EngineConfig + from api_app.models import Job + + job = Job.objects.get(pk=job_pk) + EngineConfig.objects.first().run(job) + + +@shared_task(base=FailureLoggedTask, soft_time_limit=300) +def execute_engine_module(job_pk: int, path: str): + from api_app.engines_manager.classes import EngineModule + from api_app.models import Job + + job = Job.objects.get(pk=job_pk) + obj: EngineModule = import_string(path)(job) + module_result = obj.run() + job.data_model.merge(module_result, append=False) diff --git a/api_app/engines_manager/urls.py b/api_app/engines_manager/urls.py new file mode 100644 index 0000000000..6faa1ba81d --- /dev/null +++ b/api_app/engines_manager/urls.py @@ -0,0 +1,18 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from django.urls import include, path +from rest_framework import routers + +from api_app.engines_manager.views import EngineViewSet + +# Routers provide an easy way of automatically determining the URL conf. + + +router = routers.DefaultRouter(trailing_slash=False) +router.register(r"engine", EngineViewSet, basename="engine") + +urlpatterns = [ + # Viewsets + path(r"", include(router.urls)), +] diff --git a/api_app/engines_manager/validators.py b/api_app/engines_manager/validators.py new file mode 100644 index 0000000000..a12ba5efdf --- /dev/null +++ b/api_app/engines_manager/validators.py @@ -0,0 +1,11 @@ +from django.conf import settings +from django.core.exceptions import ValidationError +from django.utils.module_loading import import_string + + +def validate_engine_module(value): + path = f"{settings.BASE_ENGINE_MODULES_PYTHON_PATH}.{value}" + try: + import_string(path) + except ImportError: + raise ValidationError(f"Path {path} does not exist") diff --git a/api_app/engines_manager/views.py b/api_app/engines_manager/views.py new file mode 100644 index 0000000000..8f24277cd0 --- /dev/null +++ b/api_app/engines_manager/views.py @@ -0,0 +1,30 @@ +from django.core.exceptions import ValidationError +from rest_framework import status, viewsets +from rest_framework.decorators import action +from rest_framework.permissions import IsAuthenticated +from rest_framework.response import Response + +from api_app.engines_manager.models import EngineConfig +from api_app.mixins import PaginationMixin +from api_app.permissions import IsObjectOwnerOrSameOrgPermission + + +class EngineViewSet(PaginationMixin, viewsets.ReadOnlyModelViewSet): + permission_classes = [IsAuthenticated, IsObjectOwnerOrSameOrgPermission] + queryset = EngineConfig.objects.all() + + @action( + methods=["POST"], + detail=False, + ) + def run(self, request): + from api_app.models import Job + + if "job" not in request.data: + raise ValidationError( + {"detail": "You should set the `job` argument in the data"} + ) + job_pk = request.data["job"] + job = Job.objects.get(pk=job_pk) + EngineConfig.objects.first().run(job) + return Response({"success": True}, status=status.HTTP_200_OK) diff --git a/api_app/filters.py b/api_app/filters.py index 22d1e4a2d0..b0a2b3522d 100644 --- a/api_app/filters.py +++ b/api_app/filters.py @@ -2,9 +2,8 @@ # See the file 'LICENSE' for copying permission. import rest_framework_filters as filters -from django.db.models import Q -from .analyzers_manager.constants import ObservableTypes +from .choices import Classification from .models import Job __all__ = [ @@ -32,10 +31,16 @@ class JobFilter(filters.FilterSet): """ is_sample = filters.BooleanFilter() - md5 = filters.CharFilter(lookup_expr="icontains") - observable_name = filters.CharFilter(lookup_expr="icontains") - file_name = filters.CharFilter(lookup_expr="icontains") - file_mimetype = filters.CharFilter(lookup_expr="icontains") + md5 = filters.CharFilter(field_name="analyzable__md5", lookup_expr="icontains") + observable_name = filters.CharFilter( + field_name="analyzable__name", lookup_expr="icontains" + ) + file_name = filters.CharFilter( + field_name="analyzable__name", lookup_expr="icontains" + ) + file_mimetype = filters.CharFilter( + field_name="analyzable__mimetype", lookup_expr="icontains" + ) tags = filters.BaseInFilter(field_name="tags__label", lookup_expr="in") playbook_to_execute = filters.CharFilter( field_name="playbook_to_execute__name", lookup_expr="icontains" @@ -45,7 +50,7 @@ class JobFilter(filters.FilterSet): user = filters.CharFilter(method="filter_for_user") id = filters.CharFilter(method="filter_for_id") type = filters.CharFilter(method="filter_for_type") - name = filters.CharFilter(method="filter_for_name") + name = filters.CharFilter(field_name="analyzable__name") @staticmethod def filter_for_user(queryset, value, user, *args, **kwargs): @@ -96,33 +101,15 @@ def filter_for_type(queryset, value, _type, *args, **kwargs): Returns: QuerySet: The filtered queryset. """ - if _type in ObservableTypes.values: - return queryset.filter(observable_classification=_type) - return queryset.filter(file_mimetype__icontains=_type) - - @staticmethod - def filter_for_name(queryset, value, name, *args, **kwargs): - """ - Filters the queryset by observable name or file name. - - Args: - queryset (QuerySet): The queryset to filter. - value (str): The filter value. - name (str): The name to filter by (observable or file name). - - Returns: - QuerySet: The filtered queryset. - """ - return queryset.filter( - Q(observable_name__icontains=name) | Q(file_name__icontains=name) - ) + if _type in Classification.values: + return queryset.filter(analyzavle__classification=_type) + return queryset.filter(analyzable__mimetype__icontains=_type) class Meta: model = Job fields = { "received_request_time": ["lte", "gte"], "finished_analysis_time": ["lte", "gte"], - "observable_classification": ["exact"], "tlp": ["exact"], "status": ["exact"], } diff --git a/api_app/helpers.py b/api_app/helpers.py index c2ecf5d180..bd33747f35 100644 --- a/api_app/helpers.py +++ b/api_app/helpers.py @@ -29,15 +29,15 @@ def gen_random_colorhex() -> str: return "#%02X%02X%02X" % (r(), r(), r()) -def calculate_md5(value) -> str: +def calculate_md5(value: bytes) -> str: return hashlib.md5(value).hexdigest() # skipcq BAN-B324 -def calculate_sha1(value) -> str: +def calculate_sha1(value: bytes) -> str: return hashlib.sha1(value).hexdigest() # skipcq BAN-B324 -def calculate_sha256(value) -> str: +def calculate_sha256(value: bytes) -> str: return hashlib.sha256(value).hexdigest() # skipcq BAN-B324 diff --git a/api_app/ingestors_manager/models.py b/api_app/ingestors_manager/models.py index 2d483b4d91..fe8ec58aa1 100644 --- a/api_app/ingestors_manager/models.py +++ b/api_app/ingestors_manager/models.py @@ -10,7 +10,7 @@ from api_app.choices import PythonModuleBasePaths from api_app.ingestors_manager.exceptions import IngestorConfigurationException -from api_app.ingestors_manager.queryset import IngestorReportQuerySet +from api_app.ingestors_manager.queryset import IngestorQuerySet, IngestorReportQuerySet from api_app.interfaces import CreateJobsFromPlaybookInterface from api_app.models import ( AbstractReport, @@ -20,7 +20,6 @@ PythonModule, ) from api_app.playbooks_manager.models import PlaybookConfig -from api_app.queryset import IngestorQuerySet logger = logging.getLogger(__name__) diff --git a/api_app/ingestors_manager/queryset.py b/api_app/ingestors_manager/queryset.py index 5b1d2c385e..b89f8cdbf6 100644 --- a/api_app/ingestors_manager/queryset.py +++ b/api_app/ingestors_manager/queryset.py @@ -1,6 +1,9 @@ from typing import TYPE_CHECKING, Type -from api_app.queryset import AbstractReportQuerySet +from django.db.models import Exists, OuterRef + +from api_app.queryset import AbstractReportQuerySet, PythonConfigQuerySet +from certego_saas.apps.user.models import User if TYPE_CHECKING: from api_app.ingestors_manager.serializers import IngestorReportBISerializer @@ -12,3 +15,30 @@ def _get_bi_serializer_class(cls) -> Type["IngestorReportBISerializer"]: from api_app.ingestors_manager.serializers import IngestorReportBISerializer return IngestorReportBISerializer + + +class IngestorQuerySet(PythonConfigQuerySet): + """ + Custom queryset for Ingestor model, providing methods for annotating configurations specific to ingestors. + + Methods: + - annotate_runnable: Annotates ingestors indicating if they are runnable. + """ + + def annotate_runnable(self, user: User = None) -> "PythonConfigQuerySet": + """ + Annotates ingestors indicating if they are runnable. + + Args: + user (User, optional): The user to check. Defaults to None. + + Returns: + PythonConfigQuerySet: The annotated queryset. + """ + # the plugin is runnable IF + # - it is not disabled + qs = self.filter( + pk=OuterRef("pk"), + ).exclude(disabled=True) + + return self.annotate(runnable=Exists(qs)) diff --git a/api_app/migrations/0061_job_depth_analysis.py b/api_app/migrations/0061_job_depth_analysis.py index 85d8667a4a..27d85f8393 100644 --- a/api_app/migrations/0061_job_depth_analysis.py +++ b/api_app/migrations/0061_job_depth_analysis.py @@ -9,6 +9,9 @@ def migrate(apps, schema_editor): PivotMap = apps.get_model("pivots_manager", "PivotMap") Investigation = apps.get_model("investigations_manager", "Investigation") + Job = apps.get_model("api_app", "Job") + if not Job.objects.count(): + return # I have to use the import here because i really need the class methods from api_app.models import Job as JobNonStoric diff --git a/api_app/migrations/0066_remove_lastelasticreportupdate_singleton_and_more.py b/api_app/migrations/0066_remove_lastelasticreportupdate_singleton_and_more.py new file mode 100644 index 0000000000..becbbdaa50 --- /dev/null +++ b/api_app/migrations/0066_remove_lastelasticreportupdate_singleton_and_more.py @@ -0,0 +1,43 @@ +# Generated by Django 4.2.17 on 2025-01-09 11:19 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("contenttypes", "0002_remove_content_type_name"), + ("api_app", "0065_job_mpnodesearch"), + ] + + operations = [ + migrations.RemoveConstraint( + model_name="lastelasticreportupdate", + name="singleton", + ), + migrations.AddField( + model_name="job", + name="data_model_content_type", + field=models.ForeignKey( + editable=False, + limit_choices_to={"app_label": "data_model_manager"}, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="contenttypes.contenttype", + blank=True, + ), + ), + migrations.AddField( + model_name="job", + name="data_model_object_id", + field=models.IntegerField(editable=False, null=True, blank=True), + ), + migrations.AddIndex( + model_name="job", + index=models.Index( + fields=["data_model_content_type", "data_model_object_id"], + name="api_app_job_data_mo_8b5e5f_idx", + ), + ), + ] diff --git a/api_app/migrations/0067_add_analyzable.py b/api_app/migrations/0067_add_analyzable.py new file mode 100644 index 0000000000..65e8d63ff3 --- /dev/null +++ b/api_app/migrations/0067_add_analyzable.py @@ -0,0 +1,28 @@ +# Generated by Django 4.2.17 on 2025-01-22 08:59 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("contenttypes", "0002_remove_content_type_name"), + ("api_app", "0066_remove_lastelasticreportupdate_singleton_and_more"), + ("analyzables_manager", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="job", + name="analyzable", + field=models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="jobs", + to="analyzables_manager.analyzable", + ), + ), + ] diff --git a/api_app/migrations/0068_remove_job_api_app_job_md5_4d2c5e_idx_and_more.py b/api_app/migrations/0068_remove_job_api_app_job_md5_4d2c5e_idx_and_more.py new file mode 100644 index 0000000000..ecf8499ed9 --- /dev/null +++ b/api_app/migrations/0068_remove_job_api_app_job_md5_4d2c5e_idx_and_more.py @@ -0,0 +1,61 @@ +# Generated by Django 4.2.17 on 2025-01-22 08:59 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("contenttypes", "0002_remove_content_type_name"), + ("api_app", "0067_add_analyzable"), + ("analyzables_manager", "0002_migrate_data"), + ] + + operations = [ + migrations.RemoveIndex( + model_name="job", + name="api_app_job_md5_4d2c5e_idx", + ), + migrations.RemoveField( + model_name="job", + name="file", + ), + migrations.RemoveField( + model_name="job", + name="file_mimetype", + ), + migrations.RemoveField( + model_name="job", + name="file_name", + ), + migrations.RemoveField( + model_name="job", + name="is_sample", + ), + migrations.RemoveField( + model_name="job", + name="md5", + ), + migrations.RemoveField( + model_name="job", + name="observable_classification", + ), + migrations.RemoveField( + model_name="job", + name="observable_name", + ), + migrations.AlterField( + model_name="job", + name="analyzable", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="jobs", + to="analyzables_manager.analyzable", + ), + ), + migrations.AddIndex( + model_name="job", + index=models.Index(fields=["status"], name="api_app_job_status_27bc7f_idx"), + ), + ] diff --git a/api_app/migrations/0069_remove_comment_job_comment_analyzable.py b/api_app/migrations/0069_remove_comment_job_comment_analyzable.py new file mode 100644 index 0000000000..ebd512ea96 --- /dev/null +++ b/api_app/migrations/0069_remove_comment_job_comment_analyzable.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.17 on 2025-01-23 14:38 + +import django.db.models.deletion +from django.db import migrations, models + + +def migrate(apps, schema_editor): + Comment = apps.get_model("api_app", "Comment") + for comment in Comment.objects.all(): + comment.analyzable = comment.job.analyzable + comment.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ( + "analyzables_manager", + "0003_analyzable_analyzables_classif_adf7ca_idx_and_more", + ), + ("api_app", "0068_remove_job_api_app_job_md5_4d2c5e_idx_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="comment", + name="analyzable", + field=models.ForeignKey( + null=True, + blank=True, + default=None, + on_delete=django.db.models.deletion.CASCADE, + related_name="comments", + to="analyzables_manager.analyzable", + ), + ), + migrations.RunPython(migrate, migrations.RunPython.noop), + ] diff --git a/api_app/migrations/0070_remove_comment_job_comment_analyzable.py b/api_app/migrations/0070_remove_comment_job_comment_analyzable.py new file mode 100644 index 0000000000..cbbccb48bf --- /dev/null +++ b/api_app/migrations/0070_remove_comment_job_comment_analyzable.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.17 on 2025-01-23 14:38 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ( + "analyzables_manager", + "0003_analyzable_analyzables_classif_adf7ca_idx_and_more", + ), + ("api_app", "0069_remove_comment_job_comment_analyzable"), + ] + + operations = [ + migrations.RemoveField( + model_name="comment", + name="job", + ), + migrations.AlterField( + model_name="comment", + name="analyzable", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="comments", + to="analyzables_manager.analyzable", + ), + ), + ] diff --git a/api_app/mixins.py b/api_app/mixins.py index 96eb96f9cf..6d3bdbfba8 100644 --- a/api_app/mixins.py +++ b/api_app/mixins.py @@ -10,9 +10,8 @@ from rest_framework.response import Response from api_app.analyzers_manager.classes import BaseAnalyzerMixin -from api_app.analyzers_manager.constants import ObservableTypes from api_app.analyzers_manager.exceptions import AnalyzerRunException -from api_app.choices import ObservableClassification +from api_app.choices import Classification from certego_saas.ext.pagination import CustomPageNumberPagination logger = logging.getLogger(__name__) @@ -84,7 +83,6 @@ class VirusTotalv3BaseMixin(metaclass=abc.ABCMeta): # If you want to query a specific subpath of the base endpoint, i.e: `analyses` url_sub_path: str _api_key_name: str - ObservableTypes = ObservableTypes @property def headers(self) -> dict: @@ -147,7 +145,7 @@ def _perform_request( @classmethod def _get_relationship_for_classification(cls, obs_clfn: str, iocs: bool) -> List: # reference: https://developers.virustotal.com/reference/metadata - if obs_clfn == cls.ObservableTypes.DOMAIN: + if obs_clfn == Classification.DOMAIN: relationships = [ "communicating_files", "historical_whois", @@ -158,7 +156,7 @@ def _get_relationship_for_classification(cls, obs_clfn: str, iocs: bool) -> List "collections", "historical_ssl_certificates", ] - elif obs_clfn == cls.ObservableTypes.IP: + elif obs_clfn == Classification.IP: relationships = [ "communicating_files", "historical_whois", @@ -167,13 +165,13 @@ def _get_relationship_for_classification(cls, obs_clfn: str, iocs: bool) -> List "collections", "historical_ssl_certificates", ] - elif obs_clfn == cls.ObservableTypes.URL: + elif obs_clfn == Classification.URL: relationships = [ "last_serving_ip_address", "collections", "network_location", ] - elif obs_clfn == cls.ObservableTypes.HASH: + elif obs_clfn == Classification.HASH: if iocs: relationships = [ "contacted_domains", @@ -216,16 +214,16 @@ def _get_requests_params_and_uri( relationships_requested = self._get_relationship_for_classification( obs_clfn, iocs ) - if obs_clfn == self.ObservableTypes.DOMAIN: + if obs_clfn == Classification.DOMAIN: uri = f"domains/{observable_name}" - elif obs_clfn == self.ObservableTypes.IP: + elif obs_clfn == Classification.IP: uri = f"ip_addresses/{observable_name}" - elif obs_clfn == self.ObservableTypes.URL: + elif obs_clfn == Classification.URL: url_id = ( base64.urlsafe_b64encode(observable_name.encode()).decode().strip("=") ) uri = f"urls/{url_id}" - elif obs_clfn == self.ObservableTypes.HASH: + elif obs_clfn == Classification.HASH: uri = f"files/{observable_name}" else: raise AnalyzerRunException( @@ -292,7 +290,7 @@ def _vt_intelligence_search( def _vt_get_iocs_from_file(self, sample_hash: str) -> Dict: try: params, uri, relationships_requested = self._get_requests_params_and_uri( - self.ObservableTypes.HASH, sample_hash, True + Classification.HASH, sample_hash, True ) logger.info(f"Requesting IOCs {relationships_requested} from {uri}") result, response = self._perform_get_request( @@ -409,14 +407,14 @@ def _vt_get_relationships( ) def _get_url_prefix_postfix(self, result: Dict) -> Tuple[str, str]: - uri_postfix = self._job.observable_name - if self._job.observable_classification == ObservableClassification.DOMAIN.value: + uri_postfix = self._job.analyzable.name + if self._job.analyzable.classification == Classification.DOMAIN.value: uri_prefix = "domain" - elif self._job.observable_classification == ObservableClassification.IP.value: + elif self._job.analyzable.classification == Classification.IP.value: uri_prefix = "ip-address" - elif self._job.observable_classification == ObservableClassification.URL.value: + elif self._job.analyzable.classification == Classification.URL.value: uri_prefix = "url" - uri_postfix = result.get("data", {}).get("id", self._job.sha256) + uri_postfix = result.get("data", {}).get("id", self._job.analyzable.sha256) else: # hash uri_prefix = "search" return uri_prefix, uri_postfix @@ -431,8 +429,7 @@ def _vt_scan_file(self, md5: str, rescan_instead: bool = False) -> Dict: else: logger.info(f"(Job: {self.job_id}, {md5}) -> VT analyzer requested scan") try: - self._job.file.seek(0) - binary = self._job.file.read() + binary = self._job.analyzable.read() logger.debug(f"BINARY: {binary}") except Exception: raise AnalyzerRunException( @@ -481,7 +478,7 @@ def _vt_scan_file(self, md5: str, rescan_instead: bool = False) -> Dict: if got_result: # retrieve the FULL report, not only scans results. # If it's a new sample, it's free of charge. - result = self._vt_get_report(self.ObservableTypes.HASH, md5) + result = self._vt_get_report(Classification.HASH, md5) else: message = ( f"[POLLING] (Job: {self.job_id}, {md5}) -> " @@ -515,7 +512,7 @@ def _vt_poll_for_report( ) # if it is not a file, we don't need to perform any scan - if obs_clfn != self.ObservableTypes.HASH: + if obs_clfn != Classification.HASH: break # this is an option to force active scan... @@ -647,7 +644,7 @@ def _vt_get_report( obs_clfn, ) - if obs_clfn == self.ObservableTypes.HASH: + if obs_clfn == Classification.HASH: # Include behavioral report, if flag enabled # Attention: this will cost additional quota! if self.include_behaviour_summary: diff --git a/api_app/models.py b/api_app/models.py index 3c2ea0f66d..131ebefd09 100644 --- a/api_app/models.py +++ b/api_app/models.py @@ -1,6 +1,5 @@ # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl # See the file 'LICENSE' for copying permission. -import base64 import datetime import json import logging @@ -12,8 +11,11 @@ from django.contrib.contenttypes.models import ContentType from django.utils.timezone import now from django_celery_beat.models import ClockedSchedule, CrontabSchedule, PeriodicTask +from solo.models import SingletonModel from treebeard.mp_tree import MP_Node +from api_app.analyzables_manager.models import Analyzable +from api_app.data_model_manager.queryset import BaseDataModelQuerySet from api_app.interfaces import OwnershipAbstractModel if TYPE_CHECKING: @@ -35,7 +37,6 @@ from api_app.choices import ( TLP, - ObservableClassification, ParamTypes, PythonModuleBasePaths, ReportStatus, @@ -46,11 +47,12 @@ if typing.TYPE_CHECKING: from api_app.classes import Plugin -from api_app.defaults import default_runtime, file_directory_path -from api_app.helpers import calculate_sha1, calculate_sha256, deprecated, get_now +from api_app.defaults import default_runtime +from api_app.helpers import deprecated, get_now from api_app.queryset import ( AbstractConfigQuerySet, AbstractReportQuerySet, + CommentQuerySet, JobQuerySet, OrganizationPluginConfigurationQuerySet, ParameterQuerySet, @@ -285,11 +287,8 @@ class Comment(models.Model): related_name="comment", ) - class Meta: - ordering = ["created_at"] - - job = models.ForeignKey( - "Job", + analyzable = models.ForeignKey( + "analyzables_manager.Analyzable", on_delete=models.CASCADE, related_name="comments", ) @@ -297,6 +296,11 @@ class Meta: created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) + objects = CommentQuerySet.as_manager() + + class Meta: + ordering = ["created_at"] + class Job(MP_Node): """ @@ -325,9 +329,9 @@ class Job(MP_Node): class Meta: indexes = [ + models.Index(fields=["data_model_content_type", "data_model_object_id"]), models.Index( fields=[ - "md5", "status", ] ), @@ -359,14 +363,11 @@ class Meta: on_delete=models.CASCADE, null=True, # for backwards compatibility ) - is_sample = models.BooleanField(blank=False, default=False) - md5 = models.CharField(max_length=32, blank=False) - observable_name = models.CharField(max_length=512, blank=True) - observable_classification = models.CharField( - max_length=12, blank=True, choices=ObservableClassification.choices + + analyzable = models.ForeignKey( + Analyzable, related_name="jobs", on_delete=models.CASCADE ) - file_name = models.CharField(max_length=512, blank=True) - file_mimetype = models.CharField(max_length=80, blank=True) + status = models.CharField( max_length=32, blank=False, choices=STATUSES.choices, default="pending" ) @@ -423,7 +424,6 @@ class Meta: warnings = pg_fields.ArrayField( models.TextField(), blank=True, default=list, null=True ) - file = models.FileField(blank=True, upload_to=file_directory_path) tags = models.ManyToManyField(Tag, related_name="jobs", blank=True) scan_mode = models.IntegerField( @@ -436,9 +436,21 @@ class Meta: null=True, blank=True, default=datetime.timedelta(hours=24) ) sent_to_bi = models.BooleanField(editable=False, default=False) + data_model_content_type = models.ForeignKey( + ContentType, + on_delete=models.CASCADE, + limit_choices_to={ + "app_label": "data_model_manager", + }, + null=True, + editable=False, + blank=True, + ) + data_model_object_id = models.IntegerField(null=True, editable=False, blank=True) + data_model = GenericForeignKey("data_model_content_type", "data_model_object_id") def __str__(self): - return f'{self.__class__.__name__}(#{self.pk}, "{self.analyzed_object_name}")' + return f'{self.__class__.__name__}(#{self.pk}, "{self.analyzable.name}")' def get_root(self): if self.is_root(): @@ -450,22 +462,9 @@ def get_root(self): # this is not a really valid solution, but it will work for now return self.objects.filter(path=self.path[0 : self.steplen]).first() # noqa - @property - def analyzed_object_name(self): - return self.file_name if self.is_sample else self.observable_name - - @property - def analyzed_object(self): - return self.file if self.is_sample else self.observable_name - @cached_property - def sha256(self) -> str: - """ - Calculate and return the SHA-256 hash of the file or observable name. - """ - return calculate_sha256( - self.file.read() if self.is_sample else self.observable_name.encode("utf-8") - ) + def is_sample(self) -> bool: + return self.analyzable.is_sample @cached_property def parent_job(self) -> Optional["Job"]: @@ -474,24 +473,6 @@ def parent_job(self) -> Optional["Job"]: """ return self.get_parent() - @cached_property - def sha1(self) -> str: - """ - Calculate and return the SHA-1 hash of the file or observable name. - """ - return calculate_sha1( - self.file.read() if self.is_sample else self.observable_name.encode("utf-8") - ) - - @cached_property - def b64(self) -> str: - """ - Return the Base64 encoded string of the file or observable name. - """ - return base64.b64encode( - self.file.read() if self.is_sample else self.observable_name.encode("utf-8") - ).decode("utf-8") - def get_absolute_url(self): """ Return the absolute URL for the job details. @@ -676,6 +657,18 @@ def _final_status_signature(self) -> Signature: def priority(self): return self.user.profile.task_priority + def _get_engine_signature(self) -> Signature: + from api_app.engines_manager.tasks import execute_engine + + return execute_engine.signature( + args=[self.pk], + kwargs={}, + queue=get_queue_name(settings.CONFIG_QUEUE), + immutable=True, + MessageGroupId=str(uuid.uuid4()), + priority=self.priority, + ) + def _get_pipeline( self, analyzers: PythonConfigQuerySet, @@ -698,6 +691,7 @@ def _get_pipeline( runner |= self._get_signatures(pivots_connectors) if visualizers.exists(): runner |= self._get_signatures(visualizers) + runner |= self._get_engine_signature() runner |= self._final_status_signature return runner @@ -712,6 +706,9 @@ def execute(self): ) runner() + def get_analyzers_data_models(self) -> BaseDataModelQuerySet: + return self.analyzerreports.get_data_models(self) + def get_config_runtime_configuration(self, config: "AbstractConfig") -> typing.Dict: try: self.__get_config_to_execute(config.__class__).get(name=config.name) @@ -1798,27 +1795,5 @@ def generate_health_check_periodic_task(self): self.save() -class SingletonModel(models.Model): - """Singleton base class. - Singleton is a desing pattern that allow only one istance of a class. - """ - - class Meta: - abstract = True - constraints = [ - models.CheckConstraint( - check=Q(pk=1), - name="singleton", - violation_error_message="This class is a singleton: only one object is allowed", - ), - ] - - def save(self, *args, **kwargs): - # check required to delete the singleton instance and create a new one - if type(self).objects.count() == 0: - self.pk = 1 - super().save(*args, **kwargs) - - class LastElasticReportUpdate(SingletonModel): last_update_datetime = models.DateTimeField() diff --git a/api_app/pivots_manager/migrations/0035_pivot_config_phishingextractortoanalysis.py b/api_app/pivots_manager/migrations/0035_pivot_config_phishingextractortoanalysis.py index c7b64d6dda..818ebc9a5a 100644 --- a/api_app/pivots_manager/migrations/0035_pivot_config_phishingextractortoanalysis.py +++ b/api_app/pivots_manager/migrations/0035_pivot_config_phishingextractortoanalysis.py @@ -151,6 +151,7 @@ class Migration(migrations.Migration): ("api_app", "0063_singleton_and_elastic_report"), ("pivots_manager", "0034_changed_resubmitdownloadedfile_playbook_to_execute"), ("playbooks_manager", "0054_playbook_config_phishinganalysis"), + ("analyzers_manager", "0129_analyzer_config_phishing_extractor"), ] operations = [migrations.RunPython(migrate, reverse_migrate)] diff --git a/api_app/pivots_manager/pivots/self_analyzable.py b/api_app/pivots_manager/pivots/self_analyzable.py index 877f3f6f97..c15241cded 100644 --- a/api_app/pivots_manager/pivots/self_analyzable.py +++ b/api_app/pivots_manager/pivots/self_analyzable.py @@ -30,6 +30,5 @@ def should_run(self) -> Tuple[bool, Optional[str]]: def get_value_to_pivot_to(self) -> Any: if self._job.is_sample: - return File(self._job.analyzed_object, name=self._job.analyzed_object_name) - else: - return self._job.analyzed_object_name + return File(self._job.analyzable.file, name=self._job.analyzable.name) + return self._job.analyzable.name diff --git a/api_app/playbooks_manager/migrations/0055_playbook_config_phishingextractor.py b/api_app/playbooks_manager/migrations/0055_playbook_config_phishingextractor.py index a49bc6e3d2..c21d881d84 100644 --- a/api_app/playbooks_manager/migrations/0055_playbook_config_phishingextractor.py +++ b/api_app/playbooks_manager/migrations/0055_playbook_config_phishingextractor.py @@ -121,6 +121,7 @@ class Migration(migrations.Migration): dependencies = [ ("playbooks_manager", "0054_playbook_config_phishinganalysis"), ("analyzers_manager", "0129_analyzer_config_phishing_extractor"), + ("pivots_manager", "0035_pivot_config_phishingextractortoanalysis"), ] operations = [migrations.RunPython(migrate, reverse_migrate)] diff --git a/api_app/playbooks_manager/migrations/0056_download_sample_vt.py b/api_app/playbooks_manager/migrations/0056_download_sample_vt.py index ffde7af1f9..c4f1836a75 100644 --- a/api_app/playbooks_manager/migrations/0056_download_sample_vt.py +++ b/api_app/playbooks_manager/migrations/0056_download_sample_vt.py @@ -2,7 +2,6 @@ from django.db import migrations -from api_app.analyzers_manager.constants import AllTypes from api_app.choices import TLP @@ -12,7 +11,7 @@ def migrate(apps, schema_editor): playbook_download_sample_vt, _ = PlaybookConfig.objects.get_or_create( name="Download_File_VT", description="Download a sample from VT", - type=[AllTypes.HASH.value], + type=["hash"], tlp=TLP.AMBER.value, scan_check_time=datetime.timedelta(days=14), ) diff --git a/api_app/playbooks_manager/migrations/0057_alter_phishing_extractor_add_domain.py b/api_app/playbooks_manager/migrations/0057_alter_phishing_extractor_add_domain.py index 0f70e6f039..d5bd9d3737 100644 --- a/api_app/playbooks_manager/migrations/0057_alter_phishing_extractor_add_domain.py +++ b/api_app/playbooks_manager/migrations/0057_alter_phishing_extractor_add_domain.py @@ -1,14 +1,12 @@ from django.db import migrations -from api_app.analyzers_manager.constants import ObservableTypes - def migrate(apps, schema_editor): PlaybookConfig = apps.get_model("playbooks_manager", "PlaybookConfig") config = PlaybookConfig.objects.get(name="PhishingExtractor") config.type = [ - ObservableTypes.URL, - ObservableTypes.DOMAIN, + "url", + "domain", ] config.full_clean() config.save() @@ -18,7 +16,7 @@ def reverse_migrate(apps, schema_editor): PlaybookConfig = apps.get_model("playbooks_manager", "PlaybookConfig") config = PlaybookConfig.objects.get(name="PhishingExtractor") config.type = [ - ObservableTypes.URL, + "url", ] config.full_clean() config.save() diff --git a/api_app/playbooks_manager/models.py b/api_app/playbooks_manager/models.py index f31b56502e..b84e3fac2c 100644 --- a/api_app/playbooks_manager/models.py +++ b/api_app/playbooks_manager/models.py @@ -5,8 +5,7 @@ from django.core.exceptions import ValidationError from django.db import models -from api_app.analyzers_manager.constants import AllTypes -from api_app.choices import TLP, ScanMode +from api_app.choices import TLP, Classification, ScanMode from api_app.defaults import default_runtime from api_app.fields import ChoiceArrayField from api_app.interfaces import OwnershipAbstractModel @@ -18,7 +17,7 @@ class PlaybookConfig(AbstractConfig, OwnershipAbstractModel): objects = PlaybookConfigQuerySet.as_manager() type = ChoiceArrayField( - models.CharField(choices=AllTypes.choices, null=False, max_length=50) + models.CharField(choices=Classification.choices, null=False, max_length=50) ) analyzers = models.ManyToManyField( @@ -113,4 +112,4 @@ def clean(self) -> None: self.clean_starting() def is_sample(self) -> bool: - return AllTypes.FILE.value in self.type + return Classification.FILE.value in self.type diff --git a/api_app/queryset.py b/api_app/queryset.py index 77a88c9dac..2d1d71cfa0 100644 --- a/api_app/queryset.py +++ b/api_app/queryset.py @@ -969,28 +969,17 @@ def get_signatures(self, job) -> Generator[Signature, None, None]: ) -class IngestorQuerySet(PythonConfigQuerySet): - """ - Custom queryset for Ingestor model, providing methods for annotating configurations specific to ingestors. - - Methods: - - annotate_runnable: Annotates ingestors indicating if they are runnable. - """ +class CommentQuerySet(QuerySet): - def annotate_runnable(self, user: User = None) -> "PythonConfigQuerySet": - """ - Annotates ingestors indicating if they are runnable. - - Args: - user (User, optional): The user to check. Defaults to None. - - Returns: - PythonConfigQuerySet: The annotated queryset. - """ - # the plugin is runnable IF - # - it is not disabled - qs = self.filter( - pk=OuterRef("pk"), - ).exclude(disabled=True) + def visible_for_user(self, user): + from api_app.analyzables_manager.models import Analyzable - return self.annotate(runnable=Exists(qs)) + analyzables = Analyzable.objects.visible_for_user(user) + qs = self.filter(analyzable__in=analyzables.values_list("pk", flat=True)) + if user.has_membership(): + qs = qs.filter( + user__membership__organization__pk=user.membership.organization.pk + ) + else: + qs = qs.filter(user=user) + return qs diff --git a/api_app/serializers/job.py b/api_app/serializers/job.py index 9d9a34db3d..b84f537542 100644 --- a/api_app/serializers/job.py +++ b/api_app/serializers/job.py @@ -16,9 +16,10 @@ from rest_framework.fields import empty from rest_framework.serializers import ModelSerializer -from api_app.analyzers_manager.constants import ObservableTypes, TypeChoices +from api_app.analyzables_manager.models import Analyzable +from api_app.analyzers_manager.constants import TypeChoices from api_app.analyzers_manager.models import AnalyzerConfig, MimeTypes -from api_app.choices import TLP, ScanMode +from api_app.choices import TLP, Classification, ScanMode from api_app.connectors_manager.exceptions import NotRunnableConnector from api_app.connectors_manager.models import ConnectorConfig from api_app.defaults import default_runtime @@ -36,6 +37,14 @@ logger = logging.getLogger(__name__) +class JobRelatedField(rfs.PrimaryKeyRelatedField): + def get_queryset(self): + if "request" in self.context: + request = self.context.get("request") + return super().get_queryset().filter(owner=request.user) + return super().get_queryset() + + class UserSerializer(rfs.ModelSerializer): class Meta: model = User @@ -54,6 +63,10 @@ class JobRecentScanSerializer(rfs.ModelSerializer): ) user = rfs.CharField(source="user.username", allow_null=False, read_only=True) importance = rfs.IntegerField(allow_null=True, read_only=True) + observable_name = rfs.CharField( + source="analyzable.name", read_only=True, allow_null=True + ) + file_name = rfs.CharField(source="analyzable.name", read_only=True, allow_null=True) class Meta: model = Job @@ -88,7 +101,6 @@ class Meta: "id", "user", "delay", - "is_sample", "tlp", "runtime_configuration", "analyzers_requested", @@ -102,7 +114,6 @@ class Meta: ) md5 = rfs.HiddenField(default=None) - is_sample = rfs.HiddenField(write_only=True, default=False) user = rfs.HiddenField(default=rfs.CurrentUserDefault()) delay = rfs.IntegerField(default=0) scan_mode = rfs.ChoiceField( @@ -330,7 +341,7 @@ def check_previous_jobs(self, validated_data: Dict) -> Job: .filter( received_request_time__gte=now() - validated_data["scan_check_time"] ) - .filter(Q(md5=validated_data["md5"])) + .filter(Q(analyzable__pk=validated_data["analyzable"].pk)) ) for analyzer in validated_data.get("analyzers_to_execute", []): qs = qs.filter(analyzers_requested__in=[analyzer]) @@ -423,6 +434,8 @@ def validate(self, attrs: dict) -> dict: def create(self, validated_data: dict) -> Comment: validated_data["user"] = self.context["request"].user + job = validated_data.pop("job") + validated_data["analyzable"] = job.analyzable return super().create(validated_data) @@ -433,19 +446,28 @@ class JobListSerializer(_AbstractJobViewSerializer): class Meta: model = Job - exclude = ( - "file", - "errors", - "scan_mode", - "scan_check_time", - "runtime_configuration", - "sent_to_bi", - "warnings", - "analyzers_requested", - "connectors_requested", - "path", - "numchild", - "depth", + fields = ( + "id", + "user", + "tags", + "status", + "file_name", + "file_mimetype", + "is_sample", + "observable_name", + "observable_classification", + "md5", + "pivots_to_execute", + "analyzers_to_execute", + "connectors_to_execute", + "playbook_to_execute", + "playbook_requested", + "visualizers_to_execute", + "received_request_time", + "finished_analysis_time", + "process_time", + "tlp", + "investigation", ) pivots_to_execute = rfs.SerializerMethodField(read_only=True) @@ -459,6 +481,14 @@ class Meta: read_only=True, slug_field="name", many=True ) playbook_to_execute = rfs.SlugRelatedField(read_only=True, slug_field="name") + file_name = rfs.CharField(source="analyzable.name", read_only=True) + file_mimetype = rfs.CharField(source="analyzable.mimetype", read_only=True) + is_sample = rfs.BooleanField(read_only=True) + observable_name = rfs.CharField(source="analyzable.name", read_only=True) + observable_classification = rfs.CharField( + source="analyzable.classification", read_only=True + ) + md5 = rfs.CharField(source="analyzable.md5", read_only=True) def get_pivots_to_execute(self, obj: Job): # skipcq: PYL-R0201 return obj.pivots_to_execute.all().values_list("name", flat=True) @@ -469,6 +499,20 @@ class JobTreeSerializer(ModelSerializer): source="pivot_parent.pivot_config.name", allow_null=True, read_only=True ) + evaluation = rfs.CharField( + source="data_model.evaluation", allow_null=True, read_only=True + ) + is_sample = rfs.BooleanField(read_only=True) + + playbook = rfs.SlugRelatedField( + source="playbook_to_execute", + slug_field="name", + queryset=PlaybookConfig.objects.all(), + many=False, + required=False, + ) + analyzed_object_name = rfs.CharField(source="analyzable.name", read_only=True) + class Meta: model = Job fields = [ @@ -479,16 +523,9 @@ class Meta: "status", "received_request_time", "is_sample", + "evaluation", ] - playbook = rfs.SlugRelatedField( - source="playbook_to_execute", - slug_field="name", - queryset=PlaybookConfig.objects.all(), - many=False, - required=False, - ) - def to_representation(self, instance): instance: Job data = super().to_representation(instance) @@ -507,15 +544,44 @@ class JobSerializer(_AbstractJobViewSerializer): class Meta: model = Job - exclude = ( - "file", - "depth", - "path", - "numchild", - "sent_to_bi", + fields = ( + "id", + "user", + "tags", + "comments", + "status", + "pivots_to_execute", + "analyzers_to_execute", + "analyzers_requested", + "connectors_to_execute", + "connectors_requested", + "visualizers_to_execute", + "playbook_requested", + "playbook_to_execute", + "investigation", + "permissions", + "data_model", + "analyzers_data_model", + "file_name", + "file_mimetype", + "is_sample", + "observable_name", + "observable_classification", + "md5", + "analyzer_reports", + "connector_reports", + "pivot_reports", + "visualizer_reports", + "received_request_time", + "finished_analysis_time", + "process_time", + "warnings", + "errors", ) - comments = CommentSerializer(many=True, read_only=True) + comments = CommentSerializer( + many=True, read_only=True, source="analyzable.comments" + ) pivots_to_execute = rfs.SlugRelatedField( many=True, read_only=True, slug_field="name" ) @@ -538,8 +604,16 @@ class Meta: playbook_to_execute = rfs.SlugRelatedField(read_only=True, slug_field="name") investigation = rfs.SerializerMethodField(read_only=True, default=None) permissions = rfs.SerializerMethodField() - + data_model = rfs.SerializerMethodField() analyzers_data_model = rfs.SerializerMethodField(read_only=True) + file_name = rfs.CharField(source="analyzable.name", read_only=True) + file_mimetype = rfs.CharField(source="analyzable.mimetype", read_only=True) + is_sample = rfs.BooleanField(read_only=True) + observable_name = rfs.CharField(source="analyzable.name", read_only=True) + observable_classification = rfs.CharField( + source="analyzable.classification", read_only=True + ) + md5 = rfs.CharField(source="analyzable.md5", read_only=True) def get_pivots_to_execute(self, obj: Job): # skipcq: PYL-R0201 # this cast is required or serializer doesn't work with websocket @@ -569,9 +643,12 @@ def get_fields(self): return super().get_fields() def get_analyzers_data_model(self, instance: Job): - if instance.observable_classification == ObservableTypes.GENERIC: - return [] - return instance.analyzerreports.get_data_models(instance).serialize() + return instance.get_analyzers_data_models().serialize() + + def get_data_model(self, instance: Job): + if instance.data_model: + return instance.data_model.serialize() + return {} class RestJobSerializer(JobSerializer): @@ -619,11 +696,11 @@ def save(self, parent: Job = None, **kwargs): if parent.playbook_to_execute: investigation_name = ( f"{parent.playbook_to_execute.name}:" - f" {parent.analyzed_object_name}" + f" {parent.analyzable.name}" ) else: investigation_name = ( - f"Pivot investigation: {parent.analyzed_object_name}" + f"Pivot investigation: {parent.analyzable.name}" ) investigation = Investigation.objects.create( @@ -740,7 +817,6 @@ class FileJobSerializer(_AbstractJobCreateSerializer): file = rfs.FileField(required=True) file_name = rfs.CharField(required=False) file_mimetype = rfs.CharField(required=False) - is_sample = rfs.HiddenField(write_only=True, default=True) class Meta: model = Job @@ -766,6 +842,24 @@ def validate(self, attrs: dict) -> dict: logger.debug(f"after attrs: {attrs}") return attrs + def create(self, validated_data): + md5 = validated_data.pop("md5") + sample, created = Analyzable.objects.get_or_create( + md5=md5, + defaults={ + "name": validated_data.pop("file_name"), + "file": validated_data.pop("file"), + "mimetype": validated_data.pop("file_mimetype"), + "md5": md5, + "classification": Classification.FILE.value, + }, + ) + if created: + sample.full_clean() + sample.save() + validated_data["analyzable"] = sample + return super().create(validated_data) + def set_analyzers_to_execute( self, analyzers_requested: List[AnalyzerConfig], @@ -860,7 +954,6 @@ class ObservableAnalysisSerializer(_AbstractJobCreateSerializer): observable_name = rfs.CharField(required=True) observable_classification = rfs.CharField(required=False) - is_sample = rfs.HiddenField(write_only=True, default=False) class Meta: model = Job @@ -870,6 +963,25 @@ class Meta: ) list_serializer_class = MultipleObservableJobSerializer + def create(self, validated_data): + observable_classification = validated_data.pop("observable_classification") + md5 = validated_data.pop("md5") + obs, created = Analyzable.objects.get_or_create( + defaults={ + "name": validated_data.pop("observable_name"), + "md5": md5, + "classification": observable_classification, + }, + md5=md5, + ) + if created: + obs.full_clean() + obs.save() + + validated_data["analyzable"] = obs + + return super().create(validated_data) + def validate(self, attrs: dict) -> dict: logger.debug(f"before attrs: {attrs}") attrs["observable_name"] = self.defanged_values_removal( @@ -877,18 +989,18 @@ def validate(self, attrs: dict) -> dict: ) # calculate ``observable_classification`` if not attrs.get("observable_classification", None): - attrs["observable_classification"] = ObservableTypes.calculate( + attrs["observable_classification"] = Classification.calculate_observable( attrs["observable_name"] ) if attrs["observable_classification"] in [ - ObservableTypes.HASH, - ObservableTypes.DOMAIN, + Classification.HASH, + Classification.DOMAIN, ]: # force lowercase in ``observable_name``. # Ref: https://github.com/intelowlproject/IntelOwl/issues/658 attrs["observable_name"] = attrs["observable_name"].lower() - if attrs["observable_classification"] == ObservableTypes.IP.value: + if attrs["observable_classification"] == Classification.IP.value: ip = ipaddress.ip_address(attrs["observable_name"]) if ip.is_loopback: raise ValidationError({"detail": "Loopback address"}) @@ -1075,7 +1187,9 @@ def create(self, validated_data): # check availability of the case where all # analyzers were run but no playbooks were # triggered. - query = Q(md5=validated_data["md5"]) & Q(status__in=statuses_to_check) + query = Q(analyzable__md5=validated_data["md5"]) & Q( + status__in=statuses_to_check + ) if validated_data.get("playbooks", []): query &= Q(playbook_requested__name__in=validated_data["playbooks"]) else: @@ -1104,6 +1218,7 @@ class JobBISerializer(AbstractBIInterface, ModelSerializer): end_time = rfs.DateTimeField(source="finished_analysis_time") playbook = rfs.SerializerMethodField(source="get_playbook") job_id = rfs.CharField(source="pk") + is_sample = rfs.BooleanField(read_only=True) class Meta: model = Job diff --git a/api_app/serializers/test.py b/api_app/serializers/test.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api_app/signals.py b/api_app/signals.py index aec583359f..8a941f87d0 100644 --- a/api_app/signals.py +++ b/api_app/signals.py @@ -10,7 +10,7 @@ from django.dispatch import receiver from api_app.decorators import prevent_signal_recursion -from api_app.helpers import calculate_md5 +from api_app.investigations_manager.models import Investigation from api_app.models import ( Job, ListCachable, @@ -25,25 +25,6 @@ logger = logging.getLogger(__name__) -@receiver(models.signals.pre_save, sender=Job) -def pre_save_job(sender, instance: Job, **kwargs): - """ - Signal receiver for the pre_save signal of the Job model. - Calculates the MD5 hash for the job file or observable name if not already set. - - Args: - sender (Model): The model class sending the signal. - instance (Job): The instance of the model being saved. - **kwargs: Additional keyword arguments. - """ - if not instance.md5: - instance.md5 = calculate_md5( - instance.file.read() - if instance.is_sample - else instance.observable_name.encode("utf-8") - ) - - @receiver(models.signals.post_save, sender=Job) @prevent_signal_recursion def post_save_job(sender, instance: Job, *args, **kwargs): @@ -62,21 +43,6 @@ def post_save_job(sender, instance: Job, *args, **kwargs): instance.process_time = round(td.total_seconds(), 2) -@receiver(models.signals.pre_delete, sender=Job) -def pre_delete_job(sender, instance: Job, **kwargs): - """ - Signal receiver for the pre_delete signal of the Job model. - Deletes the associated file if it exists. - - Args: - sender (Model): The model class sending the signal. - instance (Job): The instance of the model being deleted. - **kwargs: Additional keyword arguments. - """ - if instance.file: - instance.file.delete() - - @receiver(models.signals.post_delete, sender=Job) def post_delete_job(sender, instance: Job, **kwargs): """ @@ -88,8 +54,13 @@ def post_delete_job(sender, instance: Job, **kwargs): instance (Job): The instance of the model being deleted. **kwargs: Additional keyword arguments. """ - if instance.investigation and instance.investigation.jobs.count() == 0: - instance.investigation.delete() + # Try/catch is needed for multiple delete of jobs in the same investigation + # because the signals is called _after_ every deletion + try: + if instance.investigation_id and instance.investigation.jobs.count() == 0: + instance.investigation.delete() + except Investigation.DoesNotExist: + pass @receiver(migrate_finished) diff --git a/api_app/views.py b/api_app/views.py index 473a26d77c..61e14df48b 100644 --- a/api_app/views.py +++ b/api_app/views.py @@ -25,7 +25,7 @@ from rest_framework.response import Response from rest_framework.viewsets import ModelViewSet -from api_app.choices import ScanMode +from api_app.choices import Classification, ScanMode from api_app.exceptions import NotImplementedException from api_app.websocket import JobConsumer from certego_saas.apps.organization.permissions import ( @@ -41,8 +41,6 @@ from intel_owl.celery import app as celery_app from intel_owl.settings._util import get_environment -from .analyzers_manager.constants import ObservableTypes -from .choices import ObservableClassification from .decorators import deprecated_endpoint from .filters import JobFilter from .mixins import PaginationMixin @@ -405,10 +403,8 @@ def get_queryset(self): - Filtered queryset of comments. """ queryset = super().get_queryset() - jobs = Job.objects.visible_for_user(self.request.user).values_list( - "pk", flat=True - ) - return queryset.filter(job__id__in=jobs) + + return queryset.visible_for_user(self.request.user) @add_docs( @@ -507,7 +503,7 @@ def recent_scans(self, request): raise ValidationError({"detail": "md5 is required"}) max_temporal_distance = request.data.get("max_temporal_distance", 14) jobs = ( - Job.objects.filter(md5=request.data["md5"]) + Job.objects.filter(analyzable__md5=request.data["md5"]) .visible_for_user(self.request.user) .filter( finished_analysis_time__gte=now() @@ -535,12 +531,15 @@ def recent_scans_user(self, request): limit = request.data.get("limit", 5) if "is_sample" not in request.data: raise ValidationError({"detail": "is_sample is required"}) - jobs = ( - Job.objects.filter(user__pk=request.user.pk) - .filter(is_sample=request.data["is_sample"]) - .annotate_importance(request.user) - .order_by("-importance", "-finished_analysis_time")[:limit] - ) + is_sample = request.data["is_sample"] + jobs = Job.objects.filter(user__pk=request.user.pk) + if is_sample == "True": + jobs = jobs.filter(analyzable__classification=Classification.FILE) + else: + jobs = jobs.exclude(analyzable__classification=Classification.FILE) + jobs = jobs.annotate_importance(request.user).order_by( + "-importance", "-finished_analysis_time" + )[:limit] return Response( JobRecentScanSerializer(jobs, many=True).data, status=status.HTTP_200_OK ) @@ -577,12 +576,12 @@ def rescan(self, request, pk=None): data["analyzers_requested"] = existing_job.analyzers_requested.all() data["connectors_requested"] = existing_job.connectors_requested.all() if existing_job.is_sample: - data["file"] = existing_job.file - data["file_name"] = existing_job.file_name + data["file"] = existing_job.analyzable.file + data["file_name"] = existing_job.analyzable.name job_serializer = FileJobSerializer(data=data, context={"request": request}) else: - data["observable_classification"] = existing_job.observable_classification - data["observable_name"] = existing_job.observable_name + data["observable_classification"] = existing_job.analyzable.classification + data["observable_name"] = existing_job.analyzable.name job_serializer = ObservableAnalysisSerializer( data=data, context={"request": request} ) @@ -645,9 +644,9 @@ def download_sample(self, request, pk=None): {"detail": "Requested job does not have a sample associated with it."} ) return FileResponse( - job.file, - filename=job.file_name, - content_type=job.file_mimetype, + job.analyzable.file, + filename=job.analyzable.name, + content_type=job.analyzable.mimetype, as_attachment=True, ) @@ -733,8 +732,12 @@ def aggregate_type(self, request): - Aggregated count of jobs for each type. """ annotations = { - "file": Count("is_sample", filter=Q(is_sample=True)), - "observable": Count("is_sample", filter=Q(is_sample=False)), + "file": Count( + "pk", filter=Q(analyzable__classification=Classification.FILE.value) + ), + "observable": Count( + "pk", filter=~Q(analyzable__classification=Classification.FILE.value) + ), } return self.__aggregation_response_static( annotations, users=self.get_org_members(request) @@ -755,9 +758,15 @@ def aggregate_observable_classification(self, request): """ annotations = { oc.lower(): Count( - "observable_classification", filter=Q(observable_classification=oc) + "analyzable__classification", filter=Q(analyzable__classification=oc) ) - for oc in ObservableTypes.values + for oc in [ + Classification.DOMAIN, + Classification.IP, + Classification.HASH, + Classification.URL, + Classification.GENERIC, + ] } return self.__aggregation_response_static( annotations, users=self.get_org_members(request) @@ -777,7 +786,7 @@ def aggregate_file_mimetype(self, request): - Aggregated count of jobs for each MIME type. """ return self.__aggregation_response_dynamic( - "file_mimetype", users=self.get_org_members(request) + "analyzable__mimetype", users=self.get_org_members(request) ) @action( @@ -913,8 +922,6 @@ def __aggregation_response_dynamic( filter_kwargs = {"received_request_time__gte": delta} if users: filter_kwargs["user__in"] = users - if field_name == "md5": - filter_kwargs["is_sample"] = True most_frequent_values = ( Job.objects.filter(**filter_kwargs) @@ -922,9 +929,9 @@ def __aggregation_response_dynamic( .exclude(**{f"{field_name}__exact": ""}) # excluding those because they could lead to SQL query errors .exclude( - observable_classification__in=[ - ObservableClassification.URL, - ObservableClassification.GENERIC, + analyzable__classification__in=[ + Classification.URL, + Classification.GENERIC, ] ) .annotate(count=Count(field_name)) diff --git a/api_app/visualizers_manager/classes.py b/api_app/visualizers_manager/classes.py index c800c48afa..d854b1da70 100644 --- a/api_app/visualizers_manager/classes.py +++ b/api_app/visualizers_manager/classes.py @@ -534,24 +534,23 @@ def copy_report(self) -> VisualizerReport: report.save() return report - def analyzer_reports(self) -> QuerySet: + def get_analyzer_reports(self) -> QuerySet: from api_app.analyzers_manager.models import AnalyzerReport return AnalyzerReport.objects.filter(job=self._job) - def connector_reports(self) -> QuerySet: + def get_connector_reports(self) -> QuerySet: from api_app.connectors_manager.models import ConnectorReport return ConnectorReport.objects.filter(job=self._job) - def pivots_reports(self) -> QuerySet: + def get_pivots_reports(self) -> QuerySet: from api_app.pivots_manager.models import PivotReport return PivotReport.objects.filter(job=self._job) - def data_models(self) -> QuerySet: - from api_app.analyzers_manager.models import AnalyzerReport + def get_data_models(self) -> QuerySet: - data_model_class = AnalyzerReport.get_data_model_class(self._job) - analyzer_reports_pk = [report.pk for report in self.analyzer_reports()] + data_model_class = self._job.analyzable.get_data_model_class() + analyzer_reports_pk = [report.pk for report in self.get_analyzer_reports()] return data_model_class.objects.filter(analyzers_report__in=analyzer_reports_pk) diff --git a/api_app/visualizers_manager/migrations/0038_visualizer_config_passive_dns.py b/api_app/visualizers_manager/migrations/0038_visualizer_config_passive_dns.py index 5ab1390693..0d0269683a 100644 --- a/api_app/visualizers_manager/migrations/0038_visualizer_config_passive_dns.py +++ b/api_app/visualizers_manager/migrations/0038_visualizer_config_passive_dns.py @@ -104,6 +104,7 @@ class Migration(migrations.Migration): atomic = False dependencies = [ ("api_app", "0062_alter_parameter_python_module"), + ("playbooks_manager", "0045_playbook_config_passive_dns"), ("visualizers_manager", "0037_4_change_primary_key"), ] diff --git a/api_app/visualizers_manager/visualizers/data_model.py b/api_app/visualizers_manager/visualizers/data_model.py index ced5cc1ac5..4826c8386b 100644 --- a/api_app/visualizers_manager/visualizers/data_model.py +++ b/api_app/visualizers_manager/visualizers/data_model.py @@ -1,7 +1,6 @@ from logging import getLogger from typing import Dict, List -from api_app.analyzers_manager.models import AnalyzerReport from api_app.data_model_manager.enums import DataModelEvaluations from api_app.data_model_manager.models import ( DomainDataModel, @@ -305,7 +304,7 @@ def run(self) -> List[Dict]: suspicious_data_models = [] malicious_data_models = [] noeval_data_models = [] - data_models = self.data_models() + data_models = self.get_data_models() for data_model in data_models: printable_analyzer_name = ( @@ -409,7 +408,7 @@ def run(self) -> List[Dict]: ) ) - data_model_class = AnalyzerReport.get_data_model_class(self._job) + data_model_class = self._job.analyzable.get_data_model_class() if data_model_class == DomainDataModel: self.get_domain_data_elements(page, data_models) elif data_model_class == IPDataModel: diff --git a/api_app/visualizers_manager/visualizers/dns.py b/api_app/visualizers_manager/visualizers/dns.py index a79d7c4b7b..79849ad523 100644 --- a/api_app/visualizers_manager/visualizers/dns.py +++ b/api_app/visualizers_manager/visualizers/dns.py @@ -27,7 +27,7 @@ from api_app.analyzers_manager.observable_analyzers.dns.dns_resolvers.quad9_dns_resolver import ( # noqa: E501 Quad9DNSResolver, ) -from api_app.choices import ObservableClassification +from api_app.choices import Classification from api_app.models import Job from api_app.visualizers_manager.classes import VisualizableObject, Visualizer from api_app.visualizers_manager.decorators import ( @@ -69,8 +69,7 @@ def _dns_resolution(self, analyzer_report: AnalyzerReport) -> VisualizableObject self.Base( value=( dns_resolution["data"] - if self._job.observable_classification - == ObservableClassification.DOMAIN + if self._job.analyzable.classification == Classification.DOMAIN else dns_resolution ), disable=False, @@ -95,7 +94,7 @@ def run(self) -> List[Dict]: first_level_elements = [] second_level_elements = [] - for analyzer_report in self.analyzer_reports(): + for analyzer_report in self.get_analyzer_reports(): if "dns.dns_resolvers" in analyzer_report.config.python_module: first_level_elements.append( self._dns_resolution(analyzer_report=analyzer_report) diff --git a/api_app/visualizers_manager/visualizers/domain_reputation_services.py b/api_app/visualizers_manager/visualizers/domain_reputation_services.py index 165f32998b..da348d385e 100644 --- a/api_app/visualizers_manager/visualizers/domain_reputation_services.py +++ b/api_app/visualizers_manager/visualizers/domain_reputation_services.py @@ -22,7 +22,7 @@ def update(cls) -> bool: @visualizable_error_handler_with_params("VirusTotal") def _vt3(self): try: - analyzer_report = self.analyzer_reports().get( + analyzer_report = self.get_analyzer_reports().get( config__name="VirusTotal_v3_Get_Observable" ) except AnalyzerReport.DoesNotExist: @@ -58,7 +58,7 @@ def _vt3(self): @visualizable_error_handler_with_params("URLhaus") def _urlhaus(self): try: - analyzer_report = self.analyzer_reports().get(config__name="URLhaus") + analyzer_report = self.get_analyzer_reports().get(config__name="URLhaus") except AnalyzerReport.DoesNotExist: logger.warning("URLhaus report does not exist") else: @@ -86,7 +86,7 @@ def _urlhaus(self): @visualizable_error_handler_with_params("ThreatFox") def _threatfox(self): try: - analyzer_report = self.analyzer_reports().get(config__name="ThreatFox") + analyzer_report = self.get_analyzer_reports().get(config__name="ThreatFox") except AnalyzerReport.DoesNotExist: logger.warning("Threatfox report does not exist") else: @@ -112,7 +112,7 @@ def _threatfox(self): @visualizable_error_handler_with_params("Tranco") def _tranco(self): try: - analyzer_report = self.analyzer_reports().get(config__name="Tranco") + analyzer_report = self.get_analyzer_reports().get(config__name="Tranco") except AnalyzerReport.DoesNotExist: logger.warning("Tranco report does not exist") else: @@ -134,7 +134,7 @@ def _tranco(self): @visualizable_error_handler_with_params("Phishtank") def _phishtank(self): try: - analyzer_report = self.analyzer_reports().get(config__name="Phishtank") + analyzer_report = self.get_analyzer_reports().get(config__name="Phishtank") except AnalyzerReport.DoesNotExist: logger.warning("Phishtank report does not exist") else: @@ -155,7 +155,9 @@ def _phishtank(self): @visualizable_error_handler_with_params("PhishingArmy") def _phishing_army(self): try: - analyzer_report = self.analyzer_reports().get(config__name="PhishingArmy") + analyzer_report = self.get_analyzer_reports().get( + config__name="PhishingArmy" + ) except AnalyzerReport.DoesNotExist: logger.warning("PhishingArmy report does not exist") else: @@ -175,7 +177,9 @@ def _phishing_army(self): @visualizable_error_handler_with_params("InQuest") def _inquest_repdb(self): try: - analyzer_report = self.analyzer_reports().get(config__name="InQuest_REPdb") + analyzer_report = self.get_analyzer_reports().get( + config__name="InQuest_REPdb" + ) except AnalyzerReport.DoesNotExist: logger.warning("InQuest_REPdb report does not exist") else: @@ -200,7 +204,7 @@ def _inquest_repdb(self): @visualizable_error_handler_with_params("OTX Alienvault") def _otxquery(self): try: - analyzer_report = self.analyzer_reports().get(config__name="OTXQuery") + analyzer_report = self.get_analyzer_reports().get(config__name="OTXQuery") except AnalyzerReport.DoesNotExist: logger.warning("OTXQuery report does not exist") else: @@ -230,7 +234,7 @@ def run(self) -> List[Dict]: second_level_elements = [] third_level_elements = [] - for analyzer_report in self.analyzer_reports().filter( + for analyzer_report in self.get_analyzer_reports().filter( Q(config__name__endswith="Malicious_Detector") | Q(config__name="GoogleSafebrowsing") ): diff --git a/api_app/visualizers_manager/visualizers/ip_reputation_services.py b/api_app/visualizers_manager/visualizers/ip_reputation_services.py index 0eb8da30a0..005ab4f037 100644 --- a/api_app/visualizers_manager/visualizers/ip_reputation_services.py +++ b/api_app/visualizers_manager/visualizers/ip_reputation_services.py @@ -20,7 +20,7 @@ class IPReputationServices(Visualizer): @visualizable_error_handler_with_params("VirusTotal") def _vt3(self): try: - analyzer_report = self.analyzer_reports().get( + analyzer_report = self.get_analyzer_reports().get( config__name="VirusTotal_v3_Get_Observable" ) except AnalyzerReport.DoesNotExist: @@ -46,7 +46,7 @@ def _vt3(self): @visualizable_error_handler_with_params("Greynoise") def _greynoise(self): try: - analyzer_report = self.analyzer_reports().get( + analyzer_report = self.get_analyzer_reports().get( config__name="GreyNoiseCommunity" ) except AnalyzerReport.DoesNotExist: @@ -80,7 +80,7 @@ def _greynoise(self): @visualizable_error_handler_with_params("URLhaus") def _urlhaus(self): try: - analyzer_report = self.analyzer_reports().get(config__name="URLhaus") + analyzer_report = self.get_analyzer_reports().get(config__name="URLhaus") except AnalyzerReport.DoesNotExist: logger.warning("URLhaus report does not exist") else: @@ -108,7 +108,7 @@ def _urlhaus(self): @visualizable_error_handler_with_params("ThreatFox") def _threatfox(self): try: - analyzer_report = self.analyzer_reports().get(config__name="ThreatFox") + analyzer_report = self.get_analyzer_reports().get(config__name="ThreatFox") except AnalyzerReport.DoesNotExist: logger.warning("Threatfox report does not exist") else: @@ -132,7 +132,9 @@ def _threatfox(self): @visualizable_error_handler_with_params("InQuest") def _inquest_repdb(self): try: - analyzer_report = self.analyzer_reports().get(config__name="InQuest_REPdb") + analyzer_report = self.get_analyzer_reports().get( + config__name="InQuest_REPdb" + ) except AnalyzerReport.DoesNotExist: logger.warning("InQuest_REPdb report does not exist") else: @@ -157,7 +159,7 @@ def _inquest_repdb(self): @visualizable_error_handler_with_params("AbuseIPDB Categories") def _abuse_ipdb(self): try: - analyzer_report = self.analyzer_reports().get(config__name="AbuseIPDB") + analyzer_report = self.get_analyzer_reports().get(config__name="AbuseIPDB") except AnalyzerReport.DoesNotExist: logger.warning("AbuseIPDB report does not exist") return None, None @@ -206,7 +208,7 @@ def _abuse_ipdb(self): @visualizable_error_handler_with_params("GreedyBear Honeypots") def _greedybear(self): try: - analyzer_report = self.analyzer_reports().get(config__name="GreedyBear") + analyzer_report = self.get_analyzer_reports().get(config__name="GreedyBear") except AnalyzerReport.DoesNotExist: logger.warning("GreedyBear report does not exist") else: @@ -241,7 +243,7 @@ def _greedybear(self): ) def _crowdsec(self): try: - analyzer_report = self.analyzer_reports().get(config__name="Crowdsec") + analyzer_report = self.get_analyzer_reports().get(config__name="Crowdsec") except AnalyzerReport.DoesNotExist: logger.warning("Crowdsec report does not exist") return None, None @@ -291,7 +293,7 @@ def _crowdsec(self): @visualizable_error_handler_with_params("OTX Alienvault") def _otxquery(self): try: - analyzer_report = self.analyzer_reports().get(config__name="OTXQuery") + analyzer_report = self.get_analyzer_reports().get(config__name="OTXQuery") except AnalyzerReport.DoesNotExist: logger.warning("OTXQuery report does not exist") else: @@ -323,7 +325,9 @@ def _otxquery(self): @visualizable_error_handler_with_params("FireHol") def _firehol(self): try: - analyzer_report = self.analyzer_reports().get(config__name="FireHol_IPList") + analyzer_report = self.get_analyzer_reports().get( + config__name="FireHol_IPList" + ) except AnalyzerReport.DoesNotExist: logger.warning("FireHol_IPList report does not exist") else: @@ -349,7 +353,7 @@ def _firehol(self): @visualizable_error_handler_with_params("Tor Exit Node") def _tor(self): try: - analyzer_report = self.analyzer_reports().get(config__name="TorProject") + analyzer_report = self.get_analyzer_reports().get(config__name="TorProject") except AnalyzerReport.DoesNotExist: logger.warning("TorProject report does not exist") else: @@ -363,7 +367,7 @@ def _tor(self): @visualizable_error_handler_with_params("Talos Reputation") def _talos(self): try: - analyzer_report = self.analyzer_reports().get( + analyzer_report = self.get_analyzer_reports().get( config__name="TalosReputation" ) except AnalyzerReport.DoesNotExist: diff --git a/api_app/visualizers_manager/visualizers/passive_dns/analyzer_extractor.py b/api_app/visualizers_manager/visualizers/passive_dns/analyzer_extractor.py index 32319a02c5..45f8241257 100644 --- a/api_app/visualizers_manager/visualizers/passive_dns/analyzer_extractor.py +++ b/api_app/visualizers_manager/visualizers/passive_dns/analyzer_extractor.py @@ -83,7 +83,7 @@ def extract_threatminer_reports( report.get("first_seen").split(" ")[0], "A", report.get("ip", None) or report.get("domain", None), - job.observable_name, + job.analyzable.name, threatminer_analyzer.config.name.replace("_", " "), threatminer_analyzer.config.description, ) diff --git a/api_app/visualizers_manager/visualizers/passive_dns/visualizer.py b/api_app/visualizers_manager/visualizers/passive_dns/visualizer.py index f0bd3f4b18..7319592530 100644 --- a/api_app/visualizers_manager/visualizers/passive_dns/visualizer.py +++ b/api_app/visualizers_manager/visualizers/passive_dns/visualizer.py @@ -27,21 +27,25 @@ def update(cls) -> bool: def run(self) -> List[Dict]: raw_pdns_data = [] raw_pdns_data.extend( - extract_otxquery_reports(self.analyzer_reports(), self._job) + extract_otxquery_reports(self.get_analyzer_reports(), self._job) ) raw_pdns_data.extend( - extract_threatminer_reports(self.analyzer_reports(), self._job) + extract_threatminer_reports(self.get_analyzer_reports(), self._job) ) raw_pdns_data.extend( - extract_validin_reports(self.analyzer_reports(), self._job) + extract_validin_reports(self.get_analyzer_reports(), self._job) ) - raw_pdns_data.extend(extract_dnsdb_reports(self.analyzer_reports(), self._job)) raw_pdns_data.extend( - extract_circlpdns_reports(self.analyzer_reports(), self._job) + extract_dnsdb_reports(self.get_analyzer_reports(), self._job) ) - raw_pdns_data.extend(extract_robtex_reports(self.analyzer_reports(), self._job)) raw_pdns_data.extend( - extract_mnemonicpdns_reports(self.analyzer_reports(), self._job) + extract_circlpdns_reports(self.get_analyzer_reports(), self._job) + ) + raw_pdns_data.extend( + extract_robtex_reports(self.get_analyzer_reports(), self._job) + ) + raw_pdns_data.extend( + extract_mnemonicpdns_reports(self.get_analyzer_reports(), self._job) ) page = self.Page(name="Passive DNS") diff --git a/api_app/visualizers_manager/visualizers/pivot.py b/api_app/visualizers_manager/visualizers/pivot.py index 9dfaf6550b..a7a8396d84 100644 --- a/api_app/visualizers_manager/visualizers/pivot.py +++ b/api_app/visualizers_manager/visualizers/pivot.py @@ -108,7 +108,7 @@ def _create_job_ui_element( if pivot_config: label += f"{pivot_config.name}: " label += ( - f"Job #{job.pk} ({job.analyzed_object_name}, " + f"Job #{job.pk} ({job.analyzable.name}, " f"playbook: {job.playbook_to_execute})" ) return self.Base( diff --git a/api_app/visualizers_manager/visualizers/sample_download.py b/api_app/visualizers_manager/visualizers/sample_download.py index 8d09c56597..02b6bfe796 100644 --- a/api_app/visualizers_manager/visualizers/sample_download.py +++ b/api_app/visualizers_manager/visualizers/sample_download.py @@ -32,7 +32,7 @@ def update(cls) -> bool: def _download_button(self): # first attempt is download with VT try: - vt_report = self.analyzer_reports().get( + vt_report = self.get_analyzer_reports().get( config__python_module=VirusTotalv3SampleDownload.python_module ) except AnalyzerReport.DoesNotExist: @@ -48,7 +48,7 @@ def _download_button(self): # second attempt is download with VT try: - uri_report = self.analyzer_reports().get( + uri_report = self.get_analyzer_reports().get( config__python_module=DownloadFileFromUri.python_module ) except AnalyzerReport.DoesNotExist: diff --git a/api_app/visualizers_manager/visualizers/yara.py b/api_app/visualizers_manager/visualizers/yara.py index ca491b3a9a..23ed7b7166 100644 --- a/api_app/visualizers_manager/visualizers/yara.py +++ b/api_app/visualizers_manager/visualizers/yara.py @@ -51,7 +51,7 @@ def _yara_signatures(self, signatures: List[str]): ) def run(self) -> List[Dict]: - yara_report = self.analyzer_reports().get(config__name="Yara") + yara_report = self.get_analyzer_reports().get(config__name="Yara") yara_num_matches = sum(len(matches) for matches in yara_report.report.values()) signatures = [ match["match"] diff --git a/authentication/models.py b/authentication/models.py index df27f8fc53..83c56cfc87 100644 --- a/authentication/models.py +++ b/authentication/models.py @@ -35,9 +35,6 @@ class DiscoverFromChoices(models.TextChoices): OTHER = "other", "Other" -# models - - class UserProfile(models.Model): """ Model representing a user profile. diff --git a/integrations/phishing_analyzers/analyzers/extract_phishing_site.py b/integrations/phishing_analyzers/analyzers/extract_phishing_site.py index 609f777950..f1c80251e8 100644 --- a/integrations/phishing_analyzers/analyzers/extract_phishing_site.py +++ b/integrations/phishing_analyzers/analyzers/extract_phishing_site.py @@ -65,7 +65,6 @@ def analyze_target( result: str = json.dumps(extract_driver_result(driver_wrapper), default=str) logger.debug(f"JSON dump of driver {result=}") - print(result) except Exception as e: logger.exception( f"Exception during analysis of target website {target_url}: {e}" diff --git a/intel_owl/celery.py b/intel_owl/celery.py index 0d4f07dd61..45c877d6c8 100644 --- a/intel_owl/celery.py +++ b/intel_owl/celery.py @@ -95,7 +95,7 @@ def get_queue_name(queue: str) -> str: accept_content=["application/json"], task_serializer="json", result_serializer="json", - imports=("intel_owl.tasks",), + imports=("intel_owl.tasks", "api_app.engines_manager.tasks"), worker_redirect_stdouts=False, worker_hijack_root_logger=False, # this is to avoid RAM issues caused by long usage of this tool diff --git a/intel_owl/settings/__init__.py b/intel_owl/settings/__init__.py index b6f511086c..20b9858df1 100644 --- a/intel_owl/settings/__init__.py +++ b/intel_owl/settings/__init__.py @@ -43,6 +43,8 @@ "api_app.ingestors_manager", "api_app.investigations_manager", "api_app.data_model_manager", + "api_app.engines_manager", + "api_app.analyzables_manager", # auth "rest_email_auth", # performance debugging @@ -53,6 +55,8 @@ "channels", # tree structure "treebeard", + # shell functionalities + "django_extensions", ] from .a_secrets import * # lgtm [py/polluting-import] diff --git a/intel_owl/settings/commons.py b/intel_owl/settings/commons.py index 939d2a4442..881e7ec68a 100644 --- a/intel_owl/settings/commons.py +++ b/intel_owl/settings/commons.py @@ -45,6 +45,10 @@ "BASE_ANALYZER_FILE_PYTHON_PATH", "api_app.analyzers_manager.file_analyzers" ) ) +BASE_ENGINE_MODULES_PYTHON_PATH = PosixPath( + get_secret("BASE_ENGINE_MODULES_PYTHON_PATH", "api_app.engines_manager.engines") +) + REPO_DOWNLOADER_ENABLED = get_secret("REPO_DOWNLOADER_ENABLED", "True") == "True" GIT_KEY_PATH = MEDIA_ROOT / "my_gitpython_key" GIT_SSH_SCRIPT_PATH = ( diff --git a/intel_owl/tasks.py b/intel_owl/tasks.py index 4947772b20..a3e81bec71 100644 --- a/intel_owl/tasks.py +++ b/intel_owl/tasks.py @@ -92,6 +92,9 @@ def remove_old_jobs(): num_jobs_to_delete = old_jobs.count() logger.info(f"found {num_jobs_to_delete} old jobs to delete") for old_job in old_jobs.iterator(): + # if the job that we are going to delete is the last one, and it has a file + if old_job.analyzable.jobs.count() == 1 and old_job.analyzable.file: + old_job.analyzable.file.delete() try: old_job.delete() except Job.DoesNotExist as e: diff --git a/requirements/project-requirements.txt b/requirements/project-requirements.txt index fe7a701816..4c0d75e6ff 100644 --- a/requirements/project-requirements.txt +++ b/requirements/project-requirements.txt @@ -13,7 +13,8 @@ django-prettyjson==0.4.1 django-silk==5.3.2 django-elasticsearch-dsl==8.0 django-treebeard==4.7 - +django-solo==2.4.0 +django_extensions==3.2.3 jsonschema==4.23.0 # django rest framework libs Authlib==1.4.0 @@ -99,6 +100,7 @@ pyzipper==0.3.6 # others dateparser==1.2.0 +DeepDiff==8.1.1 # phishing form compiler module lxml==5.3.0 Faker==35.2.0 \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py index 92069d2df6..dd5b7127d4 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -10,7 +10,9 @@ from django.test import TestCase from rest_framework.test import APIClient +from api_app.analyzables_manager.models import Analyzable from api_app.analyzers_manager.models import AnalyzerConfig +from api_app.choices import Classification from api_app.models import AbstractReport, Job User = get_user_model() @@ -30,14 +32,21 @@ def setUp(self) -> None: super().setUp() settings.DEBUG = True + def tearDown(self): + super().tearDown() + Analyzable.objects.all().delete() + def _create_job_from_file(self, sample, mimetype, analyzer_config) -> Job: try: with open(sample, "rb") as f: - _job = Job.objects.create( - is_sample=True, - file_name=sample, - file_mimetype=mimetype, + analyzable = Analyzable.objects.create( + name=sample, + mimetype=mimetype, + classification=Classification.FILE, file=File(f), + ) + _job = Job.objects.create( + analyzable=analyzable, user=self.superuser, ) _job.analyzers_to_execute.set([analyzer_config]) @@ -114,6 +123,11 @@ def plugin_type(self): """ raise NotImplementedError() + def tearDown(self): + super().tearDown() + Job.objects.all().delete() + Analyzable.objects.all().delete() + @abstractmethod def init_report(self, status, user): raise NotImplementedError() diff --git a/tests/api_app/analyzers_manager/test_classes.py b/tests/api_app/analyzers_manager/test_classes.py index c2a02c36fc..9bb1171f23 100644 --- a/tests/api_app/analyzers_manager/test_classes.py +++ b/tests/api_app/analyzers_manager/test_classes.py @@ -5,9 +5,10 @@ from django.core.files import File from kombu import uuid +from api_app.analyzables_manager.models import Analyzable from api_app.analyzers_manager.classes import FileAnalyzer, ObservableAnalyzer -from api_app.analyzers_manager.constants import ObservableTypes from api_app.analyzers_manager.models import AnalyzerConfig, MimeTypes +from api_app.choices import Classification from api_app.models import Job, PluginConfig from tests import CustomTestCase @@ -97,16 +98,20 @@ def _create_jobs(self): ): try: with open(f"test_files/{sample_name}", "rb") as f: - Job.objects.create( - is_sample=True, - file_name=sample_name, - file_mimetype=mimetype, + an = Analyzable.objects.create( file=File(f), + name=sample_name, + mimetype=mimetype, + classification=Classification.FILE, + ) + Job.objects.create( + analyzable=an, user=self.superuser, ) + print(f"Created job for {sample_name}, with mimetype {mimetype}") except Exception: - print(f"No defined file for mimetype {mimetype}") + self.fail(f"No defined file for mimetype {mimetype}") def test_subclasses(self): def handler(signum, frame): @@ -152,14 +157,14 @@ def handler(signum, frame): print(f"skipping {subclass.__name__} cause health check failed") skipped = True continue - jobs = Job.objects.filter(file_mimetype=mimetype) + jobs = Job.objects.filter(analyzable__mimetype=mimetype) if jobs.exists(): found_one = True for job in jobs: job.analyzers_to_execute.set([config]) print( "\t\t" - f"Testing {job.file_name} with mimetype {mimetype}" + f"Testing {job.analyzable.name} with mimetype {mimetype}" f" for {timeout_seconds} seconds" ) signal.alarm(timeout_seconds) @@ -179,9 +184,9 @@ def handler(signum, frame): f" with configuration {config.name}" ) - @staticmethod - def tearDown() -> None: + def tearDown(self) -> None: Job.objects.all().delete() + super().tearDown() class ObservableAnalyzerTestCase(CustomTestCase): @@ -191,52 +196,73 @@ class ObservableAnalyzerTestCase(CustomTestCase): def test_config(self): config = AnalyzerConfig.objects.first() - job = Job.objects.create( - observable_name="test.com", observable_classification="domain" + an1 = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, ) + job = Job.objects.create(analyzable=an1) oa = MockUpObservableAnalyzer(config) oa.job_id = job.pk oa.config({}) self.assertEqual(oa.observable_name, "test.com") self.assertEqual(oa.observable_classification, "domain") job.delete() + an1.delete() def _create_jobs(self): + an1 = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + an2 = Analyzable.objects.create( + name="8.8.8.8", + classification=Classification.IP, + ) + an3 = Analyzable.objects.create( + name="https://www.honeynet.org/projects/active/intel-owl/", + classification=Classification.URL, + ) + an4 = Analyzable.objects.create( + name="3edd95917241e9ef9bbfc805c2c5aff3", + classification=Classification.HASH, + ) + an5 = Analyzable.objects.create( + name="test@intelowl.com", + classification=Classification.GENERIC, + ) + an6 = Analyzable.objects.create( + name="CVE-2024-51181", + classification=Classification.GENERIC, + ) + Job.objects.create( user=self.superuser, - observable_name="test.com", - observable_classification="domain", + analyzable=an1, status="reported_without_fails", ) Job.objects.create( user=self.superuser, - observable_name="8.8.8.8", - observable_classification="ip", + analyzable=an2, status="reported_without_fails", ) Job.objects.create( user=self.superuser, - observable_name="https://www.honeynet.org/projects/active/intel-owl/", - observable_classification="url", + analyzable=an3, status="reported_without_fails", ) Job.objects.create( user=self.superuser, - observable_name="3edd95917241e9ef9bbfc805c2c5aff3", - observable_classification="hash", + analyzable=an4, status="reported_without_fails", - md5="3edd95917241e9ef9bbfc805c2c5aff3", ) Job.objects.create( user=self.superuser, - observable_name="test@intelowl.com", - observable_classification="generic", + analyzable=an5, status="reported_without_fails", ), Job.objects.create( user=self.superuser, - observable_name="CVE-2024-51181", - observable_classification="generic", + analyzable=an6, status="reported_without_fails", ) @@ -262,11 +288,11 @@ def handler(signum, frame): f"Testing datatype {observable_supported}" f" for {timeout_seconds} seconds" ) - if observable_supported == ObservableTypes.GENERIC.value: + if observable_supported == Classification.GENERIC.value: # generic should handle different use cases job = Job.objects.get( - observable_classification=ObservableTypes.GENERIC.value, - observable_name=( + analyzable__classification=Classification.GENERIC.value, + analyzable__name=( "CVE-2024-51181" if config.name == "NVD_CVE" else "test@intelowl.com" @@ -274,7 +300,7 @@ def handler(signum, frame): ) else: job = Job.objects.get( - observable_classification=observable_supported + analyzable__classification=observable_supported ) job.analyzers_to_execute.set([config]) sub = subclass( @@ -296,3 +322,4 @@ def handler(signum, frame): @staticmethod def tearDown() -> None: Job.objects.all().delete() + Analyzable.objects.all().delete() diff --git a/tests/api_app/analyzers_manager/test_models.py b/tests/api_app/analyzers_manager/test_models.py index d14dd3086c..52e84a6662 100644 --- a/tests/api_app/analyzers_manager/test_models.py +++ b/tests/api_app/analyzers_manager/test_models.py @@ -5,8 +5,9 @@ from django.core.exceptions import ValidationError from kombu import uuid +from api_app.analyzables_manager.models import Analyzable from api_app.analyzers_manager.models import AnalyzerConfig, AnalyzerReport -from api_app.choices import PythonModuleBasePaths +from api_app.choices import Classification, PythonModuleBasePaths from api_app.data_model_manager.models import DomainDataModel, IPDataModel from api_app.models import Job, PythonModule from tests import CustomTestCase @@ -15,9 +16,13 @@ class AnalyzerReportTestCase(CustomTestCase): def test_get_data_models(self): + an1 = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=an1, status=Job.STATUSES.ANALYZERS_RUNNING.value, ) config = AnalyzerConfig.objects.first() @@ -36,11 +41,16 @@ def test_get_data_models(self): ) dm = AnalyzerReport.objects.filter(pk=ar.pk).get_data_models(job) self.assertEqual(dm.model, DomainDataModel) + an1.delete() def test_clean(self): + an1 = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=an1, status=Job.STATUSES.ANALYZERS_RUNNING.value, ) config = AnalyzerConfig.objects.first() @@ -66,11 +76,16 @@ def test_clean(self): ar.delete() job.delete() domain_data_model.delete() + an1.delete() def test_create_data_model(self): + an1 = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=an1, status=Job.STATUSES.ANALYZERS_RUNNING.value, ) config = AnalyzerConfig.objects.first() @@ -103,11 +118,16 @@ def test_create_data_model(self): data_model.delete() ar.delete() job.delete() + an1.delete() def test_get_value(self): + an1 = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=an1, status=Job.STATUSES.ANALYZERS_RUNNING.value, ) config = AnalyzerConfig.objects.first() @@ -130,6 +150,9 @@ def test_get_value(self): ar.get_value(ar.report, "urls.url".split(".")), ["www.intelowl.com", "www.intelowl.com"], ) + ar.delete() + job.delete() + an1.delete() class AnalyzerConfigTestCase(CustomTestCase): diff --git a/tests/api_app/analyzers_manager/test_views.py b/tests/api_app/analyzers_manager/test_views.py index ca330a1db1..5c27343d06 100644 --- a/tests/api_app/analyzers_manager/test_views.py +++ b/tests/api_app/analyzers_manager/test_views.py @@ -3,9 +3,9 @@ from typing import Type from unittest.mock import patch -from api_app.analyzers_manager.constants import ObservableTypes +from api_app.analyzables_manager.models import Analyzable from api_app.analyzers_manager.models import AnalyzerConfig, AnalyzerReport -from api_app.choices import PythonModuleBasePaths +from api_app.choices import Classification, PythonModuleBasePaths from api_app.models import Job, PythonModule from certego_saas.apps.organization.membership import Membership from certego_saas.apps.organization.organization import Organization @@ -188,12 +188,11 @@ def plugin_type(self): def init_report(self, status: str, user) -> AnalyzerReport: config = AnalyzerConfig.objects.get(name="HaveIBeenPwned") - _job = Job.objects.create( - user=user, - status=Job.STATUSES.RUNNING, - observable_name="8.8.8.8", - observable_classification=ObservableTypes.IP, + an = Analyzable.objects.create( + name="8.8.8.8", + classification=Classification.IP, ) + _job = Job.objects.create(user=user, status=Job.STATUSES.RUNNING, analyzable=an) _job.analyzers_to_execute.set([config]) _report, _ = AnalyzerReport.objects.get_or_create( **{ diff --git a/tests/api_app/connectors_manager/test_classes.py b/tests/api_app/connectors_manager/test_classes.py index 32161fe1f0..6710cf3e56 100644 --- a/tests/api_app/connectors_manager/test_classes.py +++ b/tests/api_app/connectors_manager/test_classes.py @@ -5,8 +5,9 @@ from kombu import uuid +from api_app.analyzables_manager.models import Analyzable from api_app.analyzers_manager.models import AnalyzerConfig, AnalyzerReport -from api_app.choices import PythonModuleBasePaths +from api_app.choices import Classification, PythonModuleBasePaths from api_app.connectors_manager.classes import Connector from api_app.connectors_manager.exceptions import ConnectorRunException from api_app.connectors_manager.models import ConnectorConfig @@ -58,9 +59,13 @@ class MockUpConnector(Connector): def run(self) -> dict: return {} + an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=an, status=Job.STATUSES.CONNECTORS_RUNNING.value, ) AnalyzerReport.objects.create( @@ -92,6 +97,7 @@ def run(self) -> dict: muc.before_run() cc.delete() job.delete() + an.delete() def test_subclasses(self): def handler(signum, frame): @@ -100,10 +106,13 @@ def handler(signum, frame): import signal signal.signal(signal.SIGALRM, handler) + an1 = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=an1, status="reported_without_fails", user=self.superuser, ) @@ -143,3 +152,4 @@ def handler(signum, frame): finally: signal.alarm(0) job.delete() + an1.delete() diff --git a/tests/api_app/connectors_manager/test_views.py b/tests/api_app/connectors_manager/test_views.py index e3779a8367..e9cf499a0b 100644 --- a/tests/api_app/connectors_manager/test_views.py +++ b/tests/api_app/connectors_manager/test_views.py @@ -2,7 +2,8 @@ # See the file 'LICENSE' for copying permission. from typing import Type -from api_app.analyzers_manager.constants import ObservableTypes +from api_app.analyzables_manager.models import Analyzable +from api_app.choices import Classification from api_app.connectors_manager.models import ConnectorConfig, ConnectorReport from api_app.models import Job, PluginConfig from tests import CustomViewSetTestCase, PluginActionViewsetTestCase @@ -61,15 +62,14 @@ def setUp(self): super().setUp() self.config = ConnectorConfig.objects.get(name="MISP") - def tearDown(self) -> None: - super().tearDown() - def init_report(self, status: str, user) -> ConnectorReport: + an1 = Analyzable.objects.create( + name="8.8.8.8", + classification=Classification.IP, + ) + _job = Job.objects.create( - user=user, - status=Job.STATUSES.REPORTED_WITHOUT_FAILS, - observable_name="8.8.8.8", - observable_classification=ObservableTypes.IP, + user=user, status=Job.STATUSES.REPORTED_WITHOUT_FAILS, analyzable=an1 ) _job.connectors_to_execute.set([self.config]) _report, _ = ConnectorReport.objects.get_or_create( diff --git a/tests/api_app/data_model_manager/test_models.py b/tests/api_app/data_model_manager/test_models.py index 41a4252da7..aa423d5036 100644 --- a/tests/api_app/data_model_manager/test_models.py +++ b/tests/api_app/data_model_manager/test_models.py @@ -1,4 +1,6 @@ -from api_app.data_model_manager.models import IPDataModel +from django.utils.timezone import now + +from api_app.data_model_manager.models import IETFReport, IPDataModel from tests import CustomTestCase @@ -8,3 +10,45 @@ def test_serialize(self): ip = IPDataModel.objects.create() results = IPDataModel.objects.filter(pk=ip.pk).serialize() self.assertEqual(1, len(results)) + + def test_merge_obj(self): + report1 = IETFReport.objects.create( + rrname="test", + rrtype="test2", + rdata=["test3"], + time_first=now(), + time_last=now(), + ) + report2 = IETFReport.objects.create( + rrname="test4", + rrtype="test5", + rdata=["test6"], + time_first=now(), + time_last=now(), + ) + ip = IPDataModel.objects.create(asn=3) + ip2 = IPDataModel.objects.create(asn=4, resolutions=["2.2.2.2"]) + ip2.ietf_report.add(report2) + ip3 = IPDataModel.objects.create(asn_rank=4, resolutions=["1.1.1.1"]) + ip3.ietf_report.add(report1) + ip.merge(ip2) + ip.merge(ip3) + self.assertEqual(ip2.asn, ip.asn) + self.assertEqual(ip3.asn_rank, ip.asn_rank) + self.assertCountEqual(ip2.resolutions + ip3.resolutions, ip.resolutions) + self.assertCountEqual( + ip.ietf_report.values_list("pk", flat=True), [report1.pk, report2.pk] + ) + + report1.delete() + report2.delete() + ip.delete() + ip2.delete() + ip3.delete() + + def test_merge_dict(self): + ip = IPDataModel.objects.create(asn=3) + ip.merge({"asn": 4, "resolutions": ["1.1.1.1"]}) + self.assertEqual(ip.asn, 4) + self.assertCountEqual(ip.resolutions, ["1.1.1.1"]) + ip.delete() diff --git a/tests/api_app/data_model_manager/test_queryset.py b/tests/api_app/data_model_manager/test_queryset.py new file mode 100644 index 0000000000..16233755a2 --- /dev/null +++ b/tests/api_app/data_model_manager/test_queryset.py @@ -0,0 +1,40 @@ +from django.utils.timezone import now + +from api_app.data_model_manager.models import IETFReport, IPDataModel +from tests import CustomTestCase + + +class BaseDataModelQuerySetTestCase(CustomTestCase): + def test_merge(self): + report1 = IETFReport.objects.create( + rrname="test", + rrtype="test2", + rdata=["test3"], + time_first=now(), + time_last=now(), + ) + report2 = IETFReport.objects.create( + rrname="test4", + rrtype="test5", + rdata=["test6"], + time_first=now(), + time_last=now(), + ) + ip = IPDataModel.objects.create(asn=3) + ip2 = IPDataModel.objects.create(asn=4, resolutions=["2.2.2.2"]) + ip2.ietf_report.add(report2) + ip3 = IPDataModel.objects.create(asn_rank=4, resolutions=["1.1.1.1"]) + ip3.ietf_report.add(report1) + result = IPDataModel.objects.filter(pk__in=[ip.pk, ip2.pk, ip3.pk]).merge() + + self.assertEqual(ip2.asn, result.asn) + self.assertEqual(ip3.asn_rank, result.asn_rank) + self.assertCountEqual(ip2.resolutions + ip3.resolutions, result.resolutions) + self.assertCountEqual( + result.ietf_report.values_list("pk", flat=True), [report1.pk, report2.pk] + ) + report1.delete() + report2.delete() + ip.delete() + ip2.delete() + ip3.delete() diff --git a/tests/api_app/data_model_manager/test_serializers.py b/tests/api_app/data_model_manager/test_serializers.py index f546fd6e48..fd93099fd2 100644 --- a/tests/api_app/data_model_manager/test_serializers.py +++ b/tests/api_app/data_model_manager/test_serializers.py @@ -1,6 +1,8 @@ from kombu import uuid +from api_app.analyzables_manager.models import Analyzable from api_app.analyzers_manager.models import AnalyzerConfig, AnalyzerReport +from api_app.choices import Classification from api_app.data_model_manager.models import DomainDataModel from api_app.data_model_manager.serializers import DomainDataModelSerializer from api_app.models import Job @@ -10,10 +12,12 @@ class TestDomainDataModelSerializer(CustomTestCase): def test_to_representation(self): + analyzable = Analyzable.objects.create( + name="test.com", classification=Classification.DOMAIN + ) job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", status=Job.STATUSES.ANALYZERS_RUNNING.value, + analyzable=analyzable, ) config = AnalyzerConfig.objects.first() dm = DomainDataModel.objects.create(evaluation="malicious") @@ -35,7 +39,9 @@ def test_to_representation(self): ser = DomainDataModelSerializer(dm) result = ser.data - print(result) + self.assertEqual(result["evaluation"], "malicious") dm.delete() ar.delete() + job.delete() + analyzable.delete() diff --git a/tests/api_app/data_model_manager/test_views.py b/tests/api_app/data_model_manager/test_views.py index 7c3a2eeb5b..46dd4ba44f 100644 --- a/tests/api_app/data_model_manager/test_views.py +++ b/tests/api_app/data_model_manager/test_views.py @@ -3,7 +3,9 @@ from django.db.models import Model from kombu import uuid +from api_app.analyzables_manager.models import Analyzable from api_app.analyzers_manager.models import AnalyzerConfig, AnalyzerReport +from api_app.choices import Classification from api_app.data_model_manager.models import ( DomainDataModel, FileDataModel, @@ -14,9 +16,13 @@ def create_report(user): + an1, _ = Analyzable.objects.get_or_create( + name="test.com", + classification=Classification.DOMAIN, + ) + job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=an1, status=Job.STATUSES.CONNECTORS_RUNNING.value, user=user, ) @@ -42,12 +48,17 @@ def test_url(self): self.fail(e) @classmethod - def setUpTestData(cls): - super().setUpTestData() + def setUpClass(cls): + super().setUpClass() report = create_report(cls.user) report.data_model = cls.model_class.objects.create() report.save() + @classmethod + def tearDownClass(cls): + super().tearDownClass() + Analyzable.objects.all().delete() + @classmethod @property def model_class(cls) -> Type[Model]: @@ -76,12 +87,17 @@ def get_object(self): return self.model_class.objects.order_by("?").first().pk @classmethod - def setUpTestData(cls): - super().setUpTestData() + def setUpClass(cls): + super().setUpClass() report = create_report(cls.user) report.data_model = cls.model_class.objects.create() report.save() + @classmethod + def tearDownClass(cls): + super().tearDownClass() + Analyzable.objects.all().delete() + def test_get_superuser(self): plugin = self.get_object() self.client.force_authenticate(self.superuser) @@ -101,12 +117,17 @@ def get_object(self): return self.model_class.objects.order_by("?").first().pk @classmethod - def setUpTestData(cls): - super().setUpTestData() + def setUpClass(cls): + super().setUpClass() report = create_report(cls.user) report.data_model = cls.model_class.objects.create() report.save() + @classmethod + def tearDownClass(cls): + super().tearDownClass() + Analyzable.objects.all().delete() + def test_get_superuser(self): plugin = self.get_object() self.client.force_authenticate(self.superuser) diff --git a/tests/api_app/engines_manager/__init__.py b/tests/api_app/engines_manager/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/api_app/engines_manager/test_models.py b/tests/api_app/engines_manager/test_models.py new file mode 100644 index 0000000000..2fa4ca003d --- /dev/null +++ b/tests/api_app/engines_manager/test_models.py @@ -0,0 +1,140 @@ +from django.core.exceptions import ValidationError +from django.db import transaction +from django.utils.timezone import now +from kombu import uuid + +from api_app.analyzables_manager.models import Analyzable +from api_app.analyzers_manager.models import AnalyzerConfig, AnalyzerReport +from api_app.choices import Classification +from api_app.data_model_manager.models import IPDataModel +from api_app.engines_manager.models import EngineConfig +from api_app.models import Job +from tests import CustomTestCase + + +class EngineConfigTestCase(CustomTestCase): + + def test_create_multiple_config(self): + with self.assertRaises(Exception), transaction.atomic(): + EngineConfig.objects.create() + self.assertEqual(EngineConfig.objects.count(), 1) + + def test_clean(self): + config = EngineConfig.objects.first() + config.modules.append("test.Test") + with self.assertRaises(ValidationError): + config.full_clean() + config.delete() + + def test_run_empty(self): + an1 = Analyzable.objects.create( + name="8.8.8.8", + classification=Classification.IP, + ) + + config = EngineConfig.objects.first() + job = Job.objects.create( + user=self.user, + status=Job.STATUSES.REPORTED_WITHOUT_FAILS.value, + analyzable=an1, + received_request_time=now(), + ) + config.run(job) + job.refresh_from_db() + self.assertIsNotNone(job.data_model) + self.assertEqual(job.data_model.evaluation, "clean") + job.delete() + config.delete() + an1.delete() + + def test_run_value(self): + an1 = Analyzable.objects.create( + name="8.8.8.8", + classification=Classification.IP, + ) + + config = EngineConfig.objects.first() + job = Job.objects.create( + user=self.user, + status=Job.STATUSES.REPORTED_WITHOUT_FAILS.value, + analyzable=an1, + received_request_time=now(), + ) + ar2 = AnalyzerReport.objects.create( + parameters={}, + report={ + "passive_dns": [ + { + "address": "195.22.26.248", + "first": "2022-03-19T17:14:00", + "last": "2022-03-19T17:16:33", + "hostname": "4ed8a7c6.ard.rr.zealbino.com", + "record_type": "A", + "indicator_link": "/indicator/hostname/4ed8a7c6.ard.rr.zealbino.com", # noqa: E501 + "flag_url": "assets/images/flags/pt.png", + "flag_title": "Portugal", + "asset_type": "hostname", + "asn": "AS8426 claranet ltd", + "suspicious": True, + "whitelisted_message": [], + "whitelisted": False, + }, + ], + }, + job=job, + task_id=uuid(), + config=AnalyzerConfig.objects.get(name="VirusTotal_v3_Get_Observable"), + ) + + ar = AnalyzerReport.objects.create( + parameters={}, + report={ + "passive_dns": [ + { + "address": "195.22.26.248", + "first": "2022-03-19T17:14:00", + "last": "2022-03-19T17:16:33", + "hostname": "4ed8a7c6.ard.rr.zealbino.com", + "record_type": "A", + "indicator_link": "/indicator/hostname/4ed8a7c6.ard.rr.zealbino.com", # noqa: E501 + "flag_url": "assets/images/flags/pt.png", + "flag_title": "Portugal", + "asset_type": "hostname", + "asn": "AS8426 claranet ltd", + "suspicious": True, + "whitelisted_message": [], + "whitelisted": False, + }, + ], + }, + job=job, + task_id=uuid(), + config=AnalyzerConfig.objects.get(name="OTXQuery"), + ) + ip1 = IPDataModel.objects.create( + evaluation=IPDataModel.EVALUATIONS.MALICIOUS.value, + resolutions=["1.2.3.4"], + ) + ip2 = IPDataModel.objects.create(resolutions=["1.2.3.5"]) + ar.data_model = ip1 + ar.save() + ar2.data_model = ip2 + ar2.save() + + job.refresh_from_db() + self.assertEqual(2, job.get_analyzers_data_models().count()) + config.run(job) + self.assertEqual( + job.data_model.evaluation, job.data_model.EVALUATIONS.MALICIOUS.value + ) + self.assertCountEqual( + job.data_model.resolutions, ip1.resolutions + ip2.resolutions + ) + ar.delete() + ar2.delete() + job.delete() + ip1.delete() + ip2.delete() + config.delete() + job.delete() + an1.delete() diff --git a/tests/api_app/engines_manager/test_modules.py b/tests/api_app/engines_manager/test_modules.py new file mode 100644 index 0000000000..84efd336f1 --- /dev/null +++ b/tests/api_app/engines_manager/test_modules.py @@ -0,0 +1,114 @@ +from django.utils.timezone import now +from kombu import uuid + +from api_app.analyzables_manager.models import Analyzable +from api_app.analyzers_manager.models import AnalyzerConfig, AnalyzerReport +from api_app.choices import Classification +from api_app.data_model_manager.models import IPDataModel +from api_app.engines_manager.engines.evaluation import EvaluationEngineModule +from api_app.engines_manager.engines.malware_family import MalwareFamilyEngineModule +from api_app.models import Job +from tests import CustomTestCase + + +class EngineModuleTestCase(CustomTestCase): + + def setUp(self) -> None: + super().setUp() + self.an = Analyzable.objects.create( + name="8.8.8.8", + classification=Classification.IP, + ) + self.job = Job.objects.create( + user=self.user, + status=Job.STATUSES.REPORTED_WITHOUT_FAILS.value, + received_request_time=now(), + analyzable=self.an, + ) + self.ars = [] + + def execute(self, module, *data_models): + for i, dm in enumerate(data_models): + ar = AnalyzerReport.objects.create( + parameters={}, + report={ + "passive_dns": [ + { + "address": "195.22.26.248", + "first": "2022-03-19T17:14:00", + "last": "2022-03-19T17:16:33", + "hostname": "4ed8a7c6.ard.rr.zealbino.com", + "record_type": "A", + "indicator_link": "/indicator/hostname/4ed8a7c6.ard.rr.zealbino.com", # noqa: E501 + "flag_url": "assets/images/flags/pt.png", + "flag_title": "Portugal", + "asset_type": "hostname", + "asn": "AS8426 claranet ltd", + "suspicious": True, + "whitelisted_message": [], + "whitelisted": False, + }, + ], + }, + job=self.job, + task_id=uuid(), + config=AnalyzerConfig.objects.filter( + observable_supported__contains=[Classification.IP.value] + )[i], + ) + ar.data_model = dm + ar.save() + self.ars.append(ar) + self.job.refresh_from_db() + return module.run() + + def tearDown(self) -> None: + self.job.delete() + self.an.delete() + for ar in self.ars: + ar.delete() + + def test_malware_family(self): + config = MalwareFamilyEngineModule(self.job) + + ip1 = IPDataModel.objects.create( + evaluation=IPDataModel.EVALUATIONS.MALICIOUS.value, + resolutions=["1.2.3.4"], + malware_family="test2", + ) + ip2 = IPDataModel.objects.create( + resolutions=["1.2.3.5"], + evaluation=IPDataModel.EVALUATIONS.SUSPICIOUS.value, + malware_family="test", + ) + ip3 = IPDataModel.objects.create( + resolutions=["1.2.3.5"], + evaluation=IPDataModel.EVALUATIONS.CLEAN.value, + malware_family="test2", + ) + + result = self.execute(config, ip1, ip2, ip3) + self.assertEqual(result["evaluation"], IPDataModel.EVALUATIONS.MALICIOUS.value) + self.assertEqual(result["malware_family"], "test2") + + ip1.delete() + ip2.delete() + ip3.delete() + + def test_evaluation(self): + config = EvaluationEngineModule(self.job) + + ip1 = IPDataModel.objects.create( + evaluation=IPDataModel.EVALUATIONS.MALICIOUS.value, + resolutions=["1.2.3.4"], + ) + ip2 = IPDataModel.objects.create( + resolutions=["1.2.3.5"], + evaluation=IPDataModel.EVALUATIONS.SUSPICIOUS.value, + ) + + result = self.execute(config, ip1, ip2) + self.assertEqual(result["evaluation"], IPDataModel.EVALUATIONS.MALICIOUS.value) + + ip1.delete() + ip2.delete() diff --git a/tests/api_app/investigations_manager/test_models.py b/tests/api_app/investigations_manager/test_models.py index 5df3d54da6..509baf3d1d 100644 --- a/tests/api_app/investigations_manager/test_models.py +++ b/tests/api_app/investigations_manager/test_models.py @@ -1,10 +1,26 @@ +from api_app.analyzables_manager.models import Analyzable +from api_app.choices import Classification from api_app.helpers import gen_random_colorhex from api_app.investigations_manager.models import Investigation from api_app.models import Job, Tag from tests import CustomTestCase -class InvetigationTestCase(CustomTestCase): +class InvestigationTestCase(CustomTestCase): + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + cls.an.delete() + def test_set_correct_status_created(self): an: Investigation = Investigation.objects.create(name="Test", owner=self.user) self.assertEqual(an.status, "created") @@ -14,8 +30,7 @@ def test_set_correct_status_created(self): def test_set_correct_status_running(self): job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=self.an, user=self.user, status=Job.STATUSES.REPORTED_WITH_FAILS, ) @@ -25,8 +40,7 @@ def test_set_correct_status_running(self): an.set_correct_status() self.assertEqual(an.status, "concluded") job.add_child( - observable_name="test.com", - observable_classification="domain", + analyzable=self.an, user=self.user, status=Job.STATUSES.PENDING, ) @@ -38,13 +52,11 @@ def test_set_correct_status_running(self): def test_set_correct_status_running2(self): job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=self.an, user=self.user, ) job2 = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=self.an, user=self.user, status="killed", ) @@ -59,8 +71,7 @@ def test_set_correct_status_running2(self): def test_set_correct_status_concluded(self): job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=self.an, user=self.user, status="killed", ) @@ -74,14 +85,12 @@ def test_set_correct_status_concluded(self): def test_jobs_count(self): job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=self.an, user=self.user, status="killed", ) j2 = job.add_child( - observable_name="test.com", - observable_classification="domain", + analyzable=self.an, user=self.user, status="killed", ) @@ -97,14 +106,12 @@ def test_jobs_count(self): def test_tlp(self): job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=self.an, user=self.user, tlp="CLEAR", ) job2 = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=self.an, user=self.user, tlp="RED", ) @@ -122,13 +129,11 @@ def test_tlp(self): def test_tags(self): job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=self.an, user=self.user, ) job2 = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=self.an, user=self.user, ) diff --git a/tests/api_app/investigations_manager/test_serializers.py b/tests/api_app/investigations_manager/test_serializers.py index 3538e7f771..169e51520f 100644 --- a/tests/api_app/investigations_manager/test_serializers.py +++ b/tests/api_app/investigations_manager/test_serializers.py @@ -1,3 +1,5 @@ +from api_app.analyzables_manager.models import Analyzable +from api_app.choices import Classification from api_app.investigations_manager.models import Investigation from api_app.investigations_manager.serializers import ( InvestigationSerializer, @@ -9,21 +11,24 @@ class InvestigationSerializerTestCase(CustomTestCase): def test_to_representation(self): + an1 = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", user=self.user, + analyzable=an1, status="killed", ) j2 = job.add_child( - observable_name="test.com", - observable_classification="domain", user=self.user, + analyzable=an1, status="killed", ) - an: Investigation = Investigation.objects.create(name="Test", owner=self.user) - an.jobs.add(job) - result = InvestigationSerializer(instance=an).data + inv: Investigation = Investigation.objects.create(name="Test", owner=self.user) + inv.jobs.add(job) + result = InvestigationSerializer(instance=inv).data self.assertIn("total_jobs", result) self.assertEqual(result["total_jobs"], 2) self.assertIn("tags", result) @@ -34,26 +39,30 @@ def test_to_representation(self): self.assertCountEqual(result["jobs"], [job.pk]) j2.delete() job.delete() - an.delete() + inv.delete() + an1.delete() class InvestigationTreeSerializerTestCase(CustomTestCase): def test_to_representation(self): + an1 = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=an1, user=self.user, status="killed", ) j2 = job.add_child( - observable_name="test.com", - observable_classification="domain", + analyzable=an1, user=self.user, status="killed", ) - an: Investigation = Investigation.objects.create(name="Test", owner=self.user) - an.jobs.add(job) - result = InvestigationTreeSerializer(instance=an).data + inv: Investigation = Investigation.objects.create(name="Test", owner=self.user) + inv.jobs.add(job) + result = InvestigationTreeSerializer(instance=inv).data self.assertIn("jobs", result) self.assertEqual(1, len(result["jobs"])) self.assertEqual(result["jobs"][0]["pk"], job.pk) @@ -62,4 +71,5 @@ def test_to_representation(self): self.assertEqual(result["jobs"][0]["children"][0]["pk"], j2.pk) j2.delete() job.delete() - an.delete() + inv.delete() + an1.delete() diff --git a/tests/api_app/investigations_manager/test_views.py b/tests/api_app/investigations_manager/test_views.py index c5eac742f4..5828f8f1f0 100644 --- a/tests/api_app/investigations_manager/test_views.py +++ b/tests/api_app/investigations_manager/test_views.py @@ -1,3 +1,5 @@ +from api_app.analyzables_manager.models import Analyzable +from api_app.choices import Classification from api_app.helpers import get_now from api_app.investigations_manager.models import Investigation from api_app.models import Job @@ -88,9 +90,11 @@ def test_add_job(self): self.assertEqual( result["errors"]["detail"], "You should set the `job` argument in the data" ) + an = Analyzable.objects.create( + name="test.com", classification=Classification.DOMAIN + ) job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=an, user=self.superuser, ) response = self.client.post( @@ -108,6 +112,7 @@ def test_add_job(self): ) self.assertEqual(response.status_code, 400) job.delete() + an.delete() def test_remove_job(self): investigation = self.get_object() @@ -117,15 +122,17 @@ def test_remove_job(self): self.assertEqual( result["errors"]["detail"], "You should set the `job` argument in the data" ) + an = Analyzable.objects.create( + name="test.com", classification=Classification.DOMAIN + ) + job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=an, user=self.superuser, finished_analysis_time=get_now(), ) job2 = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=an, user=self.user, finished_analysis_time=get_now(), ) @@ -152,6 +159,7 @@ def test_remove_job(self): job.delete() job2.delete() + an.delete() def test_get_superuser(self): plugin = self.get_object() diff --git a/tests/api_app/pivots_manager/test_classes.py b/tests/api_app/pivots_manager/test_classes.py index bd57109ccb..3cd02ce66b 100644 --- a/tests/api_app/pivots_manager/test_classes.py +++ b/tests/api_app/pivots_manager/test_classes.py @@ -1,5 +1,7 @@ from kombu import uuid +from api_app.analyzables_manager.models import Analyzable +from api_app.choices import Classification from api_app.models import Job from api_app.pivots_manager.classes import Pivot from api_app.pivots_manager.models import PivotConfig @@ -12,11 +14,15 @@ class PivotTestCase(CustomTestCase): ] def _create_jobs(self): + an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + Job.objects.create( user=self.superuser, - observable_name="test.com", - observable_classification="domain", status="reported_without_fails", + analyzable=an, ) def test_subclasses(self): @@ -41,7 +47,7 @@ def handler(signum, frame): f"Testing with config {config.name}" f" for {timeout_seconds} seconds" ) - job = Job.objects.get(observable_classification="domain") + job = Job.objects.get(analyzable__classification="domain") sub = subclass(config) signal.alarm(timeout_seconds) try: diff --git a/tests/api_app/pivots_manager/test_models.py b/tests/api_app/pivots_manager/test_models.py index 7205924148..15e18547dd 100644 --- a/tests/api_app/pivots_manager/test_models.py +++ b/tests/api_app/pivots_manager/test_models.py @@ -1,8 +1,8 @@ from django.core.exceptions import ValidationError from django.db import transaction -from api_app.analyzers_manager.constants import AllTypes from api_app.analyzers_manager.models import AnalyzerConfig +from api_app.choices import Classification from api_app.connectors_manager.models import ConnectorConfig from api_app.models import Job, PythonModule from api_app.pivots_manager.models import PivotConfig @@ -53,7 +53,6 @@ def test_create_job_multiple_generic(self): python_module__parameters__isnull=True, ).first() playbook.analyzers.set([ac2]) - job = Job(observable_name="test.com", tlp="AMBER", user=User.objects.first()) pc = PivotConfig( python_module=PythonModule.objects.filter( base_path="api_app.pivots_manager.pivots" @@ -63,22 +62,22 @@ def test_create_job_multiple_generic(self): jobs = list( pc.create_jobs( ["something", "something2"], - job.tlp, - job.user, + tlp="AMBER", + user=User.objects.first(), send_task=False, playbook_to_execute=playbook, ) ) self.assertEqual(2, len(jobs)) - self.assertEqual("something", jobs[0].observable_name) - self.assertEqual("generic", jobs[0].observable_classification) + self.assertEqual("something", jobs[0].analyzable.name) + self.assertEqual("generic", jobs[0].analyzable.classification) - self.assertEqual("something2", jobs[1].observable_name) - self.assertEqual("generic", jobs[1].observable_classification) + self.assertEqual("something2", jobs[1].analyzable.name) + self.assertEqual("generic", jobs[1].analyzable.classification) playbook.delete() def test_create_job_multiple_file(self): - job = Job(observable_name="test.com", tlp="AMBER", user=User.objects.first()) + job = Job(tlp="AMBER", user=User.objects.first()) pc = PivotConfig( name="PivotOnTest", python_module=PythonModule.objects.filter( @@ -94,18 +93,19 @@ def test_create_job_multiple_file(self): job.user, send_task=False, playbook_to_execute=PlaybookConfig.objects.filter( - disabled=False, type__icontains=AllTypes.FILE.value + disabled=False, type__icontains=Classification.FILE.value ).first(), ) ) self.assertEqual(1, len(jobs)) - self.assertEqual("PivotOnTest.0", jobs[0].file_name) + self.assertEqual("PivotOnTest.0", jobs[0].analyzable.name) self.assertEqual( - "application/vnd.microsoft.portable-executable", jobs[0].file_mimetype + "application/vnd.microsoft.portable-executable", + jobs[0].analyzable.mimetype, ) def test_create_job(self): - job = Job(observable_name="test.com", tlp="AMBER", user=User.objects.first()) + job = Job(tlp="AMBER", user=User.objects.first()) pc = PivotConfig( python_module=PythonModule.objects.filter( base_path="api_app.pivots_manager.pivots" @@ -123,5 +123,5 @@ def test_create_job(self): ) ) self.assertEqual(1, len(jobs)) - self.assertEqual("google.com", jobs[0].observable_name) - self.assertEqual("domain", jobs[0].observable_classification) + self.assertEqual("google.com", jobs[0].analyzable.name) + self.assertEqual("domain", jobs[0].analyzable.classification) diff --git a/tests/api_app/pivots_manager/test_serializers.py b/tests/api_app/pivots_manager/test_serializers.py index 101782e51b..dc420748b9 100644 --- a/tests/api_app/pivots_manager/test_serializers.py +++ b/tests/api_app/pivots_manager/test_serializers.py @@ -1,4 +1,6 @@ +from api_app.analyzables_manager.models import Analyzable from api_app.analyzers_manager.models import AnalyzerConfig +from api_app.choices import Classification from api_app.models import Job, PythonModule from api_app.pivots_manager.models import PivotConfig, PivotMap from api_app.pivots_manager.serializers import PivotConfigSerializer, PivotMapSerializer @@ -10,16 +12,23 @@ class PivotMapSerializerTestCase(CustomTestCase): def setUp(self) -> None: super().setUp() + self.an1 = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + self.an2 = Analyzable.objects.create( + name="test2.com", + classification=Classification.DOMAIN, + ) + self.j1 = Job.objects.create( user=self.user, - observable_name="test.com", - observable_classification="domain", + analyzable=self.an1, status="reported_without_fails", ) self.j2 = Job.objects.create( user=self.user, - observable_name="test2.com", - observable_classification="domain", + analyzable=self.an2, status="reported_without_fails", ) self.pc = PivotConfig.objects.create( @@ -33,6 +42,8 @@ def setUp(self) -> None: def tearDown(self) -> None: super().tearDown() self.j1.delete() + self.an1.delete() + self.an2.delete() self.j2.delete() self.pc.delete() diff --git a/tests/api_app/pivots_manager/test_views.py b/tests/api_app/pivots_manager/test_views.py index 94f704df45..af3871fc51 100644 --- a/tests/api_app/pivots_manager/test_views.py +++ b/tests/api_app/pivots_manager/test_views.py @@ -1,6 +1,8 @@ from typing import Type +from api_app.analyzables_manager.models import Analyzable from api_app.analyzers_manager.models import AnalyzerConfig +from api_app.choices import Classification from api_app.models import Job, PythonModule from api_app.pivots_manager.models import PivotConfig, PivotMap from api_app.playbooks_manager.models import PlaybookConfig @@ -38,16 +40,23 @@ def test_get(self): def setUp(self): super().setUp() + self.an1 = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + self.an2 = Analyzable.objects.create( + name="test2.com", + classification=Classification.DOMAIN, + ) + self.j1 = Job.objects.create( user=self.user, - observable_name="test.com", - observable_classification="domain", + analyzable=self.an1, status="reported_without_fails", ) self.j2 = Job.objects.create( user=self.user, - observable_name="test2.com", - observable_classification="domain", + analyzable=self.an2, status="reported_without_fails", ) self.pc = PivotConfig.objects.create( @@ -65,6 +74,8 @@ def tearDown(self) -> None: super().tearDown() self.j1.delete() self.j2.delete() + self.an1.delete() + self.an2.delete() self.pc.delete() PivotMap.objects.all().delete() diff --git a/tests/api_app/playbooks_manager/test_queryset.py b/tests/api_app/playbooks_manager/test_queryset.py index b21914ab72..d16c9c03a8 100644 --- a/tests/api_app/playbooks_manager/test_queryset.py +++ b/tests/api_app/playbooks_manager/test_queryset.py @@ -1,6 +1,8 @@ from django.utils.timezone import now +from api_app.analyzables_manager.models import Analyzable from api_app.analyzers_manager.models import AnalyzerConfig +from api_app.choices import Classification from api_app.models import Job from api_app.playbooks_manager.models import PlaybookConfig from api_app.playbooks_manager.queryset import PlaybookConfigQuerySet @@ -16,11 +18,19 @@ def setUp(self) -> None: self.pc = PlaybookConfig.objects.create( name="testplaybook", type=["ip"], description="test" ) + self.an1 = Analyzable.objects.create( + name="test3.com", + classification=Classification.DOMAIN, + ) + self.an2 = Analyzable.objects.create( + name="test_robot.com", + classification=Classification.DOMAIN, + ) + self.pc.analyzers.set([AnalyzerConfig.objects.first()]) self.j1 = Job.objects.create( user=self.superuser, - observable_name="test3.com", - observable_classification="domain", + analyzable=self.an1, status="reported_without_fails", playbook_to_execute=self.pc, finished_analysis_time=now(), @@ -28,21 +38,26 @@ def setUp(self) -> None: self.j2 = Job.objects.create( user=self.user, - observable_name="test3.com", - observable_classification="domain", + analyzable=self.an1, status="reported_without_fails", playbook_to_execute=self.pc, finished_analysis_time=now(), ) self.j3 = Job.objects.create( user=self.superuser, - observable_name="test3.com", - observable_classification="domain", + analyzable=self.an1, status="reported_without_fails", playbook_to_execute=self.pc, finished_analysis_time=now(), ) + def tearDown(self): + self.an1.delete() + self.an2.delete() + self.j1.delete() + self.j2.delete() + self.j3.delete() + def test__subquery_user(self): subq = PlaybookConfigQuerySet._subquery_weight_user(self.user) pc = PlaybookConfig.objects.annotate(weight=subq).get(name="testplaybook") @@ -99,32 +114,28 @@ def test_ordered_for_user(self): Job.objects.create( user=self.user, - observable_name="test3.com", - observable_classification="domain", + analyzable=self.an1, status="reported_without_fails", playbook_to_execute=self.pc, finished_analysis_time=now(), ) Job.objects.create( user=self.user, - observable_name="test3.com", - observable_classification="domain", + analyzable=self.an1, status="reported_without_fails", playbook_to_execute=self.pc, finished_analysis_time=now(), ) Job.objects.create( user=self.user, - observable_name="test3.com", - observable_classification="domain", + analyzable=self.an1, status="reported_without_fails", playbook_to_execute=pc3, finished_analysis_time=now(), ) Job.objects.create( user=self.user, - observable_name="test3.com", - observable_classification="domain", + analyzable=self.an1, status="reported_without_fails", playbook_to_execute=pc4, finished_analysis_time=now(), @@ -132,24 +143,21 @@ def test_ordered_for_user(self): # robot jobs Job.objects.create( user=robot, - observable_name="test_robot.com", - observable_classification="domain", + analyzable=self.an2, status="reported_without_fails", playbook_to_execute=pc2, finished_analysis_time=now(), ) Job.objects.create( user=robot, - observable_name="test_robot.com", - observable_classification="domain", + analyzable=self.an2, status="reported_without_fails", playbook_to_execute=pc2, finished_analysis_time=now(), ) Job.objects.create( user=robot, - observable_name="test_robot.com", - observable_classification="domain", + analyzable=self.an2, status="reported_without_fails", playbook_to_execute=pc2, finished_analysis_time=now(), @@ -157,16 +165,14 @@ def test_ordered_for_user(self): Job.objects.create( user=robot, - observable_name="test_robot.com", - observable_classification="domain", + analyzable=self.an2, status="reported_without_fails", playbook_to_execute=pc2, finished_analysis_time=now(), ) Job.objects.create( user=robot, - observable_name="test_robot.com", - observable_classification="domain", + analyzable=self.an2, status="reported_without_fails", playbook_to_execute=pc2, finished_analysis_time=now(), diff --git a/tests/api_app/test_api.py b/tests/api_app/test_api.py index cbebc5612c..6fe91d97fa 100644 --- a/tests/api_app/test_api.py +++ b/tests/api_app/test_api.py @@ -11,7 +11,9 @@ from django.core.files.uploadedfile import SimpleUploadedFile from api_app import models +from api_app.analyzables_manager.models import Analyzable from api_app.analyzers_manager.models import AnalyzerConfig +from api_app.choices import Classification from api_app.connectors_manager.models import ConnectorConfig from api_app.playbooks_manager.models import PlaybookConfig @@ -127,9 +129,9 @@ def test_analyze_file__pcap(self): job_id = int(content["job_id"]) job = models.Job.objects.get(pk=job_id) - self.assertEqual(file_name, job.file_name) - self.assertEqual(file_mimetype, job.file_mimetype) - self.assertEqual(md5, job.md5) + self.assertEqual(file_name, job.analyzable.name) + self.assertEqual(file_mimetype, job.analyzable.mimetype) + self.assertEqual(md5, job.analyzable.md5) self.assertCountEqual( ["Suricata", "YARAify_File_Scan", "Hfinger", "DetectItEasy", "Polyswarm"], @@ -145,14 +147,14 @@ def test_analyze_file__exe(self): job_id = int(content["job_id"]) job = models.Job.objects.get(pk=job_id) self.assertEqual(response.status_code, 200, msg=msg) - self.assertEqual(data["file_name"], job.file_name, msg=msg) - self.assertEqual(data["file_mimetype"], job.file_mimetype, msg=msg) + self.assertEqual(data["file_name"], job.analyzable.name, msg=msg) + self.assertEqual(data["file_mimetype"], job.analyzable.mimetype, msg=msg) self.assertCountEqual( data["analyzers_requested"], list(job.analyzers_requested.all().values_list("name", flat=True)), msg=msg, ) - self.assertEqual(self.file_md5, job.md5, msg=msg) + self.assertEqual(self.file_md5, job.analyzable.md5, msg=msg) def test_analyze_file__guess_optional(self): data = self.analyze_file_data.copy() @@ -165,14 +167,14 @@ def test_analyze_file__guess_optional(self): job_id = int(content["job_id"]) job = models.Job.objects.get(pk=job_id) - self.assertEqual(data["file_name"], job.file_name, msg=msg) + self.assertEqual(data["file_name"], job.analyzable.name, msg=msg) self.assertCountEqual( data["analyzers_requested"], list(job.analyzers_requested.all().values_list("name", flat=True)), msg=msg, ) - self.assertEqual(file_mimetype, job.file_mimetype, msg=msg) - self.assertEqual(self.file_md5, job.md5, msg=msg) + self.assertEqual(file_mimetype, job.analyzable.mimetype, msg=msg) + self.assertEqual(self.file_md5, job.analyzable.md5, msg=msg) def test_analyze_observable__domain(self): analyzers_requested = [ @@ -196,9 +198,9 @@ def test_analyze_observable__domain(self): job_id = int(content["job_id"]) job = models.Job.objects.get(pk=job_id) - self.assertEqual(observable_name, job.observable_name) - self.assertEqual(observable_classification, job.observable_classification) - self.assertEqual(md5, job.md5) + self.assertEqual(observable_name, job.analyzable.name) + self.assertEqual(observable_classification, job.analyzable.classification) + self.assertEqual(md5, job.analyzable.md5) self.assertCountEqual( analyzers_requested, list(job.analyzers_requested.all().values_list("name", flat=True)), @@ -217,16 +219,16 @@ def test_analyze_observable__ip(self): job_id = int(content["job_id"]) job = models.Job.objects.get(pk=job_id) - self.assertEqual(data["observable_name"], job.observable_name, msg=msg) + self.assertEqual(data["observable_name"], job.analyzable.name, msg=msg) self.assertCountEqual( data["analyzers_requested"], list(job.analyzers_requested.all().values_list("name", flat=True)), msg=msg, ) self.assertEqual( - data["observable_classification"], job.observable_classification, msg=msg + data["observable_classification"], job.analyzable.classification, msg=msg ) - self.assertEqual(self.observable_md5, job.md5, msg=msg) + self.assertEqual(self.observable_md5, job.analyzable.md5, msg=msg) def test_analyze_observable__guess_optional(self): data = self.analyze_observable_ip_data.copy() @@ -241,16 +243,16 @@ def test_analyze_observable__guess_optional(self): job_id = int(content["job_id"]) job = models.Job.objects.get(pk=job_id) - self.assertEqual(data["observable_name"], job.observable_name, msg=msg) + self.assertEqual(data["observable_name"], job.analyzable.name, msg=msg) self.assertCountEqual( data["analyzers_requested"], list(job.analyzers_requested.all().values_list("name", flat=True)), msg=msg, ) self.assertEqual( - observable_classification, job.observable_classification, msg=msg + observable_classification, job.analyzable.classification, msg=msg ) - self.assertEqual(self.observable_md5, job.md5, msg=msg) + self.assertEqual(self.observable_md5, job.analyzable.md5, msg=msg) def test_analyze_multiple_observables(self): data = self.mixed_observable_data.copy() @@ -266,7 +268,7 @@ def test_analyze_multiple_observables(self): job_id = int(content["job_id"]) job = models.Job.objects.get(pk=job_id) - self.assertEqual(data["observables"][0][1], job.observable_name, msg=msg) + self.assertEqual(data["observables"][0][1], job.analyzable.name, msg=msg) self.assertCountEqual( data["analyzers_requested"], list(job.analyzers_requested.all().values_list("name", flat=True)), @@ -282,12 +284,13 @@ def test_analyze_multiple_observables(self): job_id = int(content["job_id"]) job = models.Job.objects.get(pk=job_id) - self.assertEqual(data["observables"][1][1], job.observable_name, msg=msg) + self.assertEqual(data["observables"][1][1], job.analyzable.name, msg=msg) self.assertCountEqual( [data["analyzers_requested"][0]], list(job.analyzers_to_execute.all().values_list("name", flat=True)), msg=msg, ) + job.delete() def test_observable_no_analyzers_only_connector(self): models.PluginConfig.objects.create( @@ -329,7 +332,7 @@ def test_observable_no_analyzers_only_connector(self): job_id = int(content["job_id"]) job = models.Job.objects.get(pk=job_id) - self.assertEqual(data["observables"][0][1], job.observable_name, msg=msg) + self.assertEqual(data["observables"][0][1], job.analyzable.name, msg=msg) self.assertEqual(job.analyzers_requested.count(), 0) self.assertEqual(job.pivots_to_execute.count(), 0) @@ -337,22 +340,25 @@ def test_download_sample_200(self): self.assertEqual(models.Job.objects.count(), 0) filename = "file.exe" uploaded_file, md5 = self.__get_test_file(filename) + analyzable = models.Analyzable.objects.create( + name=filename, + file=uploaded_file, + classification="file", + md5=md5, + mimetype="application/vnd.microsoft.portable-executable", + ) job = models.Job.objects.create( - **{ - "md5": md5, - "is_sample": True, - "file_name": filename, - "file_mimetype": "application/vnd.microsoft.portable-executable", - "file": uploaded_file, - } + analyzable=analyzable, ) self.assertEqual(models.Job.objects.count(), 1) response = self.client.get(f"/api/jobs/{job.id}/download_sample") self.assertEqual(response.status_code, 200) self.assertEqual( response.get("Content-Disposition"), - f'attachment; filename="{job.file_name}"', + f'attachment; filename="{job.analyzable.name}"', ) + job.delete() + analyzable.delete() def test_download_sample_404(self): # requesting for an ID that we know does not exist in DB @@ -361,7 +367,10 @@ def test_download_sample_404(self): def test_download_sample_400(self): # requesting for job where is_sample=False - job = models.Job.objects.create(is_sample=False) + analyzable = Analyzable.objects.create( + name="test.com", classification=Classification.DOMAIN + ) + job = models.Job.objects.create(analyzable=analyzable) response = self.client.get(f"/api/jobs/{job.id}/download_sample") content = response.json() msg = (response, content) @@ -371,6 +380,8 @@ def test_download_sample_400(self): content["errors"], msg=msg, ) + job.delete() + analyzable.delete() def test_no_analyzers(self): data = self.mixed_observable_data.copy() @@ -419,15 +430,19 @@ def test_analyze_multiple_files__exe(self): job = models.Job.objects.get(pk=job_id) self.assertEqual(response.status_code, 200, msg=msg) self.assertEqual( - self.analyze_multiple_files_filenames[index], job.file_name, msg=msg + self.analyze_multiple_files_filenames[index], + job.analyzable.name, + msg=msg, ) self.assertCountEqual( data["analyzers_requested"], list(job.analyzers_requested.all().values_list("name", flat=True)), msg=msg, ) - self.assertEqual(self.file_md5, job.md5, msg=msg) - self.assertEqual(data["file_mimetypes"][index], job.file_mimetype, msg=msg) + self.assertEqual(self.file_md5, job.analyzable.md5, msg=msg) + self.assertEqual( + data["file_mimetypes"][index], job.analyzable.mimetype, msg=msg + ) def test_analyze_multiple_files__guess_optional(self): data = self.analyze_multiple_files_data.copy() @@ -444,15 +459,17 @@ def test_analyze_multiple_files__guess_optional(self): job = models.Job.objects.get(pk=job_id) self.assertEqual(response.status_code, 200, msg=msg) self.assertEqual( - self.analyze_multiple_files_filenames[index], job.file_name, msg=msg + self.analyze_multiple_files_filenames[index], + job.analyzable.name, + msg=msg, ) self.assertCountEqual( data["analyzers_requested"], list(job.analyzers_requested.all().values_list("name", flat=True)), msg=msg, ) - self.assertEqual(self.file_md5, job.md5, msg=msg) - self.assertEqual(file_mimetypes[index], job.file_mimetype, msg=msg) + self.assertEqual(self.file_md5, job.analyzable.md5, msg=msg) + self.assertEqual(file_mimetypes[index], job.analyzable.mimetype, msg=msg) def test_tlp_clear_and_white(self): data = self.analyze_observable_ip_data.copy() # tlp = "CLEAR" by default @@ -474,11 +491,15 @@ def test_tlp_clear_and_white(self): self.assertEqual(job.tlp, "CLEAR", msg=msg) def test_job_rescan__observable_analyzers(self): + an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + job = models.Job.objects.create( tlp="CLEAR", user=self.user, - observable_name="test.com", - observable_classification="domain", + analyzable=an, status="reported_without_fails", finished_analysis_time=datetime.datetime( 2024, 8, 24, 10, 10, tzinfo=datetime.timezone.utc @@ -496,7 +517,7 @@ def test_job_rescan__observable_analyzers(self): self.assertEqual(response.status_code, 202, contents) new_job_id = int(contents["id"]) new_job = models.Job.objects.get(pk=new_job_id) - self.assertEqual(new_job.observable_name, "test.com") + self.assertEqual(new_job.analyzable.name, "test.com") self.assertEqual(new_job.tlp, "CLEAR") self.assertEqual( list(new_job.analyzers_requested.all()), @@ -510,13 +531,18 @@ def test_job_rescan__observable_analyzers(self): "visualizers": {}, }, ) + an.delete() def test_job_rescan__observable_playbook(self): + an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + job = models.Job.objects.create( tlp="CLEAR", user=self.user, - observable_name="test.com", - observable_classification="domain", + analyzable=an, status="reported_without_fails", finished_analysis_time=datetime.datetime( 2024, 8, 24, 10, 10, tzinfo=datetime.timezone.utc @@ -534,7 +560,7 @@ def test_job_rescan__observable_playbook(self): self.assertEqual(response.status_code, 202, contents) new_job_id = int(contents["id"]) new_job = models.Job.objects.get(pk=new_job_id) - self.assertEqual(new_job.observable_name, "test.com") + self.assertEqual(new_job.analyzable.name, "test.com") self.assertEqual(new_job.tlp, "CLEAR") self.assertEqual( new_job.playbook_requested, PlaybookConfig.objects.get(name="Dns") @@ -547,15 +573,16 @@ def test_job_rescan__observable_playbook(self): "visualizers": {}, }, ) + an.delete() def test_job_rescan__sample_analyzers(self): + an = Analyzable.objects.create( + file=self.uploaded_file, name="file.exe", classification="file" + ) job = models.Job.objects.create( tlp="CLEAR", - md5=self.file_md5, user=self.user, - is_sample=True, - file_name="file.exe", - file=self.uploaded_file, + analyzable=an, status="reported_without_fails", finished_analysis_time=datetime.datetime( 2024, 8, 24, 10, 10, tzinfo=datetime.timezone.utc @@ -579,8 +606,8 @@ def test_job_rescan__sample_analyzers(self): self.assertEqual(response.status_code, 202, contents) new_job_id = int(contents["id"]) new_job = models.Job.objects.get(pk=new_job_id) - self.assertEqual(new_job.file_name, "file.exe") - self.assertEqual(new_job.file, job.file) + self.assertEqual(new_job.analyzable.name, "file.exe") + self.assertEqual(new_job.analyzable.file, job.analyzable.file) self.assertEqual(new_job.tlp, "CLEAR") self.assertEqual( list(new_job.analyzers_requested.all()), @@ -599,16 +626,17 @@ def test_job_rescan__sample_analyzers(self): "visualizers": {}, }, ) + job.delete() + an.delete() def test_job_rescan__sample_playbook(self): - + an = Analyzable.objects.create( + file=self.uploaded_file, name="file.exe", classification="file" + ) job = models.Job.objects.create( tlp="CLEAR", - md5=self.file_md5, user=self.user, - is_sample=True, - file_name="file.exe", - file=self.uploaded_file, + analyzable=an, status="reported_without_fails", playbook_requested=PlaybookConfig.objects.get(name="FREE_TO_USE_ANALYZERS"), finished_analysis_time=datetime.datetime( @@ -632,8 +660,8 @@ def test_job_rescan__sample_playbook(self): self.assertEqual(response.status_code, 202, contents) new_job_id = int(contents["id"]) new_job = models.Job.objects.get(pk=new_job_id) - self.assertEqual(new_job.file_name, "file.exe") - self.assertEqual(new_job.file, job.file) + self.assertEqual(new_job.analyzable.name, "file.exe") + self.assertEqual(new_job.analyzable.file, job.analyzable.file) self.assertEqual(new_job.tlp, "CLEAR") self.assertEqual( new_job.playbook_requested, @@ -652,13 +680,19 @@ def test_job_rescan__sample_playbook(self): "visualizers": {}, }, ) + job.delete() + an.delete() def test_job_rescan__permission(self): + an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + job = models.Job.objects.create( tlp="CLEAR", user=self.user, - observable_name="test.com", - observable_classification="domain", + analyzable=an, status="reported_without_fails", finished_analysis_time=datetime.datetime( 2024, 8, 24, 10, 10, tzinfo=datetime.timezone.utc @@ -681,3 +715,4 @@ def test_job_rescan__permission(self): response = self.client.post(f"/api/jobs/{job.pk}/rescan", format="json") contents = response.json() self.assertEqual(response.status_code, 403, contents) + an.delete() diff --git a/tests/api_app/test_classes.py b/tests/api_app/test_classes.py index 2247e74a09..60b8aebc42 100644 --- a/tests/api_app/test_classes.py +++ b/tests/api_app/test_classes.py @@ -5,7 +5,8 @@ from kombu import uuid -from api_app.choices import PythonModuleBasePaths +from api_app.analyzables_manager.models import Analyzable +from api_app.choices import Classification, PythonModuleBasePaths from api_app.classes import Plugin from api_app.connectors_manager.classes import Connector from api_app.connectors_manager.models import ConnectorConfig @@ -16,8 +17,14 @@ class PluginTestCase(CustomTestCase): def setUp(self) -> None: super().setUp() + self.an = Analyzable.objects.create( + name="8.8.8.8", + classification=Classification.IP, + ) self.job, _ = Job.objects.get_or_create( - user=self.user, status=Job.STATUSES.REPORTED_WITHOUT_FAILS + user=self.user, + status=Job.STATUSES.REPORTED_WITHOUT_FAILS, + analyzable=self.an, ) self.cc, _ = ConnectorConfig.objects.get_or_create( name="test", @@ -33,6 +40,7 @@ def setUp(self) -> None: def tearDown(self) -> None: self.job.delete() self.cc.delete() + self.an.delete() def test_abstract(self): with self.assertRaises(TypeError): diff --git a/tests/api_app/test_helpers.py b/tests/api_app/test_helpers.py index d5803d6218..3bba4547ad 100644 --- a/tests/api_app/test_helpers.py +++ b/tests/api_app/test_helpers.py @@ -3,40 +3,40 @@ from django.test import TestCase -from api_app.analyzers_manager.constants import ObservableTypes +from api_app.choices import Classification class HelperTests(TestCase): def test_accept_defanged_domains(self): observable = "www\.test\.com" - result = ObservableTypes.calculate(observable) - self.assertEqual(result, ObservableTypes.DOMAIN) + result = Classification.calculate_observable(observable) + self.assertEqual(result, Classification.DOMAIN) observable = "www[.]test[.]com" - result = ObservableTypes.calculate(observable) - self.assertEqual(result, ObservableTypes.DOMAIN) + result = Classification.calculate_observable(observable) + self.assertEqual(result, Classification.DOMAIN) def test_calculate_observable_classification(self): observable = "7.7.7.7" - result = ObservableTypes.calculate(observable) - self.assertEqual(result, ObservableTypes.IP) + result = Classification.calculate_observable(observable) + self.assertEqual(result, Classification.IP) observable = "www.test.com" - result = ObservableTypes.calculate(observable) - self.assertEqual(result, ObservableTypes.DOMAIN) + result = Classification.calculate_observable(observable) + self.assertEqual(result, Classification.DOMAIN) observable = ".www.test.com" - result = ObservableTypes.calculate(observable) - self.assertEqual(result, ObservableTypes.DOMAIN) + result = Classification.calculate_observable(observable) + self.assertEqual(result, Classification.DOMAIN) observable = "ftp://www.test.com" - result = ObservableTypes.calculate(observable) - self.assertEqual(result, ObservableTypes.URL) + result = Classification.calculate_observable(observable) + self.assertEqual(result, Classification.URL) observable = "b318ff1839771c22e50d316af613dc70" - result = ObservableTypes.calculate(observable) - self.assertEqual(result, ObservableTypes.HASH) + result = Classification.calculate_observable(observable) + self.assertEqual(result, Classification.HASH) observable = "iammeia" - result = ObservableTypes.calculate(observable) - self.assertEqual(result, ObservableTypes.GENERIC) + result = Classification.calculate_observable(observable) + self.assertEqual(result, Classification.GENERIC) diff --git a/tests/api_app/test_interfaces.py b/tests/api_app/test_interfaces.py index cca43e118b..bc46e488b6 100644 --- a/tests/api_app/test_interfaces.py +++ b/tests/api_app/test_interfaces.py @@ -1,3 +1,5 @@ +from api_app.analyzables_manager.models import Analyzable +from api_app.choices import Classification from api_app.interfaces import CreateJobsFromPlaybookInterface from api_app.investigations_manager.models import Investigation from api_app.models import Job @@ -19,9 +21,13 @@ def setUp(self) -> None: self.c.name = "test" def test__get_file_serializer(self): + an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + parent_job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=an, user=self.user, ) serializer = self.c._get_file_serializer( @@ -35,18 +41,23 @@ def test__get_file_serializer(self): jobs = serializer.save(send_task=False, parent=parent_job) self.assertEqual(len(jobs), 1) job = jobs[0] - self.assertEqual(job.analyzed_object_name, "test.0") + self.assertEqual(job.analyzable.name, "test.0") self.assertEqual(job.playbook_to_execute, self.c.playbooks_choice.first()) self.assertEqual(job.tlp, "CLEAR") - self.assertEqual(job.file.read(), b"test") + self.assertEqual(job.analyzable.read(), b"test") self.assertIsNone(job.investigation) self.assertIsNotNone(parent_job.investigation) parent_job.delete() + an.delete() def test__get_observable_serializer(self): + an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + parent_job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=an, user=self.user, ) serializer = self.c._get_observable_serializer( @@ -60,19 +71,24 @@ def test__get_observable_serializer(self): jobs = serializer.save(send_task=False, parent=parent_job) self.assertEqual(len(jobs), 1) job = jobs[0] - self.assertEqual(job.analyzed_object_name, "google.com") + self.assertEqual(job.analyzable.name, "google.com") self.assertEqual(job.playbook_to_execute, self.c.playbooks_choice.first()) self.assertEqual(job.tlp, "CLEAR") - self.assertEqual(job.observable_classification, "domain") + self.assertEqual(job.analyzable.classification, "domain") self.assertIsNone(job.investigation) self.assertIsNotNone(parent_job.investigation) parent_job.delete() + an.delete() def test__multiple_jobs_investigations(self): investigation_count = Investigation.objects.count() + an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + parent_job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=an, user=self.user, ) self.assertIsNone(parent_job.investigation) @@ -98,13 +114,18 @@ def test__multiple_jobs_investigations(self): parent_job.delete() job1.delete() job2.delete() + an.delete() def test__multiple_jobs_investigation_with_parent_in_investigation(self): investigation = Investigation.objects.create(owner=self.user, name="test") investigation_count = Investigation.objects.count() + an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + parent_job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=an, user=self.user, ) investigation.jobs.set([parent_job]) @@ -139,3 +160,4 @@ def test__multiple_jobs_investigation_with_parent_in_investigation(self): ) parent_job.delete() investigation.delete() + an.delete() diff --git a/tests/api_app/test_mixins.py b/tests/api_app/test_mixins.py index 24c2cfcde1..ac35338f18 100644 --- a/tests/api_app/test_mixins.py +++ b/tests/api_app/test_mixins.py @@ -3,8 +3,8 @@ import pathlib from pathlib import PosixPath -from api_app.analyzers_manager.constants import ObservableTypes from api_app.analyzers_manager.models import AnalyzerConfig +from api_app.choices import Classification from api_app.mixins import VirusTotalv3AnalyzerMixin, VirusTotalv3BaseMixin from tests import CustomTestCase from tests.mock_utils import MockUpResponse @@ -116,7 +116,7 @@ def test_get_requests_params_and_uri(self): "historical_ssl_certificates", ] params, uri, relationships_requested = self.base._get_requests_params_and_uri( - ObservableTypes.DOMAIN, "google.com", True + Classification.DOMAIN, "google.com", True ) self.assertIn("relationships", params) self.assertListEqual(relationships_requested, expected_relationships) @@ -132,7 +132,7 @@ def test_get_requests_params_and_uri(self): "historical_ssl_certificates", ] params, uri, relationships_requested = self.base._get_requests_params_and_uri( - ObservableTypes.IP, "8.8.8.8", True + Classification.IP, "8.8.8.8", True ) self.assertIn("relationships", params) self.assertListEqual(relationships_requested, expected_relationships) @@ -145,7 +145,7 @@ def test_get_requests_params_and_uri(self): "network_location", ] params, uri, relationships_requested = self.base._get_requests_params_and_uri( - ObservableTypes.URL, "https://google.com/robots.txt", True + Classification.URL, "https://google.com/robots.txt", True ) self.assertIn("relationships", params) self.assertListEqual(relationships_requested, expected_relationships) @@ -158,7 +158,7 @@ def test_get_requests_params_and_uri(self): "contacted_urls", ] params, uri, relationships_requested = self.base._get_requests_params_and_uri( - ObservableTypes.HASH, "5f423b7772a80f77438407c8b78ff305", True + Classification.HASH, "5f423b7772a80f77438407c8b78ff305", True ) self.assertIn("relationships", params) self.assertListEqual(relationships_requested, expected_relationships) @@ -181,7 +181,7 @@ def test_get_requests_params_and_uri(self): "collections", ] params, uri, relationships_requested = self.base._get_requests_params_and_uri( - ObservableTypes.HASH, "5f423b7772a80f77438407c8b78ff305", False + Classification.HASH, "5f423b7772a80f77438407c8b78ff305", False ) self.assertIn("relationships", params) self.assertListEqual(relationships_requested, expected_relationships) diff --git a/tests/api_app/test_models.py b/tests/api_app/test_models.py index ece8eb5bec..bf8ae1c245 100644 --- a/tests/api_app/test_models.py +++ b/tests/api_app/test_models.py @@ -9,8 +9,9 @@ from django.db import IntegrityError, transaction from django_celery_beat.models import PeriodicTask +from api_app.analyzables_manager.models import Analyzable from api_app.analyzers_manager.models import AnalyzerConfig -from api_app.choices import PythonModuleBasePaths +from api_app.choices import Classification, PythonModuleBasePaths from api_app.connectors_manager.models import ConnectorConfig from api_app.models import ( AbstractConfig, @@ -270,7 +271,8 @@ def test_is_runnable_disabled_by_org(self): org.delete() def test_get_signature_without_runnable(self): - job, _ = Job.objects.get_or_create(user=self.user) + an = Analyzable.objects.create(name="8.8.8.8", classification=Classification.IP) + job, _ = Job.objects.get_or_create(user=self.user, analyzable=an) muc, _ = VisualizerConfig.objects.get_or_create( name="test", description="test", @@ -288,9 +290,12 @@ def test_get_signature_without_runnable(self): self.fail("Stop iteration should not be raised") muc.delete() job.delete() + an.delete() def test_get_signature_disabled(self): - job, _ = Job.objects.get_or_create(user=self.user) + an = Analyzable.objects.create(name="8.8.8.8", classification=Classification.IP) + job, _ = Job.objects.get_or_create(user=self.user, analyzable=an) + muc, _ = VisualizerConfig.objects.get_or_create( name="test", description="test", @@ -312,9 +317,12 @@ def test_get_signature_disabled(self): self.fail("Stop iteration should not be raised") muc.delete() job.delete() + an.delete() def test_get_signature(self): - job, _ = Job.objects.get_or_create(user=self.user) + an = Analyzable.objects.create(name="8.8.8.8", classification=Classification.IP) + job, _ = Job.objects.get_or_create(user=self.user, analyzable=an) + muc, _ = VisualizerConfig.objects.get_or_create( name="test", description="test", @@ -336,6 +344,7 @@ def test_get_signature(self): self.assertIsInstance(signature, Signature) muc.delete() job.delete() + an.delete() class PluginConfigTestCase(CustomTestCase): @@ -462,11 +471,14 @@ def test_pivots_to_execute(self): ac = AnalyzerConfig.objects.first() ac2 = AnalyzerConfig.objects.exclude(pk__in=[ac.pk]).first() ac3 = AnalyzerConfig.objects.exclude(pk__in=[ac.pk, ac2.pk]).first() + an = Analyzable.objects.create( + name="test.com", + classification="domain", + md5="72cf478e87b031233091d8c00a38ce00", + ) j1 = Job.objects.create( - observable_name="test.com", - observable_classification="domain", user=self.user, - md5="72cf478e87b031233091d8c00a38ce00", + analyzable=an, status=Job.STATUSES.REPORTED_WITHOUT_FAILS, ) pc = PivotConfig.objects.create( diff --git a/tests/api_app/test_queryset.py b/tests/api_app/test_queryset.py index 401abb76fd..eca4b4408c 100644 --- a/tests/api_app/test_queryset.py +++ b/tests/api_app/test_queryset.py @@ -4,8 +4,9 @@ from django.utils.timezone import now from django_celery_beat.models import CrontabSchedule +from api_app.analyzables_manager.models import Analyzable from api_app.analyzers_manager.models import AnalyzerConfig -from api_app.choices import PythonModuleBasePaths +from api_app.choices import Classification, PythonModuleBasePaths from api_app.ingestors_manager.models import IngestorConfig from api_app.models import Job, Parameter, PluginConfig, PythonModule from api_app.playbooks_manager.models import PlaybookConfig @@ -503,62 +504,77 @@ def tearDown(self) -> None: Job.objects.all().delete() def test_annotate_importance_date_this_day(self): - Job.objects.create( + an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + + j = Job.objects.create( tlp="RED", user=self.user, - observable_name="test.com", - observable_classification="domain", + analyzable=an, status="reported_without_fails", finished_analysis_time=now() - datetime.timedelta(hours=5), ) - j = ( - Job.objects.filter(observable_name="test.com") - ._annotate_importance_date() - .first() - ) + j = Job.objects.filter(pk=j.pk)._annotate_importance_date().first() self.assertEqual(3, j.date_weight) j.delete() + an.delete() def test_annotate_importance_date_this_week(self): + an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + Job.objects.create( tlp="RED", user=self.user, - observable_name="test.com", - observable_classification="domain", + analyzable=an, status="reported_without_fails", finished_analysis_time=now() - datetime.timedelta(days=5), ) j = ( - Job.objects.filter(observable_name="test.com") + Job.objects.filter(analyzable__name="test.com") ._annotate_importance_date() .first() ) self.assertEqual(2, j.date_weight) j.delete() + an.delete() def test_annotate_importance_date_old(self): + an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + Job.objects.create( tlp="RED", user=self.user, - observable_name="test.com", - observable_classification="domain", + analyzable=an, status="reported_without_fails", finished_analysis_time=now() - datetime.timedelta(days=30), ) j = ( - Job.objects.filter(observable_name="test.com") + Job.objects.filter(analyzable__name="test.com") ._annotate_importance_date() .first() ) self.assertEqual(0, j.date_weight) j.delete() + an.delete() def test_annotate_importance_user_same_user_same_org(self): + an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + Job.objects.create( tlp="RED", user=self.user, - observable_name="test.com", - observable_classification="domain", + analyzable=an, status="reported_without_fails", ) org = Organization.objects.create(name="test_org") @@ -571,7 +587,7 @@ def test_annotate_importance_user_same_user_same_org(self): organization=org, ) j = ( - Job.objects.filter(observable_name="test.com") + Job.objects.filter(analyzable__name="test.com") ._annotate_importance_user(self.user) .first() ) @@ -580,13 +596,18 @@ def test_annotate_importance_user_same_user_same_org(self): m1.delete() m2.delete() org.delete() + an.delete() def test_annotate_importance_user_same_org(self): + an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + Job.objects.create( tlp="RED", user=self.superuser, - observable_name="test.com", - observable_classification="domain", + analyzable=an, status="reported_without_fails", ) org = Organization.objects.create(name="test_org") @@ -599,7 +620,7 @@ def test_annotate_importance_user_same_org(self): organization=org, ) j = ( - Job.objects.filter(observable_name="test.com") + Job.objects.filter(analyzable__name="test.com") ._annotate_importance_user(self.user) .first() ) @@ -608,33 +629,43 @@ def test_annotate_importance_user_same_org(self): m1.delete() m2.delete() org.delete() + an.delete() def test_annotate_importance_user_valid(self): + an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + Job.objects.create( tlp="RED", user=self.user, - observable_name="test.com", - observable_classification="domain", + analyzable=an, status="reported_without_fails", ) j = ( - Job.objects.filter(observable_name="test.com") + Job.objects.filter(analyzable__name="test.com") ._annotate_importance_user(self.user) .first() ) self.assertEqual(3, j.user_weight) j.delete() + an.delete() def test_annotate_importance_user_wrong(self): + an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + Job.objects.create( tlp="RED", user=self.superuser, - observable_name="test.com", - observable_classification="domain", + analyzable=an, status="reported_without_fails", ) j = ( - Job.objects.filter(observable_name="test.com") + Job.objects.filter(analyzable__name="test.com") ._annotate_importance_user(self.user) .first() ) @@ -642,11 +673,15 @@ def test_annotate_importance_user_wrong(self): j.delete() def test_visible_for_user_tlp(self): + an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + j = Job.objects.create( tlp="RED", user=self.superuser, - observable_name="test.com", - observable_classification="domain", + analyzable=an, status="reported_without_fails", ) self.assertEqual(1, Job.objects.visible_for_user(self.superuser).count()) @@ -655,20 +690,24 @@ def test_visible_for_user_tlp(self): j = Job.objects.create( tlp="GREEN", user=self.superuser, - observable_name="test.com", - observable_classification="domain", + analyzable=an, status="reported_without_fails", ) self.assertEqual(1, Job.objects.visible_for_user(self.superuser).count()) self.assertEqual(1, Job.objects.visible_for_user(self.user).count()) j.delete() + an.delete() def test_visible_for_user_membership(self): + an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + j = Job.objects.create( tlp="RED", user=self.superuser, - observable_name="test.com", - observable_classification="domain", + analyzable=an, status="reported_without_fails", ) self.assertEqual(1, Job.objects.visible_for_user(self.superuser).count()) @@ -687,8 +726,13 @@ def test_visible_for_user_membership(self): m2.delete() org.delete() j.delete() + an.delete() def test_visible_for_user_ingestor(self): + an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) schedule = CrontabSchedule.objects.create() ingestor = IngestorConfig.objects.create( name="test", @@ -704,8 +748,7 @@ def test_visible_for_user_ingestor(self): j = Job.objects.create( tlp="RED", user=ingestor.user, - observable_name="test.com", - observable_classification="domain", + analyzable=an, status="reported_without_fails", ) self.assertEqual(1, Job.objects.visible_for_user(self.superuser).count()) @@ -713,3 +756,4 @@ def test_visible_for_user_ingestor(self): ingestor.delete() schedule.delete() j.delete() + an.delete() diff --git a/tests/api_app/test_serializers.py b/tests/api_app/test_serializers.py index c1e750f84b..f18f693944 100644 --- a/tests/api_app/test_serializers.py +++ b/tests/api_app/test_serializers.py @@ -6,9 +6,10 @@ from django.utils.timezone import now from rest_framework.exceptions import ValidationError +from api_app.analyzables_manager.models import Analyzable from api_app.analyzers_manager.models import AnalyzerConfig from api_app.analyzers_manager.serializers import AnalyzerConfigSerializer -from api_app.choices import PythonModuleBasePaths +from api_app.choices import Classification, PythonModuleBasePaths from api_app.connectors_manager.models import ConnectorConfig from api_app.models import Job, Parameter, PluginConfig, PythonModule from api_app.playbooks_manager.models import PlaybookConfig @@ -36,13 +37,12 @@ class JobRecentScanSerializerTestCase(CustomTestCase): def test_to_representation(self): + an = Analyzable.objects.create(name="gigatest.com", classification="domain") j1 = Job.objects.create( **{ "user": self.user, - "is_sample": False, - "observable_name": "gigatest.com", - "observable_classification": "domain", "finished_analysis_time": now() - datetime.timedelta(hours=2), + "analyzable": an, } ) data = JobRecentScanSerializer(j1).data @@ -192,9 +192,13 @@ def test_validate(self): class RestJobSerializerTestCase(CustomTestCase): def test_validate(self): + an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=an, user=self.user, ) js = RestJobSerializer(job) @@ -203,6 +207,7 @@ def test_validate(self): self.assertIn("visualizer_reports", js.data) self.assertIn("analyzers_data_model", js.data) job.delete() + an.delete() class AbstractJobCreateSerializerTestCase(CustomTestCase): @@ -217,11 +222,14 @@ def test_check_previous_job(self): a1 = AnalyzerConfig.objects.order_by("?").first() a2 = AnalyzerConfig.objects.order_by("?").exclude(pk=a1.pk).first() a3 = AnalyzerConfig.objects.order_by("?").exclude(pk__in=[a1.pk, a2.pk]).first() + an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + j1 = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=an, user=self.user, - md5="72cf478e87b031233091d8c00a38ce00", status=Job.STATUSES.REPORTED_WITHOUT_FAILS, received_request_time=now() - datetime.timedelta(hours=3), ) @@ -236,7 +244,7 @@ def test_check_previous_job(self): self.ajcs.check_previous_jobs( validated_data={ "scan_check_time": datetime.timedelta(days=1), - "md5": "72cf478e87b031233091d8c00a38ce00", + "analyzable": an, "analyzers_to_execute": [], } ), @@ -246,7 +254,7 @@ def test_check_previous_job(self): self.ajcs.check_previous_jobs( validated_data={ "scan_check_time": datetime.timedelta(days=1), - "md5": "72cf478e87b031233091d8c00a38ce00", + "analyzable": an, "analyzers_to_execute": [a1], } ), @@ -256,7 +264,7 @@ def test_check_previous_job(self): self.ajcs.check_previous_jobs( validated_data={ "scan_check_time": datetime.timedelta(days=1), - "md5": "72cf478e87b031233091d8c00a38ce00", + "analyzable": an, "analyzers_to_execute": [a1, a2], } ), @@ -265,10 +273,12 @@ def test_check_previous_job(self): self.ajcs.check_previous_jobs( validated_data={ "scan_check_time": datetime.timedelta(days=1), - "md5": "72cf478e87b031233091d8c00a38ce00", + "analyzable": an, "analyzers_to_execute": [a1, a2, a3], } ) + j1.delete() + an.delete() def test_set_default_value_from_playbook(self): data = {"playbook_requested": PlaybookConfig.objects.first()} @@ -574,9 +584,11 @@ def test_filter_analyzer_observable_supported(self): class CommentSerializerTestCase(CustomTestCase): def setUp(self): super().setUp() + self.an = Analyzable.objects.create( + name="test.com", classification=Classification.DOMAIN + ) self.job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=self.an, user=self.user, ) self.job.save() @@ -589,6 +601,7 @@ def setUp(self): def tearDown(self) -> None: super().tearDown() self.job.delete() + self.an.delete() def test_create(self): self.assertTrue(self.cs.is_valid()) @@ -607,8 +620,12 @@ def test_null(self): self.assertEqual(result, {"status": "not_available", "job_id": None}) def test_job(self): + an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) job = Job.objects.create( - observable_name="test.com", observable_classification="domain" + analyzable=an, ) result = JobResponseSerializer(job).data self.assertIn("status", result) @@ -616,13 +633,23 @@ def test_job(self): self.assertIn("job_id", result) self.assertEqual(result["job_id"], job.id) job.delete() + an.delete() def test_many(self): + an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + an2 = Analyzable.objects.create( + name="test2.com", + classification=Classification.DOMAIN, + ) + job1 = Job.objects.create( - observable_name="test.com", observable_classification="domain" + analyzable=an, ) job2 = Job.objects.create( - observable_name="test2.com", observable_classification="domain" + analyzable=an2, ) result = JobResponseSerializer([job1, job2], many=True).data self.assertIn("count", result) @@ -635,6 +662,8 @@ def test_many(self): self.assertEqual(result["results"][0]["job_id"], job1.id) job1.delete() job2.delete() + an.delete() + an2.delete() class AbstractListConfigSerializerTestCase(CustomTestCase): diff --git a/tests/api_app/test_views.py b/tests/api_app/test_views.py index acb24f0c3f..0a132afb39 100644 --- a/tests/api_app/test_views.py +++ b/tests/api_app/test_views.py @@ -6,15 +6,16 @@ from zoneinfo import ZoneInfo from django.contrib.auth import get_user_model +from django.core.files import File from django.test import override_settings from django.utils.timezone import now from elasticsearch_dsl.query import Bool, Exists, Range, Term from rest_framework.reverse import reverse from rest_framework.test import APIClient -from api_app.analyzers_manager.constants import ObservableTypes +from api_app.analyzables_manager.models import Analyzable from api_app.analyzers_manager.models import AnalyzerConfig -from api_app.choices import ReportStatus +from api_app.choices import Classification, ReportStatus from api_app.models import Comment, Job, Parameter, PluginConfig, Tag from api_app.playbooks_manager.models import PlaybookConfig from certego_saas.apps.organization.membership import Membership @@ -30,53 +31,52 @@ class CommentViewSetTestCase(CustomViewSetTestCase): def setUp(self): super().setUp() - self.job = Job.objects.create( - user=self.superuser, - is_sample=False, - observable_name="8.8.8.8", - observable_classification=ObservableTypes.IP, - ) - self.job2 = Job.objects.create( - user=self.superuser, - is_sample=False, - observable_name="8.8.8.8", - observable_classification=ObservableTypes.IP, + self.an1 = Analyzable.objects.create( + name="8.8.8.8", + classification=Classification.IP, ) + + self.job = Job.objects.create(user=self.superuser, analyzable=self.an1) + self.job2 = Job.objects.create(user=self.user, analyzable=self.an1) self.comment = Comment.objects.create( - job=self.job, user=self.superuser, content="test" + analyzable=self.an1, user=self.user, content="test" ) - self.comment.save() def tearDown(self) -> None: super().tearDown() + self.comment.delete() self.job.delete() self.job2.delete() - self.comment.delete() + self.an1.delete() def test_list_200(self): + self.client.force_authenticate(self.user) response = self.client.get(self.comment_url) self.assertEqual(response.status_code, 200) self.assertEqual(response.json().get("count"), 1) def test_create_201(self): - data = {"job_id": self.job.id, "content": "test2"} + self.client.force_authenticate(self.user) + data = {"job_id": self.job2.id, "content": "test2"} response = self.client.post(self.comment_url, data) self.assertEqual(response.status_code, 201) self.assertEqual(response.json().get("content"), "test2") def test_delete(self): - response = self.client.delete(f"{self.comment_url}/{self.comment.pk}") - self.assertEqual(response.status_code, 403) self.client.force_authenticate(self.superuser) response = self.client.delete(f"{self.comment_url}/{self.comment.pk}") + self.assertEqual(response.status_code, 404) + self.client.force_authenticate(self.user) + response = self.client.delete(f"{self.comment_url}/{self.comment.pk}") self.assertEqual(response.status_code, 204) self.assertEqual(0, Comment.objects.all().count()) def test_get(self): - response = self.client.get(f"{self.comment_url}/{self.comment.pk}") - self.assertEqual(response.status_code, 403) self.client.force_authenticate(self.superuser) response = self.client.get(f"{self.comment_url}/{self.comment.pk}") + self.assertEqual(response.status_code, 404) + self.client.force_authenticate(self.user) + response = self.client.get(f"{self.comment_url}/{self.comment.pk}") self.assertEqual(response.status_code, 200) @@ -109,48 +109,58 @@ def setUp(self): "django.utils.timezone.now", return_value=datetime.datetime(2024, 11, 28, tzinfo=datetime.timezone.utc), ): - self.job, _ = Job.objects.get_or_create( + self.analyzable = Analyzable.objects.create( + name="1.2.3.4", classification=Classification.IP + ) + with open("test_files/file.exe", "rb") as f: + self.an2 = Analyzable.objects.create( + name="test.file", + classification=Classification.FILE, + mimetype="application/vnd.microsoft.portable-executable", + file=File(f), + ) + + self.job = Job.objects.create( **{ "user": self.superuser, - "is_sample": False, - "observable_name": "1.2.3.4", - "observable_classification": "ip", + "analyzable": self.analyzable, "playbook_to_execute": PlaybookConfig.objects.get(name="Dns"), "tlp": Job.TLP.CLEAR.value, } ) - self.job2, _ = Job.objects.get_or_create( + self.job2 = Job.objects.create( **{ "user": self.superuser, - "is_sample": True, - "md5": "test.file", - "file_name": "test.file", - "file_mimetype": "application/vnd.microsoft.portable-executable", + "analyzable": self.an2, "playbook_to_execute": PlaybookConfig.objects.get(name="Dns"), "tlp": Job.TLP.GREEN.value, } ) + def tearDown(self): + self.job2.delete() + self.job.delete() + self.analyzable.delete() + self.an2.delete() + def test_recent_scan(self): j1 = Job.objects.create( **{ "user": self.user, - "is_sample": False, - "observable_name": "gigatest.com", - "observable_classification": "domain", + "analyzable": self.analyzable, "finished_analysis_time": now() - datetime.timedelta(days=2), } ) j2 = Job.objects.create( **{ "user": self.user, - "is_sample": False, - "observable_name": "gigatest.com", - "observable_classification": "domain", + "analyzable": self.analyzable, "finished_analysis_time": now() - datetime.timedelta(hours=2), } ) - response = self.client.post(self.jobs_recent_scans_uri, data={"md5": j1.md5}) + response = self.client.post( + self.jobs_recent_scans_uri, data={"md5": j1.analyzable.md5} + ) content = response.json() msg = (response, content) self.assertEqual(200, response.status_code, msg=msg) @@ -166,9 +176,7 @@ def test_recent_scan_user(self): j1 = Job.objects.create( **{ "user": self.user, - "is_sample": False, - "observable_name": "gigatest.com", - "observable_classification": "domain", + "analyzable": self.analyzable, "finished_analysis_time": datetime.datetime( 2024, 11, 28, tzinfo=datetime.timezone.utc ), @@ -177,9 +185,7 @@ def test_recent_scan_user(self): j2 = Job.objects.create( **{ "user": self.superuser, - "is_sample": False, - "observable_name": "gigatest.com", - "observable_classification": "domain", + "analyzable": self.analyzable, "finished_analysis_time": datetime.datetime( 2024, 11, 28, tzinfo=datetime.timezone.utc ), @@ -233,7 +239,7 @@ def test_kill(self): job = Job.objects.create( status=Job.STATUSES.RUNNING, user=self.superuser, - observable_classification="ip", + analyzable=self.analyzable, ) self.assertEqual(job.status, Job.STATUSES.RUNNING) uri = reverse("jobs-kill", args=[job.pk]) @@ -252,7 +258,7 @@ def test_kill_400(self): job = Job.objects.create( status=Job.STATUSES.REPORTED_WITHOUT_FAILS, user=self.superuser, - observable_classification="ip", + analyzable=self.analyzable, ) uri = reverse("jobs-kill", args=[job.pk]) self.client.force_authenticate(user=self.job.user) @@ -304,7 +310,7 @@ def test_agg_observable_classification_200(self): msg = (resp, content) self.assertEqual(resp.status_code, 200, msg) - for field in ["date", *ObservableTypes.values]: + for field in ["date", *Classification.values[:-1]]: self.assertIn( field, content[0], @@ -349,9 +355,7 @@ def test_agg_top_user_200(self): job, _ = Job.objects.get_or_create( **{ "user": u, - "is_sample": False, - "observable_name": "1.2.3.4", - "observable_classification": "ip", + "analyzable": self.analyzable, "playbook_to_execute": PlaybookConfig.objects.get(name="Dns"), "tlp": Job.TLP.CLEAR.value, } diff --git a/tests/api_app/test_websocket.py b/tests/api_app/test_websocket.py index ec34918246..ac7d37e74b 100644 --- a/tests/api_app/test_websocket.py +++ b/tests/api_app/test_websocket.py @@ -8,9 +8,10 @@ from django.contrib.auth import get_user_model from django.test import TransactionTestCase -from api_app.analyzers_manager.constants import ObservableTypes, TypeChoices +from api_app.analyzables_manager.models import Analyzable +from api_app.analyzers_manager.constants import TypeChoices from api_app.analyzers_manager.models import AnalyzerConfig -from api_app.choices import ParamTypes +from api_app.choices import Classification, ParamTypes from api_app.models import Job, Parameter, PluginConfig, PythonModule from intel_owl.asgi import application from intel_owl.tasks import job_set_final_status, run_plugin @@ -52,15 +53,24 @@ def _pre_setup(self): class JobConsumerTestCase(WebsocketTestCase): def setUp(self) -> None: + self.an = Analyzable.objects.create( + name="8.8.8.8", + classification=Classification.IP, + ) + self.user = User.objects.create(username="websocket_test") self.job = Job.objects.create( id=1027, user=self.user, status=Job.STATUSES.REPORTED_WITHOUT_FAILS.value, - observable_name="8.8.8.8", - observable_classification=ObservableTypes.IP, + analyzable=self.an, ) + def tearDown(self) -> None: + self.user.delete() + Job.objects.all().delete() + Analyzable.objects.all().delete() + async def test_job_unauthorized(self, *args, **kwargs): self.assertEqual(await sync_to_async(Job.objects.filter(id=1027).count)(), 1) async with self.connect_communicator(1027) as (_, connected, subprotocol): @@ -98,14 +108,16 @@ async def test_job_running(self, *args, **kwargs): # The test will be blocked waiting a response from ws that already happened. # we need a sleep to wait. # in this test happens for the functions: run_plugin set_final_status. - + analyzable = await sync_to_async(Analyzable.objects.create)( + name="test.com", + classification=Classification.DOMAIN, + ) # setup db job = await sync_to_async(Job.objects.create)( id=1029, user=self.user, status=Job.STATUSES.PENDING.value, - observable_name="test.com", - observable_classification=ObservableTypes.DOMAIN, + analyzable=analyzable, ) class_dns_python_module, _ = await sync_to_async( PythonModule.objects.get_or_create @@ -120,9 +132,9 @@ async def test_job_running(self, *args, **kwargs): python_module=class_dns_python_module, type=TypeChoices.OBSERVABLE.value, observable_supported=[ - ObservableTypes.IP.value, - ObservableTypes.DOMAIN.value, - ObservableTypes.URL.value, + Classification.IP.value, + Classification.DOMAIN.value, + Classification.URL.value, ], ) analyzer_list = [classic_dns_analyzer_config] @@ -204,12 +216,15 @@ async def test_job_running(self, *args, **kwargs): self.assertIsNotNone(job_report_terminated["finished_analysis_time"]) async def test_job_killed(self, *args, **kwargs): + analyzable = await sync_to_async(Analyzable.objects.create)( + name="test.com", + classification=Classification.DOMAIN, + ) await sync_to_async(Job.objects.create)( id=1030, user=self.user, status=Job.STATUSES.RUNNING.value, - observable_name="test.com", - observable_classification=ObservableTypes.DOMAIN, + analyzable=analyzable, ) await sync_to_async(self.client.force_login)(self.user) diff --git a/tests/api_app/visualizers_manager/passive_dns/test_analyzer_extractor.py b/tests/api_app/visualizers_manager/passive_dns/test_analyzer_extractor.py index 2fe802d7f8..05b0885d5b 100644 --- a/tests/api_app/visualizers_manager/passive_dns/test_analyzer_extractor.py +++ b/tests/api_app/visualizers_manager/passive_dns/test_analyzer_extractor.py @@ -2,8 +2,9 @@ from kombu import uuid -from api_app.analyzers_manager.constants import ObservableTypes +from api_app.analyzables_manager.models import Analyzable from api_app.analyzers_manager.models import AnalyzerConfig, AnalyzerReport +from api_app.choices import Classification from api_app.models import Job from api_app.visualizers_manager.visualizers.passive_dns.analyzer_extractor import ( PDNSReport, @@ -22,11 +23,15 @@ class TestOTXQuery(CustomTestCase): @classmethod def setUpClass(cls) -> None: super().setUpClass() + cls.an = Analyzable.objects.create( + name="195.22.26.248", + classification=Classification.IP, + ) + cls.job = Job.objects.create( user=cls.user, status=Job.STATUSES.RUNNING.value, - observable_name="195.22.26.248", - observable_classification=ObservableTypes.IP, + analyzable=cls.an, received_request_time=datetime.datetime.now(), ) cls.otx_report = None @@ -37,6 +42,7 @@ def tearDownClass(cls) -> None: cls.job.delete() if cls.otx_report: cls.otx_report.delete() + cls.an.delete() def test_no_report(self): report = extract_otxquery_reports( @@ -106,11 +112,15 @@ class TestThreatminer(CustomTestCase): @classmethod def setUpClass(cls) -> None: super().setUpClass() + cls.an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + cls.job = Job.objects.create( user=cls.user, status=Job.STATUSES.RUNNING.value, - observable_name="test.com", - observable_classification=ObservableTypes.DOMAIN, + analyzable=cls.an, received_request_time=datetime.datetime.now(), ) cls.threatminer_report = None @@ -121,6 +131,7 @@ def tearDownClass(cls) -> None: cls.job.delete() if cls.threatminer_report: cls.threatminer_report.delete() + cls.an.delete() def test_no_report(self): report = extract_threatminer_reports( @@ -196,11 +207,15 @@ class TestValidin(CustomTestCase): @classmethod def setUpClass(cls) -> None: super().setUpClass() + cls.an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + cls.job = Job.objects.create( user=cls.user, status=Job.STATUSES.RUNNING.value, - observable_name="test.com", - observable_classification=ObservableTypes.DOMAIN, + analyzable=cls.an, received_request_time=datetime.datetime.now(), ) cls.validin_report = None @@ -211,6 +226,7 @@ def tearDownClass(cls) -> None: cls.job.delete() if cls.validin_report: cls.validin_report.delete() + cls.an.delete() def test_no_report(self): report = extract_validin_reports( @@ -304,11 +320,15 @@ class TestDNSdb(CustomTestCase): @classmethod def setUpClass(cls) -> None: super().setUpClass() + cls.an = Analyzable.objects.create( + name="www.farsightsecurity.com", + classification=Classification.DOMAIN, + ) + cls.job = Job.objects.create( user=cls.user, status=Job.STATUSES.RUNNING.value, - observable_name="www.farsightsecurity.com", - observable_classification=ObservableTypes.DOMAIN, + analyzable=cls.an, received_request_time=datetime.datetime.now(), ) cls.dnsdb_report = None @@ -319,6 +339,7 @@ def tearDownClass(cls) -> None: cls.job.delete() if cls.dnsdb_report: cls.dnsdb_report.delete() + cls.an.delete() def test_no_report(self): report = extract_dnsdb_reports( @@ -382,11 +403,15 @@ class TestRobtex(CustomTestCase): @classmethod def setUpClass(cls) -> None: super().setUpClass() + cls.an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + cls.job = Job.objects.create( user=cls.user, status=Job.STATUSES.RUNNING.value, - observable_name="test.com", - observable_classification=ObservableTypes.DOMAIN, + analyzable=cls.an, received_request_time=datetime.datetime.now(), ) cls.robtex_report = None @@ -397,6 +422,7 @@ def tearDownClass(cls) -> None: cls.job.delete() if cls.robtex_report: cls.robtex_report.delete() + cls.an.delete() def test_no_report(self): report = extract_robtex_reports( @@ -457,11 +483,15 @@ class TestMnemonicPDNS(CustomTestCase): @classmethod def setUpClass(cls) -> None: super().setUpClass() + cls.an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + cls.job = Job.objects.create( user=cls.user, status=Job.STATUSES.RUNNING.value, - observable_name="test.com", - observable_classification=ObservableTypes.DOMAIN, + analyzable=cls.an, received_request_time=datetime.datetime.now(), ) cls.mnemonicpdns_report = None @@ -472,6 +502,7 @@ def tearDownClass(cls) -> None: cls.job.delete() if cls.mnemonicpdns_report: cls.mnemonicpdns_report.delete() + cls.an.delete() def test_no_report(self): report = extract_mnemonicpdns_reports( @@ -532,11 +563,15 @@ class TestCIRCLPassiveDNS(CustomTestCase): @classmethod def setUpClass(cls) -> None: super().setUpClass() + cls.an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) + cls.job = Job.objects.create( user=cls.user, status=Job.STATUSES.RUNNING.value, - observable_name="test.com", - observable_classification=ObservableTypes.DOMAIN, + analyzable=cls.an, received_request_time=datetime.datetime.now(), ) cls.circlpdns_report = None @@ -547,6 +582,7 @@ def tearDownClass(cls) -> None: cls.job.delete() if cls.circlpdns_report: cls.circlpdns_report.delete() + cls.an.delete() def test_no_report(self): report = extract_circlpdns_reports( diff --git a/tests/api_app/visualizers_manager/test_classes.py b/tests/api_app/visualizers_manager/test_classes.py index 094f447b53..8112b62f51 100644 --- a/tests/api_app/visualizers_manager/test_classes.py +++ b/tests/api_app/visualizers_manager/test_classes.py @@ -4,8 +4,9 @@ from kombu import uuid +from api_app.analyzables_manager.models import Analyzable from api_app.analyzers_manager.models import AnalyzerReport -from api_app.choices import PythonModuleBasePaths +from api_app.choices import Classification, PythonModuleBasePaths from api_app.models import Job, PythonModule from api_app.playbooks_manager.models import PlaybookConfig from api_app.visualizers_manager.classes import ( @@ -473,9 +474,12 @@ def run(self) -> dict: return {} pc = PlaybookConfig.objects.first() + an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=an, status="reported_without_fails", ) vc = VisualizerConfig.objects.create( @@ -493,10 +497,11 @@ def run(self) -> dict: ) v = MockUpVisualizer(vc) v.job_id = job.pk - self.assertEqual(list(v.analyzer_reports()), [ar]) + self.assertEqual(list(v.get_analyzer_reports()), [ar]) ar.delete() job.delete() vc.delete() + an.delete() def test_subclasses(self): def handler(signum, frame): @@ -506,9 +511,12 @@ def handler(signum, frame): signal.signal(signal.SIGALRM, handler) + an = Analyzable.objects.create( + name="test.com", + classification=Classification.DOMAIN, + ) job = Job.objects.create( - observable_name="test.com", - observable_classification="domain", + analyzable=an, status="reported_without_fails", user=self.superuser, ) @@ -547,6 +555,7 @@ def handler(signum, frame): signal.alarm(0) job.delete() + an.delete() class ErrorHandlerTestCase(CustomTestCase): diff --git a/tests/auth/test_oauth.py b/tests/auth/test_oauth.py index d668e45eee..69bf9c3b39 100644 --- a/tests/auth/test_oauth.py +++ b/tests/auth/test_oauth.py @@ -67,8 +67,7 @@ def test_google_callback(self, mock_validate_and_return_user: Mock): response = self.client.get(self.google_auth_callback_uri, follow=False) msg = response.url self.assertEqual(response.status_code, 302, msg) - response_redirect = urlparse(response.url) - print(response_redirect) + urlparse(response.url) self.assertEqual(Session.objects.count(), 1) session = Session.objects.all().first() session_data = session.get_decoded() diff --git a/tests/intel_owl/test_tasks.py b/tests/intel_owl/test_tasks.py index d4cac66242..37a69cf419 100644 --- a/tests/intel_owl/test_tasks.py +++ b/tests/intel_owl/test_tasks.py @@ -4,8 +4,9 @@ from django.test import override_settings from kombu import uuid +from api_app.analyzables_manager.models import Analyzable from api_app.analyzers_manager.models import AnalyzerConfig, AnalyzerReport -from api_app.choices import PythonModuleBasePaths +from api_app.choices import Classification, PythonModuleBasePaths from api_app.connectors_manager.models import ConnectorConfig, ConnectorReport from api_app.ingestors_manager.models import IngestorConfig, IngestorReport from api_app.models import Job, LastElasticReportUpdate, PythonModule @@ -36,8 +37,13 @@ def setUp(self): self.membership, _ = Membership.objects.get_or_create( user=self.user, organization=self.organization, is_owner=True ) + self.analyzable = Analyzable.objects.create( + name="dns.google.com", classification=Classification.DOMAIN + ) self.job = Job.objects.create( - observable_name="dns.google.com", tlp="AMBER", user=self.user + tlp="AMBER", + user=self.user, + analyzable=self.analyzable, ) AnalyzerReport.objects.create( # valid config=AnalyzerConfig.objects.get( @@ -185,6 +191,8 @@ def tearDown(self): self.user.delete() self.organization.delete() self.membership.delete() + self.job.delete() + self.analyzable.delete() @override_settings(ELASTICSEARCH_DSL_ENABLED=True) @override_settings(ELASTICSEARCH_DSL_HOST="https://elasticsearch:9200") diff --git a/tests/test_crons.py b/tests/test_crons.py index 3d182d5168..82f10a3d9f 100644 --- a/tests/test_crons.py +++ b/tests/test_crons.py @@ -5,7 +5,7 @@ from django.conf import settings from django.utils.timezone import now -from api_app.analyzers_manager.constants import ObservableTypes +from api_app.analyzables_manager.models import Analyzable from api_app.analyzers_manager.file_analyzers import quark_engine, yara_scan from api_app.analyzers_manager.models import AnalyzerConfig from api_app.analyzers_manager.observable_analyzers import ( @@ -18,7 +18,7 @@ tor, tweetfeeds, ) -from api_app.choices import PythonModuleBasePaths +from api_app.choices import Classification, PythonModuleBasePaths from api_app.models import Job, Parameter, PluginConfig, PythonModule from intel_owl.tasks import check_stuck_analysis, remove_old_jobs @@ -32,11 +32,14 @@ class CronTests(CustomTestCase): def test_check_stuck_analysis(self): import datetime + an = Analyzable.objects.create( + name="8.8.8.8", + classification=Classification.IP, + ) _job = Job.objects.create( user=self.user, status=Job.STATUSES.RUNNING.value, - observable_name="8.8.8.8", - observable_classification=ObservableTypes.IP, + analyzable=an, received_request_time=now(), ) self.assertCountEqual(check_stuck_analysis(), []) @@ -54,15 +57,20 @@ def test_check_stuck_analysis(self): _job.save() self.assertCountEqual(check_stuck_analysis(check_pending=False), [_job.pk]) _job.delete() + an.delete() def test_remove_old_jobs(self): import datetime + an = Analyzable.objects.create( + name="8.8.8.8", + classification=Classification.IP, + ) + _job = Job.objects.create( user=self.user, status=Job.STATUSES.FAILED.value, - observable_name="8.8.8.8", - observable_classification=ObservableTypes.IP, + analyzable=an, received_request_time=now(), finished_analysis_time=now(), ) @@ -73,6 +81,7 @@ def test_remove_old_jobs(self): self.assertEqual(remove_old_jobs(), 1) _job.delete() + an.delete() @if_mock_connections(skip("not working without connection")) def test_maxmind_updater(self): diff --git a/tests/test_files.zip b/tests/test_files.zip index 55cb1d19b3..8447c1828a 100644 Binary files a/tests/test_files.zip and b/tests/test_files.zip differ