Deprecate and remove TalosReputation. Closes #3451

Author: cipherBT

api_app/analyzers_manager/migrations/0180_remove_talos_reputation.py

@@ -0,0 +1,86 @@
+# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
+# See the file 'LICENSE' for copying permission.
+
+
+from django.db import migrations
+
+
+def _cleanup_schedule(schedule, field_name, PeriodicTask, PythonModule, is_crontab):
+    """Delete a schedule if it is no longer referenced by any task or module."""
+    if schedule is None:
+        return
+    still_used = PeriodicTask.objects.filter(**{field_name: schedule}).exists()
+    if not still_used and is_crontab:
+        still_used = (
+            PythonModule.objects.filter(update_schedule=schedule).exists()
+            or PythonModule.objects.filter(health_check_schedule=schedule).exists()
+        )
+    if not still_used:
+        try:
+            schedule.delete()
+        except Exception:
+            pass  # Another model (e.g. IngestorConfig) may still reference this schedule
+
+
+def remove_talos_forward(apps, schema_editor):
+    AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig")
+    PythonModule = apps.get_model("api_app", "PythonModule")
+    PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask")
+    IntervalSchedule = apps.get_model("django_celery_beat", "IntervalSchedule")
+    CrontabSchedule = apps.get_model("django_celery_beat", "CrontabSchedule")
+    SolarSchedule = apps.get_model("django_celery_beat", "SolarSchedule")
+    ClockedSchedule = apps.get_model("django_celery_beat", "ClockedSchedule")
+
+    # Identify Talos PythonModule instances first
+    module_qs = PythonModule.objects.filter(
+        module="talos.Talos",
+        base_path="api_app.analyzers_manager.observable_analyzers",
+    )
+
+    # Remove all analyzer configs referencing these modules
+    AnalyzerConfig.objects.filter(python_module__in=module_qs).delete()
+
+    for module in module_qs:
+        update_task_id = getattr(module, "update_task_id", None)
+        if update_task_id is not None:
+            try:
+                task = PeriodicTask.objects.get(id=update_task_id)
+            except PeriodicTask.DoesNotExist:
+                task = None
+            if task is not None:
+                schedule_data = (
+                    ("interval", IntervalSchedule, task.interval),
+                    ("crontab", CrontabSchedule, task.crontab),
+                    ("solar", SolarSchedule, task.solar),
+                    ("clocked", ClockedSchedule, task.clocked),
+                )
+                task.delete()
+                for field_name, schedule_model, schedule in schedule_data:
+                    _cleanup_schedule(
+                        schedule,
+                        field_name,
+                        PeriodicTask,
+                        PythonModule,
+                        is_crontab=(schedule_model is CrontabSchedule),
+                    )
+        # Capture modules own schedule before deletion
+        module_schedules = [
+            getattr(module, attr, None)
+            for attr in ("update_schedule", "health_check_schedule")
+        ]
+        module.delete()
+        # Clean up orphaned CrontabSchedules from the module
+        for sched in module_schedules:
+            _cleanup_schedule(sched, "crontab", PeriodicTask, PythonModule, is_crontab=True)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("analyzers_manager", "0179_add_local_db_models_tor_danmeuk"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            remove_talos_forward, reverse_code=migrations.RunPython.noop
+        ),
+    ]

api_app/analyzers_manager/observable_analyzers/talos.py

@@ -1,67 +1,21 @@
 # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
 # See the file 'LICENSE' for copying permission.
 
-import logging
-import os
-
-import requests
-from django.conf import settings
-
-from api_app.analyzers_manager import classes
+"""
+Minimal stub kept so that historical data migration
+0002_0116_analyzer_config_talosreputation can still import the module
+path during full_clean().  The analyzer itself is removed; migration
+0180_remove_talos_reputation deletes the DB rows.
+"""
+
+from api_app.analyzers_manager.classes import ObservableAnalyzer
 from api_app.analyzers_manager.exceptions import AnalyzerRunException
 
-logger = logging.getLogger(__name__)
-
-db_name = "talos_ip_blacklist.txt"
-database_location = f"{settings.MEDIA_ROOT}/{db_name}"
-
 
-class Talos(classes.ObservableAnalyzer):
+class Talos(ObservableAnalyzer):
     def run(self):
-        result = {"found": False}
-        if not os.path.isfile(database_location) and not self.update():
-            raise AnalyzerRunException("Failed extraction of talos db")
-
-        if not os.path.exists(database_location):
-            raise AnalyzerRunException(f"database location {database_location} does not exist")
-
-        with open(database_location, "r", encoding="utf-8") as f:
-            db = f.read()
-
-        db_list = db.split("\n")
-        if self.observable_name in db_list:
-            result["found"] = True
-
-        return result
+        raise AnalyzerRunException("TalosReputation has been deprecated and removed.")
 
     @classmethod
-    def update(cls) -> bool:
-        try:
-            logger.info("starting download of db from talos")
-            url = "https://snort.org/downloads/ip-block-list"
-            r = requests.get(url)
-            r.raise_for_status()
-
-            with open(database_location, "w", encoding="utf-8") as f:
-                f.write(r.content.decode())
-
-            if not os.path.exists(database_location):
-                return False
-            logger.info("ended download of db from talos")
-            return True
-        except Exception as e:
-            logger.exception(e)
-
+    def update(cls):
         return False
-
-    def _do_create_data_model(self):
-        return super()._do_create_data_model()
-
-    def _update_data_model(self, data_model):
-        super()._update_data_model(data_model)
-        found = self.report.report.get("found", False)
-        if found:
-            data_model.external_references.append(
-                f"https://www.talosintelligence.com/reputation_center/lookup?search={self.report.job.analyzable.name}"
-            )
-            data_model.evaluation = self.EVALUATIONS.MALICIOUS.value

api_app/playbooks_manager/migrations/0067_remove_talos_from_playbooks.py

@@ -0,0 +1,40 @@
+# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
+# See the file 'LICENSE' for copying permission.
+
+
+from django.db import migrations
+
+
+def remove_talos_from_playbooks(apps, schema_editor):
+    PlaybookConfig = apps.get_model("playbooks_manager", "PlaybookConfig")
+    AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig")
+
+    try:
+        talos_analyzer = AnalyzerConfig.objects.get(name="TalosReputation")
+    except AnalyzerConfig.DoesNotExist:
+        talos_analyzer = None
+
+    if talos_analyzer is not None:
+        for playbook in PlaybookConfig.objects.filter(analyzers=talos_analyzer):
+            playbook.analyzers.remove(talos_analyzer)
+
+    # Also scrub runtime_configuration if present
+    for playbook in PlaybookConfig.objects.all():
+        rc = playbook.runtime_configuration
+        if isinstance(rc, dict) and "analyzers" in rc and "TalosReputation" in rc["analyzers"]:
+            del rc["analyzers"]["TalosReputation"]
+            playbook.runtime_configuration = rc
+            playbook.save(update_fields=["runtime_configuration"])
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("playbooks_manager", "0066_link_crawl_visualizer_to_playbook"),
+        ("analyzers_manager", "0179_add_local_db_models_tor_danmeuk"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            remove_talos_from_playbooks, reverse_code=migrations.RunPython.noop
+        ),
+    ]

api_app/visualizers_manager/visualizers/ip_reputation_services.py

@@ -329,20 +329,6 @@ def _tor(self):
             )
             return tor_report
 
-    @visualizable_error_handler_with_params("Talos Reputation")
-    def _talos(self):
-        try:
-            analyzer_report = self.get_analyzer_reports().get(config__name="TalosReputation")
-        except AnalyzerReport.DoesNotExist:
-            logger.warning("TalosReputation report does not exist")
-        else:
-            found = analyzer_report.report.get("found", False)
-            talos_report = self.Bool(
-                value="Talos Reputation",
-                disable=not (analyzer_report.status == ReportStatus.SUCCESS and found),
-            )
-            return talos_report
-
     def run(self) -> List[Dict]:
         first_level_elements = []
         second_level_elements = []
@@ -378,8 +364,6 @@ def run(self) -> List[Dict]:
 
         third_level_elements.append(self._tor())
 
-        third_level_elements.append(self._talos())
-
         page = self.Page(name="Reputation")
         page.add_level(
             self.Level(

docker/scripts/cron/update_repositories

@@ -1,5 +1,4 @@
 0 1 * * 3  root python3 docker exec -ti $(docker ps | grep intelowl_celery_worker_default | awk '{print $1}') python3 manage.py update_analyzer MaxMindGeoIP
-5 */6 * * *  root python3 docker exec -ti $(docker ps | grep intelowl_celery_worker_default | awk '{print $1}') python3 manage.py update_analyzer TalosReputation
 5 */6 * * *  root python3 docker exec -ti $(docker ps | grep intelowl_celery_worker_default | awk '{print $1}') python3 manage.py update_analyzer PhishingArmy
 */10 * * * *  root python3 docker exec -ti $(docker ps | grep intelowl_celery_worker_default | awk '{print $1}') python3 manage.py update_analyzer TorProject
 0 4 * * *  root python3 docker exec -ti $(docker ps | grep intelowl_celery_worker_default | awk '{print $1}') python3 manage.py update_analyzer Yara

tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_talos.py

@@ -14,7 +14,6 @@
     ja4_db,
     maxmind,
     phishing_army,
-    talos,
     tor,
     tor_nodes_danmeuk,
     tweetfeeds,
@@ -90,11 +89,6 @@ def test_maxmind_updater(self):
         for db in maxmind.Maxmind.get_db_names():
             self.assertTrue(os.path.exists(db))
 
-    @if_mock_connections(patch("requests.get", return_value=MockUpResponse({}, 200, text="91.192.100.61")))
-    def test_talos_updater(self, mock_get=None):
-        db_file_path = talos.Talos.update()
-        self.assertTrue(os.path.exists(db_file_path))
-
     @if_mock_connections(
         patch(
             "requests.get",