Datamodel visualizer (#2600)

* fix abuseipdb

* added datamodel visualizer

* update datamodel visualizer

* Fix datamodel creation

* Updates visualizer

* Updates visualizer

* Updates data model visualizer

* Updates data model visualizer

* Added pdns table to data model visualizer

* Fix pdns table to data model visualizer

* Added signature table to data model visualizer

* Updates signature table

* Updates pdns table

* Fix signature table

* Fix level size

* Fix visualizer

* added _mokeypatch

* added migration

* fix signature table

* fixes deepsource

* fix

Author: cristinaascari

api_app/visualizers_manager/classes.py

@@ -548,3 +548,10 @@ def pivots_reports(self) -> QuerySet:
         from api_app.pivots_manager.models import PivotReport
 
         return PivotReport.objects.filter(job=self._job)
+
+    def data_models(self) -> QuerySet:
+        from api_app.analyzers_manager.models import AnalyzerReport
+
+        data_model_class = AnalyzerReport.get_data_model_class(self._job)
+        analyzer_reports_pk = [report.pk for report in self.analyzer_reports()]
+        return data_model_class.objects.filter(analyzers_report__in=analyzer_reports_pk)

api_app/visualizers_manager/migrations/0040_visualizer_config_data_model.py

@@ -0,0 +1,117 @@
+from django.db import migrations
+from django.db.models.fields.related_descriptors import (
+    ForwardManyToOneDescriptor,
+    ForwardOneToOneDescriptor,
+    ManyToManyDescriptor,
+    ReverseManyToOneDescriptor,
+    ReverseOneToOneDescriptor,
+)
+
+plugin = {
+    "python_module": {
+        "health_check_schedule": None,
+        "update_schedule": None,
+        "module": "data_model.DataModel",
+        "base_path": "api_app.visualizers_manager.visualizers",
+    },
+    "playbooks": ["FREE_TO_USE_ANALYZERS"],
+    "name": "Data_Model",
+    "description": "Visualizer for Data Models",
+    "disabled": False,
+    "soft_time_limit": 60,
+    "routing_key": "default",
+    "health_check_status": True,
+    "model": "visualizers_manager.VisualizerConfig",
+}
+
+params = []
+
+values = []
+
+
+def _get_real_obj(Model, field, value):
+    def _get_obj(Model, other_model, value):
+        if isinstance(value, dict):
+            real_vals = {}
+            for key, real_val in value.items():
+                real_vals[key] = _get_real_obj(other_model, key, real_val)
+            value = other_model.objects.get_or_create(**real_vals)[0]
+        # it is just the primary key serialized
+        else:
+            if isinstance(value, int):
+                if Model.__name__ == "PluginConfig":
+                    value = other_model.objects.get(name=plugin["name"])
+                else:
+                    value = other_model.objects.get(pk=value)
+            else:
+                value = other_model.objects.get(name=value)
+        return value
+
+    if (
+        type(getattr(Model, field))
+        in [
+            ForwardManyToOneDescriptor,
+            ReverseManyToOneDescriptor,
+            ReverseOneToOneDescriptor,
+            ForwardOneToOneDescriptor,
+        ]
+        and value
+    ):
+        other_model = getattr(Model, field).get_queryset().model
+        value = _get_obj(Model, other_model, value)
+    elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value:
+        other_model = getattr(Model, field).rel.model
+        value = [_get_obj(Model, other_model, val) for val in value]
+    return value
+
+
+def _create_object(Model, data):
+    mtm, no_mtm = {}, {}
+    for field, value in data.items():
+        value = _get_real_obj(Model, field, value)
+        if type(getattr(Model, field)) is ManyToManyDescriptor:
+            mtm[field] = value
+        else:
+            no_mtm[field] = value
+    try:
+        o = Model.objects.get(**no_mtm)
+    except Model.DoesNotExist:
+        o = Model(**no_mtm)
+        o.full_clean()
+        o.save()
+        for field, value in mtm.items():
+            attribute = getattr(o, field)
+            if value is not None:
+                attribute.set(value)
+        return False
+    return True
+
+
+def migrate(apps, schema_editor):
+    Parameter = apps.get_model("api_app", "Parameter")
+    PluginConfig = apps.get_model("api_app", "PluginConfig")
+    python_path = plugin.pop("model")
+    Model = apps.get_model(*python_path.split("."))
+    if not Model.objects.filter(name=plugin["name"]).exists():
+        exists = _create_object(Model, plugin)
+        if not exists:
+            for param in params:
+                _create_object(Parameter, param)
+            for value in values:
+                _create_object(PluginConfig, value)
+
+
+def reverse_migrate(apps, schema_editor):
+    python_path = plugin.pop("model")
+    Model = apps.get_model(*python_path.split("."))
+    Model.objects.get(name=plugin["name"]).delete()
+
+
+class Migration(migrations.Migration):
+    atomic = False
+    dependencies = [
+        ("api_app", "0064_vt_sample_download"),
+        ("visualizers_manager", "0039_sample_download"),
+    ]
+
+    operations = [migrations.RunPython(migrate, reverse_migrate)]