Added the ability to export the current filtered QuerySet of a 'FilterView' into the JSON format, Solves issue #1319

Signed-off-by: Aayush Kumar <[email protected]>

Author: aayushkdev

scanpipe/pipes/output.py

@@ -280,6 +280,16 @@ def get_relations(self, project):
         )
 
 
+JSON_EXCLUDE_FIELDS = [
+    "extra_data",
+    "package_data",
+    "license_detections",
+    "other_license_detections",
+    "license_clues",
+    "affected_by_vulnerabilities",
+]
+
+
 def to_json(project):
     """
     Generate output for the provided `project` in JSON format.

scanpipe/templates/scanpipe/dropdowns/list_actions_dropdown.html

@@ -10,7 +10,10 @@
   <div class="dropdown-menu" id="dropdown-menu-action" role="menu">
     <div class="dropdown-content">
       <a href="?{{ export_xlsx_url_query }}" class="dropdown-item">
-        <i class="fa-solid fa-download mr-2"></i>Export results as XLSX
+        <i class="fa-solid fa-file-excel mr-2"></i>Export results as XLSX
+      </a>
+      <a href="?{{ export_json_url_query }}" class="dropdown-item">
+        <i class="fa-solid fa-file-pdf mr-2"></i>Export results as JSON
       </a>
     </div>
   </div>

scanpipe/tests/test_views.py

@@ -29,6 +29,7 @@
 
 from django.apps import apps
 from django.core.exceptions import SuspiciousFileOperation
+from django.http import FileResponse
 from django.http.response import Http404
 from django.test import TestCase
 from django.test import override_settings
@@ -1325,3 +1326,88 @@ def test_scanpipe_policies_broken_policies_project_details(self):
         response = self.client.get(url)
         self.assertEqual(200, response.status_code)
         self.assertContains(response, "Policies file format error")
+
+    def test_scanpipe_views_export_json_returns_valid_response(self):
+        url = reverse("project_resources", args=[self.project1.slug])
+        response = self.client.get(url + "?export_json=True")
+
+        self.assertIsInstance(response, FileResponse)
+        self.assertEqual(response.get("Content-Type"), "application/json")
+        self.assertTrue(response.get("Content-Disposition").startswith("attachment"))
+
+    def test_scanpipe_views_export_json_correct_filename(self):
+        url = reverse("project_resources", args=[self.project1.slug])
+        response = self.client.get(url + "?export_json=True")
+
+        actual_filename = response.get("Content-Disposition")
+        expected_filename = (
+            f'attachment; filename="{self.project1.name}_codebaseresource.json"'
+        )
+        self.assertEqual(actual_filename, expected_filename)
+
+    def test_scanpipe_views_export_json_contains_expected_fields(self):
+        make_resource_file(self.project1, "file1.txt")
+        url = reverse("project_resources", args=[self.project1.slug])
+        response = self.client.get(url + "?export_json=True")
+
+        file_content = b"".join(response.streaming_content).decode("utf-8")
+        json_data = json.loads(file_content)
+
+        expected_fields = [
+            "path",
+            "type",
+            "name",
+            "status",
+            "for_packages",
+            "tag",
+            "extension",
+            "size",
+            "md5",
+            "sha1",
+            "sha256",
+            "sha512",
+            "mime_type",
+            "file_type",
+            "programming_language",
+            "is_binary",
+            "is_text",
+            "is_archive",
+            "is_media",
+            "is_legal",
+            "is_manifest",
+            "is_readme",
+            "is_top_level",
+            "is_key_file",
+            "detected_license_expression",
+            "detected_license_expression_spdx",
+            "percentage_of_license_text",
+            "compliance_alert",
+            "copyrights",
+            "holders",
+            "authors",
+            "emails",
+            "urls",
+        ]
+
+        for field in expected_fields:
+            self.assertIn(field, json_data)
+
+    def test_scanpipe_views_export_json_excludes_fields(self):
+        make_resource_file(self.project1, "file1.txt")
+        url = reverse("project_resources", args=[self.project1.slug])
+        response = self.client.get(url + "?export_json=True")
+
+        file_content = b"".join(response.streaming_content).decode("utf-8")
+        json_data = json.loads(file_content)
+
+        excluded_fields = [
+            "extra_data",
+            "package_data",
+            "license_detections",
+            "other_license_detections",
+            "license_clues",
+            "affected_by_vulnerabilities",
+        ]
+
+        for field in excluded_fields:
+            self.assertNotIn(field, json_data)

scanpipe/views.py

@@ -498,6 +498,69 @@ def get(self, request, *args, **kwargs):
         return response
 
 
+class ExportJSONMixin:
+    """
+    Add the ability to export the current filtered QuerySet of a `FilterView`
+     into JSON format.
+    """
+
+    export_json_query_param = "export_json"
+
+    def get_export_json_queryset(self):
+        return self.filterset.qs
+
+    def get_export_json_filename(self):
+        return f"{self.project.name}_{self.model._meta.model_name}.json"
+
+    def get_filtered_files(self, queryset):
+        from scanpipe.api.serializers import CodebaseResourceSerializer
+
+        yield from self.encode_queryset(queryset, CodebaseResourceSerializer)
+
+    def export_json_file_response(self):
+        queryset = self.get_export_json_queryset()
+
+        output_file = io.BytesIO()
+
+        for chunk in self.get_filtered_files(queryset):
+            output_file.write(chunk.encode("utf-8"))
+        output_file.seek(0)
+
+        return FileResponse(
+            output_file,
+            as_attachment=True,
+            filename=self.get_export_json_filename(),
+            content_type="application/json",
+        )
+
+    def encode_queryset(self, queryset, serializer_class):
+        for obj in queryset.iterator(chunk_size=2000):
+            serialized_obj = serializer_class(obj)
+            data = serialized_obj.data
+
+            for field in output.JSON_EXCLUDE_FIELDS:
+                data.pop(field, None)
+
+            yield json.dumps(data, indent=2)
+
+    def get_context_data(self, **kwargs):
+        context = super().get_context_data(**kwargs)
+
+        query_dict = self.request.GET.copy()
+        query_dict[self.export_json_query_param] = True
+        context["export_json_url_query"] = query_dict.urlencode()
+
+        return context
+
+    def get(self, request, *args, **kwargs):
+        response = super().get(request, *args, **kwargs)
+
+        if request.GET.get(self.export_json_query_param):
+            return self.export_json_file_response()
+
+        return response
+
+
 class FormAjaxMixin:
     def is_xhr(self):
         return self.request.META.get("HTTP_X_REQUESTED_WITH") == "XMLHttpRequest"
@@ -1542,6 +1605,7 @@ class CodebaseResourceListView(
     ProjectRelatedViewMixin,
     TableColumnsMixin,
     ExportXLSXMixin,
+    ExportJSONMixin,
     PaginatedFilterView,
 ):
     model = CodebaseResource
@@ -1615,6 +1679,7 @@ class DiscoveredPackageListView(
     ProjectRelatedViewMixin,
     TableColumnsMixin,
     ExportXLSXMixin,
+    ExportJSONMixin,
     PaginatedFilterView,
 ):
     model = DiscoveredPackage
@@ -1671,6 +1736,7 @@ class DiscoveredDependencyListView(
     ProjectRelatedViewMixin,
     TableColumnsMixin,
     ExportXLSXMixin,
+    ExportJSONMixin,
     PaginatedFilterView,
 ):
     model = DiscoveredDependency
@@ -1740,6 +1806,7 @@ class ProjectMessageListView(
     ProjectRelatedViewMixin,
     TableColumnsMixin,
     ExportXLSXMixin,
+    ExportJSONMixin,
     FilterView,
 ):
     model = ProjectMessage
@@ -1764,6 +1831,7 @@ class CodebaseRelationListView(
     PrefetchRelatedViewMixin,
     TableColumnsMixin,
     ExportXLSXMixin,
+    ExportJSONMixin,
     PaginatedFilterView,
 ):
     model = CodebaseRelation