made the implementation of ExportJSONMixin simpler by removing streaming of data and removed exclude_json_fields

Signed-off-by: Aayush Kumar <[email protected]>

Author: aayushkdev

CHANGELOG.rst

@@ -13,6 +13,9 @@ v34.10.2 (unreleased)
   if available, when the codebase has been sent for matching to MatchCode.
   https://github.com/aboutcode-org/scancode.io/pull/1656
 
+- Add the ability to export filtered QuerySet of a FilterView into the JSON format.
+  https://github.com/aboutcode-org/scancode.io/pull/1572
+
 v34.10.1 (2025-03-26)
 ---------------------
 

scanpipe/pipes/output.py

@@ -280,16 +280,6 @@ def get_relations(self, project):
         )
 
 
-JSON_EXCLUDE_FIELDS = [
-    "extra_data",
-    "package_data",
-    "license_detections",
-    "other_license_detections",
-    "license_clues",
-    "affected_by_vulnerabilities",
-]
-
-
 def to_json(project):
     """
     Generate output for the provided `project` in JSON format.

scanpipe/tests/test_views.py

@@ -1404,7 +1404,7 @@ def test_project_packages_export_json(self):
         ]
 
         for field in expected_fields:
-            self.assertIn(field, json_data)
+            self.assertIn(field, json_data[0])
 
     def test_project_dependencies_export_json(self):
         make_resource_file(self.project1, "file.ext")
@@ -1437,7 +1437,7 @@ def test_project_dependencies_export_json(self):
         ]
 
         for field in expected_fields:
-            self.assertIn(field, json_data)
+            self.assertIn(field, json_data[0])
 
     def test_project_relations_export_json(self):
         make_relation(
@@ -1465,7 +1465,7 @@ def test_project_relations_export_json(self):
         ]
 
         for field in expected_fields:
-            self.assertIn(field, json_data)
+            self.assertIn(field, json_data[0])
 
     def test_project_messages_export_json(self):
         self.project1.add_message("warning")
@@ -1491,7 +1491,7 @@ def test_project_messages_export_json(self):
         ]
 
         for field in expected_fields:
-            self.assertIn(field, json_data)
+            self.assertIn(field, json_data[0])
 
     def test_project_codebase_resources_export_json(self):
         make_resource_file(self.project1, "file.ext")
@@ -1515,13 +1515,25 @@ def test_project_codebase_resources_export_json(self):
             "tag",
             "extension",
             "size",
+            "mime_type",
+            "file_type",
+            "programming_language",
+            "detected_license_expression",
+            "detected_license_expression_spdx",
+            "license_detections",
+            "license_clues",
+            "percentage_of_license_text",
+            "compliance_alert",
+            "copyrights",
+            "holders",
+            "authors",
+            "package_data",
+            "emails",
+            "urls",
             "md5",
             "sha1",
             "sha256",
             "sha512",
-            "mime_type",
-            "file_type",
-            "programming_language",
             "is_binary",
             "is_text",
             "is_archive",
@@ -1531,37 +1543,8 @@ def test_project_codebase_resources_export_json(self):
             "is_readme",
             "is_top_level",
             "is_key_file",
-            "detected_license_expression",
-            "detected_license_expression_spdx",
-            "percentage_of_license_text",
-            "compliance_alert",
-            "copyrights",
-            "holders",
-            "authors",
-            "emails",
-            "urls",
-        ]
-
-        for field in expected_fields:
-            self.assertIn(field, json_data)
-
-    def test_scanpipe_views_export_json_excludes_fields(self):
-        make_resource_file(self.project1, "file.ext")
-
-        url = reverse("project_resources", args=[self.project1.slug])
-        response = self.client.get(url + "?export_json=True")
-
-        file_content = b"".join(response.streaming_content).decode("utf-8")
-        json_data = json.loads(file_content)
-
-        excluded_fields = [
             "extra_data",
-            "package_data",
-            "license_detections",
-            "other_license_detections",
-            "license_clues",
-            "affected_by_vulnerabilities",
         ]
 
-        for field in excluded_fields:
-            self.assertNotIn(field, json_data)
+        for field in expected_fields:
+            self.assertIn(field, json_data[0])

scanpipe/views.py

@@ -36,6 +36,7 @@
 from django.core.exceptions import SuspiciousFileOperation
 from django.core.exceptions import ValidationError
 from django.core.files.storage.filesystem import FileSystemStorage
+from django.core.serializers.json import DjangoJSONEncoder
 from django.db.models import Prefetch
 from django.db.models.manager import Manager
 from django.http import FileResponse
@@ -513,25 +514,15 @@ def get_export_json_queryset(self):
     def get_export_json_filename(self):
         return f"{self.project.name}_{self.model._meta.model_name}.json"
 
-    def get_filtered_files(self, queryset):
+    def export_json_file_response(self):
         from scanpipe.api.serializers import get_model_serializer
 
-        serializer_class = get_model_serializer(queryset.model)
-
-        yield from self.encode_queryset(queryset, serializer_class)
-
-    def export_json_file_response(self):
         queryset = self.get_export_json_queryset()
+        serializer_class = get_model_serializer(queryset.model)
+        serializer = serializer_class(queryset, many=True)
+        serialized_data = json.dumps(serializer.data, indent=2, cls=DjangoJSONEncoder)
 
-        output_file = io.BytesIO()
-
-        first = True
-        for chunk in self.get_filtered_files(queryset):
-            if not first:
-                output_file.write(b",\n")
-            output_file.write(chunk.encode("utf-8"))
-            first = False
-        output_file.seek(0)
+        output_file = io.BytesIO(serialized_data.encode("utf-8"))
 
         return FileResponse(
             output_file,
@@ -540,14 +531,6 @@ def export_json_file_response(self):
             content_type="application/json",
         )
 
-    def encode_queryset(self, queryset, serializer_class):
-        for obj in queryset.iterator(chunk_size=2000):
-            data = serializer_class(obj).data
-            for field in output.JSON_EXCLUDE_FIELDS:
-                data.pop(field, None)
-
-            yield json.dumps(data, indent=2)
-
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)