diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 17b6a9d6c6..31c9b1b847 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,9 @@ v34.10.2 (unreleased) if available, when the codebase has been sent for matching to MatchCode. https://github.com/aboutcode-org/scancode.io/pull/1656 +- Add the ability to export filtered QuerySet of a FilterView into the JSON format. + https://github.com/aboutcode-org/scancode.io/pull/1572 + v34.10.1 (2025-03-26) --------------------- diff --git a/scanpipe/templates/scanpipe/dropdowns/list_actions_dropdown.html b/scanpipe/templates/scanpipe/dropdowns/list_actions_dropdown.html index c301dbf953..463f249261 100644 --- a/scanpipe/templates/scanpipe/dropdowns/list_actions_dropdown.html +++ b/scanpipe/templates/scanpipe/dropdowns/list_actions_dropdown.html @@ -10,7 +10,10 @@ diff --git a/scanpipe/tests/test_views.py b/scanpipe/tests/test_views.py index 2db25414ec..57e43ddce4 100644 --- a/scanpipe/tests/test_views.py +++ b/scanpipe/tests/test_views.py @@ -29,6 +29,7 @@ from django.apps import apps from django.core.exceptions import SuspiciousFileOperation +from django.http import FileResponse from django.http.response import Http404 from django.test import TestCase from django.test import override_settings @@ -1340,3 +1341,210 @@ def test_scanpipe_views_codebase_resource_details_get_matched_snippet_annotation results = CodebaseResourceDetailsView.get_matched_snippet_annotations(resource1) expected_results = [{"start_line": 1, "end_line": 6}] self.assertEqual(expected_results, results) + + def test_project_packages_export_json(self): + make_package(self.project1, package_url="pkg:type/a") + + url = reverse("project_packages", args=[self.project1.slug]) + response = self.client.get(url + "?export_json=True") + + self.assertIsInstance(response, FileResponse) + self.assertEqual(response.get("Content-Type"), "application/json") + self.assertTrue(response.get("Content-Disposition").startswith("attachment")) + + file_content = b"".join(response.streaming_content).decode("utf-8") + json_data = json.loads(file_content) + + expected_fields = [ + "purl", + "type", + "namespace", + "name", + "version", + "qualifiers", + "subpath", + "tag", + "primary_language", + "description", + "notes", + "release_date", + "parties", + "keywords", + "homepage_url", + "download_url", + "bug_tracking_url", + "code_view_url", + "vcs_url", + "repository_homepage_url", + "repository_download_url", + "api_data_url", + "size", + "md5", + "sha1", + "sha256", + "sha512", + "copyright", + "holder", + "declared_license_expression", + "declared_license_expression_spdx", + "other_license_expression", + "other_license_expression_spdx", + "extracted_license_statement", + "compliance_alert", + "notice_text", + "source_packages", + "package_uid", + "is_private", + "is_virtual", + "datasource_ids", + "datafile_paths", + "file_references", + "missing_resources", + "modified_resources", + ] + + for field in expected_fields: + self.assertIn(field, json_data[0]) + + def test_project_dependencies_export_json(self): + make_resource_file(self.project1, "file.ext") + make_dependency(self.project1) + + url = reverse("project_dependencies", args=[self.project1.slug]) + response = self.client.get(url + "?export_json=True") + + self.assertIsInstance(response, FileResponse) + self.assertEqual(response.get("Content-Type"), "application/json") + self.assertTrue(response.get("Content-Disposition").startswith("attachment")) + + file_content = b"".join(response.streaming_content).decode("utf-8") + json_data = json.loads(file_content) + + expected_fields = [ + "purl", + "extracted_requirement", + "scope", + "is_runtime", + "is_optional", + "is_pinned", + "is_direct", + "dependency_uid", + "for_package_uid", + "resolved_to_package_uid", + "datafile_path", + "datasource_id", + "package_type", + ] + + for field in expected_fields: + self.assertIn(field, json_data[0]) + + def test_project_relations_export_json(self): + make_relation( + from_resource=make_resource_file(self.project1, "file1.ext"), + to_resource=make_resource_file(self.project1, "file2.ext"), + map_type="path", + ) + + url = reverse("project_relations", args=[self.project1.slug]) + response = self.client.get(url + "?export_json=True") + + self.assertIsInstance(response, FileResponse) + self.assertEqual(response.get("Content-Type"), "application/json") + self.assertTrue(response.get("Content-Disposition").startswith("attachment")) + + file_content = b"".join(response.streaming_content).decode("utf-8") + json_data = json.loads(file_content) + + expected_fields = [ + "to_resource", + "status", + "map_type", + "score", + "from_resource", + ] + + for field in expected_fields: + self.assertIn(field, json_data[0]) + + def test_project_messages_export_json(self): + self.project1.add_message("warning") + + url = reverse("project_messages", args=[self.project1.slug]) + response = self.client.get(url + "?export_json=True") + + self.assertIsInstance(response, FileResponse) + self.assertEqual(response.get("Content-Type"), "application/json") + self.assertTrue(response.get("Content-Disposition").startswith("attachment")) + + file_content = b"".join(response.streaming_content).decode("utf-8") + json_data = json.loads(file_content) + + expected_fields = [ + "uuid", + "severity", + "description", + "model", + "details", + "traceback", + "created_date", + ] + + for field in expected_fields: + self.assertIn(field, json_data[0]) + + def test_project_codebase_resources_export_json(self): + make_resource_file(self.project1, "file.ext") + + url = reverse("project_resources", args=[self.project1.slug]) + response = self.client.get(url + "?export_json=True") + + self.assertIsInstance(response, FileResponse) + self.assertEqual(response.get("Content-Type"), "application/json") + self.assertTrue(response.get("Content-Disposition").startswith("attachment")) + + file_content = b"".join(response.streaming_content).decode("utf-8") + json_data = json.loads(file_content) + + expected_fields = [ + "path", + "type", + "name", + "status", + "for_packages", + "tag", + "extension", + "size", + "mime_type", + "file_type", + "programming_language", + "detected_license_expression", + "detected_license_expression_spdx", + "license_detections", + "license_clues", + "percentage_of_license_text", + "compliance_alert", + "copyrights", + "holders", + "authors", + "package_data", + "emails", + "urls", + "md5", + "sha1", + "sha256", + "sha512", + "is_binary", + "is_text", + "is_archive", + "is_media", + "is_legal", + "is_manifest", + "is_readme", + "is_top_level", + "is_key_file", + "extra_data", + ] + + for field in expected_fields: + self.assertIn(field, json_data[0]) diff --git a/scanpipe/views.py b/scanpipe/views.py index 43d12e38e0..201ad7792b 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -36,6 +36,7 @@ from django.core.exceptions import SuspiciousFileOperation from django.core.exceptions import ValidationError from django.core.files.storage.filesystem import FileSystemStorage +from django.core.serializers.json import DjangoJSONEncoder from django.db.models import Prefetch from django.db.models.manager import Manager from django.http import FileResponse @@ -499,6 +500,55 @@ def get(self, request, *args, **kwargs): return response +class ExportJSONMixin: + """ + Add the ability to export the current filtered QuerySet of a `FilterView` + into JSON format. + """ + + export_json_query_param = "export_json" + + def get_export_json_queryset(self): + return self.filterset.qs + + def get_export_json_filename(self): + return f"{self.project.name}_{self.model._meta.model_name}.json" + + def export_json_file_response(self): + from scanpipe.api.serializers import get_model_serializer + + queryset = self.get_export_json_queryset() + serializer_class = get_model_serializer(queryset.model) + serializer = serializer_class(queryset, many=True) + serialized_data = json.dumps(serializer.data, indent=2, cls=DjangoJSONEncoder) + + output_file = io.BytesIO(serialized_data.encode("utf-8")) + + return FileResponse( + output_file, + as_attachment=True, + filename=self.get_export_json_filename(), + content_type="application/json", + ) + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + + query_dict = self.request.GET.copy() + query_dict[self.export_json_query_param] = True + context["export_json_url_query"] = query_dict.urlencode() + + return context + + def get(self, request, *args, **kwargs): + response = super().get(request, *args, **kwargs) + + if request.GET.get(self.export_json_query_param): + return self.export_json_file_response() + + return response + + class FormAjaxMixin: def is_xhr(self): return self.request.META.get("HTTP_X_REQUESTED_WITH") == "XMLHttpRequest" @@ -1543,6 +1593,7 @@ class CodebaseResourceListView( ProjectRelatedViewMixin, TableColumnsMixin, ExportXLSXMixin, + ExportJSONMixin, PaginatedFilterView, ): model = CodebaseResource @@ -1616,6 +1667,7 @@ class DiscoveredPackageListView( ProjectRelatedViewMixin, TableColumnsMixin, ExportXLSXMixin, + ExportJSONMixin, PaginatedFilterView, ): model = DiscoveredPackage @@ -1672,6 +1724,7 @@ class DiscoveredDependencyListView( ProjectRelatedViewMixin, TableColumnsMixin, ExportXLSXMixin, + ExportJSONMixin, PaginatedFilterView, ): model = DiscoveredDependency @@ -1741,6 +1794,7 @@ class ProjectMessageListView( ProjectRelatedViewMixin, TableColumnsMixin, ExportXLSXMixin, + ExportJSONMixin, FilterView, ): model = ProjectMessage @@ -1765,6 +1819,7 @@ class CodebaseRelationListView( PrefetchRelatedViewMixin, TableColumnsMixin, ExportXLSXMixin, + ExportJSONMixin, PaginatedFilterView, ): model = CodebaseRelation