Add Scan realted actions in REST API packages endpoint

Signed-off-by: tdruez <[email protected]>

Author: tdruez

CHANGELOG.rst

@@ -7,6 +7,14 @@ Release notes
   https://github.com/aboutcode-org/dejacode/pull/315
   https://github.com/aboutcode-org/dejacode/pull/312
 
+- Add REST API "actions" in package endpoint to track the scan status and fetch results:
+  * `/packages/{uuid}/scan_info/` Scan information including the current status.
+  * `/packages/{uuid}/scan_results/` Scan results.
+  * `/packages/{uuid}/scan_summary/` Scan summary.
+  * `/packages/{uuid}/scan_data_download_zip/` Download all scan data: results and
+  summary, as a zip file.
+  https://github.com/aboutcode-org/dejacode/issues/272
+
 - Add new `is_locked` "Locked inventory" field to the ProductStatus model.
   When a Product is locked through his status, its inventory cannot be modified.
   https://github.com/aboutcode-org/dejacode/issues/189

component_catalog/api.py

@@ -10,12 +10,15 @@
 
 from django.db import transaction
 from django.forms.widgets import HiddenInput
+from django.http import FileResponse
 
 import django_filters
 from packageurl.contrib import url2purl
 from packageurl.contrib.django.filters import PackageURLFilter
 from rest_framework import serializers
+from rest_framework import status
 from rest_framework.decorators import action
+from rest_framework.exceptions import APIException
 from rest_framework.fields import ListField
 from rest_framework.response import Response
 
@@ -864,6 +867,16 @@ def collect_create_scan(download_url, user):
     return package
 
 
+class ScanCodeUnavailable(APIException):
+    status_code = status.HTTP_400_BAD_REQUEST
+    default_detail = "The ScanCode.io service is not available"
+
+
+class ScanDataUnavailable(APIException):
+    status_code = status.HTTP_400_BAD_REQUEST
+    default_detail = "Scan data is not available"
+
+
 class PackageViewSet(
     SendAboutFilesMixin,
     AboutCodeFilesActionMixin,
@@ -919,6 +932,73 @@ def about(self, request, uuid):
         package = self.get_object()
         return Response({"about_data": package.as_about_yaml()})
 
+    def _get_scancodeio_project_info(self, scancodeio, package):
+        if not scancodeio.is_available():
+            raise ScanCodeUnavailable()
+
+        project_info = scancodeio.get_project_info(download_url=package.download_url)
+        if not project_info:
+            raise ScanDataUnavailable()
+
+        return project_info
+
+    @action(detail=True, name="Scan informations")
+    def scan_info(self, request, uuid):
+        """Return information about the scan from ScanCode.io."""
+        package = self.get_object()
+        dataspace = request.user.dataspace
+        scancodeio = ScanCodeIO(dataspace)
+        project_info = self._get_scancodeio_project_info(scancodeio, package)
+
+        return Response(project_info)
+
+    @action(detail=True, name="Scan results")
+    def scan_results(self, request, uuid):
+        """Return the scan results from ScanCode.io."""
+        package = self.get_object()
+        dataspace = request.user.dataspace
+        scancodeio = ScanCodeIO(dataspace)
+        project_info = self._get_scancodeio_project_info(scancodeio, package)
+
+        project_uuid = project_info.get("uuid")
+        scan_results_url = scancodeio.get_scan_action_url(project_uuid, "results")
+        scan_results = scancodeio.fetch_scan_data(scan_results_url)
+
+        return Response(scan_results)
+
+    @action(detail=True, name="Scan summary")
+    def scan_summary(self, request, uuid):
+        """Return the scan summary from ScanCode.io."""
+        package = self.get_object()
+        dataspace = request.user.dataspace
+        scancodeio = ScanCodeIO(dataspace)
+        project_info = self._get_scancodeio_project_info(scancodeio, package)
+
+        project_uuid = project_info.get("uuid")
+        scan_summary_url = scancodeio.get_scan_action_url(project_uuid, "summary")
+        scan_summary = scancodeio.fetch_scan_data(scan_summary_url)
+
+        return Response(scan_summary)
+
+    @action(detail=True, name="Scan data (download as .zip)")
+    def scan_data_download_zip(self, request, uuid):
+        """Download scan data: results and summary, as a zip file."""
+        package = self.get_object()
+        dataspace = request.user.dataspace
+        scancodeio = ScanCodeIO(dataspace)
+        project_info = self._get_scancodeio_project_info(scancodeio, package)
+
+        project_uuid = project_info.get("uuid")
+        filename = package.filename or package.package_url_filename
+
+        scan_data_as_zip = scancodeio.scan_data_as_zip(project_uuid, filename)
+        return FileResponse(
+            scan_data_as_zip,
+            filename=f"{filename}_scan.zip",
+            as_attachment=True,
+            content_type="application/zip",
+        )
+
     @action(detail=False, methods=["post"], name="Package Add")
     def add(self, request):
         """

component_catalog/tests/test_api.py

@@ -1475,6 +1475,101 @@ def test_api_package_viewset_aboutcode_files_action(self):
             'attachment; filename="package1.zip_about.zip"', response["content-disposition"]
         )
 
+    @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.get_project_info")
+    @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.is_available")
+    def test_api_package_viewset_scan_actions_errors(
+        self, mock_is_available, mock_get_project_info
+    ):
+        scan_actions_urls = [
+            reverse("api_v2:package-scan-info", args=[self.package1.uuid]),
+            reverse("api_v2:package-scan-results", args=[self.package1.uuid]),
+            reverse("api_v2:package-scan-summary", args=[self.package1.uuid]),
+            reverse("api_v2:package-scan-data-download-zip", args=[self.package1.uuid]),
+        ]
+
+        mock_get_project_info.return_value = None
+        for action_url in scan_actions_urls:
+            response = self.client.get(action_url)
+            self.assertEqual(403, response.status_code, msg=action_url)
+            response = self.client.post(action_url)
+            self.assertEqual(403, response.status_code, msg=action_url)
+
+        self.client.login(username=self.base_user.username, password="secret")
+        mock_is_available.return_value = False
+        for action_url in scan_actions_urls:
+            response = self.client.get(action_url)
+            self.assertEqual(status.HTTP_400_BAD_REQUEST, response.status_code, msg=action_url)
+            expected = "The ScanCode.io service is not available"
+            self.assertEqual(expected, str(response.data["detail"]), msg=action_url)
+
+        mock_is_available.return_value = True
+        for action_url in scan_actions_urls:
+            response = self.client.get(action_url)
+            self.assertEqual(status.HTTP_400_BAD_REQUEST, response.status_code, msg=action_url)
+            expected = "Scan data is not available"
+            self.assertEqual(expected, str(response.data["detail"]), msg=action_url)
+
+    @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.get_project_info")
+    @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.is_available")
+    def test_api_package_viewset_scan_info_action(self, mock_is_available, mock_get_project_info):
+        self.client.login(username=self.base_user.username, password="secret")
+        action_url = reverse("api_v2:package-scan-info", args=[self.package1.uuid])
+        mock_is_available.return_value = True
+        project_info = {"uuid": "abcdef"}
+        mock_get_project_info.return_value = project_info
+
+        response = self.client.get(action_url)
+        self.assertEqual(200, response.status_code)
+        self.assertEqual(project_info, response.data)
+
+    @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.get_project_info")
+    @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.fetch_scan_data")
+    @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.is_available")
+    def test_api_package_viewset_scan_results_action(
+        self, mock_is_available, mock_fetch_scan_data, mock_get_project_info
+    ):
+        self.client.login(username=self.base_user.username, password="secret")
+        action_url = reverse("api_v2:package-scan-results", args=[self.package1.uuid])
+        mock_is_available.return_value = True
+        mock_get_project_info.return_value = {"uuid": "abcdef"}
+        mock_fetch_scan_data.return_value = {"results": ""}
+        response = self.client.get(action_url)
+        self.assertEqual(200, response.status_code)
+        self.assertEqual({"results": ""}, response.data)
+
+    @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.get_project_info")
+    @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.fetch_scan_data")
+    @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.is_available")
+    def test_api_package_viewset_scan_summary_action(
+        self, mock_is_available, mock_fetch_scan_data, mock_get_project_info
+    ):
+        self.client.login(username=self.base_user.username, password="secret")
+        action_url = reverse("api_v2:package-scan-summary", args=[self.package1.uuid])
+        mock_is_available.return_value = True
+        mock_get_project_info.return_value = {"uuid": "abcdef"}
+        mock_fetch_scan_data.return_value = {"summary": ""}
+        response = self.client.get(action_url)
+        self.assertEqual(200, response.status_code)
+        self.assertEqual({"summary": ""}, response.data)
+
+    @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.get_project_info")
+    @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.fetch_scan_data")
+    @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.is_available")
+    def test_api_package_viewset_scan_data_download_zip_action(
+        self, mock_is_available, mock_fetch_scan_data, mock_get_project_info
+    ):
+        self.client.login(username=self.base_user.username, password="secret")
+        action_url = reverse("api_v2:package-scan-data-download-zip", args=[self.package1.uuid])
+        mock_is_available.return_value = True
+        mock_get_project_info.return_value = {"uuid": "abcdef"}
+        mock_fetch_scan_data.return_value = {}
+        response = self.client.get(action_url)
+        self.assertEqual(200, response.status_code)
+        self.assertEqual("application/zip", response["content-type"])
+        self.assertEqual(
+            'attachment; filename="package1.zip_scan.zip"', response["content-disposition"]
+        )
+
     def test_api_package_protected_fields_as_read_only(self):
         policy = UsagePolicy.objects.create(
             label="PackagePolicy",

component_catalog/tests/test_scancodeio.py

@@ -113,23 +113,18 @@ def test_scancodeio_fetch_scan_info(self, mock_session_get):
 
         scancodeio.fetch_scan_info(uri=uri)
         params = mock_session_get.call_args.kwargs["params"]
-        expected = {"format": "json", "name__startswith": get_hash_uid(uri)}
-        self.assertEqual(expected, params)
-
-        scancodeio.fetch_scan_info(
-            uri=uri,
-            user=self.basic_user,
-            dataspace=self.basic_user.dataspace,
-        )
-        params = mock_session_get.call_args.kwargs["params"]
         expected = {
-            "format": "json",
             "name__startswith": get_hash_uid(uri),
             "name__contains": get_hash_uid(self.basic_user.dataspace.uuid),
-            "name__endswith": get_hash_uid(self.basic_user.uuid),
+            "format": "json",
         }
         self.assertEqual(expected, params)
 
+        scancodeio.fetch_scan_info(uri=uri, user=self.basic_user)
+        params = mock_session_get.call_args.kwargs["params"]
+        expected["name__endswith"] = get_hash_uid(self.basic_user.uuid)
+        self.assertEqual(expected, params)
+
     @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.request_get")
     def test_scancodeio_find_project(self, mock_request_get):
         scancodeio = ScanCodeIO(self.dataspace)
@@ -166,9 +161,9 @@ def test_scancodeio_find_project(self, mock_request_get):
         }
         self.assertIsNone(scancodeio.find_project(name="project_name"))
 
-    @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.get_scan_results")
+    @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.get_project_info")
     @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.fetch_scan_data")
-    def test_scancodeio_update_from_scan(self, mock_fetch_scan_data, mock_get_scan_results):
+    def test_scancodeio_update_from_scan(self, mock_fetch_scan_data, mock_get_project_info):
         license_policy = make_usage_policy(self.dataspace, model=License)
         package_policy = make_usage_policy(self.dataspace, model=Package)
         make_associated_policy(license_policy, package_policy)
@@ -180,13 +175,13 @@ def test_scancodeio_update_from_scan(self, mock_fetch_scan_data, mock_get_scan_r
         self.package1.save()
         scancodeio = ScanCodeIO(self.dataspace)
 
-        mock_get_scan_results.return_value = None
+        mock_get_project_info.return_value = None
         mock_fetch_scan_data.return_value = None
 
         updated_fields = scancodeio.update_from_scan(self.package1, self.super_user)
         self.assertEqual([], updated_fields)
 
-        mock_get_scan_results.return_value = {"url": "https://scancode.io/"}
+        mock_get_project_info.return_value = {"url": "https://scancode.io/"}
         updated_fields = scancodeio.update_from_scan(self.package1, self.super_user)
         self.assertEqual([], updated_fields)
 

component_catalog/tests/test_views.py

@@ -2971,7 +2971,9 @@ def test_delete_scan_view(self, mock_fetch_scan_list, mock_delete_scan):
         self.assertEqual(404, response.status_code)
 
     @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.fetch_scan_data")
-    def test_send_scan_data_as_file_view(self, mock_fetch_scan_data):
+    @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.is_available")
+    def test_send_scan_data_as_file_view(self, mock_is_available, mock_fetch_scan_data):
+        mock_is_available.return_value = True
         mock_fetch_scan_data.return_value = {}
 
         project_uuid = "348df847-f48f-4ac7-b864-5785b44c65e2"