Log "Import ScanCode scan results" as ScanCodeProject #241

Signed-off-by: tdruez <[email protected]>

Author: tdruez

component_catalog/templates/component_catalog/includes/scan_status.html

@@ -3,6 +3,8 @@
     <div class="progress-bar {% if has_errors %}bg-warning{% else %}bg-success{% endif %}" role="progressbar" style="width: 100%" aria-label="Scan progress" aria-valuenow="100" aria-valuemin="0" aria-valuemax="100"></div>
   {% elif status == 'failure' or status == "stopped" or status == "stale" %}
     <div class="progress-bar bg-danger" role="progressbar" style="width: 100%" aria-label="Scan progress" aria-valuenow="100" aria-valuemin="0" aria-valuemax="100"></div>
+  {% elif status == 'warning' %}
+    <div class="progress-bar bg-warning" role="progressbar" style="width: 100%" aria-label="Scan progress" aria-valuenow="100" aria-valuemin="0" aria-valuemax="100"></div>
   {% elif status == 'running' %}
     <div class="progress-bar" role="progressbar" style="width: 40%" aria-label="Scan progress" aria-valuenow="40" aria-valuemin="0" aria-valuemax="100"></div>
   {% elif status == 'not_started' or status == 'queued' %}

product_portfolio/forms.py

@@ -543,7 +543,7 @@ class ImportFromScanForm(forms.Form):
         extensions=["json"],
     )
     create_codebase_resources = forms.BooleanField(
-        label=_('Create Codebase Resources (from <code>"files"</code>)'),
+        label=_('Create Codebase Resources (from "files" resources)'),
         required=False,
         initial=False,
         help_text=_(

product_portfolio/importers.py

@@ -45,6 +45,7 @@
 from product_portfolio.models import ProductItemPurpose
 from product_portfolio.models import ProductPackage
 from product_portfolio.models import ProductRelationStatus
+from product_portfolio.models import ScanCodeProject
 
 
 class CleanProductMixin(ComponentRelatedFieldImportMixin):
@@ -394,6 +395,7 @@ def __init__(
         self.created_counts = {}
         self.warnings = []
         self.product_packages_by_id = {}
+        self.scancode_project = None
 
         self.resource_additional_fields = [
             "programming_language",
@@ -408,22 +410,21 @@ def __init__(
         ]
 
     def save(self):
+        self.create_scancode_project()
         self.load_data_from_file()
         self.validate_headers()
         self.import_packages()
         if self.create_codebase_resources:
             self.import_codebase_resources()
-
+        self.update_scancode_project()
         return self.warnings, self.created_counts
 
     def load_data_from_file(self):
-        with self.upload_file.open() as f:
-            file_content = f.read()
-
         try:
-            self.data = json.loads(file_content)
-        except json.JSONDecodeError:
-            raise ValidationError("The file content is not proper JSON.")
+            with self.upload_file.open("r", encoding="utf-8") as f:
+                self.data = json.load(f)
+        except json.JSONDecodeError as e:
+            raise ValidationError(f"Invalid JSON file: {e}")
 
     def validate_headers(self):
         """
@@ -604,6 +605,37 @@ def import_codebase_resources(self):
             if codebase_resources_count:
                 self.created_counts["Codebase Resources"] = codebase_resources_count
 
+    def create_scancode_project(self):
+        """Create a ScanCodeProject entry to log this import on the Product."""
+        self.scancode_project = ScanCodeProject.objects.create(
+            product=self.product,
+            dataspace=self.product.dataspace,
+            type=ScanCodeProject.ProjectType.IMPORT_SCAN_RESULTS,
+            input_file=self.upload_file,
+            created_by=self.user,
+            status=ScanCodeProject.Status.IMPORT_STARTED,
+        )
+
+    def update_scancode_project(self):
+        """Update the ScanCodeProject entry with import results."""
+        if self.created_counts:
+            status = ScanCodeProject.Status.SUCCESS
+            import_log = [
+                f"- Imported {count} {label.lower()}"
+                for label, count in self.created_counts.items()
+            ]
+        else:
+            status = ScanCodeProject.Status.WARNING
+            import_log = ["Nothing imported."]
+
+        if self.warnings:
+            import_log.extend(self.warnings)
+
+        self.scancode_project.update(
+            status=status,
+            import_log=import_log,
+        )
+
 
 class ImportPackageFromScanCodeIO:
     """

product_portfolio/migrations/0012_alter_scancodeproject_status_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.1.6 on 2025-02-21 22:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('product_portfolio', '0011_alter_product_owner'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='scancodeproject',
+            name='status',
+            field=models.CharField(choices=[('submitted', 'Submitted'), ('importing', 'Import Started'), ('success', 'Completed'), ('failure', 'Failure'), ('warning', 'Warning')], db_index=True, max_length=10),
+        ),
+        migrations.AlterField(
+            model_name='scancodeproject',
+            name='type',
+            field=models.CharField(choices=[('IMPORT_FROM_MANIFEST', 'Import Package manifests'), ('LOAD_SBOMS', 'Import SBOM'), ('PULL_FROM_SCANCODEIO', 'Import ScanCode.io project'), ('IMPROVE_FROM_PURLDB', 'Improve from PurlDB'), ('IMPORT_SCAN_RESULTS', 'Import ScanCode scan results')], db_index=True, help_text='The type of import, for the ProjectType choices.', max_length=50),
+        ),
+    ]

product_portfolio/migrations/0012_alter_scancodeproject_type.py

@@ -1457,19 +1457,21 @@ def in_progress(self):
 
 
 class ScanCodeProject(HistoryFieldsMixin, DataspacedModel):
-    """Wrap a ScanCode.io Project."""
+    """Wrap Product imports, such as a ScanCode.io Project."""
 
     class ProjectType(models.TextChoices):
         IMPORT_FROM_MANIFEST = "IMPORT_FROM_MANIFEST", _("Import Package manifests")
         LOAD_SBOMS = "LOAD_SBOMS", _("Import SBOM")
         PULL_FROM_SCANCODEIO = "PULL_FROM_SCANCODEIO", _("Import ScanCode.io project")
         IMPROVE_FROM_PURLDB = "IMPROVE_FROM_PURLDB", _("Improve from PurlDB")
+        IMPORT_SCAN_RESULTS = "IMPORT_SCAN_RESULTS", _("Import ScanCode scan results")
 
     class Status(models.TextChoices):
         SUBMITTED = "submitted"
         IMPORT_STARTED = "importing"
         SUCCESS = "success", _("Completed")
         FAILURE = "failure"
+        WARNING = "warning"
 
     uuid = models.UUIDField(
         _("UUID"),

product_portfolio/models.py

@@ -435,7 +435,7 @@ def test_api_product_endpoint_import_from_scan_action(self):
         }
         response = self.client.post(url, data)
         self.assertEqual(status.HTTP_400_BAD_REQUEST, response.status_code)
-        expected = ["The file content is not proper JSON."]
+        expected = ["Invalid JSON file: Expecting value: line 1 column 1 (char 0)"]
         self.assertEqual(expected, response.data)
 
         scan_input_location = self.testfiles_path / "import_from_scan.json"

product_portfolio/tests/test_api.py

@@ -3090,15 +3090,16 @@ def test_product_portfolio_load_sbom_view(self, mock_submit):
         expected = "Import SBOM"
         self.assertContains(response, expected)
 
-        data = {"input_file": ContentFile("{}", name="file.json")}
+        data = {"input_file": ContentFile('{"data": "data"}', name="file.json")}
         response = self.client.post(url, data=data, follow=True)
         expected = "SBOM file submitted to ScanCode.io for inspection."
         self.assertContains(response, expected)
         self.assertEqual(1, ScanCodeProject.objects.count())
 
+        data = {"input_file": ContentFile('{"data": "data"}', name="file2.json")}
         with override_settings(CLAMD_ENABLED=True):
             with mock.patch("dje.fields.SmartFileField.scan_file_for_virus") as scan:
-                self.client.post(url, data=data, follow=True)
+                response = self.client.post(url, data=data, follow=True)
                 scan.assert_called_once()
 
     @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.submit_project")

product_portfolio/tests/test_views.py

@@ -2016,7 +2016,7 @@ def import_from_scan_view(request, dataspace, name, version=""):
                 messages.success(request, format_html(msg))
             if warnings:
                 messages.warning(request, format_html("<br>".join(warnings)))
-            return redirect(product)
+            return redirect(f"{product.get_absolute_url()}#imports")
     else:
         form = form_class(request.user)