Update ProductPackage license unknown during Scan all Packages (#420)

Signed-off-by: tdruez <[email protected]>

Author: tdruez

CHANGELOG.rst

@@ -1,6 +1,14 @@
 Release notes
 =============
 
+### Version 5.5.0-dev
+
+- Update ProductPackage "unknown" license during "Scan all Packages".
+  Only "unknown" licenses are updated.
+  Products with a is_locked configuration status are excluded.
+  Inactive is_active=False products are excluded.
+  https://github.com/aboutcode-org/dejacode/issues/388
+
 ### Version 5.4.2
 
 - Migrate the LDAP testing from using mockldap to slapdtest.

component_catalog/models.py

@@ -2630,19 +2630,28 @@ def update_from_purldb(self, user):
         )
         return updated_fields
 
-    def update_from_scan(self, user):
-        scancodeio = ScanCodeIO(self.dataspace)
+    def update_from_scan(self, user, update_products=False):
+        package = self
+        dataspace = self.dataspace
+        scancodeio = ScanCodeIO(dataspace)
+
         can_update_from_scan = all(
             [
-                self.dataspace.enable_package_scanning,
-                self.dataspace.update_packages_from_scan,
+                dataspace.enable_package_scanning,
+                dataspace.update_packages_from_scan,
                 scancodeio.is_configured(),
             ]
         )
+        if not can_update_from_scan:
+            return
+
+        updated_fields = scancodeio.update_from_scan(package=package, user=user)
 
-        if can_update_from_scan:
-            updated_fields = scancodeio.update_from_scan(package=self, user=user)
-            return updated_fields
+        if update_products:
+            if "declared_license_expression" in updated_fields:
+                package.productpackages.update_license_unknown()
+
+        return updated_fields
 
     def get_related_packages_qs(self):
         """

component_catalog/tests/test_models.py

@@ -1769,22 +1769,33 @@ def test_package_model_create_from_url_enable_purldb_access(
 
     @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.is_configured")
     @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.update_from_scan")
-    def test_package_model_update_from_scan(self, mock_update_from_scan, mock_is_configured):
+    def test_package_model_update_from_scan(self, mock_scio_update_from_scan, mock_is_configured):
         mock_is_configured.return_value = True
-        package1 = make_package(self.dataspace)
+        package1 = make_package(self.dataspace, declared_license_expression="mit")
+        product1 = make_product(self.dataspace, inventory=[package1])
+
+        pp1 = product1.productpackages.get()
+        self.assertEqual("", pp1.license_expression)
+        pp1.update(license_expression="unknown")
 
         results = package1.update_from_scan(user=self.user)
-        mock_update_from_scan.assert_not_called()
+        mock_scio_update_from_scan.assert_not_called()
         self.assertIsNone(results)
 
         self.dataspace.enable_package_scanning = True
         self.dataspace.update_packages_from_scan = True
         self.dataspace.save()
 
-        mock_update_from_scan.return_value = ["updated_field"]
-        results = package1.update_from_scan(user=self.user)
-        mock_update_from_scan.assert_called()
-        self.assertEqual(["updated_field"], results)
+        mock_scio_update_from_scan.return_value = ["declared_license_expression"]
+        results = package1.update_from_scan(user=self.user, update_products=False)
+        mock_scio_update_from_scan.assert_called()
+        self.assertEqual(["declared_license_expression"], results)
+        pp1.refresh_from_db()
+        self.assertEqual("unknown", pp1.license_expression)
+
+        results = package1.update_from_scan(user=self.user, update_products=True)
+        pp1.refresh_from_db()
+        self.assertEqual("mit", pp1.license_expression)
 
     def test_package_model_get_url_methods(self):
         package = Package(

component_catalog/views.py

@@ -1761,7 +1761,7 @@ def send_scan_notification(request, key):
     run = json_data.get("run")
     scan_status = run.get("status")
     if scan_status.lower() == "success":
-        updated_fields = package.update_from_scan(user)
+        updated_fields = package.update_from_scan(user, update_products=True)
 
     if updated_fields:
         description = (

dje/models.py

@@ -749,6 +749,12 @@ def product(self, product):
         """Filter based on the provided ``product`` object."""
         return self.filter(product=product)
 
+    def exclude_locked_products(self):
+        """Filter out the non-active and locked Products."""
+        return self.exclude(product__configuration_status__is_locked=True).filter(
+            product__is_active=True
+        )
+
 
 class DataspacedModel(models.Model):
     """Abstract base model for all models that are keyed by Dataspace."""

product_portfolio/models.py

@@ -579,13 +579,8 @@ def improve_packages_from_purldb(self, user):
         # Update the Product Package relationship `license_expression` if the
         # Package.declared_license_expression was updated from "unknwon" value using
         # PurlDB data.
-        productpackages_unknown_licenses = self.productpackages.filter(
-            package__in=updated_packages, license_expression="unknown"
-        )
-        for product_package in productpackages_unknown_licenses:
-            package_license_expression = product_package.package.declared_license_expression
-            if package_license_expression and package_license_expression != "unknown":
-                product_package.update(license_expression=package_license_expression)
+        productpackages_qs = self.productpackages.filter(package__in=updated_packages)
+        productpackages_qs.update_license_unknown()
 
         return updated_packages
 
@@ -707,6 +702,13 @@ class ProductPackageQuerySet(ProductSecuredQuerySet):
     def vulnerable(self):
         return self.filter(weighted_risk_score__isnull=False)
 
+    def license_unknown(self):
+        return self.filter(license_expression="unknown")
+
+    def update_license_unknown(self):
+        for product_package in self.exclude_locked_products().license_unknown():
+            product_package.update_license_unknown()
+
     def annotate_weighted_risk_score(self):
         """Annotate the Queeryset with the weighted_risk_score computed value."""
         purpose = ProductItemPurpose.objects.filter(productpackage=OuterRef("pk"))
@@ -1100,6 +1102,20 @@ def __str__(self):
     def permission_protected_fields(self):
         return {"review_status": "change_review_status_on_productpackage"}
 
+    def update_license_unknown(self):
+        """
+        Update this Product Package relationship `license_expression` from "unknown"
+        if the related Package `declared_license_expression` has known value.
+        """
+        package_license_expression = self.package.declared_license_expression
+        conditions = [
+            self.license_expression == "unknown",
+            package_license_expression,
+            package_license_expression != "unknown",
+        ]
+        if all(conditions):
+            self.update(license_expression=package_license_expression)
+
 
 class ProductAssignedLicense(DataspacedModel):
     product = models.ForeignKey(

product_portfolio/templates/product_portfolio/tables/scan_action_cell.html

@@ -1,6 +1,6 @@
 <span{% if package.download_url %} data-bs-toggle="modal" data-bs-target="#scan-package-modal" data-package-scan-url="{% url 'component_catalog:package_scan' user.dataspace package.uuid %}"{% endif %}>
   <span data-bs-toggle="tooltip" title="{% if package.download_url %}Submit Scan Request{% else %}Download URL not available{% endif %}">
-    <button type="button" class="btn btn-outline-dark btn-sm{% if not package.download_url %} disabled{% endif %}">
+    <button type="button" style="width:max-content;" class="btn btn-outline-dark btn-sm{% if not package.download_url %} disabled{% endif %}">
       <i class="fas fa-barcode"></i> Scan
     </button>
   </span>

product_portfolio/templates/product_portfolio/tabs/tab_imports.html

@@ -18,7 +18,7 @@
     <thead>
       <tr>
         <th>{% trans 'Import type' %}</th>
-        <th style="width:115px;">{% trans 'Status' %}</th>
+        <th style="width:140px;">{% trans 'Status' %}</th>
         <th>{% trans 'Input' %}</th>
         <th colspan="2">{% trans 'Log' %}</th>
       </tr>
@@ -41,17 +41,17 @@
             </div>
           </td>
           <td>
+            {% if scancode_project.project_uuid %}
+              <a href="#" role="button" class="ms-1 float-end" data-bs-toggle="modal" data-bs-target="#scancode-project-status-modal" data-fetch-status-url="{% url 'product_portfolio:scancodeio_project_status' scancode_project.uuid %}">
+                <i class="fas fa-info-circle"></i>
+              </a>
+            {% endif %}
             <strong>
               {{ scancode_project.get_status_display|title }}
               {% if scancode_project.has_errors %}
                 <span class="float-start"> with errors</span>
               {% endif %}
             </strong>
-            {% if scancode_project.project_uuid %}
-              <a href="#" role="button" class="ms-1 float-end" data-bs-toggle="modal" data-bs-target="#scancode-project-status-modal" data-fetch-status-url="{% url 'product_portfolio:scancodeio_project_status' scancode_project.uuid %}">
-                <i class="fas fa-info-circle"></i>
-              </a>
-            {% endif %}
             {% include 'component_catalog/includes/scan_status.html' with status=scancode_project.status has_errors=scancode_project.has_errors only %}
           </td>
           <td>

product_portfolio/tests/test_models.py

@@ -780,6 +780,19 @@ def test_productcomponent_model_is_custom_component(self):
         pc1.save()
         self.assertFalse(pc1.is_custom_component)
 
+    def test_productpackage_model_update_license_unknown(self):
+        package1 = make_package(self.dataspace, declared_license_expression="mit")
+        pp1 = make_product_package(self.product1, package=package1)
+
+        pp1.update_license_unknown()
+        pp1.refresh_from_db()
+        self.assertEqual("", pp1.license_expression)
+
+        pp1.update(license_expression="unknown")
+        pp1.update_license_unknown()
+        pp1.refresh_from_db()
+        self.assertEqual("mit", pp1.license_expression)
+
     def test_product_relationship_queryset_vulnerable(self):
         pp1 = make_product_package(self.product1)
         product_package_qs = ProductPackage.objects.vulnerable()
@@ -833,6 +846,70 @@ def test_product_relationship_queryset_update_weighted_risk_score(self):
         pp1.refresh_from_db()
         self.assertIsNone(pp1.weighted_risk_score)
 
+    def test_productpackage_queryset_exclude_locked_products(self):
+        active_product = make_product(self.dataspace)
+        pp1 = make_product_package(active_product)
+        inactive_product = make_product(self.dataspace, is_active=False)
+        make_product_package(inactive_product)
+
+        qs = ProductPackage.objects.exclude_locked_products()
+        self.assertQuerySetEqual(qs, [pp1])
+
+        locked_status = make_product_status(self.dataspace, is_locked=True)
+        active_product.update(configuration_status=locked_status)
+        qs = ProductPackage.objects.exclude_locked_products()
+        self.assertQuerySetEqual(qs, [])
+
+    def test_productpackage_queryset_license_unknown(self):
+        package1 = make_package(self.dataspace, declared_license_expression="unknown")
+        package2 = make_package(self.dataspace, declared_license_expression="mit")
+        pp1 = make_product_package(self.product1, package=package1)
+        pp2 = make_product_package(self.product1, package=package2)
+
+        qs = ProductPackage.objects.license_unknown()
+        self.assertQuerySetEqual([], qs)
+
+        pp1.update(license_expression="unknown")
+        pp2.update(license_expression="mit")
+        qs = ProductPackage.objects.license_unknown()
+        self.assertQuerySetEqual(qs, [pp1])
+
+    def test_productpackage_queryset_update_license_unknown(self):
+        package1 = make_package(self.dataspace, declared_license_expression="mit")
+        package2 = make_package(self.dataspace, declared_license_expression="mit")
+        pp1 = make_product_package(self.product1, package=package1)
+        pp2 = make_product_package(self.product1, package=package2)
+
+        ProductPackage.objects.update_license_unknown()
+        pp1.refresh_from_db()
+        pp2.refresh_from_db()
+        self.assertEqual("", pp1.license_expression)
+        self.assertEqual("", pp2.license_expression)
+
+        pp1.update(license_expression="unknown")
+        ProductPackage.objects.update_license_unknown()
+        pp1.refresh_from_db()
+        pp2.refresh_from_db()
+        self.assertEqual("mit", pp1.license_expression)
+        self.assertEqual("", pp2.license_expression)
+
+    def test_productpackage_queryset_update_license_unknown_exclude_locked_products(self):
+        locked_status = make_product_status(self.dataspace, is_locked=True)
+        self.product1.update(configuration_status=locked_status)
+
+        package1 = make_package(self.dataspace, declared_license_expression="mit")
+        pp1 = make_product_package(self.product1, package=package1, license_expression="unknown")
+
+        ProductPackage.objects.update_license_unknown()
+        pp1.refresh_from_db()
+        # Product is locked
+        self.assertEqual("unknown", pp1.license_expression)
+
+        self.product1.update(configuration_status=None)
+        ProductPackage.objects.update_license_unknown()
+        pp1.refresh_from_db()
+        self.assertEqual("mit", pp1.license_expression)
+
     def test_productrelation_model_compute_weighted_risk_score(self):
         purpose1 = make_product_item_purpose(self.dataspace)