Prevent the creation of duplicated "resolved" dependencies during import #297 (#299)

Signed-off-by: tdruez <[email protected]>

Author: tdruez

CHANGELOG.rst

@@ -128,11 +128,15 @@ Release notes
 - Add the ability to download Product "Imports" input file.
   https://github.com/aboutcode-org/dejacode/issues/156
 
-- Fix a logic issue in the ImportPackageFromScanCodeIO.import_package that occurs when
+- Fix a logic issue in the ``ImportPackageFromScanCodeIO.import_package`` that occurs when
   multiple packages with the same PURL, but different download_url or filename,
   are present in the Dataspace.
   https://github.com/aboutcode-org/dejacode/issues/295
 
+- Fix a logic issue in the ``ImportPackageFromScanCodeIO.import_dependencies`` to
+  prevent the creation of duplicated "resolved" dependencies.
+  https://github.com/aboutcode-org/dejacode/issues/297
+
 ### Version 5.2.1
 
 - Fix the models documentation navigation.

dje/tasks.py

@@ -218,9 +218,12 @@ def pull_project_data_from_scancodeio(scancodeproject_uuid):
         scancode_project.append_to_log(msg)
 
     for object_type, values in existing.items():
-        msg = (
-            f"- {len(values)} {object_type}{pluralize(values)} already available in the Dataspace."
-        )
+        object_type_plural = f"{object_type}{pluralize(values)}"
+        object_type_plural = object_type_plural.replace("dependencys", "dependencies")
+        reason = "already available in the dataspace"
+        if object_type == "dependency":
+            reason = "already defined on the product"
+        msg = f"- {len(values)} {object_type_plural} skipped: {reason}."
         scancode_project.append_to_log(msg)
 
     for object_type, values in errors.items():

product_portfolio/importers.py

@@ -747,18 +747,30 @@ def import_package(self, package_data):
     def import_dependency(self, dependency_data):
         dependency_uid = dependency_data.get("dependency_uid")
 
-        dependency_qs = ProductDependency.objects.scope(self.user.dataspace)
-        if dependency_qs.filter(product=self.product, dependency_uid=dependency_uid).exists():
+        dependency_qs = self.product.dependencies.all()
+        if dependency_qs.filter(dependency_uid=dependency_uid).exists():
             self.existing["dependency"].append(str(dependency_uid))
             return
 
+        for_package = None
+        resolved_to_package = None
         dependency_data["product"] = self.product
         if for_package_uid := dependency_data.get("for_package_uid"):
-            dependency_data["for_package"] = self.package_uid_mapping.get(for_package_uid)
+            for_package = self.package_uid_mapping.get(for_package_uid)
+            dependency_data["for_package"] = for_package
         if resolved_to_package_uid := dependency_data.get("resolved_to_package_uid"):
-            dependency_data["resolved_to_package"] = self.package_uid_mapping.get(
-                resolved_to_package_uid
+            resolved_to_package = self.package_uid_mapping.get(resolved_to_package_uid)
+            dependency_data["resolved_to_package"] = resolved_to_package
+
+        # Check if the dependency entry already exists in the product.
+        if for_package and resolved_to_package:
+            resolved_dependency_qs = dependency_qs.filter(
+                for_package=for_package,
+                resolved_to_package=resolved_to_package,
             )
+            if resolved_dependency_qs.exists():
+                self.existing["dependency"].append(str(dependency_uid))
+                return
 
         if purl := dependency_data.get("purl"):
             dependency_data["declared_dependency"] = purl

product_portfolio/templates/product_portfolio/tabs/tab_inventory.html

@@ -75,7 +75,7 @@
         {{ filter_productcomponent.form.is_modified }}
       </th>
       {% if product.dataspace.enable_vulnerablecodedb_access %}
-        <th style="width: 75;">
+        <th style="width: 70px;">
           <span class="help_text" data-bs-toggle="tooltip" data-bs-placement="bottom" data-bs-title="{{ help_texts.risk_score }}">
             {% trans 'Risk' %}
           </span>

product_portfolio/tests/test_importers.py

@@ -1120,3 +1120,92 @@ def test_product_portfolio_import_packages_from_scio_importer_multiple_package_o
         )
         self.assertEqual({}, existing)
         self.assertEqual({}, errors)
+
+    @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.fetch_project_dependencies")
+    @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.fetch_project_packages")
+    def test_product_portfolio_import_packages_from_scio_importer_duplicate_dependency(
+        self, mock_fetch_packages, mock_fetch_dependencies
+    ):
+        for_package_purl = "pkg:pypi/[email protected]"
+        for_package_uid = "6eb9d787-2f2a-40f3-8815-fbf8f6c373de"
+        resolved_to_package_purl = "pkg:pypi/[email protected]"
+        resolved_to_package_uid = "895de5cb-2d40-4532-ae46-56104cd6a1bf"
+        mock_fetch_packages.return_value = [
+            {
+                "type": "pypi",
+                "name": "package",
+                "version": "1.0",
+                "purl": for_package_purl,
+                "package_uid": for_package_uid,
+            },
+            {
+                "type": "pypi",
+                "name": "dep",
+                "version": "0.5",
+                "purl": resolved_to_package_purl,
+                "package_uid": resolved_to_package_uid,
+            },
+        ]
+
+        dependency_uid = "12a4113b-99d2-455a-a96d-468ca29861d6"
+        mock_fetch_dependencies.return_value = [
+            {
+                "dependency_uid": dependency_uid,
+                "for_package_uid": for_package_uid,
+                "resolved_to_package_uid": resolved_to_package_uid,
+            }
+        ]
+
+        importer = ImportPackageFromScanCodeIO(
+            user=self.super_user,
+            project_uuid=uuid.uuid4(),
+            product=self.product1,
+        )
+        created, existing, errors = importer.save()
+        expected = {
+            "package": ["pkg:pypi/[email protected]", "pkg:pypi/[email protected]"],
+            "dependency": ["12a4113b-99d2-455a-a96d-468ca29861d6"],
+        }
+        self.assertEqual(expected, created)
+        self.assertEqual({}, existing)
+        self.assertEqual({}, errors)
+        self.assertEqual(2, self.product1.packages.count())
+        self.assertEqual(1, self.product1.dependencies.count())
+
+        # Re-run the importer and make sure no duplicate entries are created
+        importer = ImportPackageFromScanCodeIO(
+            user=self.super_user,
+            project_uuid=uuid.uuid4(),
+            product=self.product1,
+        )
+        created, existing, errors = importer.save()
+        self.assertEqual({}, created)
+        self.assertEqual(expected, existing)
+        self.assertEqual({}, errors)
+        self.assertEqual(2, self.product1.packages.count())
+        self.assertEqual(1, self.product1.dependencies.count())
+
+        # Change the dependency_uid to make sure the match happen on the FKs as well.
+        dependency_uid = "a46c682f-51a7-44a4-a00f-ccfc826befc6"
+        mock_fetch_dependencies.return_value = [
+            {
+                "dependency_uid": dependency_uid,
+                "for_package_uid": for_package_uid,
+                "resolved_to_package_uid": resolved_to_package_uid,
+            }
+        ]
+        importer = ImportPackageFromScanCodeIO(
+            user=self.super_user,
+            project_uuid=uuid.uuid4(),
+            product=self.product1,
+        )
+        created, existing, errors = importer.save()
+        self.assertEqual({}, created)
+        expected = {
+            "package": ["pkg:pypi/[email protected]", "pkg:pypi/[email protected]"],
+            "dependency": ["a46c682f-51a7-44a4-a00f-ccfc826befc6"],
+        }
+        self.assertEqual(expected, existing)
+        self.assertEqual({}, errors)
+        self.assertEqual(2, self.product1.packages.count())
+        self.assertEqual(1, self.product1.dependencies.count())

product_portfolio/tests/test_views.py

@@ -3283,7 +3283,7 @@ def test_product_portfolio_pull_project_data_from_scancodeio_task(self, mock_imp
         self.assertEqual(ScanCodeProject.Status.SUCCESS, scancode_project.status)
         expected = [
             "- Imported 1 package.",
-            "- 1 package already available in the Dataspace.",
+            "- 1 package skipped: already available in the dataspace.",
             "- 1 package error occurred during import.",
         ]
         self.assertEqual(expected, scancode_project.import_log)