Create function to check for existing duplicates

Signed-off-by: Jono Yang <[email protected]>

Author: JonoYang

minecode/indexing.py

@@ -119,29 +119,74 @@ def index_package_files(package, scan_data, reindex=False):
     return scan_index_errors
 
 
-def index_package(
-    scannable_uri, package, scan_data, summary_data, project_extra_data, reindex=False
-):
+def update_package_relationships(package, existing_package):
+    """
+    Update the relations of `existing_package` to point at `package`
+    """
+    existing_package.approximatedirectorycontentindex_set.update(package=package)
+    existing_package.approximatedirectorystructureindex_set.update(package=package)
+    existing_package.approximateresourcecontentindex_set.update(package=package)
+    existing_package.exactfileindex_set.update(package=package)
+    existing_package.snippetindex_set.update(package=package)
+    existing_package.stemmedsnippetindex_set.update(package=package)
+    existing_package.resources.update(package=package)
+    existing_package.is_duplicate = True
+    existing_package.save()
+    package.is_duplicate = False
+    package.save()
+
+
+def check_for_duplicate_packages(package):
+    """
+    Given a `package`, check to see if it has already been indexed already. If
+    so, then check to see if `package` is a better candidate than the existing
+    package for being the ultimate source for that package.
+
+    Return True if a duplicate package already exists and relations have been
+    updated, otherwise return False.
+    """
     from packagedb.models import Package
 
+    if not package.sha1:
+        return False
+
+    repo_types = [
+        "maven",
+        "pypi",
+        "npm",
+        "crate",
+    ]
+    git_repo_types = [
+        "github",
+        "gitlab",
+        "bitbucket",
+    ]
+
     # Check for dupes
-    existing_packages = Package.objects.filter(sha1=package.sha1)
-    if existing_packages:
-        for existing_package in existing_packages:
-            # see if the package we are indexing is older than the package we have
-            if existing_package.package.release_date > package.release_date:
-                existing_package.approximatedirectorycontentindex_set.update(package=package)
-                existing_package.approximatedirectorystructureindex_set.update(package=package)
-                existing_package.approximateresourcecontentindex_set.update(package=package)
-                existing_package.exactfileindex_set.update(package=package)
-                existing_package.snippetindex_set.update(package=package)
-                existing_package.stemmedsnippetindex_set.update(package=package)
-                existing_package.resources.update(package=package)
-                existing_package.is_duplicate = True
-                existing_package.save()
-                package.is_duplicate = False
-                package.save()
-                return
+    existing_packages = Package.objects.filter(sha1=package.sha1, is_duplicate=False)
+    for existing_package in existing_packages:
+        # see if the package we are indexing is older than the package we have
+        # TODO: This will probably have to be a task
+        if (
+            (package.type in repo_types and existing_package.type not in repo_types)
+            or (package.type in git_repo_types and package.type not in git_repo_types)
+            or (
+                (existing_package.release_date and package.release_date)
+                and (existing_package.release_date > package.release_date)
+            )
+        ):
+            update_package_relationships(
+                package=package, existing_package=existing_package
+            )
+
+    return bool(package.sha1) and bool(existing_packages)
+
+
+def index_package(
+    scannable_uri, package, scan_data, summary_data, project_extra_data, reindex=False
+):
+    if check_for_duplicate_packages(package):
+        return
 
     scan_index_errors = []
     try:

minecode/tests/test_indexing.py

@@ -184,3 +184,41 @@ def test_indexing_index_package_dwarf(self):
         extra_data = result.first().extra_data
         expected_extra_data = scan_data["files"][0]["extra_data"]
         self.assertEqual(expected_extra_data, extra_data)
+
+    def test_update_package_relationships(self):
+        test_package1 = Package.objects.create(
+            download_url="https://github.com//wagon-api/wagon-api-20040705.181715.jar",
+            type="github",
+            namespace="",
+            name="wagon-api",
+            version="20040705.181715",
+            sha1="12345",
+        )
+        test_package2 = Package.objects.create(
+            download_url="https://repo1.maven.org/wagon-api-20040705.181715.jar",
+            type="maven",
+            namespace="",
+            name="wagon-api",
+            version="20040705.181715",
+            sha1="12345",
+        )
+        scan_data_loc = self.get_test_loc(
+            "indexing/scancodeio_wagon-api-20040705.181715.json"
+        )
+        with open(scan_data_loc, "rb") as f:
+            scan_data = json.loads(f.read())
+
+        indexing_errors = indexing.index_package_files(test_package1, scan_data)
+        indexing.update_package_relationships(
+            package=test_package2, existing_package=test_package1
+        )
+
+        resources = Resource.objects.filter(package=test_package2)
+        self.assertEqual(64, len(resources))
+        resource_data = [r.to_dict() for r in resources]
+        expected_resources_loc = self.get_test_loc(
+            "indexing/scancodeio_wagon-api-20040705.181715-expected.json"
+        )
+        self.check_expected_results(
+            resource_data, expected_resources_loc, regen=FIXTURES_REGEN
+        )

packagedb/models.py

@@ -555,7 +555,8 @@ class Package(
         ),
     )
     is_duplicate = models.BooleanField(
-        default=False, help_text=_("True if this Package is a duplicate of another Package")
+        default=False,
+        help_text=_("True if this Package is a duplicate of another Package"),
     )
 
     objects = PackageQuerySet.as_manager()