Merge pull request #591 from aboutcode-org/index-time-matching

Index time matching

Author: JonoYang

minecode/indexing.py

@@ -119,9 +119,95 @@ def index_package_files(package, scan_data, reindex=False):
     return scan_index_errors
 
 
+def update_package_relationships(package, existing_package):
+    """
+    Update the relations of `existing_package` to point at `package`
+    """
+    existing_package.approximatedirectorycontentindex_set.update(package=package)
+    existing_package.approximatedirectorystructureindex_set.update(package=package)
+    existing_package.approximateresourcecontentindex_set.update(package=package)
+    existing_package.exactfileindex_set.update(package=package)
+    existing_package.snippetindex_set.update(package=package)
+    existing_package.stemmedsnippetindex_set.update(package=package)
+    existing_package.resources.update(package=package)
+    existing_package.is_duplicate = True
+    existing_package.save()
+    package.is_duplicate = False
+    package.save()
+
+
+def check_for_duplicate_packages(package):
+    """
+    Given a `package`, check to see if it has already been indexed already. If
+    so, then check to see if `package` is a better candidate than the existing
+    package for being the ultimate source for that package.
+
+    Return True if a duplicate package already exists and relations have been
+    updated, otherwise return False.
+    """
+    from packagedb.models import Package
+
+    if not package.sha1:
+        return False
+
+    repo_types = [
+        "apache",
+        "bower",
+        "composer",
+        "cpan",
+        "cran",
+        "crate",
+        "deb",
+        "docker",
+        "eclipse",
+        "fdroid",
+        "gem",
+        "golang",
+        "gstreamer",
+        "maven",
+        "npm",
+        "nuget",
+        "openwrt",
+        "pypi",
+        "rpm",
+    ]
+    source_repo_types = [
+        "bitbucket",
+        "github",
+        "gitlab",
+        "googlecode",
+        "sourceforge",
+    ]
+
+    # Check for dupes
+    existing_packages = Package.objects.filter(sha1=package.sha1, is_duplicate=False)
+    for existing_package in existing_packages:
+        # see if the package we are indexing is older than the package we have
+        # TODO: This will probably have to be a task
+        if (
+            (package.type in repo_types and existing_package.type not in repo_types)
+            or (
+                package.type in source_repo_types
+                and existing_package.type not in source_repo_types
+            )
+            or (
+                (existing_package.release_date and package.release_date)
+                and (existing_package.release_date > package.release_date)
+            )
+        ):
+            update_package_relationships(
+                package=package, existing_package=existing_package
+            )
+
+    return bool(existing_packages)
+
+
 def index_package(
     scannable_uri, package, scan_data, summary_data, project_extra_data, reindex=False
 ):
+    if check_for_duplicate_packages(package):
+        return
+
     scan_index_errors = []
     try:
         indexing_errors = index_package_files(package, scan_data, reindex=reindex)

minecode/tests/test_indexing.py

@@ -9,6 +9,7 @@
 
 import json
 import os
+from datetime import datetime
 
 from matchcode.models import ApproximateDirectoryContentIndex
 from matchcode.models import ApproximateDirectoryStructureIndex
@@ -184,3 +185,74 @@ def test_indexing_index_package_dwarf(self):
         extra_data = result.first().extra_data
         expected_extra_data = scan_data["files"][0]["extra_data"]
         self.assertEqual(expected_extra_data, extra_data)
+
+    def test_update_check_for_duplicate_packages(self):
+        test_package1 = Package.objects.create(
+            download_url="https://github.com//wagon-api/wagon-api-20040705.181715.jar",
+            type="github",
+            namespace="",
+            name="wagon-api",
+            version="20040705.181715",
+            sha1="12345",
+        )
+        test_package2 = Package.objects.create(
+            download_url="https://repo1.maven.org/wagon-api-20040705.181715.jar",
+            type="maven",
+            namespace="",
+            name="wagon-api",
+            version="20040705.181715",
+            sha1="12345",
+        )
+        scan_data_loc = self.get_test_loc(
+            "indexing/scancodeio_wagon-api-20040705.181715.json"
+        )
+        with open(scan_data_loc, "rb") as f:
+            scan_data = json.loads(f.read())
+
+        # Test that resources
+        indexing.index_package_files(test_package1, scan_data)
+        indexing.update_package_relationships(
+            package=test_package2, existing_package=test_package1
+        )
+        resources = Resource.objects.filter(package=test_package2)
+        self.assertEqual(64, len(resources))
+        resource_data = [r.to_dict() for r in resources]
+        expected_resources_loc = self.get_test_loc(
+            "indexing/scancodeio_wagon-api-20040705.181715-expected.json"
+        )
+        self.check_expected_results(
+            resource_data, expected_resources_loc, regen=FIXTURES_REGEN
+        )
+
+    def test_update_check_for_duplicate_packages_release_date(self):
+        test_package1 = Package.objects.create(
+            download_url="https://bitbucket.com//wagon-api/wagon-api-20040705.181715.jar",
+            type="bitbucket",
+            namespace="",
+            name="wagon-api",
+            version="20040705.181715",
+            sha1="12345",
+            release_date=datetime.now(),
+        )
+        test_package2 = Package.objects.create(
+            download_url="https://github.com/wagon-api-20040705.181715.jar",
+            type="github",
+            namespace="",
+            name="wagon-api",
+            version="20040705.181715",
+            sha1="12345",
+            release_date=datetime.now(),
+        )
+        scan_data_loc = self.get_test_loc(
+            "indexing/scancodeio_wagon-api-20040705.181715.json"
+        )
+        with open(scan_data_loc, "rb") as f:
+            scan_data = json.loads(f.read())
+
+        # Test that resources are updated to use the older package
+        indexing.index_package_files(test_package2, scan_data)
+        indexing.update_package_relationships(
+            package=test_package1, existing_package=test_package2
+        )
+        resources = Resource.objects.filter(package=test_package1)
+        self.assertEqual(64, len(resources))

packagedb/migrations/0091_package_is_duplicate.py

@@ -0,0 +1,21 @@
+# Generated by Django 5.1.5 on 2025-03-12 21:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("packagedb", "0090_alter_packageactivity_uuid"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="package",
+            name="is_duplicate",
+            field=models.BooleanField(
+                default=False,
+                help_text="True if this Package is a duplicate of another Package",
+            ),
+        ),
+    ]

packagedb/models.py

@@ -554,6 +554,10 @@ class Package(
             "A mapping containing a summary and license clarity score for this Package"
         ),
     )
+    is_duplicate = models.BooleanField(
+        default=False,
+        help_text=_("True if this Package is a duplicate of another Package"),
+    )
 
     objects = PackageQuerySet.as_manager()