Add additional repo types

    * Test that older packages are selected in check_for_duplicate_packages

Signed-off-by: Jono Yang <[email protected]>

Author: JonoYang

minecode/indexing.py

@@ -151,15 +151,32 @@ def check_for_duplicate_packages(package):
         return False
 
     repo_types = [
+        "apache",
+        "bower",
+        "composer",
+        "cpan",
+        "cran",
+        "crate",
+        "deb",
+        "docker",
+        "eclipse",
+        "fdroid",
+        "gem",
+        "golang",
+        "gstreamer",
         "maven",
-        "pypi",
         "npm",
-        "crate",
+        "nuget",
+        "openwrt",
+        "pypi",
+        "rpm",
     ]
-    git_repo_types = [
+    source_repo_types = [
+        "bitbucket",
         "github",
         "gitlab",
-        "bitbucket",
+        "googlecode",
+        "sourceforge",
     ]
 
     # Check for dupes
@@ -169,7 +186,10 @@ def check_for_duplicate_packages(package):
         # TODO: This will probably have to be a task
         if (
             (package.type in repo_types and existing_package.type not in repo_types)
-            or (package.type in git_repo_types and package.type not in git_repo_types)
+            or (
+                package.type in source_repo_types
+                and existing_package.type not in source_repo_types
+            )
             or (
                 (existing_package.release_date and package.release_date)
                 and (existing_package.release_date > package.release_date)
@@ -179,7 +199,7 @@ def check_for_duplicate_packages(package):
                 package=package, existing_package=existing_package
             )
 
-    return bool(package.sha1) and bool(existing_packages)
+    return bool(existing_packages)
 
 
 def index_package(

minecode/tests/test_indexing.py

@@ -9,6 +9,7 @@
 
 import json
 import os
+from datetime import datetime
 
 from matchcode.models import ApproximateDirectoryContentIndex
 from matchcode.models import ApproximateDirectoryStructureIndex
@@ -185,7 +186,7 @@ def test_indexing_index_package_dwarf(self):
         expected_extra_data = scan_data["files"][0]["extra_data"]
         self.assertEqual(expected_extra_data, extra_data)
 
-    def test_update_package_relationships(self):
+    def test_update_check_for_duplicate_packages(self):
         test_package1 = Package.objects.create(
             download_url="https://github.com//wagon-api/wagon-api-20040705.181715.jar",
             type="github",
@@ -208,11 +209,11 @@ def test_update_package_relationships(self):
         with open(scan_data_loc, "rb") as f:
             scan_data = json.loads(f.read())
 
-        indexing_errors = indexing.index_package_files(test_package1, scan_data)
+        # Test that resources
+        indexing.index_package_files(test_package1, scan_data)
         indexing.update_package_relationships(
             package=test_package2, existing_package=test_package1
         )
-
         resources = Resource.objects.filter(package=test_package2)
         self.assertEqual(64, len(resources))
         resource_data = [r.to_dict() for r in resources]
@@ -222,3 +223,36 @@ def test_update_package_relationships(self):
         self.check_expected_results(
             resource_data, expected_resources_loc, regen=FIXTURES_REGEN
         )
+
+    def test_update_check_for_duplicate_packages_release_date(self):
+        test_package1 = Package.objects.create(
+            download_url="https://bitbucket.com//wagon-api/wagon-api-20040705.181715.jar",
+            type="bitbucket",
+            namespace="",
+            name="wagon-api",
+            version="20040705.181715",
+            sha1="12345",
+            release_date=datetime.now(),
+        )
+        test_package2 = Package.objects.create(
+            download_url="https://github.com/wagon-api-20040705.181715.jar",
+            type="github",
+            namespace="",
+            name="wagon-api",
+            version="20040705.181715",
+            sha1="12345",
+            release_date=datetime.now(),
+        )
+        scan_data_loc = self.get_test_loc(
+            "indexing/scancodeio_wagon-api-20040705.181715.json"
+        )
+        with open(scan_data_loc, "rb") as f:
+            scan_data = json.loads(f.read())
+
+        # Test that resources are updated to use the older package
+        indexing.index_package_files(test_package2, scan_data)
+        indexing.update_package_relationships(
+            package=test_package1, existing_package=test_package2
+        )
+        resources = Resource.objects.filter(package=test_package1)
+        self.assertEqual(64, len(resources))