Add support for more fields in ScanCode.io scan results to package #255 (#319)

Signed-off-by: tdruez <[email protected]>

Author: tdruez

CHANGELOG.rst

@@ -19,6 +19,11 @@ Release notes
   When a Product is locked through his status, its inventory cannot be modified.
   https://github.com/aboutcode-org/dejacode/issues/189
 
+- Add support for the following fields in ScanCode.io scan results to package:
+  download_url, repository_download_url, repository_homepage_url, bug_tracking_url,
+  code_view_url, vcs_url, api_data_url, size, md5, sha1, sha256, sha512.
+  https://github.com/aboutcode-org/dejacode/issues/255
+
 ### Version 5.3.0
 
 - Rename ProductDependency is_resolved to is_pinned.

component_catalog/forms.py

@@ -490,10 +490,22 @@ class Meta:
             "copyright",
             "primary_language",
             "description",
+            "download_url",
+            "repository_download_url",
             "homepage_url",
+            "repository_homepage_url",
+            "bug_tracking_url",
+            "code_view_url",
+            "vcs_url",
+            "api_data_url",
+            "size",
             "release_date",
             "notice_text",
             "dependencies",
+            "md5",
+            "sha1",
+            "sha256",
+            "sha512",
         ]
         widgets = {
             "copyright": forms.Textarea(attrs={"rows": 2}),

component_catalog/tests/test_scancodeio.py

@@ -202,7 +202,12 @@ def test_scancodeio_update_from_scan(self, mock_fetch_scan_data, mock_get_projec
             "primary_language",
             "other_license_expression",
             "description",
+            "repository_download_url",
             "homepage_url",
+            "repository_homepage_url",
+            "bug_tracking_url",
+            "vcs_url",
+            "api_data_url",
             "keywords",
             "copyright",
         ]
@@ -227,9 +232,11 @@ def test_scancodeio_update_from_scan(self, mock_fetch_scan_data, mock_get_projec
         self.assertEqual(self.super_user, self.package1.last_modified_by)
         history_entry = History.objects.get_for_object(self.package1).get()
         expected = (
-            "Automatically updated license_expression, declared_license_expression, holder, "
-            "primary_language, other_license_expression, description, homepage_url, "
-            "keywords, copyright from scan results"
+            "Automatically updated license_expression, declared_license_expression, "
+            "holder, primary_language, other_license_expression, description, "
+            "repository_download_url, homepage_url, repository_homepage_url, "
+            "bug_tracking_url, vcs_url, api_data_url, keywords, copyright from scan "
+            "results"
         )
         self.assertEqual(expected, history_entry.change_message)
 
@@ -262,25 +269,32 @@ def test_scancodeio_map_detected_package_data(self):
                 "Development Status :: 5 - Production/Stable",
                 "Operating System :: OS Independent",
             ],
+            "homepage_url": "https://www.djangoproject.com/",
+            "repository_homepage_url": "https://www.djangoproject.com/",
+            "bug_tracking_url": "https://code.djangoproject.com/",
+            "code_view_url": "https://github.com/django/django",
+            "vcs_url": "https://github.com/django/django.git",
+            "download_url": "https://www.djangoproject.com/file.zip",
+            "repository_download_url": "https://www.djangoproject.com/file.zip",
+            "copyright": "Copyright",
+            "notice_text": "Notice",
+            "api_data_url": "https://www.djangoproject.com/api/data",
             # skipped, no values
             "description": "",
             # skipped, not a SCAN_PACKAGE_FIELD
             "is_key_file": 1,
         }
 
-        expected = {
-            "package_url": "pkg:maven/aopalliance/[email protected]",
-            "purl": "pkg:maven/aopalliance/[email protected]",
-            "license_expression": "mit",
-            "declared_license_expression": "mit",
-            "other_license_expression": "apache-2.0",
-            "primary_language": "Java",
-            "keywords": [
-                "json",
-                "Development Status :: 5 - Production/Stable",
-                "Operating System :: OS Independent",
-            ],
-        }
+        expected = detected_package.copy()
+        expected["package_url"] = "pkg:maven/aopalliance/[email protected]"
+        # Licenses are simplified
+        expected["license_expression"] = "mit"
+        expected["declared_license_expression"] = "mit"
+        expected["other_license_expression"] = "apache-2.0"
+        # Skipped
+        del expected["description"]
+        del expected["is_key_file"]
+
         mapped_data = ScanCodeIO.map_detected_package_data(detected_package)
         self.assertEqual(expected, mapped_data)
 

component_catalog/tests/test_views.py

@@ -2649,7 +2649,8 @@ def test_package_details_view_scan_to_package(self, mock_fetch_scan_info, mock_f
         messages = list(response.context["messages"])
         expected = (
             "Values for package_url, license_expression, copyright, primary_language, "
-            "description, homepage_url, release_date, notice_text assigned to the package."
+            "description, download_url, homepage_url, release_date, notice_text "
+            "assigned to the package."
         )
         self.assertEqual(expected, str(messages[0]))
 
@@ -2665,7 +2666,7 @@ def test_package_details_view_scan_to_package(self, mock_fetch_scan_info, mock_f
         history = History.objects.get_for_object(self.package1, action_flag=History.CHANGE).get()
         expected = (
             "Changed Package URL, Concluded license expression, Copyright, Primary language, "
-            "Description, Homepage URL, Release date and Notice text."
+            "Description, Download URL, Homepage URL, Release date and Notice text."
         )
         self.assertEqual(expected, history.get_change_message())
 

dejacode_toolkit/scancodeio.py

@@ -300,10 +300,22 @@ def scan_data_as_zip(self, project_uuid, filename):
         ("Holder", "holder"),
         ("Description", "description"),
         ("Primary language", "primary_language"),
+        ("Download URL", "download_url"),
+        ("Repository download URL", "repository_download_url"),
         ("Homepage URL", "homepage_url"),
+        ("Repository homepage URL", "repository_homepage_url"),
+        ("Bug tracking URL", "bug_tracking_url"),
+        ("Code view URL", "code_view_url"),
+        ("VCS URL", "vcs_url"),
+        ("API data URL", "api_data_url"),
+        ("Size", "size"),
         ("Keywords", "keywords"),
         ("Release date", "release_date"),
         ("Notice text", "notice_text"),
+        ("MD5", "md5"),
+        ("SHA1", "sha1"),
+        ("SHA256", "sha256"),
+        ("SHA512", "sha512"),
         # ('Dependencies', 'dependencies'),
     ]