add on demand package data collection for nuget #597 (#606)

* Fixed #597 - Add on-demand package data collection for nuget

Signed-off-by: Chin Yeung Li <[email protected]>

* #597 - ran `make valid` to fix code fomat

Signed-off-by: Chin Yeung Li <[email protected]>

* Update minecode/tests/collectors/test_nuget.py

Signed-off-by: Chin Yeung Li <[email protected]>

Co-authored-by: Jono Yang <[email protected]>

* Update minecode/collectors/nuget.py

Signed-off-by: Chin Yeung Li <[email protected]>

Co-authored-by: Jono Yang <[email protected]>

---------

Signed-off-by: Chin Yeung Li <[email protected]>
Co-authored-by: Jono Yang <[email protected]>

Author: chinyeungli

minecode/collectors/nuget.py

@@ -0,0 +1,109 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import logging
+
+import requests
+from packageurl import PackageURL
+
+from minecode import priority_router
+from minecode.miners.nuget import build_packages_with_json
+from packagedb.models import PackageContentType
+
+"""
+Collect Nuget packages from nuget registries.
+"""
+
+logger = logging.getLogger(__name__)
+handler = logging.StreamHandler()
+logger.addHandler(handler)
+logger.setLevel(logging.INFO)
+
+
+def get_package_json(name):
+    """
+    Return the contents of the JSON file of the package.
+    """
+    # Create URLs using project name.
+    url = f"https://api.nuget.org/v3/registration5-gz-semver2/{name}/index.json"
+
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        return response.json()
+    except requests.exceptions.HTTPError as err:
+        logger.error(f"HTTP error occurred: {err}")
+
+
+def map_nuget_package(package_url, pipelines, priority=0):
+    """
+    Add a nuget `package_url` to the PackageDB.
+
+    Return an error string if any errors are encountered during the process
+    """
+    from minecode.model_utils import add_package_to_scan_queue
+    from minecode.model_utils import merge_or_create_package
+
+    error = ""
+    package_json = get_package_json(name=package_url.name.lower())
+
+    if not package_json:
+        error = f"Package does not exist on nuget.org: {package_url}"
+        logger.error(error)
+        return error
+
+    packages_metadata = package_json.get("items")[0].get("items")
+
+    for package_metadata in packages_metadata:
+        metadata = package_metadata["catalogEntry"]
+        if package_url.version:
+            if metadata.get("version") == package_url.version:
+                built_package = build_packages_with_json(metadata, package_url)
+            else:
+                continue
+        else:
+            built_package = build_packages_with_json(metadata, package_url)
+
+        for package in built_package:
+            package.extra_data["package_content"] = PackageContentType.SOURCE_ARCHIVE
+            db_package, _, _, error = merge_or_create_package(package, visit_level=0)
+            if error:
+                break
+
+            # Submit package for scanning
+            if db_package:
+                add_package_to_scan_queue(
+                    package=db_package, pipelines=pipelines, priority=priority
+                )
+
+    return error
+
+
+@priority_router.route("pkg:nuget/.*")
+def process_request(purl_str, **kwargs):
+    """
+    Process `priority_resource_uri` containing a nuget Package URL (PURL) as a
+    URI.
+
+    This involves obtaining Package information for the PURL from nuget and
+    using it to create a new PackageDB entry. The package is then added to the
+    scan queue afterwards.
+    """
+    from minecode.model_utils import DEFAULT_PIPELINES
+
+    addon_pipelines = kwargs.get("addon_pipelines", [])
+    pipelines = DEFAULT_PIPELINES + tuple(addon_pipelines)
+    priority = kwargs.get("priority", 0)
+
+    package_url = PackageURL.from_string(purl_str)
+
+    error_msg = map_nuget_package(package_url, pipelines, priority)
+
+    if error_msg:
+        return error_msg

minecode/miners/nuget.py

@@ -217,6 +217,8 @@ def build_packages_with_json(metadata, purl=None):
             copyright=copyr,
             parties=authors,
             keywords=keywords,
+            declared_license_expression=metadata.get("licenseExpression"),
+            download_url=metadata.get("packageContent"),
         )
         package = scan_models.PackageData.from_data(package_data=package_mapping)
         package.set_purl(purl)

minecode/tests/collectors/test_nuget.py

@@ -0,0 +1,47 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import json
+import os
+
+from django.test import TestCase as DjangoTestCase
+
+from packageurl import PackageURL
+
+import packagedb
+from minecode.collectors import nuget
+from minecode.utils_test import JsonBasedTesting
+
+
+class NugetPriorityQueueTests(JsonBasedTesting, DjangoTestCase):
+    test_data_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "testfiles")
+
+    def setUp(self):
+        super().setUp()
+        self.expected_json_loc = self.get_test_loc("nuget/entityframework2.json")
+        with open(self.expected_json_loc) as f:
+            self.expected_json_contents = json.load(f)
+
+    def test_get_package_json(self):
+        json_contents = nuget.get_package_json(name="entityframework")
+        expected_id = "https://api.nuget.org/v3/registration5-gz-semver2/entityframework/index.json"
+        self.assertEqual(json_contents.get("@id"), expected_id)
+
+    def test_map_nuget_package(self):
+        package_count = packagedb.models.Package.objects.all().count()
+        self.assertEqual(0, package_count)
+        package_url = PackageURL.from_string("pkg:nuget/[email protected]")
+        nuget.map_nuget_package(package_url, ("test_pipeline"))
+        package_count = packagedb.models.Package.objects.all().count()
+        self.assertEqual(1, package_count)
+        package = packagedb.models.Package.objects.all().first()
+        expected_purl_str = "pkg:nuget/[email protected]"
+        expected_download_url = "https://api.nuget.org/v3-flatcontainer/entityframework/6.1.3/entityframework.6.1.3.nupkg"
+        self.assertEqual(expected_purl_str, package.purl)
+        self.assertEqual(expected_download_url, package.download_url)

packagedb/api.py

@@ -975,6 +975,7 @@ def _reindex_package(package, reindexed_packages, **kwargs):
             "pypi",
             "cargo",
             "gem",
+            "nuget",
         ]
 
         unique_packages, unsupported_packages, unsupported_vers = get_resolved_packages(