Merge pull request #603 from aboutcode-org/595-add_on-demand_package_data_collection_for_gem

JonoYang · web-flow · commit 57d5409c76c5 · 2025-04-08T10:54:47.000-07:00
Collect Rubygems PURL ondemand #595
diff --git a/minecode/collectors/rubygems.py b/minecode/collectors/rubygems.py
@@ -0,0 +1,134 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import logging
+
+import requests
+from packageurl import PackageURL
+
+from minecode import priority_router
+from minecode.miners.rubygems import build_rubygem_packages_from_api_v2_data
+from packagedb.models import PackageContentType
+
+"""
+Collect GEM packages from gem registries.
+"""
+
+logger = logging.getLogger(__name__)
+handler = logging.StreamHandler()
+logger.addHandler(handler)
+logger.setLevel(logging.INFO)
+
+
+def get_package_json(name, version):
+    """
+    Return the contents of the JSON file of the package.
+    """
+    # Create the RubyGems API URL
+    url = f"https://rubygems.org/api/v2/rubygems/{name}/versions/{version}.json"
+
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        return response.json()
+    except requests.exceptions.HTTPError as err:
+        logger.error(f"HTTP error occurred: {err}")
+
+
+def get_all_package_version(name):
+    """
+    Return a list of all version numbers for the package name.
+    """
+    url = f"https://rubygems.org/api/v1/versions/{name}.json"
+    try:
+        versions = []
+        response = requests.get(url)
+        response.raise_for_status()
+        data = response.json()
+        # Get all available versions
+        for item in data:
+            versions.append(item["number"])
+        return versions
+    except requests.exceptions.HTTPError as err:
+        logger.error(f"HTTP error occurred: {err}")
+
+
+def map_gem_package(package_url, pipelines, priority=0):
+    """
+    Add a gem `package_url` to the PackageDB.
+
+    Return an error string if any errors are encountered during the process
+    """
+    from minecode.model_utils import add_package_to_scan_queue
+    from minecode.model_utils import merge_or_create_package
+
+    error = ""
+    package_json = get_package_json(
+        name=package_url.name,
+        version=package_url.version,
+    )
+
+    if not package_json:
+        error = f"Package does not exist on rubygems.org: {package_url}"
+        logger.error(error)
+        return error
+
+    metadata = package_json
+    packages = build_rubygem_packages_from_api_v2_data(metadata, package_url)
+
+    for package in packages:
+        package.extra_data["package_content"] = PackageContentType.SOURCE_ARCHIVE
+        db_package, _, _, error = merge_or_create_package(package, visit_level=0)
+        if error:
+            break
+
+        # Submit package for scanning
+        if db_package:
+            add_package_to_scan_queue(
+                package=db_package, pipelines=pipelines, priority=priority
+            )
+
+    return error
+
+
+@priority_router.route("pkg:gem/.*")
+def process_request(purl_str, **kwargs):
+    """
+    Process `priority_resource_uri` containing a gem Package URL (PURL) as a
+    URI.
+
+    This involves obtaining Package information for the PURL from rubygem and
+    using it to create a new PackageDB entry. The package is then added to the
+    scan queue afterwards.
+    """
+    from minecode.model_utils import DEFAULT_PIPELINES
+
+    addon_pipelines = kwargs.get("addon_pipelines", [])
+    pipelines = DEFAULT_PIPELINES + tuple(addon_pipelines)
+    priority = kwargs.get("priority", 0)
+
+    package_url = PackageURL.from_string(purl_str)
+
+    if not package_url.version:
+        versions = get_all_package_version(package_url.name)
+        for version in versions:
+            # package_url.version cannot be set as it will raise
+            # AttributeError: can't set attribute
+            # package_url.version = version
+            purl = purl_str + "@" + version
+            package_url = PackageURL.from_string(purl)
+            error_msg = map_gem_package(package_url, pipelines, priority)
+
+            if error_msg:
+                return error_msg
+    else:
+        error_msg = map_gem_package(package_url, pipelines, priority)
+
+        if error_msg:
+            return error_msg
diff --git a/minecode/miners/rubygems.py b/minecode/miners/rubygems.py
@@ -213,6 +213,58 @@ def build_rubygem_packages_from_api_data(metadata, name, purl=None):
         yield package
 
 
+def build_rubygem_packages_from_api_v2_data(metadata_dict, purl):
+    """
+    Yield ScannedPackage built from RubyGems API v2.
+    purl: String value of the package url of the ResourceURI object
+    """
+    name = metadata_dict["name"]
+    version = metadata_dict["version"]
+    description = metadata_dict["description"]
+    homepage_url = metadata_dict["homepage_uri"]
+    repository_homepage_url = metadata_dict["project_uri"]
+    release_date = metadata_dict["version_created_at"]
+
+    extracted_license_statement = []
+    lic_list = metadata_dict["licenses"]
+    if lic_list:
+        extracted_license_statement = lic_list
+
+    # mapping of information that are common to all the downloads of a
+    # version
+    common_data = dict(
+        name=name,
+        version=version,
+        description=description,
+        homepage_url=homepage_url,
+        repository_homepage_url=repository_homepage_url,
+        release_date=release_date,
+        extracted_license_statement=extracted_license_statement,
+    )
+
+    author = metadata_dict["authors"]
+    if author:
+        parties = common_data.get("parties")
+        if not parties:
+            common_data["parties"] = []
+        common_data["parties"].append(scan_models.Party(name=author, role="author"))
+
+    download_url = metadata_dict["gem_uri"]
+
+    download_data = dict(
+        datasource_id="gem_pkginfo",
+        type="gem",
+        download_url=download_url,
+        sha256=metadata_dict["sha"],
+    )
+    download_data.update(common_data)
+    package = scan_models.PackageData.from_data(download_data)
+
+    package.datasource_id = "gem_api_metadata"
+    package.set_purl(purl)
+    yield package
+
+
 @map_router.route(r"https?://rubygems.org/downloads/[\w\-\.]+.gem")
 class RubyGemsPackageArchiveMetadataMapper(Mapper):
     """Mapper to build on e Package from the metadata file found inside a gem."""
diff --git a/minecode/tests/collectors/test_rubygems.py b/minecode/tests/collectors/test_rubygems.py
@@ -0,0 +1,58 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import json
+import os
+
+from django.test import TestCase as DjangoTestCase
+
+from packageurl import PackageURL
+
+import packagedb
+from minecode.collectors import rubygems
+from minecode.tests import FIXTURES_REGEN
+from minecode.utils_test import JsonBasedTesting
+
+
+class RubyGemsPriorityQueueTests(JsonBasedTesting, DjangoTestCase):
+    test_data_dir = os.path.join(
+        os.path.dirname(os.path.dirname(__file__)), "testfiles"
+    )
+
+    def setUp(self):
+        super().setUp()
+        self.expected_json_loc = self.get_test_loc("rubygems/apiv2/rails-8.0.2.json")
+        with open(self.expected_json_loc) as f:
+            self.expected_json_contents = json.load(f)
+
+    def test_get_package_json(self, regen=FIXTURES_REGEN):
+        # As certain fields, such as "downloads," "versions_downloads," and
+        # "downloads_count," may vary over time, we cannot rely on
+        # "assertEqual" for comparison. Instead, we will verify that the
+        # response includes some essential data such as "name" and "version"
+        # to make sure json data is collected.
+        json_contents = rubygems.get_package_json(
+            name="rails",
+            version="8.0.2",
+        )
+        self.assertEqual(json_contents["name"], "rails")
+        self.assertEqual(json_contents["version"], "8.0.2")
+
+    def test_map_gem_package(self):
+        package_count = packagedb.models.Package.objects.all().count()
+        self.assertEqual(0, package_count)
+        package_url = PackageURL.from_string("pkg:gem/rails@8.0.2")
+        rubygems.map_gem_package(package_url, ("test_pipeline"))
+        package_count = packagedb.models.Package.objects.all().count()
+        self.assertEqual(1, package_count)
+        package = packagedb.models.Package.objects.all().first()
+        expected_purl_str = "pkg:gem/rails@8.0.2"
+        expected_download_url = "https://rubygems.org/gems/rails-8.0.2.gem"
+        self.assertEqual(expected_purl_str, package.purl)
+        self.assertEqual(expected_download_url, package.download_url)
diff --git a/minecode/tests/miners/test_rubygems.py b/minecode/tests/miners/test_rubygems.py
@@ -16,6 +16,7 @@
 from django.test import TestCase as DjangoTestCase
 
 from commoncode.fileutils import file_name
+from packageurl import PackageURL
 
 from minecode import miners
 from minecode import route
@@ -26,6 +27,7 @@
 from minecode.miners.rubygems import RubyGemsPackageArchiveMetadataMapper
 from minecode.miners.rubygems import RubyGemsPackageArchiveMetadataVisitor
 from minecode.miners.rubygems import build_rubygem_packages_from_api_data
+from minecode.miners.rubygems import build_rubygem_packages_from_api_v2_data
 from minecode.miners.rubygems import build_rubygem_packages_from_metadata
 from minecode.miners.rubygems import get_gem_metadata
 from minecode.models import ResourceURI
@@ -169,6 +171,15 @@ def test_RubyGemsApiVersionsJsonMapper(self):
         )
         self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN)
 
+    def test_build_rubygem_packages_from_api_v2_data(self):
+        with open(self.get_test_loc("rubygems/apiv2/rails-8.0.2.json")) as gem_data:
+            metadata = json.load(gem_data)
+        package_url = PackageURL.from_string("pkg:gem/rails@8.0.2")
+        packages = build_rubygem_packages_from_api_v2_data(metadata, package_url)
+        packages = [p.to_dict() for p in packages]
+        expected_loc = self.get_test_loc("rubygems/apiv2/expected-rails-8.0.2.json")
+        self.check_expected_results(packages, expected_loc, regen=FIXTURES_REGEN)
+
 
 class RubyGemsArchiveMapperTest(JsonBasedTesting):
     test_data_dir = os.path.join(
diff --git a/minecode/tests/testfiles/rubygems/apiv2/expected-rails-8.0.2.json b/minecode/tests/testfiles/rubygems/apiv2/expected-rails-8.0.2.json
@@ -0,0 +1,77 @@
+[
+    {
+      "type": "gem",
+      "namespace": null,
+      "name": "rails",
+      "version": "8.0.2",
+      "qualifiers": {},
+      "subpath": null,
+      "primary_language": null,
+      "description": "Ruby on Rails is a full-stack web framework optimized for programmer happiness and sustainable productivity. It encourages beautiful code by favoring convention over configuration.",
+      "release_date": "2025-03-12T03:09:11.097Z",
+      "parties": [
+        {
+          "type": null,
+          "role": "author",
+          "name": "David Heinemeier Hansson",
+          "email": null,
+          "url": null
+        }
+      ],
+      "keywords": [],
+      "homepage_url": "https://rubyonrails.org",
+      "download_url": "https://rubygems.org/gems/rails-8.0.2.gem",
+      "size": null,
+      "sha1": null,
+      "md5": null,
+      "sha256": "fdfaa5a83ec0388e02864e88d515959caedc88053b5f701c4deb1652d8f164c6",
+      "sha512": null,
+      "bug_tracking_url": null,
+      "code_view_url": null,
+      "vcs_url": null,
+      "copyright": null,
+      "holder": null,
+      "declared_license_expression": "mit",
+      "declared_license_expression_spdx": "MIT",
+      "license_detections": [
+        {
+          "license_expression": "mit",
+          "license_expression_spdx": "MIT",
+          "matches": [
+            {
+              "license_expression": "mit",
+              "license_expression_spdx": "MIT",
+              "from_file": null,
+              "start_line": 1,
+              "end_line": 1,
+              "matcher": "1-spdx-id",
+              "score": 100,
+              "matched_length": 1,
+              "match_coverage": 100,
+              "rule_relevance": 100,
+              "rule_identifier": "spdx-license-identifier-mit-5da48780aba670b0860c46d899ed42a0f243ff06",
+              "rule_url": null,
+              "matched_text": "MIT"
+            }
+          ],
+          "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf"
+        }
+      ],
+      "other_license_expression": null,
+      "other_license_expression_spdx": null,
+      "other_license_detections": [],
+      "extracted_license_statement": "- MIT\n",
+      "notice_text": null,
+      "source_packages": [],
+      "file_references": [],
+      "is_private": false,
+      "is_virtual": false,
+      "extra_data": {},
+      "dependencies": [],
+      "repository_homepage_url": "https://rubygems.org/gems/rails",
+      "repository_download_url": null,
+      "api_data_url": null,
+      "datasource_id": "gem_api_metadata",
+      "purl": "pkg:gem/rails@8.0.2"
+    }
+  ]
diff --git a/minecode/tests/testfiles/rubygems/apiv2/rails-8.0.2.json b/minecode/tests/testfiles/rubygems/apiv2/rails-8.0.2.json
diff --git a/packagedb/api.py b/packagedb/api.py
