Add support cpan on demand collection (#693)

* Add support for cpan on demand collection

Signed-off-by: Tushar Goel <[email protected]>

* Fix license header file

Signed-off-by: Tushar Goel <[email protected]>

---------

Signed-off-by: Tushar Goel <[email protected]>

Author: TG1999

minecode/collectors/cpan.py

@@ -0,0 +1,95 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0
+# See https://github.com/aboutcode-org/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import logging
+import requests
+from packageurl import PackageURL
+
+from minecode import priority_router
+from packagedb.models import PackageContentType
+
+logger = logging.getLogger(__name__)
+handler = logging.StreamHandler()
+logger.addHandler(handler)
+logger.setLevel(logging.INFO)
+
+
+def get_cpan_release_json(distribution, version):
+    """
+    Return the MetaCPAN release JSON for a given distribution@version.
+
+    Example:
+    https://fastapi.metacpan.org/v1/release/_search?q=distribution:Mojolicious%20AND%20version:9.22
+
+    """
+    url = (
+        f"https://fastapi.metacpan.org/v1/release/_search?"
+        f"q=distribution:{distribution}%20AND%20version:{version}"
+    )
+
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        results = response.json()
+        hits = results.get("hits", {}).get("hits", [])
+        if not hits:
+            return None
+        return hits[0].get("_source")
+    except requests.exceptions.HTTPError as err:
+        logger.error(f"HTTP error occurred: {err}")
+        return None
+
+
+def map_cpan_package(package_url, pipelines, priority=0):
+    """
+    Add a CPAN distribution `package_url` to the PackageDB.
+    """
+    from minecode.model_utils import add_package_to_scan_queue, merge_or_create_package
+    from minecode.miners.cpan import build_packages
+
+    name = package_url.name
+    version = package_url.version
+    release_json = get_cpan_release_json(name, version)
+
+    if not release_json:
+        error = f"Distribution does not exist on CPAN: {package_url}"
+        logger.error(error)
+        return error
+
+    packages = build_packages(release_json, package_url)
+
+    error = None
+    for package in packages:
+        package.extra_data["package_content"] = PackageContentType.SOURCE_ARCHIVE
+        db_package, _, _, error = merge_or_create_package(package, visit_level=0)
+        if error:
+            break
+
+        if db_package:
+            add_package_to_scan_queue(package=db_package, pipelines=pipelines, priority=priority)
+
+    return error
+
+
+@priority_router.route("pkg:cpan/.*")
+def process_request(purl_str, **kwargs):
+    """
+    Process CPAN Package URL (PURL).
+    """
+    from minecode.model_utils import DEFAULT_PIPELINES
+
+    addon_pipelines = kwargs.get("addon_pipelines", [])
+    pipelines = DEFAULT_PIPELINES + tuple(addon_pipelines)
+    priority = kwargs.get("priority", 0)
+
+    package_url = PackageURL.from_string(purl_str)
+    error_msg = map_cpan_package(package_url, pipelines, priority)
+
+    if error_msg:
+        return error_msg

minecode/miners/cpan.py

@@ -484,3 +484,84 @@ def get_vcs_repo_fromstring(content):
         return "git", repo[repo.index("<") + 1 : repo.index(">")]
     else:
         return None, None
+
+
+def build_packages(release_json, purl):
+    """
+    Yield ScannedPackage built from MetaCPAN release API.
+
+    Example release_json (_source):
+    {
+      "name": "Mojolicious-9.22",
+      "distribution": "Mojolicious",
+      "version": "9.22",
+      "abstract": "A next-generation web framework for Perl",
+      "license": ["perl_5"],
+      "author": "SRI",
+      "resources": {
+        "homepage": "https://mojolicious.org",
+        "repository": { "url": "https://github.com/mojolicious/mojo" }
+      },
+      "download_url": "https://cpan.metacpan.org/authors/id/S/SR/SRI/Mojolicious-9.22.tar.gz"
+    }
+    """
+    name = release_json.get("distribution") or purl.name
+    version = release_json.get("version")
+    description = release_json.get("abstract")
+    release_date = release_json.get("date")
+    license_list = release_json.get("license", [])
+
+    resources = release_json.get("resources", {})
+    homepage_url = resources.get("homepage")
+    repo = resources.get("repository", {})
+    bugtracker = resources.get("bugtracker", {})
+
+    vcs_url = None
+    if repo and repo.get("url"):
+        vcs_url = repo.get("url")
+
+    parties = []
+    author = release_json.get("author")
+    if author:
+        parties.append(scan_models.Party(name=author, role="author"))
+
+    download_url = release_json.get("download_url")
+    size = release_json.get("stat", {}).get("size")
+    md5 = release_json.get("checksum_md5")
+    sha256 = release_json.get("checksum_sha256")
+
+    keywords = release_json.get("keywords") or []
+
+    common_data = dict(
+        name=name,
+        version=version,
+        primary_language="Perl",
+        description=description,
+        release_date=release_date,
+        homepage_url=homepage_url,
+        vcs_url=vcs_url,
+        bug_tracking_url=bugtracker.get("web"),
+        code_view_url=repo.get("web"),
+        repository_homepage_url=f"https://metacpan.org/release/{name}",
+        repository_download_url=download_url,
+        api_data_url=f"https://fastapi.metacpan.org/v1/release/{name}",
+        extracted_license_statement=license_list,
+        declared_license_expression=" OR ".join(license_list) if license_list else None,
+        parties=parties,
+        keywords=keywords,
+        size=size,
+        md5=md5,
+        sha256=sha256,
+    )
+
+    download_data = dict(
+        datasource_id="cpan_pkginfo",
+        type="cpan",
+        download_url=download_url,
+    )
+    download_data.update(common_data)
+
+    package = scan_models.PackageData.from_data(download_data)
+    package.datasource_id = "cpan_api_metadata"
+    package.set_purl(purl)
+    yield package

minecode/tests/collectors/test_cpan.py

@@ -0,0 +1,60 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0
+# See https://github.com/nexB/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import json
+import os
+
+from django.test import TestCase as DjangoTestCase
+from packageurl import PackageURL
+
+import packagedb
+from minecode.collectors import cpan
+from minecode.utils_test import JsonBasedTesting
+
+
+class CpanPriorityQueueTests(JsonBasedTesting, DjangoTestCase):
+    test_data_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "testfiles")
+
+    def setUp(self):
+        super().setUp()
+        self.expected_json_loc = self.get_test_loc("cpan/Mojolicious-9.22.json")
+        with open(self.expected_json_loc) as f:
+            self.expected_json_contents = json.load(f)
+
+    def test_get_cpan_release_json(self):
+        """
+        Verify get_cpan_release_json() returns expected keys for CPAN distribution.
+        """
+        json_contents = cpan.get_cpan_release_json(distribution="Mojolicious", version="9.22")
+        self.assertIn("distribution", json_contents)
+        self.assertEqual("Mojolicious", json_contents["distribution"])
+        self.assertEqual("9.22", json_contents["version"])
+
+    def test_map_cpan_package(self):
+        """
+        Verify map_cpan_package() creates a Package in the DB with correct PURL
+        and download URL.
+        """
+        package_count = packagedb.models.Package.objects.all().count()
+        self.assertEqual(0, package_count)
+
+        package_url = PackageURL.from_string("pkg:cpan/[email protected]")
+        cpan.map_cpan_package(package_url, ("test_pipeline",))
+
+        package_count = packagedb.models.Package.objects.all().count()
+        self.assertEqual(1, package_count)
+
+        package = packagedb.models.Package.objects.all().first()
+        expected_purl_str = "pkg:cpan/[email protected]"
+        expected_download_url = (
+            "https://cpan.metacpan.org/authors/id/S/SR/SRI/Mojolicious-9.22.tar.gz"
+        )
+
+        self.assertEqual(expected_purl_str, package.purl)
+        self.assertEqual(expected_download_url, package.download_url)