Add hackage collector for on demand collection (#716)

* Add hackage collector for on demand collection

Signed-off-by: Tushar Goel <[email protected]>

* Fix codestyle error

Signed-off-by: Tushar Goel <[email protected]>

---------

Signed-off-by: Tushar Goel <[email protected]>

Author: TG1999

minecode/collectors/hackage.py

@@ -0,0 +1,91 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import logging
+from packageurl import PackageURL
+import requests
+from minecode import priority_router
+
+from packagedb.models import PackageContentType
+from packagedcode import models as scan_models
+
+logger = logging.getLogger(__name__)
+handler = logging.StreamHandler()
+logger.addHandler(handler)
+logger.setLevel(logging.INFO)
+
+
+def get_hackage_package_json(name):
+    """
+    Return the contents of the JSON file of the package from Hackage.
+    Example: https://hackage.haskell.org/package/dplyr.json
+    """
+    url = f"https://hackage.haskell.org/package/{name}.json"
+
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        return response.json()
+    except requests.RequestException as err:
+        logger.error(f"Error fetching package data from Hackage: {err}")
+        return None
+
+
+def map_hackage_package(package_url, pipelines, priority=0):
+    """
+    Add a hackage `package_url` to the PackageDB.
+    """
+    from minecode.model_utils import add_package_to_scan_queue, merge_or_create_package
+
+    name = package_url.name
+    version = package_url.version
+
+    versions = get_hackage_package_json(name=name)
+    if version not in versions:
+        error = f"Version {version} not found for {name} on hackage"
+        logger.error(error)
+        return error
+
+    download_url = f"https://hackage.haskell.org/package/{name}-{version}/{name}-{version}.tar.gz"
+    homepage_url = f"https://hackage.haskell.org/package/{name}-{version}"
+
+    package = scan_models.Package(
+        type="hackage",
+        name=name,
+        version=version,
+        download_url=download_url,
+        homepage_url=homepage_url,
+        primary_language="haskell",
+    )
+
+    package.extra_data["package_content"] = PackageContentType.SOURCE_ARCHIVE
+    db_package, _, _, error = merge_or_create_package(package, visit_level=0)
+
+    if db_package:
+        add_package_to_scan_queue(package=db_package, pipelines=pipelines, priority=priority)
+
+    return error
+
+
+@priority_router.route("pkg:hackage/.*")
+def process_request(purl_str, **kwargs):
+    """
+    Process Hackage Package URL (PURL).
+    """
+    from minecode.model_utils import DEFAULT_PIPELINES
+
+    addon_pipelines = kwargs.get("addon_pipelines", [])
+    pipelines = DEFAULT_PIPELINES + tuple(addon_pipelines)
+    priority = kwargs.get("priority", 0)
+
+    package_url = PackageURL.from_string(purl_str)
+    error_msg = map_hackage_package(package_url, pipelines, priority)
+
+    if error_msg:
+        return error_msg

minecode/tests/collectors/test_hackage.py

@@ -0,0 +1,54 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import pytest
+from unittest.mock import patch
+
+from packageurl import PackageURL
+import requests
+
+import minecode.collectors.hackage as hackage
+
+
+@pytest.fixture
+def package_url():
+    return PackageURL.from_string("pkg:hackage/[email protected]")
+
+
+def test_map_hackage_package_success(package_url):
+    with (
+        patch("minecode.collectors.hackage.get_hackage_package_json") as mock_get_json,
+        patch("minecode.model_utils.merge_or_create_package") as mock_merge,
+        patch("minecode.model_utils.add_package_to_scan_queue") as mock_add,
+    ):
+        mock_get_json.return_value = {"1.1.1": "normal", "1.2.1": "normal"}
+        mock_merge.return_value = ("db_package", None, None, None)
+
+        error = hackage.map_hackage_package(package_url, pipelines=["p1"], priority=1)
+
+        assert error is None
+        mock_get_json.assert_called_once_with(name="ac-halfinteger")
+        mock_merge.assert_called_once()
+        mock_add.assert_called_once_with(package="db_package", pipelines=["p1"], priority=1)
+
+
+def test_map_hackage_package_version_not_found(package_url):
+    with patch("minecode.collectors.hackage.get_hackage_package_json") as mock_get_json:
+        mock_get_json.return_value = {"2.0.0": "normal"}
+
+        error = hackage.map_hackage_package(package_url, pipelines=[])
+        assert "not found" in error
+
+
+def test_map_hackage_package_network_error(package_url):
+    with patch("minecode.collectors.hackage.requests.get") as mock_get:
+        mock_get.side_effect = requests.RequestException("Network down")
+
+        data = hackage.get_hackage_package_json(package_url.name)
+        assert data is None