feat: check PyPI registry when deps.dev fails to find a source repository (#982)

This PR adds a fallback option for PyPI PURLs in cases where deps.dev does not report their repositories. Instead, the PyPI registry is used to find the appropriate repository URL.

Signed-off-by: Ben Selwyn-Smith <[email protected]>

Author: benmss

src/macaron/__main__.py

@@ -276,6 +276,8 @@ def perform_action(action_args: argparse.Namespace) -> None:
             try:
                 for git_service in GIT_SERVICES:
                     git_service.load_defaults()
+                for package_registry in PACKAGE_REGISTRIES:
+                    package_registry.load_defaults()
             except ConfigurationError as error:
                 logger.error(error)
                 sys.exit(os.EX_USAGE)

src/macaron/json_tools.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
 
 """This module provides utility functions for JSON data."""
@@ -53,5 +53,5 @@ def json_extract(entry: dict | list, keys: Sequence[str | int], type_: type[T])
     if isinstance(entry, type_):
         return entry
 
-    logger.debug("Found value of incorrect type: %s instead of %s.", type(entry), type(type_))
+    logger.debug("Found value of incorrect type: %s instead of %s.", type(entry), type_)
     return None

src/macaron/malware_analyzer/pypi_heuristics/metadata/closer_release_join_date.py

@@ -95,7 +95,7 @@ def analyze(self, pypi_package_json: PyPIPackageJsonAsset) -> tuple[HeuristicRes
             The result and related information collected during the analysis.
         """
         maintainers_join_date: list[datetime] | None = self._get_maintainers_join_date(
-            pypi_package_json.pypi_registry, pypi_package_json.component.name
+            pypi_package_json.pypi_registry, pypi_package_json.component_name
         )
         latest_release_date: datetime | None = self._get_latest_release_date(pypi_package_json)
         detail_info: dict[str, JsonType] = {

src/macaron/malware_analyzer/pypi_heuristics/metadata/source_code_repo.py

@@ -41,6 +41,6 @@ def analyze(self, pypi_package_json: PyPIPackageJsonAsset) -> tuple[HeuristicRes
             The result and related information collected during the analysis.
         """
         # If a sourcecode repo exists, then this will have already been validated
-        if not pypi_package_json.component.repository:
+        if not pypi_package_json.has_repository:
             return HeuristicResult.FAIL, {}
         return HeuristicResult.PASS, {}

src/macaron/malware_analyzer/pypi_heuristics/metadata/wheel_absence.py

@@ -61,7 +61,7 @@ def analyze(self, pypi_package_json: PyPIPackageJsonAsset) -> tuple[HeuristicRes
             logger.debug(error_msg)
             raise HeuristicAnalyzerValueError(error_msg)
 
-        version = pypi_package_json.component.version
+        version = pypi_package_json.component_version
         if version is None:  # check latest release version
             version = pypi_package_json.get_latest_version()
 

src/macaron/repo_finder/repo_finder.py

@@ -43,7 +43,7 @@
 from macaron.config.defaults import defaults
 from macaron.config.global_config import global_config
 from macaron.errors import CloneError, RepoCheckOutError
-from macaron.repo_finder import to_domain_from_known_purl_types
+from macaron.repo_finder import repo_finder_pypi, to_domain_from_known_purl_types
 from macaron.repo_finder.commit_finder import find_commit, match_tags
 from macaron.repo_finder.repo_finder_base import BaseRepoFinder
 from macaron.repo_finder.repo_finder_deps_dev import DepsDevRepoFinder
@@ -66,11 +66,16 @@
     list_remote_references,
     resolve_local_path,
 )
+from macaron.slsa_analyzer.specs.package_registry_spec import PackageRegistryInfo
 
 logger: logging.Logger = logging.getLogger(__name__)
 
 
-def find_repo(purl: PackageURL, check_latest_version: bool = True) -> tuple[str, RepoFinderInfo]:
+def find_repo(
+    purl: PackageURL,
+    check_latest_version: bool = True,
+    package_registries_info: list[PackageRegistryInfo] | None = None,
+) -> tuple[str, RepoFinderInfo]:
     """Retrieve the repository URL that matches the given PURL.
 
     Parameters
@@ -79,6 +84,9 @@ def find_repo(purl: PackageURL, check_latest_version: bool = True) -> tuple[str,
         The parsed PURL to convert to the repository path.
     check_latest_version: bool
         A flag that determines whether the latest version of the PURL is also checked.
+    package_registries_info: list[PackageRegistryInfo] | None
+        The list of package registry information if available.
+        If no package registries are loaded, this can be set to None.
 
     Returns
     -------
@@ -103,6 +111,9 @@ def find_repo(purl: PackageURL, check_latest_version: bool = True) -> tuple[str,
     logger.debug("Analyzing %s with Repo Finder: %s", purl, type(repo_finder))
     found_repo, outcome = repo_finder.find_repo(purl)
 
+    if not found_repo:
+        found_repo, outcome = find_repo_alternative(purl, outcome, package_registries_info)
+
     if check_latest_version and not defaults.getboolean("repofinder", "try_latest_purl", fallback=True):
         check_latest_version = False
 
@@ -117,13 +128,49 @@ def find_repo(purl: PackageURL, check_latest_version: bool = True) -> tuple[str,
         return "", RepoFinderInfo.NO_NEWER_VERSION
 
     found_repo, outcome = DepsDevRepoFinder().find_repo(latest_version_purl)
+    if found_repo:
+        return found_repo, outcome
+
+    if not found_repo:
+        found_repo, outcome = find_repo_alternative(latest_version_purl, outcome, package_registries_info)
+
     if not found_repo:
         logger.debug("Could not find repo from latest version of PURL: %s", latest_version_purl)
         return "", RepoFinderInfo.LATEST_VERSION_INVALID
 
     return found_repo, outcome
 
 
+def find_repo_alternative(
+    purl: PackageURL, outcome: RepoFinderInfo, package_registries_info: list[PackageRegistryInfo] | None = None
+) -> tuple[str, RepoFinderInfo]:
+    """Use PURL type specific methods to find the repository when the standard methods have failed.
+
+    Parameters
+    ----------
+    purl : PackageURL
+        The parsed PURL to convert to the repository path.
+    outcome: RepoFinderInfo
+        A previous outcome to report if this method does nothing.
+    package_registries_info: list[PackageRegistryInfo] | None
+        The list of package registry information if available.
+        If no package registries are loaded, this can be set to None.
+
+    Returns
+    -------
+    tuple[str, RepoFinderOutcome] :
+        The repository URL for the passed package, if found, and the outcome to report.
+    """
+    found_repo = ""
+    if purl.type == "pypi":
+        found_repo, outcome = repo_finder_pypi.find_repo(purl, package_registries_info)
+
+    if not found_repo:
+        logger.debug("Could not find repository using type specific (%s) methods for PURL: %s", purl.type, purl)
+
+    return found_repo, outcome
+
+
 def to_repo_path(purl: PackageURL, available_domains: list[str]) -> str | None:
     """Return the repository path from the PURL string.
 

src/macaron/repo_finder/repo_finder_enums.py

@@ -57,6 +57,18 @@ class RepoFinderInfo(Enum):
     #: Reported if deps.dev returns data that does not contain the desired SCM URL. E.g. The repository URL.
     DDEV_NO_URLS = "deps.dev no URLs"
 
+    #: Reported if there was an error with the request sent to the PyPI registry.
+    PYPI_HTTP_ERROR = "PyPI HTTP error"
+
+    #: Reported if there was an error parsing the JSON returned by the PyPI registry.
+    PYPI_JSON_ERROR = "PyPI JSON error"
+
+    #: Reported if there was no matching URLs in the JSON returned by the PyPI registry.
+    PYPI_NO_URLS = "PyPI no matching URLs"
+
+    #: Reported if the PyPI registry is disabled or not present in the list of package registries.
+    PYPI_NO_REGISTRY = "PyPI registry disabled or absent"
+
     #: Reported if the provided PURL did not produce a result, but a more recent version could not be found.
     NO_NEWER_VERSION = "No newer version than provided which failed"
 
@@ -70,7 +82,10 @@ class RepoFinderInfo(Enum):
     FOUND_FROM_PARENT = "Found from parent"
 
     #: Reported when a repository is found from a more recent version than was provided by the user.
-    FOUND_FROM_LATEST = "Found form latest"
+    FOUND_FROM_LATEST = "Found from latest"
+
+    #: Reported when a repository could only be found by checking the PyPI registry JSON.
+    FOUND_FROM_PYPI = "Found from PyPI"
 
     #: Default value. Reported if the Repo Finder was not called. E.g. Because the repository URL was already present.
     NOT_USED = "Not used"

src/macaron/repo_finder/repo_finder_pypi.py

@@ -0,0 +1,90 @@
+# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
+
+"""This module contains the logic for finding repositories of PyPI projects."""
+import logging
+
+from packageurl import PackageURL
+
+from macaron.repo_finder.repo_finder_enums import RepoFinderInfo
+from macaron.repo_finder.repo_validator import find_valid_repository_url
+from macaron.slsa_analyzer.package_registry import PACKAGE_REGISTRIES, PyPIRegistry
+from macaron.slsa_analyzer.package_registry.pypi_registry import PyPIPackageJsonAsset
+from macaron.slsa_analyzer.specs.package_registry_spec import PackageRegistryInfo
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+def find_repo(
+    purl: PackageURL, package_registries_info: list[PackageRegistryInfo] | None = None
+) -> tuple[str, RepoFinderInfo]:
+    """Retrieve the repository URL that matches the given PyPI PURL.
+
+    Parameters
+    ----------
+    purl : PackageURL
+        The parsed PURL to convert to the repository path.
+    package_registries_info: list[PackageRegistryInfo] | None
+        The list of package registry information if available.
+        If no package registries are loaded, this can be set to None.
+
+    Returns
+    -------
+    tuple[str, RepoFinderOutcome] :
+        The repository URL for the passed package, if found, and the outcome to report.
+    """
+    pypi_info = None
+    if package_registries_info:
+        # Find the package registry info object that contains the PyPI registry and has the pypi build tool.
+        pypi_info = next(
+            (
+                info
+                for info in package_registries_info
+                if isinstance(info.package_registry, PyPIRegistry) and info.build_tool_name in {"poetry", "pip"}
+            ),
+            None,
+        )
+
+    if not pypi_info or not isinstance(pypi_info.package_registry, PyPIRegistry):
+        pypi_registry = next((registry for registry in PACKAGE_REGISTRIES if isinstance(registry, PyPIRegistry)), None)
+    else:
+        pypi_registry = pypi_info.package_registry
+
+    if not pypi_registry:
+        logger.debug("PyPI package registry not available.")
+        return "", RepoFinderInfo.PYPI_NO_REGISTRY
+
+    pypi_asset = None
+    from_metadata = False
+    if pypi_info:
+        for existing_asset in pypi_info.metadata:
+            if not isinstance(existing_asset, PyPIPackageJsonAsset):
+                continue
+
+            if existing_asset.component_name == purl.name and existing_asset.component_version == purl.version:
+                pypi_asset = existing_asset
+                from_metadata = True
+                break
+
+    if not pypi_asset:
+        pypi_asset = PyPIPackageJsonAsset(purl.name, purl.version, False, pypi_registry, {})
+
+    if not pypi_asset.package_json and not pypi_asset.download(dest=""):
+        return "", RepoFinderInfo.PYPI_HTTP_ERROR
+
+    if not from_metadata and pypi_info:
+        # Save the asset for later use.
+        pypi_info.metadata.append(pypi_asset)
+
+    url_dict = pypi_asset.get_project_links()
+    if not url_dict:
+        return "", RepoFinderInfo.PYPI_JSON_ERROR
+
+    # Look for the repository URL.
+    fixed_url = find_valid_repository_url(url_dict.values())
+    if not fixed_url:
+        return "", RepoFinderInfo.PYPI_NO_URLS
+
+    logger.debug("Found repository URL from PyPI: %s", fixed_url)
+    pypi_asset.has_repository = True
+    return fixed_url, RepoFinderInfo.FOUND_FROM_PYPI

src/macaron/repo_finder/repo_validator.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2023 - 2024, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2023 - 2025, Oracle and/or its affiliates. All rights reserved.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
 
 """This module exists to validate URLs in terms of their use as a repository that can be analyzed."""