feat: add support for JFrog Artifactory and witness provenances produced on GitLab CI (#349)

In this PR (#349):
* Introduce the concept of package registries, which is a place where artifacts and provenances are potentially published.
* Add support for JFrog Artifactory as a package registry. Note that
  (1) only Artifactory repos following the Maven layout are supported, and
  (2) artifact & provenance discovery only works for projects built using Gradle.
* Add support for Witness provenances, specifically the `witness_provenance_l1_check`. Note that only projects built on GitLab CI are supported.

---------

Signed-off-by: Nathan Nguyen <[email protected]>

Author: nathanwn

src/macaron/__main__.py

@@ -22,6 +22,7 @@
 from macaron.policy_engine.policy_engine import run_policy_engine, show_prelude
 from macaron.slsa_analyzer.analyzer import Analyzer
 from macaron.slsa_analyzer.git_service import GIT_SERVICES
+from macaron.slsa_analyzer.package_registry import PACKAGE_REGISTRIES
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -142,6 +143,8 @@ def perform_action(action_args: argparse.Namespace) -> None:
             try:
                 for git_service in GIT_SERVICES:
                     git_service.load_defaults()
+                for package_registry in PACKAGE_REGISTRIES:
+                    package_registry.load_defaults()
             except ConfigurationError as error:
                 logger.error(error)
                 sys.exit(os.EX_USAGE)

src/macaron/config/defaults.ini

@@ -337,3 +337,20 @@ provenance_extensions =
 max_download_size = 70000000
 # This is the timeout (in seconds) to run the SLSA verifier.
 timeout = 120
+
+# Witness provenance. See: https://github.com/testifysec/witness.
+[provenance.witness]
+# The allowed values of the `predicateType` field in the provenance (data type: list).
+# For more details, see:
+# https://github.com/in-toto/attestation/tree/main/spec/v0.1.0#statement
+predicate_types =
+    https://witness.testifysec.com/attestation-collection/v0.1
+artifact_extensions =
+    jar
+
+# Package registries.
+# [package_registry.jfrog.maven]
+# In this example, the Maven repo can be accessed at `https://internal.registry.org/repo-name`.
+# hostname = internal.registry.org
+# repo = repo-name
+# download_timeout = 120

src/macaron/slsa_analyzer/analyze_context.py

@@ -19,6 +19,7 @@
 from macaron.slsa_analyzer.slsa_req import ReqName, SLSAReq, get_requirements_dict
 from macaron.slsa_analyzer.specs.build_spec import BuildSpec
 from macaron.slsa_analyzer.specs.ci_spec import CIInfo
+from macaron.slsa_analyzer.specs.package_registry_spec import PackageRegistryInfo
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -38,6 +39,8 @@ class ChecksOutputs(TypedDict):
     # class uses inlined functions, which is not supported by Protocol.
     expectation: Expectation | None
     """The expectation to verify the provenance for this repository."""
+    package_registries: list[PackageRegistryInfo]
+    """The package registries for this repository."""
 
 
 class AnalyzeContext:
@@ -82,6 +85,7 @@ def __init__(
             git_service=NoneGitService(),
             build_spec=BuildSpec(tools=[]),
             ci_services=[],
+            package_registries=[],
             is_inferred_prov=True,
             expectation=None,
         )
@@ -93,12 +97,19 @@ def provenances(self) -> dict:
         Returns
         -------
         dict
+            A dictionary in which each key is a CI service's name and each value is
+            the corresponding provenance payload.
         """
         try:
             ci_services = self.dynamic_data["ci_services"]
             result = {}
             for ci_info in ci_services:
-                result[ci_info["service"].name] = ci_info["provenances"]
+                result[ci_info["service"].name] = [payload.statement for payload in ci_info["provenances"]]
+            package_registry_entries = self.dynamic_data["package_registries"]
+            for package_registry_entry in package_registry_entries:
+                result[package_registry_entry.package_registry.name] = [
+                    provenance.payload.statement for provenance in package_registry_entry.provenances
+                ]
             return result
         except KeyError:
             return {}

src/macaron/slsa_analyzer/analyzer.py

@@ -44,10 +44,13 @@
 from macaron.slsa_analyzer.database_store import store_analyze_context_to_db
 from macaron.slsa_analyzer.git_service import GIT_SERVICES, BaseGitService
 from macaron.slsa_analyzer.git_service.base_git_service import NoneGitService
+from macaron.slsa_analyzer.package_registry import PACKAGE_REGISTRIES
 from macaron.slsa_analyzer.provenance.expectations.expectation_registry import ExpectationRegistry
+from macaron.slsa_analyzer.provenance.intoto import InTotoV01Payload
 from macaron.slsa_analyzer.registry import registry
 from macaron.slsa_analyzer.specs.ci_spec import CIInfo
 from macaron.slsa_analyzer.specs.inferred_provenance import Provenance
+from macaron.slsa_analyzer.specs.package_registry_spec import PackageRegistryInfo
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -808,7 +811,10 @@ def perform_checks(self, analyze_ctx: AnalyzeContext) -> dict[str, CheckResult]:
                 ci_service.load_defaults()
                 ci_service.set_api_client()
 
-                if ci_service.is_detected(analyze_ctx.component.repository.fs_path):
+                if ci_service.is_detected(
+                    repo_path=analyze_ctx.component.repository.fs_path,
+                    git_service=analyze_ctx.dynamic_data["git_service"],
+                ):
                     logger.info("The repo uses %s CI service.", ci_service.name)
 
                     # Parse configuration files and generate IRs.
@@ -825,7 +831,20 @@ def perform_checks(self, analyze_ctx: AnalyzeContext) -> dict[str, CheckResult]:
                             callgraph=callgraph,
                             provenance_assets=[],
                             latest_release={},
-                            provenances=[Provenance().payload],
+                            provenances=[InTotoV01Payload(statement=Provenance().payload)],
+                        )
+                    )
+
+        # Determine the package registries.
+        # We match the repo against package registries through build tools.
+        build_tools = analyze_ctx.dynamic_data["build_spec"]["tools"]
+        for package_registry in PACKAGE_REGISTRIES:
+            for build_tool in build_tools:
+                if package_registry.is_detected(build_tool):
+                    analyze_ctx.dynamic_data["package_registries"].append(
+                        PackageRegistryInfo(
+                            build_tool=build_tool,
+                            package_registry=package_registry,
                         )
                     )
 

src/macaron/slsa_analyzer/asset/__init__.py

@@ -0,0 +1,40 @@
+# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
+
+"""This module defines classes and interfaces related to assets.
+
+Assets are files published from some build.
+"""
+
+from typing import Protocol
+
+
+class AssetLocator(Protocol):
+    """Interface of an asset locator."""
+
+    @property
+    def name(self) -> str:
+        """Get the name (file name) of the asset."""
+
+    @property
+    def url(self) -> str:
+        """Get the url to the asset."""
+
+    @property
+    def size_in_bytes(self) -> int:
+        """Get the size of the asset in bytes."""
+
+    def download(self, dest: str) -> bool:
+        """Download the asset.
+
+        Parameters
+        ----------
+        dest : str
+            The local destination where the asset is downloaded to.
+            Note that this must include the file name.
+
+        Returns
+        -------
+        bool
+            ``True`` if the asset is downloaded successfully; ``False`` if not.
+        """

src/macaron/slsa_analyzer/build_tool/gradle.py

@@ -8,7 +8,9 @@
 
 import logging
 import os
+import subprocess  # nosec B404
 
+import macaron
 from macaron.config.defaults import defaults
 from macaron.config.global_config import global_config
 from macaron.dependency_analyzer import DependencyAnalyzer, DependencyAnalyzerError, DependencyTools
@@ -135,3 +137,107 @@ def get_dep_analyzer(self, repo_path: str) -> CycloneDxGradle:
             )
 
         raise DependencyAnalyzerError(f"Unsupported SBOM generator for Gradle: {tool_name}.")
+
+    def get_gradle_exec(self, repo_path: str) -> str:
+        """Get the Gradle executable for the repo.
+
+        Parameters
+        ----------
+        repo_path: str
+            The absolute path to a repository containing Gradle projects.
+
+        Returns
+        -------
+        str
+            The absolute path to the Gradle executable.
+        """
+        # We try to use the gradlew that comes with the repository first.
+        repo_gradlew = os.path.join(repo_path, "gradlew")
+        if os.path.isfile(repo_gradlew) and os.access(repo_gradlew, os.X_OK):
+            return repo_gradlew
+
+        # We use Macaron's built-in gradlew as a fallback option.
+        return os.path.join(os.path.join(macaron.MACARON_PATH, "resources"), "gradlew")
+
+    def get_group_ids(self, repo_path: str) -> set[str]:
+        """Get the group ids of all Gradle projects in a repository.
+
+        A Gradle project is a directory containing a ``build.gradle`` file.
+        According to the Gradle's documentation, there is a one-to-one mapping between
+        a "project" and a ``build.gradle`` file.
+        See: https://docs.gradle.org/current/javadoc/org/gradle/api/Project.html.
+
+        Note: This method makes the assumption that projects nested in a parent project
+        directory has the same group id with the parent. This behavior is consistent with
+        the behavior of the ``get_build_dirs`` method.
+
+        Parameters
+        ----------
+        repo_path: str
+            The absolute path to a repository containing Gradle projects.
+
+        Returns
+        -------
+        set[str]
+            The set of group ids of all Gradle projects in the repository.
+        """
+        gradle_exec = self.get_gradle_exec(repo_path)
+        group_ids = set()
+
+        for gradle_project_relpath in self.get_build_dirs(repo_path):
+            gradle_project_path = os.path.join(repo_path, gradle_project_relpath)
+            group_id = self.get_group_id(
+                gradle_exec=gradle_exec,
+                project_path=gradle_project_path,
+            )
+            if group_id:
+                group_ids.add(group_id)
+
+        return group_ids
+
+    def get_group_id(self, gradle_exec: str, project_path: str) -> str | None:
+        """Get the group id of a Gradle project.
+
+        A Gradle project is a directory containing a ``build.gradle`` file.
+        According to the Gradle's documentation, there is a one-to-one mapping between
+        a "project" and a ``build.gradle`` file.
+        See: https://docs.gradle.org/current/javadoc/org/gradle/api/Project.html.
+
+        Parameters
+        ----------
+        gradle_exec: str
+            The absolute path to the Gradle executable.
+
+        project_path : str
+            The absolute path to the Gradle project.
+
+        Returns
+        -------
+        str | None
+            The group id of the project, if exists.
+        """
+        try:
+            result = subprocess.run(  # nosec B603
+                [gradle_exec, "properties"],
+                capture_output=True,
+                cwd=project_path,
+                check=False,
+            )
+        except (subprocess.CalledProcessError, OSError) as error:
+            logger.debug("Could not capture the group id of the Gradle project at %s", project_path)
+            logger.debug("Error: %s", error)
+            return None
+
+        if result.returncode == 0:
+            lines = result.stdout.decode().split("\n")
+            for line in lines:
+                if line.startswith("group: "):
+                    group = line.replace("group: ", "")
+                    # The value of group here can be an empty string.
+                    if group:
+                        return group
+                    break
+
+        logger.debug("Could not capture the group id of the repo at %s", project_path)
+        logger.debug("Stderr:\n%s", result.stderr)
+        return None

src/macaron/slsa_analyzer/checks/build_as_code_check.py

@@ -5,6 +5,7 @@
 
 import logging
 import os
+from typing import Any
 
 from sqlalchemy import ForeignKey
 from sqlalchemy.orm import Mapped, mapped_column
@@ -22,6 +23,7 @@
 from macaron.slsa_analyzer.ci_service.gitlab_ci import GitLabCI
 from macaron.slsa_analyzer.ci_service.jenkins import Jenkins
 from macaron.slsa_analyzer.ci_service.travis import Travis
+from macaron.slsa_analyzer.provenance.intoto import InTotoV01Payload
 from macaron.slsa_analyzer.registry import registry
 from macaron.slsa_analyzer.slsa_req import ReqName
 from macaron.slsa_analyzer.specs.ci_spec import CIInfo
@@ -202,8 +204,12 @@ def _check_build_tool(
                                 else "However, could not find a passing workflow run.",
                             ]
                             check_result["justification"].extend(justification)
-                            if ctx.dynamic_data["is_inferred_prov"] and ci_info["provenances"]:
-                                predicate = ci_info["provenances"][0]["predicate"]
+                            if (
+                                ctx.dynamic_data["is_inferred_prov"]
+                                and ci_info["provenances"]
+                                and isinstance(ci_info["provenances"][0], InTotoV01Payload)
+                            ):
+                                predicate: Any = ci_info["provenances"][0].statement["predicate"]
                                 predicate["buildType"] = f"Custom {ci_service.name}"
                                 predicate["builder"]["id"] = deploy_action_source_link
                                 predicate["invocation"]["configSource"]["uri"] = (
@@ -261,8 +267,12 @@ def _check_build_tool(
                             else "However, could not find a passing workflow run.",
                         ]
                         check_result["justification"].extend(justification_cmd)
-                        if ctx.dynamic_data["is_inferred_prov"] and ci_info["provenances"]:
-                            predicate = ci_info["provenances"][0]["predicate"]
+                        if (
+                            ctx.dynamic_data["is_inferred_prov"]
+                            and ci_info["provenances"]
+                            and isinstance(ci_info["provenances"][0], InTotoV01Payload)
+                        ):
+                            predicate = ci_info["provenances"][0].statement["predicate"]
                             predicate["buildType"] = f"Custom {ci_service.name}"
                             predicate["builder"]["id"] = bash_source_link
                             predicate["invocation"]["configSource"]["uri"] = (
@@ -300,8 +310,13 @@ def _check_build_tool(
                                 f"The target repository uses build tool {build_tool.name}"
                                 + f" in {ci_service.name} using {deploy_kw} to deploy."
                             )
-                            if ctx.dynamic_data["is_inferred_prov"] and ci_info["provenances"]:
-                                predicate = ci_info["provenances"][0]["predicate"]
+
+                            if (
+                                ctx.dynamic_data["is_inferred_prov"]
+                                and ci_info["provenances"]
+                                and isinstance(ci_info["provenances"][0], InTotoV01Payload)
+                            ):
+                                predicate = ci_info["provenances"][0].statement["predicate"]
                                 predicate["buildType"] = f"Custom {ci_service.name}"
                                 predicate["builder"]["id"] = config_name
                                 predicate["invocation"]["configSource"]["uri"] = (

src/macaron/slsa_analyzer/checks/build_service_check.py

@@ -5,6 +5,7 @@
 
 import logging
 import os
+from typing import Any
 
 from sqlalchemy import ForeignKey
 from sqlalchemy.orm import Mapped, mapped_column
@@ -20,6 +21,7 @@
 from macaron.slsa_analyzer.ci_service.gitlab_ci import GitLabCI
 from macaron.slsa_analyzer.ci_service.jenkins import Jenkins
 from macaron.slsa_analyzer.ci_service.travis import Travis
+from macaron.slsa_analyzer.provenance.intoto import InTotoV01Payload
 from macaron.slsa_analyzer.registry import registry
 from macaron.slsa_analyzer.slsa_req import ReqName
 from macaron.slsa_analyzer.specs.ci_spec import CIInfo
@@ -183,8 +185,12 @@ def _check_build_tool(
                         )
                     ]
 
-                    if ctx.dynamic_data["is_inferred_prov"] and ci_info["provenances"]:
-                        predicate = ci_info["provenances"][0]["predicate"]
+                    if (
+                        ctx.dynamic_data["is_inferred_prov"]
+                        and ci_info["provenances"]
+                        and isinstance(ci_info["provenances"][0], InTotoV01Payload)
+                    ):
+                        predicate: Any = ci_info["provenances"][0].statement["predicate"]
                         predicate["buildType"] = f"Custom {ci_service.name}"
                         predicate["builder"]["id"] = bash_source_link
                         predicate["invocation"]["configSource"]["uri"] = (
@@ -219,8 +225,12 @@ def _check_build_tool(
                             )
                         ]
 
-                        if ctx.dynamic_data["is_inferred_prov"] and ci_info["provenances"]:
-                            predicate = ci_info["provenances"][0]["predicate"]
+                        if (
+                            ctx.dynamic_data["is_inferred_prov"]
+                            and ci_info["provenances"]
+                            and isinstance(ci_info["provenances"][0], InTotoV01Payload)
+                        ):
+                            predicate = ci_info["provenances"][0].statement["predicate"]
                             predicate["buildType"] = f"Custom {ci_service.name}"
                             predicate["builder"]["id"] = config_name
                             predicate["invocation"]["configSource"]["uri"] = (