WIP - verify GH action tag/SHA combinations

This change introduces a new function `verify_actions` to validate the contents against GitHub.

TL;DR
The function verifies that the SHAs specified in `actions.yml` exist in the GH repo.
Also ensures that the SHA exists on the Git tag, if the `tag` attribute is specified.
The rest of the (currently spaghetti code) function is a lot of output and error(failure) and warning collection.

Although it issues quite a few GH API requests, the rate limiter should not kick in (with an authenticated GH token).
I opted to rely on the HTTP/1.1 `urllib.request` stuff, which has no connection-reuse. The alternative would have been to add a dependency.

The algorithm roughly works like this, for each action specified in `actions.yml`:
* Issue a warning and stop, if the name is like `OWNER/*` ("wildcard" repository).
  Can't verify Git SHAs in this case.
* Issue a warning and stop, if the name is like `docker:*` (not implemented)
* Issue an error and stop, if the name doesn't start with an `OWNER/REPO` pattern.
* Each expired entry is just skipped
* If there is a wildcard reference and a SHA reference, issue an error.

Then, for each reference for an action:
* If no `tag` is specified, let GH resolve the commit SHA.
  Emit a warning to add the value of the `tag` attribute, if the SHA can be resolved.
  Otherwise, emit an error.
* If `tag` is specified:
  * Add the SHA to the set of requested-shas-by-tag
  * Call GH's "matching-refs" endpoint for the 'tag' value
    * Emit en error, if the object type is not a tag or commit.
    * Also resolve 'tag' object types to 'commit' object types.
    * Add each returned SHA to the set of valid-shas-by-tag.
* For each "requested tag" verify that the sets of valid and requested shas intersect. If not, emit an error.

Author: snazy

.github/workflows/update_actions.yml

@@ -32,7 +32,9 @@ jobs:
       - run: pip install ruyaml 
 
       - name: Update actions.yml
-        shell: python 
+        shell: python
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           import sys
           sys.path.append("./gateway/")
@@ -41,6 +43,11 @@ jobs:
           g.update_actions(".github/workflows/dummy.yml", "actions.yml")
           g.update_patterns("approved_patterns.yml", "actions.yml")
 
+          import action_tags as at
+          result = at.verify_actions("actions.yml")
+          if result.has_failures:
+            raise Exception(f"Failed to verify actions:\n{result}")
+
       - name: Commit and push changes
         if: ${{ github.event_name != 'pull_request' }}
         run: |

CONTRIBUTING.md

@@ -0,0 +1,15 @@
+
+# Updating code in `gateway/`
+
+## Prerequisites
+
+1. Python 3.13+
+2. `pipx install uv`
+
+## Running tests
+
+`uvx --with ruyaml pytest`
+
+To print stdout/stderr to the console when running pytest:
+
+`uvx --with ruyaml pytest -s`

gateway/action_tags.py

@@ -0,0 +1,299 @@
+# /// script
+# requires-python = ">=3.13"
+# dependencies = [
+#     "ruyaml",
+# ]
+# ///
+
+import os
+import re
+from urllib.error import HTTPError
+
+import ruyaml
+
+from datetime import date
+from urllib.request import Request, urlopen
+from pathlib import Path
+from ruyaml import CommentedMap, CommentedSeq
+from gateway import ActionsYAML, load_yaml, on_gha
+
+re_github_actions_repo_wildcard = r"^[A-Za-z0-9-_.]+/[*]$"
+re_github_actions_repo = r"^([A-Za-z0-9-_.]+/[A-Za-z0-9-_.]+)(/.+)?$"
+# Something like 'pytooling/actions/with-post-step' or 'readthedocs/actions/preview'.
+re_docker_image = r"^docker://.+"
+re_git_sha = r"^[a-f0-9]{7,}$"
+
+class ActionTagsCheckResult(object):
+    def __init__(self, log_to_console: bool = True):
+        self.log_to_console = log_to_console
+        self.logs = []
+        self.failures = []
+        self.warnings = []
+
+    def log(self, message: str) -> None:
+        if self.log_to_console:
+            print(message)
+        self.logs.append(message)
+
+    def failure(self, message: str) -> None:
+        self.failures.append(message)
+
+    def warning(self, message: str) -> None:
+        self.warnings.append(message)
+
+    def has_failures(self) -> bool:
+        return len(self.failures) > 0
+
+    def has_warnings(self) -> bool:
+        return len(self.warnings) > 0
+
+    def __str__(self):
+        return (
+            '\n'.join([f"FAILURE: {failure}" for failure in self.failures])
+          + '\n'.join([f"WARNING: {warning}" for warning in self.warnings]))
+
+
+class ApiResponse(object):
+    def __init__(self, req_url: str, status: int, reason: str, headers: dict[str, str], body: str):
+        self.req_url = req_url
+        self.status = status
+        self.reason = reason
+        self.headers = headers
+        self.body = body
+
+
+def _gh_api_get(url_abspath: str) -> ApiResponse:
+    headers: dict[str, str] = {
+        'Accept': 'application/vnd.github.v3+json',
+    }
+    # Use GH_TOKEN, if available.
+    # Unauthorized GH API requests are quite rate-limited.
+    # Tip: add an extra space before 'export' to prevent adding the line to the shell history.
+    #    export GH_TOKEN=$(gh auth token)
+    gh_token = os.environ['GH_TOKEN']
+    if gh_token:
+        headers['Authorization'] = f"Bearer {gh_token}"
+    req_url = f"https://api.github.com{url_abspath}"
+    request = Request(url=req_url, headers=headers)
+    try:
+        with urlopen(request) as response:
+            return ApiResponse(req_url, response.status, response.reason, dict(response.headers), response.read().decode('utf-8'))
+    except HTTPError as e:
+        return ApiResponse(req_url, e.code, e.reason, dict(e.headers), e.read().decode('utf-8'))
+    except Exception as e:
+        print(f"Failed to fetch '{req_url}' from GitHub API")
+        raise e
+
+def _gh_get_commit_object(owner_repo: str, sha: str) -> ApiResponse:
+    return _gh_api_get(f"/repos/{owner_repo}/git/commits/{sha}")
+
+def _gh_get_tag(owner_repo: str, tag_sha: str) -> ApiResponse:
+    return _gh_api_get(f"/repos/{owner_repo}/git/tags/{tag_sha}")
+
+def _gh_matching_tags(owner_repo: str, tag: str) -> ApiResponse:
+    return _gh_api_get(f"/repos/{owner_repo}/git/matching-refs/tags/{tag}")
+
+def verify_actions(actions_path: Path, log_to_console: bool = True) -> ActionTagsCheckResult:
+    """
+    Validates the contents of the actions file against GitHub.
+
+    The function verifies that the SHAs specified in `actions.yml` exist in the GH repo.
+    Also ensures that the SHA exists on the Git tag, if the `tag` attribute is specified.
+
+    The algorithm roughly works like this, for each action specified in `actions.yml`:
+    * Issue a warning and stop, if the name is like `OWNER/*` ("wildcard" repository).
+      Can't verify Git SHAs in this case.
+    * Issue a warning and stop, if the name is like `docker:*` (not implemented)
+    * Issue an error and stop, if the name doesn't start with an `OWNER/REPO` pattern.
+    * Each expired entry is just skipped
+    * If there is a wildcard reference and a SHA reference, issue an error.
+
+    Then, for each reference for an action:
+    * If no `tag` is specified, let GH resolve the commit SHA.
+      Emit a warning to add the value of the `tag` attribute, if the SHA can be resolved.
+      Otherwise, emit an error.
+    * If `tag` is specified:
+      * Add the SHA to the set of requested-shas-by-tag
+      * Call GH's "matching-refs" endpoint for the 'tag' value
+        * Emit en error, if the object type is not a tag or commit.
+        * Also resolve 'tag' object types to 'commit' object types.
+        * Add each returned SHA to the set of valid-shas-by-tag.
+    * For each "requested tag" verify that the sets of valid and requested shas intersect. If not, emit an error.
+
+    Args:
+        actions_path: Path to the actions list file (mandatory)
+        log_to_console: Whether to log messages immediately to the console (default: True)
+    """
+    if on_gha():
+        print(f"::group::Verfiy GitHub Actions")
+        gh_token = os.environ['GH_TOKEN']
+        if not gh_token or len(gh_token) == 0:
+            raise Exception("GH_TOKEN environment variable is not set or empty")
+
+    actions: ActionsYAML = load_yaml(actions_path)
+
+    result = ActionTagsCheckResult(log_to_console=log_to_console or on_gha())
+
+    for name, action in actions.items():
+        gh_repo_matcher = re.match(re_github_actions_repo, name)
+        if gh_repo_matcher is not None:
+            owner_repo = gh_repo_matcher.group(1)
+            result.log(f"Checking GitHub action 'https://github.com/{owner_repo}' ...")
+            valid_shas_by_tag: dict[str, set[str]] = {}
+            requested_shas_by_tag: dict[str, set[str]] = {}
+            has_wildcard = False
+            for ref, details in action.items():
+                if details and 'expires_at' in details:
+                    expires_at: date = details.get('expires_at')
+                    # TODO consider the 'keep=true' flag?
+                    if expires_at < date.today():
+                        # skip expired entries
+                        result.log(f"  .. ref '{ref}' is expired, skipping")
+                        continue
+
+                if ref == '*':
+                    # "wildcard" SHA - what would we...
+                    result.log(f"  .. detected wildcard ref")
+                    if len(requested_shas_by_tag) > 0:
+                        m = f"GitHub action 'https://github.com/{owner_repo}' references a wildcard SHA but also has specific SHAs"
+                        result.log(f"    .. ❌ {m}")
+                        result.failure(m)
+                    has_wildcard = True
+                    continue
+                elif re.match(re_git_sha, ref):
+                    result.log(f"  .. detected entry with Git SHA '{ref}'")
+                    if has_wildcard:
+                        m = f"GitHub action 'https://github.com/{owner_repo}' references a wildcard SHA but also has specific SHAs"
+                        result.log(f"    .. ⚡ {m}")
+                        result.warning(m)
+
+                    if not details or not 'tag' in details:
+                        result.log(f"    .. no Git tag")
+                        # https://docs.github.com/en/rest/git/commits?apiVersion=2022-11-28#get-a-commit-object
+                        response = _gh_get_commit_object(owner_repo, ref)
+                        match response.status:
+                            case 200:
+                                m = f"GitHub action 'https://github.com/{owner_repo}' references existing commit SHA '{ref}' but but does specify the tag name for it."
+                                result.log(f"    .. ⚡ {m}")
+                                result.warning(m)
+                            case 404:
+                                m = f"GitHub action 'https://github.com/{owner_repo}' references non existing commit SHA '{ref}': HTTP/{response.status}: {response.reason}, API URL: {response.req_url}"
+                                result.log(f"    .. ❌ {m}")
+                                result.failure(m)
+                            case _:
+                                m = f"Failed to fetch Git SHA '{ref}' from GitHub repo 'https://github.com/{owner_repo}': HTTP/{response.status}: {response.reason}, API URL: {response.req_url}\n{response.body}"
+                                result.log(f"    .. ❌ {m}")
+                                result.failure(m)
+                    else:
+                        tag: str = details.get('tag')
+                        result.log(f"    .. collecting Git SHAs for tag {tag}")
+
+                        if not tag in requested_shas_by_tag:
+                            requested_shas_by_tag[tag] = set()
+                        requested_shas_by_tag[tag].add(ref)
+
+                        if not tag in valid_shas_by_tag:
+                            valid_shas_by_tag[tag] = set()
+                        valid_shas_for_tag = valid_shas_by_tag[tag]
+
+                        # https://docs.github.com/en/rest/git/refs?apiVersion=2022-11-28#list-matching-references
+                        response = _gh_matching_tags(owner_repo, tag)
+                        match response.status:
+                            case 200:
+                                response_json: CommentedSeq = ruyaml.YAML().load(response.body)
+                                for msg in response_json:
+                                    tag_ref_map: CommentedMap = msg
+                                    tag_object: CommentedMap = tag_ref_map["object"]
+                                    tab_object_type: str = tag_object["type"]
+                                    tag_object_sha: str = tag_object["sha"]
+                                    result.log(f"      .. GH yields {tab_object_type} SHA '{tag_object_sha}' for '{tag_ref_map['ref']}'")
+                                    match tab_object_type:
+                                        case "tag":
+                                            valid_shas_for_tag.add(tag_object_sha)
+                                            # https://docs.github.com/en/rest/git/tags?apiVersion=2022-11-28#get-a-tag
+                                            response2 = _gh_get_tag(owner_repo, tag_object_sha)
+                                            match response2.status:
+                                                case 200:
+                                                    tag_object_sha = ruyaml.YAML().load(response2.body)["object"]["sha"]
+                                                    valid_shas_for_tag.add(tag_object_sha)
+                                                    result.log(f"        .. GH returns commit SHA '{tag_object_sha}' for previous tag SHA")
+                                                case 404:
+                                                    result.log(f"        .. commit SHA '{tag_object_sha}' does not exist")
+                                                case _:
+                                                    m = f"Failed to fetch details for Git tag '{tag}' from GitHub repo 'https://github.com/{owner_repo}': HTTP/{response2.status}: {response2.reason}, API URL: {response2.req_url}\n{response2.body}"
+                                                    result.log(f"        .. ❌ {m}")
+                                                    result.failure(m)
+                                        case "commit":
+                                            valid_shas_for_tag.add(tag_object_sha)
+                                        case "branch":
+                                            m = f"Branch references mentioned for Git tag '{tag}' for GitHub action 'https://github.com/{owner_repo}'"
+                                            result.log(f"        .. ❌ {m}")
+                                            result.failure(m)
+                                        case _:
+                                            m = f"Invalid Git object type '{tag_object['type']}' for Git tag '{tag}' in GitHub repo 'https://github.com/{owner_repo}'"
+                                            result.log(f"        .. ❌ {m}")
+                                            result.failure(m)
+                            case _:
+                                m = f"Failed to fetch matching Git tags for '{tag}' from GitHub repo 'https://github.com/{owner_repo}': HTTP/{response.status}: {response.reason}, API URL: {response.req_url}\n{response.body}"
+                                result.log(f"      .. ❌ {m}")
+                                result.failure(m)
+                else:
+                    m = f"GitHub action 'https://github.com/{owner_repo}' references an invalid Git SHA '{ref}'"
+                    result.log(f"      .. ❌ {m}")
+                    result.failure(m)
+
+            for req_tag, req_shas in requested_shas_by_tag.items():
+                result.log(f"  .. checking tag '{req_tag}'")
+                result.log(f"    .. referenced SHAs: {req_shas}")
+                valid_shas = valid_shas_by_tag.get(req_tag)
+                result.log(f"    .. verified SHAs: {valid_shas if len(valid_shas)>0 else '(none)'}")
+                if not valid_shas:
+                    m = f"GitHub action 'https://github.com/{owner_repo}' references Git tag '{req_tag}' via SHAs '{req_shas}' but no SHAs for tag could be found - does the Git tag exist?"
+                    result.failure(m)
+                    result.log(f"  ❌ {m}")
+                elif req_shas.isdisjoint(valid_shas):
+                    m = f"GitHub action 'https://github.com/{owner_repo}' references Git tag '{req_tag}' via SHAs '{req_shas}' but none of those matches the valid SHAs '{valid_shas}'"
+                    result.failure(m)
+                    result.log(f"  ❌ {m}")
+                else:
+                    result.log(f"  ✅ GitHub action 'https://github.com/{owner_repo}' definition for tag '{req_tag}' is good!")
+
+        elif re.match(re_github_actions_repo_wildcard, name):
+            m =f"Ignoring '{name}' because it uses a GitHub repository wildcard ..."
+            result.warning(m)
+            result.log(f"⚡ {m}")
+
+        elif re.match(re_docker_image, name):
+            m =f"Ignoring '{name}' because it references a Docker image ..."
+            result.warning(m)
+            result.log(f"⚡ {m}")
+
+        else:
+            m = f"Cannot determine action kind for '{name}'"
+            result.failure(m)
+            result.log(f"❌ {m}")
+
+    if on_gha():
+        if result.has_failures() or result.has_warnings():
+            with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:
+                f.write(f"# GitHub Actions verification result\n")
+                if len(result.failures) > 0:
+                    f.write(f"## Failures ({len(result.failures)})\n")
+                    f.write('```\n')
+                    for msg in result.failures:
+                        f.write(f"{msg}\n\n")
+                    f.write('```\n')
+                if len(result.warnings) > 0:
+                    f.write(f"## Warnings ({len(result.warnings)})\n")
+                    f.write('```\n')
+                    for msg in result.warnings:
+                        f.write(f"{msg}\n\n")
+                f.write('```\n')
+                f.write(f"## Log\n")
+                f.write('```\n')
+                for msg in result.logs:
+                    f.write(f"{msg}\n")
+                f.write('```\n')
+        print("::endgroup::")
+
+    return result

gateway/gateway.py

@@ -24,6 +24,7 @@ class RefDetails(TypedDict):
 
     expires_at: date
     keep: NotRequired[bool]
+    tag: NotRequired[str]
 
 
 ActionRefs = Dict[str, RefDetails]

gateway/test_action_tags.py

@@ -0,0 +1,21 @@
+from action_tags import *
+
+
+# TODO add some more test cases
+def test_patterns():
+    assert re.match(re_github_actions_repo, "foo/bar")
+    assert not re.match(re_github_actions_repo, "foo/*")
+    assert re.match(re_github_actions_repo, "foo/bar/.github/actions/*")
+    assert re.match(re_github_actions_repo, "foo/bar/.github/actions/some.yml")
+    assert re.match(re_docker_image, "docker://foo/bar")
+
+
+# TODO this is not a test, but a utility to run verify_actions manually.
+def test_verify_actions():
+    gh_token = os.environ['GH_TOKEN']
+    if not gh_token:
+        raise Exception("GH_TOKEN environment variable should be set for this test as it issues GitHub API requests.")
+
+    this_dir = os.path.dirname(os.path.realpath(__file__))
+    actions_path = this_dir + "/../actions.yml"
+    verify_actions(actions_path)