Merge pull request #12 from techofourown/feat/converge-catalog-tooling

refactor: converge scripts with canonical catalog-tooling

Author: johnbenac

.github/workflows/ci.yml

@@ -11,15 +11,5 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: rhysd/actionlint@v1.7.11
       - uses: oras-project/setup-oras@v1
-      - name: Validate script syntax
-        run: bash -n scripts/render-catalog-bundle.sh scripts/check-catalog-bundle-smoke.sh scripts/check-image-refs-exist.sh scripts/check-publish-workflow.sh
-      - name: Validate catalog-row helper syntax
-        run: python3 -m py_compile scripts/render-catalog-rows.py
-      - name: Validate publish workflow invariants
-        run: bash scripts/check-publish-workflow.sh
-      - name: Validate and render catalog bundle
-        run: bash scripts/check-catalog-bundle-smoke.sh
-      - name: Validate referenced images exist
-        run: bash scripts/check-image-refs-exist.sh
+      - run: bash scripts/validate-catalog-repo.sh

.github/workflows/publish-catalog-bundle.yml

@@ -27,92 +27,14 @@ jobs:
           set -euo pipefail
           digest="$(oras resolve ghcr.io/techofourown/sw-ourbox-os/platform-contract:edge)"
           echo "OURBOX_PLATFORM_CONTRACT_DIGEST=${digest}" >> "${GITHUB_ENV}"
-      - name: Render catalog bundle
-        run: bash scripts/render-catalog-bundle.sh
       - name: Login to GHCR
         run: echo "${GITHUB_TOKEN}" | oras login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Publish bundle
-        run: |
-          set -euo pipefail
-          REF="ghcr.io/techofourown/sw-ourbox-catalog-hello-world:latest"
-          CATALOG_REF="ghcr.io/techofourown/sw-ourbox-catalog-hello-world:catalog-amd64"
-          CATALOG_ARTIFACT_TYPE="application/vnd.techofourown.ourbox.application-catalog.catalog.v1"
-          IMMUTABLE_TAG="sha-${GITHUB_SHA}-run-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          IMMUTABLE_REF="ghcr.io/techofourown/sw-ourbox-catalog-hello-world:${IMMUTABLE_TAG}"
-          VERSION_TAG="main-${GITHUB_SHA::12}"
-          CREATED="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
-          oras push \
-            --artifact-type application/vnd.techofourown.ourbox.application-catalog.v1.tar+gzip \
-            "${IMMUTABLE_REF}" \
-            dist/application-catalog-bundle.tar.gz:application/vnd.techofourown.ourbox.application-catalog.v1.tar+gzip
-          DIGEST="$(oras resolve "${IMMUTABLE_REF}")"
-          EXISTING_CATALOG=""
-          rm -rf dist/catalog-existing
-          if oras pull "${CATALOG_REF}" -o dist/catalog-existing >"${RUNNER_TEMP}/catalog-pull.out" 2>"${RUNNER_TEMP}/catalog-pull.err"; then
-            EXISTING_CATALOG="$(find dist/catalog-existing -maxdepth 4 -type f -name 'catalog.tsv' | head -n 1 || true)"
-          elif grep -Eiq 'MANIFEST_UNKNOWN|NAME_UNKNOWN|not found|404' "${RUNNER_TEMP}/catalog-pull.err"; then
-            echo "No existing catalog index found at ${CATALOG_REF}; creating a new catalog.tsv"
-          else
-            echo "Failed to pull existing catalog index ${CATALOG_REF}" >&2
-            cat "${RUNNER_TEMP}/catalog-pull.err" >&2
-            exit 1
-          fi
-          python3 scripts/render-catalog-rows.py \
-            --catalog-json catalog/catalog.json \
-            --profile-env catalog/profile.env \
-            --images-lock dist/images.lock.json \
-            --existing-catalog "${EXISTING_CATALOG}" \
-            --out-catalog dist/catalog.tsv \
-            --channel stable \
-            --tag "${IMMUTABLE_TAG}" \
-            --created "${CREATED}" \
-            --version "${VERSION_TAG}" \
-            --revision "${GITHUB_SHA}" \
-            --arch amd64 \
-            --artifact-digest "${DIGEST}" \
-            --pinned-ref "ghcr.io/techofourown/sw-ourbox-catalog-hello-world@${DIGEST}"
-          oras push \
-            --artifact-type "${CATALOG_ARTIFACT_TYPE}" \
-            "${CATALOG_REF}" \
-            dist/catalog.tsv:text/tab-separated-values
-          oras tag "${IMMUTABLE_REF}" latest >/dev/null
-          oras tag "${IMMUTABLE_REF}" stable >/dev/null
-          oras tag "${IMMUTABLE_REF}" "${VERSION_TAG}" >/dev/null
-          LATEST_DIGEST="$(oras resolve "${REF}")"
-          [[ "${LATEST_DIGEST}" == "${DIGEST}" ]] || {
-            echo "latest tag did not resolve to the published immutable digest" >&2
-            echo "expected: ${DIGEST}" >&2
-            echo "actual:   ${LATEST_DIGEST}" >&2
-            exit 1
-          }
-          DIGEST="${DIGEST}" MUTABLE_REF="${REF}" IMMUTABLE_REF="${IMMUTABLE_REF}" python3 - <<'PY'
-          import json
-          import os
-          from pathlib import Path
-
-          record = {
-              "schema": 1,
-              "kind": "ourbox-application-catalog-bundle-publish-record",
-              "catalog_id": "hello-world",
-              "catalog_name": "Hello World Catalog",
-              "mutable_ref": os.environ["MUTABLE_REF"],
-              "immutable_ref": os.environ["IMMUTABLE_REF"],
-              "artifact_ref": f"ghcr.io/techofourown/sw-ourbox-catalog-hello-world@{os.environ['DIGEST']}",
-              "artifact_digest": os.environ["DIGEST"],
-              "github_sha": os.environ["GITHUB_SHA"],
-              "github_run_id": os.environ["GITHUB_RUN_ID"],
-              "github_run_attempt": os.environ["GITHUB_RUN_ATTEMPT"],
-          }
-          Path("dist/catalog-bundle.publish-record.json").write_text(
-              json.dumps(record, indent=2) + "\n",
-              encoding="utf-8",
-          )
-          PY
+      - run: bash scripts/publish-catalog-bundle.sh
       - uses: actions/upload-artifact@v4
         with:
-          name: hello-world-catalog-bundle
+          name: application-catalog-bundle
           path: |
             dist/application-catalog-bundle.tar.gz
             dist/application-catalog-bundle.tar.gz.sha256

.gitignore

@@ -1,2 +1,3 @@
 .DS_Store
 dist/
+__pycache__/

scripts/check-catalog-bundle-smoke.sh

@@ -19,7 +19,25 @@ need_cmd sha256sum
 
 python3 -m py_compile "${ROOT}/scripts/render-catalog-rows.py"
 
-python3 - <<'PY' "${ROOT}/catalog/catalog.json" "${ROOT}/catalog/image-sources.json" "${ROOT}/catalog/profile.env"
+WORK_ROOT="${TMP_ROOT}/template-copy"
+mkdir -p "${WORK_ROOT}"
+cp -R "${ROOT}/catalog" "${WORK_ROOT}/catalog"
+cp -R "${ROOT}/scripts" "${WORK_ROOT}/scripts"
+
+python3 - <<'PY' "${WORK_ROOT}/catalog/image-sources.json"
+import json
+import sys
+from pathlib import Path
+
+path = Path(sys.argv[1])
+payload = json.loads(path.read_text(encoding="utf-8"))
+for entry in payload.get("images", []):
+    name = str(entry["name"])
+    entry["ref"] = f"ghcr.io/example-org/example-apps/{name}@sha256:" + ("1" * 64)
+path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+PY
+
+python3 - <<'PY' "${WORK_ROOT}/catalog/catalog.json" "${WORK_ROOT}/catalog/image-sources.json" "${WORK_ROOT}/catalog/profile.env"
 import json
 import re
 import sys
@@ -100,31 +118,31 @@ PY
 
 export OURBOX_PLATFORM_CONTRACT_DIGEST="sha256:0000000000000000000000000000000000000000000000000000000000000000"
 
-bash "${ROOT}/scripts/render-catalog-bundle.sh"
+bash "${WORK_ROOT}/scripts/render-catalog-bundle.sh"
 
-test -f "${ROOT}/dist/application-catalog-bundle.tar.gz"
-test -f "${ROOT}/dist/application-catalog-bundle.tar.gz.sha256"
-test -f "${ROOT}/dist/images.lock.json"
+test -f "${WORK_ROOT}/dist/application-catalog-bundle.tar.gz"
+test -f "${WORK_ROOT}/dist/application-catalog-bundle.tar.gz.sha256"
+test -f "${WORK_ROOT}/dist/images.lock.json"
 
-expected_sha="$(awk 'NF>=1 {print $1; exit}' "${ROOT}/dist/application-catalog-bundle.tar.gz.sha256")"
-actual_sha="$(sha256sum "${ROOT}/dist/application-catalog-bundle.tar.gz" | awk '{print $1}')"
+expected_sha="$(awk 'NF>=1 {print $1; exit}' "${WORK_ROOT}/dist/application-catalog-bundle.tar.gz.sha256")"
+actual_sha="$(sha256sum "${WORK_ROOT}/dist/application-catalog-bundle.tar.gz" | awk '{print $1}')"
 [[ "${expected_sha}" == "${actual_sha}" ]] || {
   echo "bundle sha mismatch" >&2
   exit 1
 }
 
 mkdir -p "${TMP_ROOT}/extract"
-tar -xzf "${ROOT}/dist/application-catalog-bundle.tar.gz" -C "${TMP_ROOT}/extract"
-cmp -s "${ROOT}/catalog/catalog.json" "${TMP_ROOT}/extract/catalog.json"
-cmp -s "${ROOT}/dist/images.lock.json" "${TMP_ROOT}/extract/images.lock.json"
-cmp -s "${ROOT}/catalog/profile.env" "${TMP_ROOT}/extract/profile.env"
+tar -xzf "${WORK_ROOT}/dist/application-catalog-bundle.tar.gz" -C "${TMP_ROOT}/extract"
+cmp -s "${WORK_ROOT}/catalog/catalog.json" "${TMP_ROOT}/extract/catalog.json"
+cmp -s "${WORK_ROOT}/dist/images.lock.json" "${TMP_ROOT}/extract/images.lock.json"
+cmp -s "${WORK_ROOT}/catalog/profile.env" "${TMP_ROOT}/extract/profile.env"
 
 python3 - <<'PY' \
   "${TMP_ROOT}/extract/manifest.env" \
   "${TMP_ROOT}/extract/profile.env" \
-  "${ROOT}/catalog/catalog.json" \
-  "${ROOT}/catalog/image-sources.json" \
-  "${ROOT}/dist/images.lock.json"
+  "${WORK_ROOT}/catalog/catalog.json" \
+  "${WORK_ROOT}/catalog/image-sources.json" \
+  "${WORK_ROOT}/dist/images.lock.json"
 import json
 import os
 import re
@@ -183,10 +201,10 @@ for image in images_lock["images"]:
         raise SystemExit(f"generated image lock used_by mismatch for {image['name']}")
 PY
 
-python3 "${ROOT}/scripts/render-catalog-rows.py" \
-  --catalog-json "${ROOT}/catalog/catalog.json" \
-  --profile-env "${ROOT}/catalog/profile.env" \
-  --images-lock "${ROOT}/dist/images.lock.json" \
+python3 "${WORK_ROOT}/scripts/render-catalog-rows.py" \
+  --catalog-json "${WORK_ROOT}/catalog/catalog.json" \
+  --profile-env "${WORK_ROOT}/catalog/profile.env" \
+  --images-lock "${WORK_ROOT}/dist/images.lock.json" \
   --out-catalog "${TMP_ROOT}/catalog.tsv" \
   --channel stable \
   --tag "sha-test-run-1" \
@@ -195,10 +213,11 @@ python3 "${ROOT}/scripts/render-catalog-rows.py" \
   --revision "deadbeefcafedeadbeefcafedeadbeefcafedead" \
   --arch amd64 \
   --artifact-digest "sha256:1111111111111111111111111111111111111111111111111111111111111111" \
-  --pinned-ref "ghcr.io/example/sw-ourbox-catalog-hello-world@sha256:1111111111111111111111111111111111111111111111111111111111111111"
+  --pinned-ref "ghcr.io/example/sw-ourbox-catalog-example@sha256:1111111111111111111111111111111111111111111111111111111111111111"
 
-python3 - <<'PY' "${TMP_ROOT}/catalog.tsv"
+python3 - <<'PY' "${TMP_ROOT}/catalog.tsv" "${WORK_ROOT}/catalog/profile.env" "${WORK_ROOT}/catalog/catalog.json"
 import csv
+import json
 import os
 import sys
 from pathlib import Path
@@ -207,6 +226,14 @@ rows = list(csv.DictReader(Path(sys.argv[1]).open("r", encoding="utf-8"), delimi
 if len(rows) != 1:
     raise SystemExit(f"expected one rendered catalog row, got {len(rows)}")
 row = rows[0]
+profile = {}
+for raw_line in Path(sys.argv[2]).read_text(encoding="utf-8").splitlines():
+    line = raw_line.strip()
+    if not line or line.startswith("#"):
+        continue
+    key, value = line.split("=", 1)
+    profile[key] = value
+catalog = json.loads(Path(sys.argv[3]).read_text(encoding="utf-8"))
 expected = {
     "channel": "stable",
     "tag": "sha-test-run-1",
@@ -215,9 +242,9 @@ expected = {
     "revision": "deadbeefcafedeadbeefcafedeadbeefcafedead",
     "arch": "amd64",
     "platform_contract_digest": os.environ["OURBOX_PLATFORM_CONTRACT_DIGEST"],
-    "platform_profile": "hello-world",
+    "platform_profile": catalog["catalog_id"],
     "artifact_digest": "sha256:1111111111111111111111111111111111111111111111111111111111111111",
-    "pinned_ref": "ghcr.io/example/sw-ourbox-catalog-hello-world@sha256:1111111111111111111111111111111111111111111111111111111111111111",
+    "pinned_ref": "ghcr.io/example/sw-ourbox-catalog-example@sha256:1111111111111111111111111111111111111111111111111111111111111111",
 }
 for key, expected_value in expected.items():
     if row.get(key) != expected_value:

scripts/check-image-refs-exist.sh

@@ -3,14 +3,23 @@ set -euo pipefail
 
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 WORKFLOW="${ROOT}/.github/workflows/publish-catalog-bundle.yml"
+SCRIPT="${ROOT}/scripts/publish-catalog-bundle.sh"
 
-python3 - <<'PY' "${WORKFLOW}"
+python3 - <<'PY' "${WORKFLOW}" "${SCRIPT}"
 import sys
 from pathlib import Path
 
 workflow = Path(sys.argv[1]).read_text(encoding="utf-8")
-lines = {line.strip() for line in workflow.splitlines()}
-required = [
+script = Path(sys.argv[2]).read_text(encoding="utf-8")
+
+workflow_lines = {line.strip() for line in workflow.splitlines()}
+script_lines = {line.strip() for line in script.splitlines()}
+
+workflow_required = [
+    "bash scripts/publish-catalog-bundle.sh",
+]
+
+script_required = [
     'IMMUTABLE_TAG="sha-${GITHUB_SHA}-run-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"',
     'VERSION_TAG="main-${GITHUB_SHA::12}"',
     '"${IMMUTABLE_REF}" \\',
@@ -20,25 +29,34 @@ required = [
     'oras tag "${IMMUTABLE_REF}" "${VERSION_TAG}" >/dev/null',
     'LATEST_DIGEST="$(oras resolve "${REF}")"',
     '[[ "${LATEST_DIGEST}" == "${DIGEST}" ]] || {',
-    'python3 scripts/render-catalog-rows.py \\',
+    'python3 "${ROOT}/scripts/render-catalog-rows.py" \\',
     'dist/catalog.tsv:text/tab-separated-values',
 ]
-banned = [
+
+script_banned = [
     'DIGEST="$(oras resolve "${REF}")"',
     'oras tag "${REF}" "${IMMUTABLE_TAG}" >/dev/null',
 ]
 
-missing = [item for item in required if item not in lines]
-unexpected = [item for item in banned if item in lines]
-if missing or unexpected:
-    lines = []
-    if missing:
-        lines.append("missing expected workflow invariants:")
-        lines.extend(f"  - {item}" for item in missing)
-    if unexpected:
-        lines.append("found banned workflow patterns:")
-        lines.extend(f"  - {item}" for item in unexpected)
-    raise SystemExit("\n".join(lines))
+messages = []
+
+wf_missing = [i for i in workflow_required if not any(i in l for l in workflow_lines)]
+if wf_missing:
+    messages.append("workflow missing expected lines:")
+    messages.extend(f"  - {item}" for item in wf_missing)
+
+sc_missing = [i for i in script_required if i not in script_lines]
+if sc_missing:
+    messages.append("publish-catalog-bundle.sh missing expected invariants:")
+    messages.extend(f"  - {item}" for item in sc_missing)
+
+sc_banned = [i for i in script_banned if i in script_lines]
+if sc_banned:
+    messages.append("publish-catalog-bundle.sh has banned patterns:")
+    messages.extend(f"  - {item}" for item in sc_banned)
+
+if messages:
+    raise SystemExit("\n".join(messages))
 PY
 
-echo "Workflow publish invariants look correct: ${WORKFLOW}"
+echo "Workflow and publish script invariants look correct"