e2e #61
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: e2e | |
| # Attested-build roundtrip against the LIVE kettle-orchestrator | |
| # (build.confidential.ai): submit a build -> the orchestrator launches a CVM, | |
| # builds + hardware-signs SLSA provenance -> download the output -> verify it | |
| # with the kettle CLI BUILT FROM THIS CHECKOUT, fail-closed. So: merge to main | |
| # -> the verifier you just changed is exercised against a real attested build. | |
| # | |
| # P1 (required gate): `kettle verify --nonce` — signature + provenance + | |
| # artifact checksums + nonce/freshness. | |
| # P2 (required gate): also oras-pull the pinned CVM image and run | |
| # `kettle verify --igvm --image` (launch measurement + dm-verity roothash) — | |
| # binds the build to the EXACT pinned CVM image. FAIL-CLOSED. | |
| # History: P2 first ran as a non-blocking probe and surfaced real deployment | |
| # drift (the orchestrator advertised image 8c1a825 / golden 82e291e0 but its | |
| # CVM attested 1932a2f5). A redeploy realigned it and P2 became a required | |
| # gate. If the live service drifts again, P2 goes red — which is the point. | |
| # | |
| # The TEE is REMOTE (the orchestrator's CVM), so this job is a pure HTTP | |
| # client on a GitHub-hosted runner. No secrets: POST /build is public and the | |
| # pinned CVM image is anonymously pullable. | |
| # | |
| # PLANNED second job: kettle's own TEE-gated tests (bin/test-integration = | |
| # `cargo nextest run --features attest --ignored all`) inside a real SEV-SNP | |
| # CVM via the confidential-ci primitive | |
| # (confidential-dot-ai/confidential-ci/.github/workflows/cvm-e2e.yml, | |
| # flavor: base-cpu) — lands once the attestation-rs lane proves that vehicle. | |
| on: | |
| push: | |
| branches: [main] | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '47 8 * * *' # daily: catch orchestrator deployment drift (P2's purpose) | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: e2e-${{ github.sha }} | |
| cancel-in-progress: false | |
| env: | |
| ORCH: https://build.confidential.ai | |
| ORAS_VER: 1.2.0 | |
| # A real Cargo project with a committed Cargo.lock (required for auto-detection) | |
| # and git history (kettle records `git rev-parse HEAD` in the provenance). | |
| # Pinned to ripgrep 14.1.1's commit so the build + provenance are reproducible. | |
| REPO_URL: https://github.com/BurntSushi/ripgrep | |
| REPO_REF: 4649aa9700619f94cf9c66876e9549d83420e16c | |
| jobs: | |
| roundtrip: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1.16.1 | |
| - name: build kettle from this checkout (the verifier under test) + oras | |
| run: | | |
| set -euo pipefail | |
| cargo build --release --features cli --bin kettle | |
| mkdir -p bin-e2e | |
| install -m 0755 target/release/kettle bin-e2e/kettle | |
| # oras — to pull the pinned CVM image for the P2 launch-measurement bind | |
| curl -fsSL "https://github.com/oras-project/oras/releases/download/v${ORAS_VER}/oras_${ORAS_VER}_linux_amd64.tar.gz" -o oras.tgz | |
| tar -xzf oras.tgz oras | |
| install -m 0755 oras bin-e2e/oras | |
| echo "$PWD/bin-e2e" >> "$GITHUB_PATH" | |
| NO_COLOR=1 bin-e2e/kettle --version | |
| bin-e2e/oras version | |
| - name: submit attested build (nonce-bound) | |
| run: | | |
| set -euo pipefail | |
| NONCE=$(openssl rand -hex 16) | |
| echo "NONCE=$NONCE" >> "$GITHUB_ENV" | |
| echo "nonce=$NONCE source=$REPO_URL @ $REPO_REF" | |
| printf '{"nonce":"%s","repo_url":"%s","repo_ref":"%s"}' \ | |
| "$NONCE" "$REPO_URL" "$REPO_REF" > body.json | |
| http=$(curl -sS -m 60 -o post.json -w '%{http_code}' \ | |
| -X POST "$ORCH/build" -H 'content-type: application/json' --data @body.json) | |
| echo "POST /build -> $http"; cat post.json; echo | |
| [ "$http" = "200" ] || { echo "::error::POST /build failed ($http)"; exit 1; } | |
| JOB=$(jq -r '.job_id' post.json) | |
| [ -n "$JOB" ] && [ "$JOB" != "null" ] || { echo "::error::no job_id in response"; exit 1; } | |
| echo "JOB=$JOB" >> "$GITHUB_ENV" | |
| echo "job_id=$JOB" | |
| - name: wait for the CVM build to complete | |
| run: | | |
| set -uo pipefail | |
| # /events is an SSE stream that does NOT auto-close on completion, so we | |
| # POLL: each iteration grabs the event backlog (a terminal job returns it | |
| # and closes immediately; a running job yields the backlog in the first | |
| # seconds) to fail fast on a build error, and treats /result 200 as the | |
| # authoritative "done + artifact ready" gate. | |
| code=000 | |
| for i in $(seq 1 140); do | |
| curl -sS -N --max-time 6 "$ORCH/build/$JOB/events?from=0" > events.log 2>/dev/null || true | |
| if grep -q '"status":"failed"' events.log; then | |
| echo "::error::orchestrator reported the build FAILED" | |
| grep -oE '"error[^"]*":"[^"]*"' events.log | head; exit 1 | |
| fi | |
| code=$(curl -sS -o build.tar.gz -w '%{http_code}' "$ORCH/build/$JOB/result") | |
| [ "$code" = "200" ] && { echo "[$i] /result ready (200)"; break; } | |
| echo "[$i] /result=$code last events:"; tail -n 3 events.log | |
| sleep 10 | |
| done | |
| [ "$code" = "200" ] || { echo "::error::build did not produce a result in time ($code)"; tail -n 30 events.log; exit 1; } | |
| - name: 'P1 — verify attestation (fail-closed, REQUIRED)' | |
| run: | | |
| set -euo pipefail | |
| mkdir -p out && tar -xzf build.tar.gz -C out | |
| DIR=$(dirname "$(find out -name evidence.json | head -1)") | |
| [ -n "$DIR" ] && [ -d "$DIR" ] || { echo "::error::no evidence.json in the result tarball"; find out | head -40; exit 1; } | |
| echo "DIR=$DIR" >> "$GITHUB_ENV" | |
| echo "build dir: $DIR"; ls -la "$DIR" | |
| # NOTE: `kettle verify` EXITS 0 even on a FAILED verdict (it only prints | |
| # the result table), so we MUST assert on the verdict text, not $?. | |
| NO_COLOR=1 kettle verify "$DIR" --nonce "$NONCE" 2>&1 | tee verify.out | |
| grep -q "Verification PASSED" verify.out || { echo "::error::kettle verify did not report PASSED"; exit 1; } | |
| if grep -q "Verification FAILED" verify.out; then echo "::error::kettle verify reported FAILED"; exit 1; fi | |
| echo "✅ P1 VERIFIED — built in a real TEE; signature + provenance + checksum + nonce(freshness) all pass." | |
| - name: 'P2 — launch-measurement bind (--igvm/--image, fail-closed, REQUIRED)' | |
| run: | | |
| set -uo pipefail | |
| # Bind the build to the EXACT pinned CVM image: report launch measurement | |
| # == digest of the pinned IGVM, and dm-verity roothash == disk.raw. | |
| # Infra hiccups (service/registry unreachable) are non-fatal; a genuine | |
| # verification mismatch is fail-closed. kettle verify exits 0 even on a | |
| # FAILED verdict, so we classify on the verdict text. | |
| CFG=$(curl -fsS -m 30 "$ORCH/config") || { echo "::warning title=P2 skipped::/config unreachable (infra)"; exit 0; } | |
| IMG=$(echo "$CFG" | jq -r '.image.reference') | |
| IGVM=$(echo "$CFG" | jq -r '.image.igvm') | |
| DISK=$(echo "$CFG" | jq -r '.image.disk') | |
| echo "pinned image: $IMG (igvm=$IGVM disk=$DISK)" | |
| if ! oras pull "$IMG"; then echo "::warning title=P2 skipped::oras pull failed (infra)"; exit 0; fi | |
| NO_COLOR=1 kettle verify "$DIR" --nonce "$NONCE" --igvm "$IGVM" --image "$DISK" 2>&1 | tee p2.out || true | |
| if grep -q "Verification PASSED" p2.out; then | |
| echo "✅ P2 FULL-CHAIN VERIFIED — bound to the exact pinned CVM image (launch measurement + dm-verity roothash)." | |
| elif grep -q "IGVM launch measurement mismatch" p2.out; then | |
| echo "::error title=P2 measurement mismatch::Attested launch digest != the pinned image's. Usually orchestrator deployment drift (it advertises an image whose IGVM it isn't booting)." | |
| exit 1 | |
| else | |
| echo "::error title=P2 verification failed::Launch-measurement bind FAILED:" | |
| cat p2.out | |
| exit 1 | |
| fi | |
| # ── PER-JOB ATTESTATION OF THE STANDING RUNNER ─────────────────────────── | |
| # Runs alongside `tee`, not in front of it. A green run therefore means both | |
| # that the suite passed and that an attested environment answered to this | |
| # label for this run, rather than only the first. | |
| # | |
| # Wired for tdx-metal first because it is the runner with no other | |
| # instrument. The TDX node image is console-dead by design and the | |
| # provisioner discards the operator key, so a signed report is the only | |
| # evidence anyone can get about that guest after provisioning. | |
| attest-snp-metal: | |
| uses: ./.github/workflows/attest-runner.yml | |
| with: | |
| runner: snp-metal-cvm | |
| platform: snp | |
| # Same value the ephemeral c8s lane pins, so it is already known good. | |
| expected_launch_digest: '15bc995383ddcf1fee2107753e6c26bc65358602519a342b8ff8fbfdfa4b40865d90b11cef11b7d34934e6c5eed19115' | |
| attest-tdx-metal: | |
| uses: ./.github/workflows/attest-runner.yml | |
| with: | |
| runner: tdx-metal-cvm | |
| platform: tdx | |
| # Published in the tdx-rke2-image-refs ConfigMap alongside the node image. | |
| expected_mrtd: '9309eaae9c151e766de0f97b1d1aaeb76b8c8c366080803943fb566521c8f0cf00a142d8b7b0683ed1d42c5a27198ba1' | |
| tee: | |
| strategy: | |
| fail-fast: false # one platform failing must not hide the other | |
| matrix: | |
| runs_on: [snp-metal-cvm, azure-snp-cvm, azure-tdx-cvm, tdx-metal-cvm] | |
| runs-on: ${{ matrix.runs_on }} | |
| timeout-minutes: 45 | |
| steps: | |
| - name: prove the runner is in a real TEE before trusting any result | |
| run: | | |
| # /dev/tdx_guest is checked AFTER /dev/tpmrm0 on purpose: an Azure TDX | |
| # guest exposes BOTH, and it attests through the vTPM + IMDS path, so it | |
| # must report vTPM. Only bare-metal TDX reaches the tdx_guest branch. | |
| # The device node is an underscore; the hyphen spelling is qemu's | |
| # -object tdx-guest, which is a host-side string. | |
| if [ -e /dev/sev-guest ]; then echo "TEE: native SNP (/dev/sev-guest)" | |
| elif [ -e /dev/tpmrm0 ]; then echo "TEE: vTPM (/dev/tpmrm0)" | |
| elif [ -e /dev/tdx_guest ]; then echo "TEE: native TDX (/dev/tdx_guest)" | |
| else echo "::error::no TEE device (/dev/sev-guest, /dev/tpmrm0 or /dev/tdx_guest); this runner is not in a CVM"; exit 1; fi | |
| - name: "build deps (root runner pod: libtss2 + tpm2-tools for --features attest)" | |
| run: | | |
| set -euo pipefail | |
| export DEBIAN_FRONTEND=noninteractive | |
| apt-get update -qq | |
| apt-get install -y -qq --no-install-recommends git curl ca-certificates build-essential pkg-config libtss2-dev tpm2-tools jq | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1.16.1 | |
| - name: install cargo-nextest | |
| run: | | |
| set -euo pipefail | |
| mkdir -p /usr/local/bin | |
| curl -fsSL --connect-timeout 10 --max-time 120 --retry 3 --retry-all-errors https://get.nexte.st/0.9/linux | tar -xz -C /usr/local/bin cargo-nextest | |
| cargo-nextest --version | |
| - name: "TEE integration suite (kettle attest → verify, in the guest)" | |
| env: | |
| # 4-core/16Gi guest with rke2-on-tmpfs (~5Gi free): cap parallelism + | |
| # drop debuginfo so the compile fits (attestation-rs run 29801365953 | |
| # OOM'd unconstrained). | |
| CARGO_BUILD_JOBS: "2" | |
| CARGO_PROFILE_DEV_DEBUG: "0" | |
| CARGO_INCREMENTAL: "0" | |
| run: | | |
| cargo nextest run --features attest --run-ignored all --no-fail-fast \ | |
| -E 'not (test(stored_roothash_from_real_disk) | test(cli_attest_alejandra))' \ | |
| --test-threads 2 |