AI Smoke #55
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: AI Smoke | |
| # Runs the AI-verdict tests that talk to the real Anthropic API. Kept out of | |
| # the required-checks matrix (ci.yml) so a flaky model call or | |
| # Anthropic outage cannot block PRs. Triggers: | |
| # - workflow_dispatch (run manually from the Actions tab) | |
| # - nightly schedule (catches regressions in main against the live API) | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| strict: | |
| description: "Hard-fail on the model-dependent audit test (otherwise it is informational)" | |
| type: boolean | |
| default: false | |
| schedule: | |
| - cron: "0 8 * * *" | |
| permissions: | |
| contents: read | |
| issues: write | |
| concurrency: | |
| group: ai-smoke-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| ai-smoke: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 | |
| - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 | |
| with: | |
| node-version: "24" | |
| cache: "pnpm" | |
| - run: pnpm install --frozen-lockfile | |
| - name: pnpm test (AI tests enabled) | |
| env: | |
| # workflow_dispatch can be triggered from any branch, and this step runs | |
| # that branch's (untrusted) tests and build. Only expose the Anthropic | |
| # secret when the ref is main, so a write-capable actor can't dispatch a | |
| # modified branch to exfiltrate the key. The real-AI tests gate on the | |
| # key being present, so a branch dispatch simply skips them. For | |
| # pre-merge AI testing on a branch, use a protected environment with | |
| # required reviewers rather than widening this. | |
| BREAK_CHECK_ANTHROPIC_API_KEY: ${{ github.ref == 'refs/heads/main' && secrets.BREAK_CHECK_ANTHROPIC_API_KEY || '' }} | |
| BREAK_CHECK_RUN_REAL_AI_TESTS: ${{ github.ref == 'refs/heads/main' && '1' || '' }} | |
| # Off by default: the missed-break audit is a model judgment call that | |
| # flickers, so nightly red is reserved for the robust contract checks. | |
| # A manual dispatch can opt into hard-gating it. | |
| BREAK_CHECK_STRICT_AI_TESTS: ${{ inputs.strict && '1' || '' }} | |
| run: pnpm test | |
| - name: Open an issue when the nightly run fails | |
| if: ${{ failure() && github.event_name == 'schedule' }} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| TITLE: "Nightly AI smoke failed" | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| body=$(printf '%s\n\n%s\n' \ | |
| "The nightly AI smoke run failed: $RUN_URL" \ | |
| "This job exercises the live Anthropic API contract through the CLI. The model-dependent audit assertion is non-gating, so a failure here points at something robust: an SDK or submit_review tool-call shape change, the configured model going away, an auth/secret problem, or the model getting a clearly obvious verdict wrong.") | |
| # Reuse a single open issue instead of filing a fresh one every night. | |
| number=$(gh issue list --state open --search "$TITLE in:title" --json number --jq '.[0].number // empty') | |
| if [ -n "$number" ]; then | |
| gh issue comment "$number" --body "$body" | |
| else | |
| gh issue create --title "$TITLE" --body "$body" | |
| fi |