code-review-model-benchmark.yml

name: "Benchmark: tier-0 model code review"

# Per-model code-review benchmark against the QA cloud. For each recommended
# model in the BYOK catalog (curated-models.json, tier="recommended") it points
# ONE QA benchmark tenant's BYOK at the model and opens the fixed 5-PR set on the
# shared kodus-e2e benchmark repos, waiting for Kody to review each.
#
# Gate (mechanical): `benchmark:models` exits non-zero if any review fails to
# complete — that is the pass/fail. The precision/recall scorecard
# (`benchmark:scorecard`, judged by Sonnet vs golden_comments) is uploaded as an
# ARTIFACT for regression tracking against main — it is NOT a gate.
#
# Triggers (locked spec 2026-05-28, revised 2026-05-28):
#   - push to main on engine paths → FULL run (5 models, 5 PRs each = 25
#     reviews, ~$17, ~40min). Notify-only.
#   - workflow_dispatch → manual: pin one model (only_model).
# NO pull_request trigger: the benchmark drives reviews on qa.web.kodus.io,
# which runs main's image — a PR-time LIGHT run would benchmark MAIN's
# engine quality (not the PR's), defeating the purpose. The right signal
# at PR-time is the kody rule "engine sem benchmark" reminding the dev to
# update the dataset; the FULL run after merge measures the actual change.
# NO nightly cron: full FULL run only when the engine actually changes,
# to keep cost bounded. Opus is dropped at the driver level
# (tests/e2e/benchmark/run.ts DEFAULT_EXCLUDED) — ~$23/run extra and not
# worth the CI spend. Run ad-hoc via dispatch with only_model="opus-4-7"
# if you ever need it.
#
# Required secrets (add in repo settings — only GH_TEST_TOKEN exists today):
#   GH_TEST_TOKEN            PAT that can see + write the kodus-e2e/* bench repos
#   BYOK_ANTHROPIC_API_KEY   Claude (sonnet-4-6, opus-4-7)
#   BYOK_OPENAI_API_KEY      gpt-5.4
#   BYOK_GOOGLE_API_KEY      gemini-3.1-pro
#   BYOK_MOONSHOT_API_KEY    kimi-k2.6 (openai_compatible)
#   BYOK_ZHIPU_API_KEY       glm-5.1 (openai_compatible)
#   ANTHROPIC_API_KEY        the scorecard judge (Sonnet)
# Optional: BENCH_TENANT_EMAIL / BENCH_TENANT_PASSWORD for a dedicated tenant.

permissions:
    contents: read

on:
    workflow_dispatch:
        inputs:
            only_model:
                description: "Restrict to one model slug (e.g. sonnet-4-6, or opus-4-7 to run the otherwise-excluded model)"
                required: false
                default: ""
                type: string
    push:
        branches: [main]
        paths:
            # Engine paths — anything that can change review OUTPUT quality.
            # Stages/agents/prompts/rules/sandbox/BYOK + the model catalog.
            - "libs/code-review/**"
            - "libs/ee/codeBase/**"
            - "libs/ee/codeReview/**"
            - "libs/ee/kodyRules/**"
            - "libs/kodyRules/**"
            - "libs/sandbox/**"
            - "libs/core/infrastructure/services/tokenTracking/**"
            - "apps/web/src/features/ee/byok/_data/curated-models.json"
            - "tests/e2e/benchmark/**"
            - "scripts/benchmark/tier0-bench-prs.json"

concurrency:
    # Single benchmark tenant + shared repos → never run two at once.
    # cancel-in-progress: rapid successive engine merges collapse into one
    # full run instead of queueing $17×N jobs. The latest commit's result is
    # what matters; intermediate runs would just spend money.
    group: code-review-model-benchmark
    cancel-in-progress: true

jobs:
    benchmark:
        name: tier-0 model benchmark
        runs-on: ubuntu-latest
        # QA environment: home of QA_WAF_BYPASS_HEADER (the WAF Allow-rule
        # header — runner IPs intermittently land on the QA WAF's block
        # rules and every login 403s, the exact symptom described below).
        # All triggers run from main, which the QA branch policy allows.
        environment: QA
        # 6 models x 5 PRs, each review polled up to 25 min. Full runs are long;
        # GitHub's job ceiling is 6h. PR (light) runs are ~1 model.
        timeout-minutes: 350
        steps:
            - name: Checkout monorepo
              uses: actions/checkout@v6.0.2

            - name: Setup Node
              uses: actions/setup-node@v4
              with:
                  node-version: "22"

            - name: Install e2e deps
              working-directory: tests/e2e
              run: npm install --no-audit --no-fund

            - name: Decide scope (push main = full, dispatch = manual override)
              id: scope
              run: |
                  set -euo pipefail
                  ONLY_MODEL_INPUT="${{ github.event.inputs.only_model }}"

                  if [ "${{ github.event_name }}" = "push" ]; then
                      # FULL on merge: all recommended models EXCEPT the ones
                      # excluded at the driver level (Opus, see run.ts
                      # DEFAULT_EXCLUDED) — 5 models, ~$17.
                      echo "only_model=" >> "$GITHUB_OUTPUT"
                      echo "Full run (engine merged to main): 5 models (Opus excluded by driver)"
                  else
                      # workflow_dispatch — pass through whatever was given,
                      # including "opus-4-7" to opt into the otherwise-excluded model.
                      echo "only_model=$ONLY_MODEL_INPUT" >> "$GITHUB_OUTPUT"
                      echo "Manual dispatch: only_model='$ONLY_MODEL_INPUT'"
                  fi

            - name: Wait for QA backend to settle on this build (avoid deploy-race)
              env:
                  CLOUD_WEB_BASE_URL: ${{ vars.CLOUD_QA_WEB_URL || 'https://qa.web.kodus.io' }}
                  QA_WAF_BYPASS_HEADER: ${{ secrets.QA_WAF_BYPASS_HEADER }}
                  # The benchmark fires on the SAME push that triggers the QA GitOps
                  # deploy of github.sha. Gate on the live /health version matching
                  # this SHA so the benchmark BLOCKS until its own rollout has fully
                  # landed — and stays settled — before opening 25 PRs. Empty on
                  # workflow_dispatch (no deploy in flight) → pure readiness probe.
                  EXPECTED_VERSION: ${{ github.event_name == 'push' && github.sha || '' }}
              run: |
                  set -uo pipefail
                  BASE="${CLOUD_WEB_BASE_URL%/}/api/proxy/api"
                  WAF_HDR=()
                  if [ -n "${QA_WAF_BYPASS_HEADER:-}" ]; then
                      WAF_HDR=(-H "x-kodus-e2e: ${QA_WAF_BYPASS_HEADER}")
                  fi
                  # On push this runs alongside the QA GitOps deploy, so it can hit
                  # the backend mid-rollout: the gateway answers 403/5xx/000, OR the
                  # API answers app-level for a moment and then restarts again as the
                  # rollout rolls across replicas — and a single momentary 200 is
                  # exactly how the review WORKER ends up not-yet-consuming when the
                  # 25 reviews fire (every model × repo then "review timeout", the
                  # 2026-06-12 red). So we do NOT proceed on the first healthy probe:
                  # we require the backend to stay app-ready AND on THIS build for
                  # several CONSECUTIVE checks, proving the rollout has settled.
                  echo "Probing $BASE/auth/login for app-level readiness (settle gate)…"
                  NEED_OK=3          # consecutive clean probes required to proceed
                  SETTLE_SLEEP=10    # spacing between probes (s)
                  ok_streak=0
                  for i in $(seq 1 90); do
                      code=$(curl -sS -o /dev/null -w '%{http_code}' \
                          -X POST "$BASE/auth/login" \
                          "${WAF_HDR[@]}" \
                          -H 'Content-Type: application/json' \
                          -d '{"email":"readiness-probe@invalid.kodus","password":"x"}' \
                          2>/dev/null || echo 000)
                      # If we know the target build, the live version must ALSO still
                      # match on each probe — a flip back to the prior build means a
                      # replica is mid-restart, so the rollout hasn't settled. Stays
                      # best-effort: an empty/'unknown' version can't be gated on, so
                      # it never blocks here.
                      ver_ok=1
                      if [ -n "${EXPECTED_VERSION:-}" ]; then
                          ver=$(curl -sS "${WAF_HDR[@]}" "$BASE/health" 2>/dev/null \
                              | node -e "let s='';process.stdin.on('data',d=>s+=d).on('end',()=>{try{console.log(JSON.parse(s).version||'')}catch{console.log('')}})" \
                              2>/dev/null || echo "")
                          case "$ver" in
                              ""|unknown) ver_ok=1 ;;
                              *"$EXPECTED_VERSION"*|*"${EXPECTED_VERSION:0:7}"*) ver_ok=1 ;;
                              *) ver_ok=0 ;;
                          esac
                      fi
                      case "$code" in
                          200|201|400|401|404|422)
                              if [ "$ver_ok" = 1 ]; then
                                  ok_streak=$((ok_streak + 1))
                                  echo "attempt $i/90: app-ready (HTTP $code), settle streak $ok_streak/$NEED_OK"
                                  if [ "$ok_streak" -ge "$NEED_OK" ]; then
                                      echo "QA settled on this build — proceeding with the benchmark."
                                      exit 0
                                  fi
                              else
                                  echo "attempt $i/90: app-ready (HTTP $code) but version='$ver' (rollout still moving) — resetting streak"
                                  ok_streak=0
                              fi
                              sleep "$SETTLE_SLEEP" ;;
                          *)
                              echo "attempt $i/90: HTTP $code (gateway/rollout) — resetting streak, waiting ${SETTLE_SLEEP}s…"
                              ok_streak=0
                              sleep "$SETTLE_SLEEP" ;;
                      esac
                  done
                  echo "::error::QA backend never settled (app-ready + this build, $NEED_OK consecutive) in ~15min — aborting before the benchmark to avoid deploy-race false failures."
                  exit 1

            - name: Run benchmark (mechanical gate — fails if any review fails)
              working-directory: tests/e2e
              env:
                  CLOUD_WEB_BASE_URL: ${{ vars.CLOUD_QA_WEB_URL || 'https://qa.web.kodus.io' }}
                  BENCH_ONLY_MODEL: ${{ steps.scope.outputs.only_model }}
                  BENCH_TENANT_EMAIL: ${{ secrets.BENCH_TENANT_EMAIL }}
                  BENCH_TENANT_PASSWORD: ${{ secrets.BENCH_TENANT_PASSWORD }}
                  # WAF Allow-rule header — lib/http.ts injects it on every
                  # qa.*.kodus.io request (see e2e-cloud.yml for rationale).
                  QA_WAF_BYPASS_HEADER: ${{ secrets.QA_WAF_BYPASS_HEADER }}
                  GH_TEST_TOKEN: ${{ secrets.GH_TEST_TOKEN }}
                  BYOK_ANTHROPIC_API_KEY: ${{ secrets.BYOK_ANTHROPIC_API_KEY }}
                  BYOK_OPENAI_API_KEY: ${{ secrets.BYOK_OPENAI_API_KEY }}
                  BYOK_GOOGLE_API_KEY: ${{ secrets.BYOK_GOOGLE_API_KEY }}
                  BYOK_MOONSHOT_API_KEY: ${{ secrets.BYOK_MOONSHOT_API_KEY }}
                  BYOK_ZHIPU_API_KEY: ${{ secrets.BYOK_ZHIPU_API_KEY }}
              run: npm run benchmark:models

            - name: Re-collect settled findings (repair late-posted findings)
              if: always()
              working-directory: tests/e2e
              env:
                  GH_TEST_TOKEN: ${{ secrets.GH_TEST_TOKEN }}
              run: npm run benchmark:recollect || echo "recollect step failed (non-gating)"

            - name: Scorecard (artifact, not a gate)
              id: scorecard
              if: always()
              working-directory: tests/e2e
              env:
                  ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
              # Never fail the job on the quality scorecard — the gate is the
              # mechanical "all reviews completed" step above. Capture the printed
              # table into a step output so the success notification can post it.
              run: |
                  npm run benchmark:scorecard 2>&1 | tee /tmp/scorecard.txt || echo "scorecard step failed (non-gating)"
                  {
                      echo 'table<<SCORECARD_EOF'
                      grep -vE '^> |^npm |^$' /tmp/scorecard.txt | tail -n 40 || echo '(scorecard unavailable)'
                      echo 'SCORECARD_EOF'
                  } >> "$GITHUB_OUTPUT"

            - name: Upload results + scorecard
              if: always()
              uses: actions/upload-artifact@v4
              with:
                  name: tier0-benchmark-${{ github.run_id }}
                  path: |
                      tests/e2e/benchmark/results.json
                      tests/e2e/benchmark/scorecard.json
                  if-no-files-found: warn
                  retention-days: 30

            - name: Notify Discord on failure
              if: failure()
              uses: ./.github/actions/discord-notify
              with:
                  # Bench has its own audience (model quality watchers, not
                  # infra). Falls back to general so missing channel doesn't
                  # silence regressions.
                  webhook: ${{ secrets.DISCORD_WEBHOOK_INTERNAL || secrets.DISCORD_WEBHOOK }}
                  status: failure
                  title: "❌ Model code-review benchmark failed"
                  description: |
                      Engine code merged to main but the per-model benchmark didn't complete.
                      Scope: ${{ steps.scope.outputs.only_model != '' && format('only_model={0}', steps.scope.outputs.only_model) || 'FULL (5 models)' }}
                      Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
                      Trigger: ${{ github.event_name }}

            - name: Notify Discord on success (scorecard)
              if: success()
              uses: ./.github/actions/discord-notify
              with:
                  webhook: ${{ secrets.DISCORD_WEBHOOK_INTERNAL || secrets.DISCORD_WEBHOOK }}
                  status: success
                  title: "✅ Model code-review benchmark — scorecard"
                  description: |
                      Scope: ${{ steps.scope.outputs.only_model != '' && format('only_model={0}', steps.scope.outputs.only_model) || 'FULL (5 models)' }}
                      ```
                      ${{ steps.scorecard.outputs.table }}
                      ```
                      Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}