Docs Agent Eval #1102
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Docs Agent Eval — label-triggered CI check (V1, informational) | |
| # | |
| # Add the `run-docs-tests` label to a docs PR to run an agent-buildability | |
| # eval against the PR's Vercel docs preview: a deterministic router picks | |
| # no-eval / Smoke (1 agent) / Deep (5 agents), fresh isolated agents follow | |
| # the proposed docs through a rewrite proxy, and the result is posted back | |
| # here as a PR comment. `run-docs-tests-deep` forces Deep. | |
| # | |
| # Two entry paths, so label-then-push and push-then-label both work: | |
| # - deployment_status (Vercel finished a docs preview): runs if the PR | |
| # already carries a run-docs-tests label. | |
| # - pull_request labeled/reopened: runs if the docs preview already | |
| # succeeded; otherwise skips, and the deployment_status event re-enters. | |
| # | |
| # The eval logic lives in the PRIVATE repo ComposioHQ/docs-agent-eval-ci. | |
| # A public repo cannot `uses:` a private repo's reusable workflow, so instead | |
| # a GitHub App token checks that repo out and this job runs it inline. See its | |
| # ARCHITECTURE.md. Informational check — must NOT be a required status in V1. | |
| # | |
| # Required repo/org secrets: COMPOSIO_ORG_API_KEY_FOR_CODING_AGENT_EVAL, DEEPSEEK_API_KEY, | |
| # DOCS_EVAL_APP_ID, DOCS_EVAL_APP_PRIVATE_KEY (GitHub App that can read the | |
| # private engine repo). | |
| name: Docs Agent Eval | |
| on: | |
| pull_request: | |
| types: [labeled, reopened] | |
| deployment_status: | |
| env: | |
| ENGINE_REPO: ComposioHQ/docs-agent-eval-ci | |
| # Full-SHA pin of the engine code — bump on docs-agent-eval-ci releases. | |
| ENGINE_REF: 8bd5993a85d60d4b4b70f171d41816b91ab67295 | |
| jobs: | |
| # Decides IF an eval should start and FOR WHICH PR. deployment_status | |
| # events know the commit but not the PR (deployments attach to commits), | |
| # so this job maps SHA -> open PR before any label check is possible. | |
| route: | |
| if: | | |
| (github.event_name == 'pull_request' && | |
| (contains(github.event.pull_request.labels.*.name, 'run-docs-tests') || | |
| contains(github.event.pull_request.labels.*.name, 'run-docs-tests-deep')) && | |
| (github.event.action != 'labeled' || startsWith(github.event.label.name, 'run-docs-tests'))) || | |
| (github.event_name == 'deployment_status' && | |
| github.event.deployment_status.state == 'success' && | |
| startsWith(github.event.deployment_status.environment_url, 'https://docs-') && | |
| contains(github.event.deployment_status.environment_url, '.preview.composio.dev')) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| permissions: | |
| contents: read | |
| deployments: read | |
| pull-requests: read | |
| outputs: | |
| should_run: ${{ steps.decide.outputs.should_run }} | |
| pr_number: ${{ steps.decide.outputs.pr_number }} | |
| mode: ${{ steps.decide.outputs.mode }} | |
| steps: | |
| - name: Decide whether to run, and for which PR | |
| id: decide | |
| shell: bash | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| REPO: ${{ github.repository }} | |
| run: | | |
| set -euo pipefail | |
| pr="" assoc="" labels="" preview_ok=false | |
| if [ "$GITHUB_EVENT_NAME" = deployment_status ]; then | |
| # Trap 2: the event payload has no PR number — map commit -> PR. | |
| SHA=$(jq -r .deployment.sha "$GITHUB_EVENT_PATH") | |
| PRJSON=$(gh api "repos/$REPO/commits/$SHA/pulls" \ | |
| --jq '[.[] | select(.state=="open")][0] // empty') | |
| if [ -n "$PRJSON" ]; then | |
| pr=$(jq -r .number <<<"$PRJSON") | |
| assoc=$(jq -r .author_association <<<"$PRJSON") | |
| labels=$(jq -r '[.labels[].name] | join(",")' <<<"$PRJSON") | |
| fi | |
| preview_ok=true # this event IS the successful preview | |
| else | |
| pr=$(jq -r .pull_request.number "$GITHUB_EVENT_PATH") | |
| assoc=$(jq -r .pull_request.author_association "$GITHUB_EVENT_PATH") | |
| labels=$(jq -r '[.pull_request.labels[].name] | join(",")' "$GITHUB_EVENT_PATH") | |
| SHA=$(jq -r .pull_request.head.sha "$GITHUB_EVENT_PATH") | |
| # Trap 1: label may predate the preview. Only run now if a docs | |
| # preview already succeeded for this exact commit; otherwise skip | |
| # — the deployment_status event re-enters when the preview lands. | |
| for d in $(gh api "repos/$REPO/deployments?sha=$SHA&per_page=10" --jq '.[].id'); do | |
| URL=$(gh api "repos/$REPO/deployments/$d/statuses" \ | |
| --jq '[.[] | select(.state=="success")][0].environment_url // empty') | |
| case "$URL" in | |
| https://docs-*.preview.composio.dev*) preview_ok=true; break ;; | |
| esac | |
| done | |
| fi | |
| haslabel=false | |
| case ",$labels," in | |
| *,run-docs-tests,*|*,run-docs-tests-deep,*) haslabel=true ;; | |
| esac | |
| # Employee gate: PR author must be org-affiliated (public repo). | |
| trusted=false | |
| case "$assoc" in MEMBER|OWNER|COLLABORATOR) trusted=true ;; esac | |
| mode=Auto | |
| case ",$labels," in *,run-docs-tests-deep,*) mode=Deep ;; esac | |
| should=false | |
| if [ -n "$pr" ] && $haslabel && $trusted && $preview_ok; then | |
| should=true | |
| fi | |
| echo "event=$GITHUB_EVENT_NAME pr=${pr:-none} labels=[$labels] assoc=${assoc:-none} preview_ok=$preview_ok -> should_run=$should" | |
| { | |
| echo "should_run=$should" | |
| echo "pr_number=$pr" | |
| echo "mode=$mode" | |
| } >> "$GITHUB_OUTPUT" | |
| # Runs the eval inline: mint an App token, check out the private engine with | |
| # it, then run the engine's code on this runner. (A public repo can't `uses:` | |
| # a private reusable workflow, so the App token + checkout replaces it.) | |
| eval: | |
| needs: route | |
| if: needs.route.outputs.should_run == 'true' | |
| concurrency: | |
| group: docs-agent-eval-pr-${{ needs.route.outputs.pr_number }} | |
| cancel-in-progress: true | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 55 | |
| permissions: | |
| contents: read | |
| deployments: read | |
| pull-requests: write # the result comment (posted with the built-in token) | |
| env: | |
| SOURCE_REPO: ${{ github.repository }} | |
| PR_NUMBER: ${{ needs.route.outputs.pr_number }} | |
| MODE: ${{ needs.route.outputs.mode }} | |
| COMPOSIO_ORG_API_KEY_FOR_CODING_AGENT_EVAL: ${{ secrets.COMPOSIO_ORG_API_KEY_FOR_CODING_AGENT_EVAL }} | |
| # Builder runs on DeepSeek V4 Flash via the Anthropic-compatible endpoint. | |
| ANTHROPIC_BASE_URL: https://api.deepseek.com/anthropic | |
| ANTHROPIC_AUTH_TOKEN: ${{ secrets.DEEPSEEK_API_KEY }} | |
| DEEP_RUNS: '5' | |
| POST_PR_COMMENT: 'true' | |
| steps: | |
| - name: Mint a GitHub App token (read the private engine repo) | |
| id: app-token | |
| uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 | |
| with: | |
| app-id: ${{ secrets.DOCS_EVAL_APP_ID }} | |
| private-key: ${{ secrets.DOCS_EVAL_APP_PRIVATE_KEY }} | |
| owner: ComposioHQ | |
| repositories: docs-agent-eval-ci | |
| - name: Check out the private engine | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| repository: ${{ env.ENGINE_REPO }} | |
| ref: ${{ env.ENGINE_REF }} | |
| token: ${{ steps.app-token.outputs.token }} | |
| path: engine | |
| # Don't write the App token into engine/.git/config — the eval then | |
| # runs untrusted agent code with filesystem access on this runner, | |
| # which could read a persisted token. We only need it for this fetch. | |
| persist-credentials: false | |
| - name: Setup Python | |
| uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: '3.12' | |
| cache: pip | |
| cache-dependency-path: engine/ci/requirements.txt | |
| - name: Install harness dependencies | |
| working-directory: engine | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install -r ci/requirements.txt | |
| - name: Cache npm downloads | |
| uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| path: ~/.npm | |
| key: npm-claude-code-${{ runner.os }} | |
| - name: Install Claude Code | |
| run: npm install --global @anthropic-ai/claude-code | |
| - name: Validate required credentials | |
| shell: bash | |
| run: | | |
| test -n "$COMPOSIO_ORG_API_KEY_FOR_CODING_AGENT_EVAL" || { echo "COMPOSIO_ORG_API_KEY_FOR_CODING_AGENT_EVAL is not configured"; exit 1; } | |
| test -n "$ANTHROPIC_AUTH_TOKEN" || { echo "DEEPSEEK_API_KEY is not configured"; exit 1; } | |
| - name: Route (deterministic no-eval / Smoke / Deep) | |
| id: extent | |
| working-directory: engine | |
| shell: bash | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| run: | | |
| mkdir -p ci-output | |
| python ci/route_pr.py --repo "$SOURCE_REPO" --pr "$PR_NUMBER" > ci-output/route.json | |
| cat ci-output/route.json | |
| EXTENT=$(python -c 'import json; print(json.load(open("ci-output/route.json"))["validation_extent"])') | |
| HEAD=$(python -c 'import json; print(json.load(open("ci-output/route.json"))["head_sha"])') | |
| # run-docs-tests-deep forces Deep regardless of the router. | |
| [ "$MODE" = Deep ] && EXTENT=deep | |
| echo "extent=$EXTENT" >> "$GITHUB_OUTPUT" | |
| echo "head_sha=$HEAD" >> "$GITHUB_OUTPUT" | |
| - name: Record no-eval outcome | |
| if: steps.extent.outputs.extent == 'none' | |
| working-directory: engine | |
| shell: bash | |
| run: | | |
| python - <<'EOF' | |
| import json | |
| route = json.load(open("ci-output/route.json")) | |
| json.dump({"status": "NO_EVAL", "reasons": route["reasons"]}, open("ci-output/outcome.json", "w"), indent=2) | |
| lines = "\n".join(f"- {r}" for r in route["reasons"]) | |
| open("ci-output/comment.md", "w").write( | |
| f"## Docs Agent Eval — ✓ NOT APPLICABLE\n\nNo agent-impacting docs change detected; no eval run.\n\n{lines}\n") | |
| EOF | |
| - name: Run exact-HEAD docs agent eval | |
| if: steps.extent.outputs.extent != 'none' | |
| continue-on-error: true | |
| working-directory: engine | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| run: | | |
| RUNMODE=Smoke | |
| [ '${{ steps.extent.outputs.extent }}' = deep ] && RUNMODE=Deep | |
| python ci/run_full_eval.py \ | |
| --repo "$SOURCE_REPO" \ | |
| --head-sha '${{ steps.extent.outputs.head_sha }}' \ | |
| --mode "$RUNMODE" | |
| - name: Upload eval artifacts | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: docs-agent-eval-artifacts | |
| path: engine/ci-output/ | |
| if-no-files-found: warn | |
| retention-days: 14 | |
| - name: Publish job summary | |
| if: always() | |
| working-directory: engine | |
| shell: bash | |
| run: | | |
| if [ -f ci-output/comment.md ]; then | |
| cat ci-output/comment.md >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo '## Docs Agent Eval — ✕ ERROR' >> "$GITHUB_STEP_SUMMARY" | |
| echo 'The CI spine failed before it could produce a report.' >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| - name: Post result comment on the PR | |
| if: always() | |
| working-directory: engine | |
| shell: bash | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| test -f ci-output/comment.md || exit 0 | |
| gh pr comment "$PR_NUMBER" --repo "$SOURCE_REPO" --body-file ci-output/comment.md | |
| - name: Set check result | |
| if: always() | |
| working-directory: engine | |
| shell: bash | |
| run: | | |
| test -f ci-output/outcome.json || exit 1 | |
| STATUS=$(python -c 'import json; print(json.load(open("ci-output/outcome.json"))["status"])') | |
| test "$STATUS" = PASS || test "$STATUS" = NO_EVAL |