Blog pipeline failed: Unexpected error (line 329) — AI Agents Onchain #10186
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sentry Auto-Fix | |
| on: | |
| issues: | |
| types: [labeled] | |
| workflow_dispatch: | |
| inputs: | |
| issue_number: | |
| description: 'Issue number to triage' | |
| required: true | |
| pull_request_review: | |
| types: [submitted] | |
| issue_comment: | |
| types: [created, edited] | |
| schedule: | |
| - cron: '0 */2 * * *' | |
| permissions: | |
| contents: write | |
| issues: write | |
| pull-requests: write | |
| id-token: write | |
| jobs: | |
| triage-and-fix: | |
| concurrency: | |
| group: sentry-triage-${{ github.event.issue.number || github.event.inputs.issue_number }} | |
| cancel-in-progress: false | |
| if: > | |
| ( | |
| (github.event_name == 'issues' && | |
| github.event.action == 'labeled' && | |
| (contains(github.event.issue.labels.*.name, 'sentry') || | |
| contains(github.event.issue.labels.*.name, 'worker-crash')) && | |
| !contains(github.event.issue.labels.*.name, 'needs-human')) | |
| || | |
| github.event_name == 'workflow_dispatch' | |
| ) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Get issue info | |
| id: issue_info | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| ISSUE_NUM="${{ github.event.inputs.issue_number }}" | |
| else | |
| ISSUE_NUM="${{ github.event.issue.number }}" | |
| fi | |
| echo "number=$ISSUE_NUM" >> $GITHUB_OUTPUT | |
| ISSUE_DATA=$(gh issue view "$ISSUE_NUM" --repo ${{ github.repository }} --json title,body) | |
| echo "title=$(echo "$ISSUE_DATA" | jq -r '.title')" >> $GITHUB_OUTPUT | |
| # Save body to file to avoid escaping issues | |
| - name: Verify worker-crash trigger source | |
| if: contains(github.event.issue.labels.*.name, 'worker-crash') | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| if [ "${{ github.actor }}" != "github-actions[bot]" ]; then | |
| gh issue comment ${{ steps.issue_info.outputs.number }} \ | |
| --repo ${{ github.repository }} \ | |
| --body "Skipped: worker-crash auto-fix can only be triggered by the automated watchdog (github-actions[bot])." | |
| echo "::error::Unauthorized actor ${{ github.actor }} added worker-crash label" | |
| exit 1 | |
| fi | |
| echo "$ISSUE_DATA" | jq -r '.body' > /tmp/issue-body.txt | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: develop | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 18 | |
| - name: Install backend deps | |
| run: cd run && npm install | |
| - name: Install frontend deps | |
| run: yarn install | |
| - uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1 # v1.6 | |
| - name: Setup SSH | |
| uses: webfactory/ssh-agent@v0.9.0 | |
| with: | |
| ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| - name: Add known hosts | |
| run: | | |
| mkdir -p ~/.ssh | |
| ssh-keyscan -H 157.90.154.200 >> ~/.ssh/known_hosts | |
| - name: Configure git | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| - name: Circuit breaker - check for active incident | |
| id: circuit_breaker | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| # Count sentry-labeled issues created in the last 30 minutes | |
| CUTOFF=$(date -u -d '30 minutes ago' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-30M +%Y-%m-%dT%H:%M:%SZ) | |
| RECENT_COUNT=$(gh issue list --repo ${{ github.repository }} --label sentry --state open --json number,createdAt \ | |
| --jq "[.[] | select(.createdAt > \"$CUTOFF\")] | length") | |
| # Also check for incident-labeled issues (created by scanner correlation) | |
| HAS_INCIDENT=$(gh issue list --repo ${{ github.repository }} --label incident --state open --limit 1 --json number \ | |
| --jq 'length') | |
| if [ "$HAS_INCIDENT" -gt 0 ]; then | |
| echo "tripped=true" >> $GITHUB_OUTPUT | |
| echo "Circuit breaker: active incident issue exists. Skipping auto-fix." | |
| gh issue edit ${{ steps.issue_info.outputs.number }} --add-label "needs-human" --repo ${{ github.repository }} | |
| gh issue comment ${{ steps.issue_info.outputs.number }} \ | |
| --body "⚡ **Circuit breaker tripped**: An active incident issue exists. Auto-fix paused to avoid treating individual symptoms. See the open \`incident\` issue for context. @antoinedc please investigate the root cause." \ | |
| --repo ${{ github.repository }} | |
| elif [ "$RECENT_COUNT" -ge 3 ]; then | |
| echo "tripped=true" >> $GITHUB_OUTPUT | |
| echo "Circuit breaker: $RECENT_COUNT sentry issues in last 30 min (threshold: 3). Escalating." | |
| gh issue edit ${{ steps.issue_info.outputs.number }} --add-label "needs-human" --repo ${{ github.repository }} | |
| gh issue comment ${{ steps.issue_info.outputs.number }} \ | |
| --body "⚡ **Circuit breaker tripped**: $RECENT_COUNT sentry issues opened in the last 30 minutes, indicating a systemic incident. Auto-fix paused to avoid fixing symptoms individually. @antoinedc please investigate the root cause." \ | |
| --repo ${{ github.repository }} | |
| else | |
| echo "tripped=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Check for existing fix PR | |
| id: existing_pr | |
| if: steps.circuit_breaker.outputs.tripped != 'true' | |
| run: | | |
| EXISTING_PR=$(gh pr list --repo ${{ github.repository }} --head "fix/sentry-${{ steps.issue_info.outputs.number }}" --state open --json number -q '.[0].number' 2>/dev/null || true) | |
| if [ -n "$EXISTING_PR" ]; then | |
| echo "exists=true" >> $GITHUB_OUTPUT | |
| echo "PR #$EXISTING_PR already exists for this issue, skipping" | |
| else | |
| echo "exists=false" >> $GITHUB_OUTPUT | |
| fi | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Notify dashboard - triaging | |
| if: always() | |
| continue-on-error: true | |
| run: | | |
| curl -s -X POST "${{ secrets.APP_URL }}/webhooks/github-actions" \ | |
| -H "Authorization: Bearer ${{ secrets.ETHERNAL_WEBHOOK_SECRET }}" \ | |
| -H "Content-Type: application/json" \ | |
| -d "{ | |
| \"workflowRunId\": ${{ github.run_id }}, | |
| \"githubIssueNumber\": ${{ steps.issue_info.outputs.number }}, | |
| \"status\": \"triaging\", | |
| \"currentStep\": \"Claude is investigating the error\" | |
| }" | |
| - name: Check for regression | |
| id: regression | |
| env: | |
| SENTRY_API_TOKEN: ${{ secrets.SENTRY_API_TOKEN }} | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| # Extract Sentry issue ID from issue body | |
| SENTRY_ISSUE_ID=$(grep -oP 'sentry(?:\.tryethernal\.com|\.io)/organizations/[^/]+/issues/\K\d+' /tmp/issue-body.txt | head -1) | |
| CONTEXT="" | |
| if [ -n "$SENTRY_ISSUE_ID" ]; then | |
| # Check if Sentry marks this as a regression | |
| IS_REGRESSION=$(curl -s \ | |
| "https://sentry.io/api/0/issues/$SENTRY_ISSUE_ID/" \ | |
| -H "Authorization: Bearer $SENTRY_API_TOKEN" | jq -r '.isRegression // false') | |
| if [ "$IS_REGRESSION" = "true" ]; then | |
| # Search for previous closed issues with the same Sentry issue ID | |
| PREV_ISSUES=$(gh issue list --label sentry --state closed --limit 20 --json number,title,body \ | |
| --jq "[.[] | select(.body | contains(\"issues/$SENTRY_ISSUE_ID/\")) | {number, title}]") | |
| # Find linked PRs for those issues | |
| PREV_PRS="" | |
| for ISSUE_NUM in $(echo "$PREV_ISSUES" | jq -r '.[].number'); do | |
| PR=$(gh pr list --state merged --search "Fixes #$ISSUE_NUM" --json number,title,url \ | |
| --jq '.[0] | "PR #\(.number): \(.title) (\(.url))"' 2>/dev/null) | |
| if [ -n "$PR" ]; then | |
| PREV_PRS="$PREV_PRS\n- $PR (fixed issue #$ISSUE_NUM)" | |
| fi | |
| done | |
| if [ -n "$PREV_PRS" ]; then | |
| CONTEXT="⚠️ **REGRESSION**: This error was previously fixed and has recurred. | |
| Previous fix attempts: | |
| $(echo -e "$PREV_PRS") | |
| Review the previous PRs to understand what was tried before. The previous fix was insufficient — identify WHY it regressed and ensure your new fix addresses the root cause more thoroughly." | |
| else | |
| CONTEXT="⚠️ **REGRESSION**: Sentry reports this error was previously resolved but has recurred. Search git history for related fixes." | |
| fi | |
| fi | |
| fi | |
| # Save to file to avoid shell escaping issues | |
| echo "$CONTEXT" > /tmp/regression-context.txt | |
| if [ -n "$CONTEXT" ]; then | |
| echo "is_regression=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "is_regression=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Start conversation streamer | |
| if: steps.circuit_breaker.outputs.tripped != 'true' && steps.existing_pr.outputs.exists != 'true' | |
| continue-on-error: true | |
| run: | | |
| chmod +x .github/scripts/stream-conversation.sh | |
| .github/scripts/stream-conversation.sh & | |
| echo $! > /tmp/streamer-pid | |
| env: | |
| WEBHOOK_URL: ${{ secrets.APP_URL }}/webhooks/github-actions | |
| WEBHOOK_SECRET: ${{ secrets.ETHERNAL_WEBHOOK_SECRET }} | |
| WORKFLOW_RUN_ID: ${{ github.run_id }} | |
| GITHUB_ISSUE_NUMBER: ${{ steps.issue_info.outputs.number }} | |
| - name: Claude Code - Triage and Fix | |
| if: steps.circuit_breaker.outputs.tripped != 'true' && steps.existing_pr.outputs.exists != 'true' | |
| uses: anthropics/claude-code-action@v1 | |
| with: | |
| claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} | |
| label_trigger: sentry | |
| allowed_bots: "ethernal-sentry[bot],claude[bot]" | |
| claude_args: "--model claude-sonnet-4-20250514 --max-turns 50 --allowedTools Bash,Read,Write,Edit,Glob,Grep" | |
| show_full_output: true | |
| prompt: | | |
| You are an autonomous error-fixing agent for the Ethernal project — an EVM block explorer. | |
| A Sentry error has been auto-reported as GitHub issue #${{ steps.issue_info.outputs.number }}. | |
| ## Issue Details | |
| Title: ${{ steps.issue_info.outputs.title }} | |
| Body: | |
| $(cat /tmp/issue-body.txt) | |
| ## Regression Context | |
| $(cat /tmp/regression-context.txt) | |
| ## Your Task | |
| ### 1. Investigate | |
| - Parse the Sentry issue from the body to extract error type, stack trace, and context | |
| - Use the Sentry API to fetch full event details: | |
| ``` | |
| curl -H "Authorization: Bearer $SENTRY_API_TOKEN" \ | |
| "https://sentry.io/api/0/issues/{sentry_issue_id}/events/latest/" | |
| ``` | |
| The Sentry issue ID may be in the issue body URL. The org slug is "sentry". | |
| Backend project slug: "ethernal-backend", Frontend project slug: "ethernal-frontend" | |
| - Check production logs if needed: `flyctl logs -a ethernal --no-tail | tail -100` | |
| - You have NO database access. Production is not reachable from CI and | |
| there is no read-only credential any more: it would have meant an | |
| internet-facing database password stored here, and read access to all | |
| customer data, to serve an occasional convenience. If an issue can only | |
| be resolved by inspecting production data, ESCALATE it — say plainly | |
| that it needs a human with database access, and stop. | |
| ### 2. Triage (choose ONE action) | |
| **IMPORTANT**: If the issue already has the `needs-human` label, it was flagged as a sustained infrastructure error (high volume of connection errors outside of a deploy). In this case, ALWAYS choose **AUTO-FIX** — investigate the root cause (check DB health, connection pool stats, Fly.io status) and attempt a fix. Also notify @antoinedc: | |
| ``` | |
| gh issue comment ${{ steps.issue_info.outputs.number }} --body "🚨 Sustained infra errors detected (not correlated with a deploy). Investigating and attempting a fix. @antoinedc please monitor." | |
| ``` | |
| **CLOSE** the issue if: | |
| - It's a duplicate of an existing issue | |
| - It's expected behavior (e.g., user input validation errors) | |
| - It's a third-party/network error we can't control | |
| - It's already fixed on the develop branch | |
| - It's a transient infrastructure error (database connection drops, ECONNRESET, "Connection terminated unexpectedly", SequelizeConnectionError, SequelizeConnectionRefusedError) — BUT first verify it correlates with a recent deploy/restart by checking: `flyctl releases -a ethernal --json | jq '.[0:3]'`. If the error timestamps align with a release within the last 10 minutes, it's transient → close. If there's NO recent deploy and connection errors persist, ESCALATE instead — it may be a real infra issue. Do NOT add retry logic or try-catch wrappers for these — BullMQ already handles retries at the job level. | |
| Close with: `gh issue close ${{ steps.issue_info.outputs.number }} --comment "Closing: [reason]"` | |
| **ESCALATE** if: | |
| - It involves security, data corruption, or data loss | |
| - It touches billing, auth, Stripe, or payment code | |
| - The root cause is unclear after investigation | |
| - The fix would require architectural changes | |
| Escalate with: | |
| ``` | |
| gh issue edit ${{ steps.issue_info.outputs.number }} --add-label "needs-human" | |
| gh issue comment ${{ steps.issue_info.outputs.number }} --body "Escalating: [reason]. @antoinedc please review." | |
| ``` | |
| **AUTO-FIX** if: | |
| - It's a null/undefined check, missing error handling, off-by-one error | |
| - It's a simple logic bug with clear root cause | |
| - The fix is contained (1-3 files) and low risk | |
| **ESCALATE** (not auto-fix) if the issue is an **infrastructure/config problem** (WebSocket failures, Fly.io service instability, DNS, routing, health checks). These require human review — do NOT create code PRs for infra issues. | |
| For **performance issues specifically**, apply this triage checklist BEFORE writing any code: | |
| 1. **Check event volume thresholds FIRST — this is mandatory**: | |
| - **User-facing endpoints** (API routes, HTTP handlers): need **50+ events/24h**. Below that → CLOSE. | |
| - **Background jobs** (BullMQ workers, cron jobs): need **100+ events/24h**. Below that → CLOSE. | |
| If the event count is below the threshold, CLOSE the issue immediately with a comment explaining the threshold. Do NOT proceed to investigation or fix. | |
| 2. **Check for missing indexes**: read the migrations in `run/migrations/` to | |
| see which indexes exist on the affected table(s). You cannot query the | |
| database, so this is the source of truth. If the slow query would be | |
| fixed by an index, the fix is just a migration — don't restructure code. | |
| If the migrations do not let you decide, ESCALATE rather than guess. | |
| 3. **Prefer simple fixes**: An index > a query tweak > code restructuring. Never split a working single query into multiple queries to "optimize" it unless you've confirmed the original query plan is actually bad. | |
| 4. **Complexity budget**: If your fix requires > 20 lines of logic changes (not counting tests), ESCALATE instead of auto-fixing. Complex performance optimizations need human review. | |
| 5. **Group related issues**: Before fixing, check if there are other open sentry issues touching the same code path. If so, note it in the issue comment — one coherent fix is better than 5 micro-PRs. | |
| ### 3. Fix (only if AUTO-FIX) | |
| - Before creating a branch, verify no PR already exists: | |
| ```bash | |
| if gh pr list --repo tryethernal/ethernal --head "fix/sentry-${{ steps.issue_info.outputs.number }}" --state open --json number -q '.[0].number' 2>/dev/null | grep -q .; then | |
| echo "A fix PR already exists for this issue. Exiting." | |
| exit 0 | |
| fi | |
| ``` | |
| - **Check for overlapping in-flight PRs**: Before writing code, identify which files you plan to change. Then check if any other open `fix/sentry-*` PRs touch those same files: | |
| ```bash | |
| # List files changed by other open sentry fix PRs (use jq filtering, not head: search which requires exact match) | |
| gh pr list --repo tryethernal/ethernal --state open --json number,headRefName \ | |
| --jq '[.[] | select(.headRefName | startswith("fix/sentry-"))] | .[].number' \ | |
| > /tmp/sentry-pr-nums.txt | |
| for PR_NUM in $(cat /tmp/sentry-pr-nums.txt); do | |
| gh pr diff "$PR_NUM" --repo tryethernal/ethernal --name-only 2>/dev/null | |
| done | sort -u > /tmp/other-pr-files.txt | |
| ``` | |
| If any of your planned files appear in `/tmp/other-pr-files.txt`, **ESCALATE instead of fixing** — another fix is already in flight for the same code. Note which PR has the overlap in the escalation comment. | |
| - Create a branch: `git checkout -b fix/sentry-${{ steps.issue_info.outputs.number }}` | |
| - Implement the minimal fix needed | |
| - Run relevant tests: | |
| - Backend: `cd run && npm test -- --testPathPattern=<relevant_test>` | |
| - Frontend: `yarn test -- <relevant_test>` | |
| - If tests fail, fix them. If you can't fix them, escalate instead. | |
| - Commit with a clear message explaining the fix | |
| - Push: `git push origin fix/sentry-${{ steps.issue_info.outputs.number }}` | |
| - Create PR: | |
| ``` | |
| gh pr create \ | |
| --title "fix: [brief description]" \ | |
| --body "## Summary | |
| Fixes #${{ steps.issue_info.outputs.number }} | |
| **Sentry Error:** [error type and message] | |
| **Root Cause:** [what caused it] | |
| **Fix:** [what was changed and why] | |
| **Regression:** [If regression, link to previous PR and explain why the previous fix was insufficient] | |
| ## Test plan | |
| - [ ] Relevant unit tests pass | |
| - [ ] Fix addresses the root cause, not just symptoms | |
| 🤖 Generated with [Claude Code](https://claude.com/claude-code)" \ | |
| --base develop | |
| ``` | |
| ### Rules | |
| - NEVER modify files in: `run/api/stripe.js`, `run/webhooks/stripe.js`, `run/lib/stripe.js`, `run/middlewares/auth.js`, `run/lib/crypto.js`, `run/config/database.js` | |
| - ESCALATE (do not auto-fix) any change that modifies database connection pool settings, connection timeouts, or Sequelize pool configuration — these affect every query system-wide and require human review | |
| - NEVER write to the production database | |
| - NEVER skip running tests | |
| - NEVER close the issue manually — GitHub auto-closes it when the PR with "Fixes #N" is merged | |
| - Keep fixes minimal — don't refactor surrounding code | |
| - If uncertain about the fix, escalate rather than guess | |
| - NEVER add retry/reconnection logic for database connection errors — BullMQ already handles retries. Adding in-job retry creates double-retry complexity. | |
| - NEVER silently catch and swallow database errors with fallback values — this masks real issues. Let errors propagate so BullMQ retries the job. | |
| - NEVER split working database queries into multiple queries to "prevent timeouts" — timeouts are solved by indexes, not query splitting. | |
| - NEVER wrap entire job functions in try-catch just to log and re-throw — this adds no value. | |
| - For performance fixes: ALWAYS check existing indexes first, by reading | |
| `run/migrations/` — there is no database to query. If the right index | |
| already exists, the query is likely fine — close the issue or look for | |
| the real problem elsewhere. | |
| - For performance fixes: prefer adding an index (migration) over restructuring code. An index is 1 file, zero risk, and fixes the root cause. Code restructuring (caching, batching, multi-path strategies) adds complexity and bugs. | |
| - NEVER create a fix PR with > 20 lines of logic changes for a performance issue. Escalate instead — complex optimizations need human review. | |
| - NEVER claim a fix was made without verifying it appears in `git diff`. Run `git diff` before writing the review processing summary and confirm each claimed change is present. | |
| - If your fix introduces a new catch block, timeout handler, or fallback path, you MUST add a test that exercises that path. Untested error-handling code is where bugs hide — the overnight incident on 2026-03-30 had 3 bugs in auto-merged PRs, all in untested timeout/fallback paths. If you can't write a test for the new path, ESCALATE instead of shipping untested error handling. | |
| env: | |
| SENTRY_API_TOKEN: ${{ secrets.SENTRY_API_TOKEN }} | |
| FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} | |
| - name: Stop conversation streamer | |
| if: always() | |
| continue-on-error: true | |
| run: | | |
| if [ -f /tmp/streamer-pid ]; then | |
| kill $(cat /tmp/streamer-pid) 2>/dev/null || true | |
| fi | |
| - name: Extract conversation log | |
| id: conversation | |
| if: always() | |
| continue-on-error: true | |
| run: | | |
| LOG_FILE="/home/runner/work/_temp/claude-execution-output.json" | |
| if [ ! -f "$LOG_FILE" ]; then | |
| echo "[]" > /tmp/conversation-log.json | |
| exit 0 | |
| fi | |
| python3 << 'PYEOF' | |
| import json | |
| with open("/home/runner/work/_temp/claude-execution-output.json") as f: | |
| data = json.load(f) | |
| log = [] | |
| messages = data if isinstance(data, list) else [data] | |
| for msg in messages: | |
| if not isinstance(msg, dict): | |
| continue | |
| role = msg.get("role", msg.get("type", "")) | |
| if role in ("system", "preset"): | |
| continue | |
| content = msg.get("content", []) | |
| if isinstance(content, str): | |
| log.append({"role": role, "text": content}) | |
| continue | |
| if not isinstance(content, list): | |
| content = [content] if isinstance(content, dict) else [] | |
| for block in content: | |
| if not isinstance(block, dict): | |
| continue | |
| btype = block.get("type", "") | |
| if btype == "text": | |
| log.append({"role": role, "text": block.get("text", "")}) | |
| elif btype == "tool_use": | |
| inp = block.get("input", {}) | |
| log.append({ | |
| "role": role, | |
| "tool": block.get("name", ""), | |
| "input": (inp.get("command") or inp.get("description") or str(inp))[:500] | |
| }) | |
| elif btype == "tool_result": | |
| cv = block.get("content", "") | |
| if isinstance(cv, list): | |
| cv = " ".join(b.get("text", "") for b in cv if isinstance(b, dict)) | |
| log.append({"role": "tool", "output": str(cv)[:1000]}) | |
| log = log[-200:] | |
| with open("/tmp/conversation-log.json", "w") as f: | |
| json.dump(log, f) | |
| print(f"Extracted {len(log)} conversation entries") | |
| PYEOF | |
| - name: Notify dashboard - post-triage | |
| if: always() | |
| continue-on-error: true | |
| run: | | |
| # Determine status based on whether a PR branch exists | |
| if git ls-remote --heads origin "fix/sentry-${{ steps.issue_info.outputs.number }}" | grep -q .; then | |
| STATUS="fixing" | |
| STEP="Claude created a fix branch" | |
| DECISION="auto-fix" | |
| else | |
| # Check if issue was closed | |
| ISSUE_STATE=$(gh issue view ${{ steps.issue_info.outputs.number }} --repo ${{ github.repository }} --json state -q '.state') | |
| if [ "$ISSUE_STATE" = "CLOSED" ]; then | |
| STATUS="closed" | |
| STEP="Issue was closed by Claude" | |
| DECISION="close" | |
| elif gh issue view ${{ steps.issue_info.outputs.number }} --repo ${{ github.repository }} --json labels -q '.labels[].name' | grep -q "needs-human"; then | |
| STATUS="escalated" | |
| STEP="Escalated to human" | |
| DECISION="escalate" | |
| else | |
| STATUS="failed" | |
| STEP="Triage completed without action" | |
| DECISION="unknown" | |
| fi | |
| fi | |
| # Build payload with conversation log | |
| CONV_LOG=$(cat /tmp/conversation-log.json 2>/dev/null || echo "[]") | |
| jq -n \ | |
| --argjson runId "${{ github.run_id }}" \ | |
| --argjson issueNum "${{ steps.issue_info.outputs.number }}" \ | |
| --arg status "$STATUS" \ | |
| --arg step "$STEP" \ | |
| --arg decision "$DECISION" \ | |
| --argjson convLog "$CONV_LOG" \ | |
| '{ | |
| workflowRunId: $runId, | |
| githubIssueNumber: $issueNum, | |
| status: $status, | |
| currentStep: $step, | |
| triageDecision: $decision, | |
| conversationLog: $convLog | |
| }' | curl -s -X POST "${{ secrets.APP_URL }}/webhooks/github-actions" \ | |
| -H "Authorization: Bearer ${{ secrets.ETHERNAL_WEBHOOK_SECRET }}" \ | |
| -H "Content-Type: application/json" \ | |
| -d @- | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Notify dashboard - PR created | |
| if: always() | |
| continue-on-error: true | |
| run: | | |
| PR_NUMBER=$(gh pr list --head "fix/sentry-${{ steps.issue_info.outputs.number }}" --json number -q '.[0].number' 2>/dev/null) | |
| if [ -n "$PR_NUMBER" ]; then | |
| curl -s -X POST "${{ secrets.APP_URL }}/webhooks/github-actions" \ | |
| -H "Authorization: Bearer ${{ secrets.ETHERNAL_WEBHOOK_SECRET }}" \ | |
| -H "Content-Type: application/json" \ | |
| -d "{ | |
| \"workflowRunId\": ${{ github.run_id }}, | |
| \"githubIssueNumber\": ${{ steps.issue_info.outputs.number }}, | |
| \"githubPrNumber\": $PR_NUMBER, | |
| \"status\": \"reviewing\", | |
| \"currentStep\": \"PR #$PR_NUMBER awaiting review\" | |
| }" | |
| fi | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Notify dashboard - failure | |
| if: failure() | |
| continue-on-error: true | |
| run: | | |
| curl -s -X POST "${{ secrets.APP_URL }}/webhooks/github-actions" \ | |
| -H "Authorization: Bearer ${{ secrets.ETHERNAL_WEBHOOK_SECRET }}" \ | |
| -H "Content-Type: application/json" \ | |
| -d "{ | |
| \"workflowRunId\": ${{ github.run_id }}, | |
| \"githubIssueNumber\": ${{ steps.issue_info.outputs.number }}, | |
| \"status\": \"failed\", | |
| \"currentStep\": \"Workflow failed\" | |
| }" | |
| process-review: | |
| concurrency: | |
| group: sentry-review-${{ github.event.issue.number || github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| if: > | |
| ( | |
| (github.event_name == 'pull_request_review' && | |
| startsWith(github.event.pull_request.head.ref, 'fix/sentry-')) | |
| || | |
| (github.event_name == 'issue_comment' && | |
| github.event.issue.pull_request && | |
| contains(github.event.comment.body, '@claude')) | |
| || | |
| (github.event_name == 'issue_comment' && | |
| github.event.issue.pull_request && | |
| github.event.sender.login == 'greptile-apps[bot]') | |
| ) && | |
| github.event.sender.login != 'github-actions[bot]' && | |
| github.event.sender.login != 'claude[bot]' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Debounce | |
| run: sleep 30 | |
| - name: Get PR info | |
| id: pr_info | |
| run: | | |
| if [ "${{ github.event_name }}" = "issue_comment" ]; then | |
| PR_NUMBER="${{ github.event.issue.number }}" | |
| BRANCH=$(gh pr view "$PR_NUMBER" --repo ${{ github.repository }} --json headRefName -q '.headRefName') | |
| else | |
| PR_NUMBER="${{ github.event.pull_request.number }}" | |
| BRANCH="${{ github.event.pull_request.head.ref }}" | |
| fi | |
| echo "pr_number=$PR_NUMBER" >> $GITHUB_OUTPUT | |
| echo "branch=$BRANCH" >> $GITHUB_OUTPUT | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Check if sentry fix PR | |
| id: check_sentry | |
| run: | | |
| BRANCH="${{ steps.pr_info.outputs.branch }}" | |
| if [[ "$BRANCH" == fix/sentry-* ]]; then | |
| echo "is_sentry=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "is_sentry=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Checkout PR branch | |
| if: steps.check_sentry.outputs.is_sentry == 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ steps.pr_info.outputs.branch }} | |
| - uses: actions/setup-node@v4 | |
| if: steps.check_sentry.outputs.is_sentry == 'true' | |
| with: | |
| node-version: 18 | |
| - name: Install backend deps | |
| if: steps.check_sentry.outputs.is_sentry == 'true' | |
| run: cd run && npm install | |
| - name: Install frontend deps | |
| if: steps.check_sentry.outputs.is_sentry == 'true' | |
| run: yarn install | |
| - uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1 # v1.6 | |
| if: steps.check_sentry.outputs.is_sentry == 'true' | |
| - name: Setup SSH | |
| if: steps.check_sentry.outputs.is_sentry == 'true' | |
| uses: webfactory/ssh-agent@v0.9.0 | |
| with: | |
| ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| - name: Add known hosts | |
| if: steps.check_sentry.outputs.is_sentry == 'true' | |
| run: | | |
| mkdir -p ~/.ssh | |
| ssh-keyscan -H 157.90.154.200 >> ~/.ssh/known_hosts | |
| - name: Configure git | |
| if: steps.check_sentry.outputs.is_sentry == 'true' | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| - name: Start conversation streamer | |
| if: steps.check_sentry.outputs.is_sentry == 'true' | |
| continue-on-error: true | |
| run: | | |
| chmod +x .github/scripts/stream-conversation.sh | |
| .github/scripts/stream-conversation.sh & | |
| echo $! > /tmp/streamer-pid | |
| env: | |
| WEBHOOK_URL: ${{ secrets.APP_URL }}/webhooks/github-actions | |
| WEBHOOK_SECRET: ${{ secrets.ETHERNAL_WEBHOOK_SECRET }} | |
| WORKFLOW_RUN_ID: ${{ github.run_id }} | |
| GITHUB_ISSUE_NUMBER: ${{ steps.pr_info.outputs.pr_number }} | |
| - name: Claude Code - Process Review | |
| id: claude_review | |
| if: steps.check_sentry.outputs.is_sentry == 'true' | |
| uses: anthropics/claude-code-action@v1 | |
| with: | |
| claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} | |
| allowed_bots: "ethernal-sentry[bot],greptile-apps[bot]" | |
| claude_args: "--model claude-sonnet-4-20250514 --max-turns 50 --allowedTools Bash,Read,Write,Edit,Glob,Grep" | |
| show_full_output: true | |
| base_branch: develop | |
| prompt: | | |
| You are processing code review feedback on PR #${{ steps.pr_info.outputs.pr_number }} (branch: ${{ steps.pr_info.outputs.branch }}). | |
| ## Step 1: Fetch All Comments | |
| ```bash | |
| # Fetch formal PR reviews | |
| gh api repos/${{ github.repository }}/pulls/${{ steps.pr_info.outputs.pr_number }}/reviews | |
| # Fetch inline review comments | |
| gh api repos/${{ github.repository }}/pulls/${{ steps.pr_info.outputs.pr_number }}/comments | |
| # Fetch issue-level comments (where Greptile posts its summary) | |
| gh api repos/${{ github.repository }}/issues/${{ steps.pr_info.outputs.pr_number }}/comments | |
| ``` | |
| ## Step 2: Verify and Process Review Comments | |
| For each review comment from `greptile-apps[bot]` that flags an issue: | |
| 1. **FIRST verify** the comment is correct by reading the actual code it references. Check ALL commits on the branch, not just the diff — a later commit may have already addressed the issue. | |
| 2. If it's a **valid issue** → fix it, commit, push, reply explaining the fix, and react with 👍: | |
| ```bash | |
| # For inline review comments: | |
| gh api repos/${{ github.repository }}/pulls/comments/{comment_id}/reactions -f content='+1' | |
| # For issue-level comments: | |
| gh api repos/${{ github.repository }}/issues/comments/{comment_id}/reactions -f content='+1' | |
| ``` | |
| 3. If it's **incorrect** → reply on the PR explaining why it's wrong, and react with 👎: | |
| ```bash | |
| # For inline review comments: | |
| gh api repos/${{ github.repository }}/pulls/comments/{comment_id}/reactions -f content='-1' | |
| # For issue-level comments: | |
| gh api repos/${{ github.repository }}/issues/comments/{comment_id}/reactions -f content='-1' | |
| ``` | |
| 4. If it was a **valid issue but already fixed** by a later commit on this branch → reply noting it was valid but already addressed, and react with 👍: | |
| ```bash | |
| # For inline review comments: | |
| gh api repos/${{ github.repository }}/pulls/comments/{comment_id}/reactions -f content='+1' | |
| # For issue-level comments: | |
| gh api repos/${{ github.repository }}/issues/comments/{comment_id}/reactions -f content='+1' | |
| ``` | |
| 5. If it's unclear → ask for clarification in a reply (no reaction yet) | |
| **IMPORTANT:** Do NOT react to comments before verifying them. Verify first, then react with the appropriate emoji based on your assessment. | |
| If the review summary is clean (no issues flagged), react with 👍 on the summary. | |
| Take comments seriously but verify they're correct before acting. | |
| NEVER remove working code to satisfy a review bot. | |
| NEVER modify: `run/api/stripe.js`, `run/webhooks/stripe.js`, `run/lib/stripe.js`, `run/middlewares/auth.js`, `run/lib/crypto.js` | |
| ## Step 3: Signal Merge Readiness | |
| After processing ALL comments, determine if the PR is ready to merge: | |
| - **READY**: All review issues have been addressed (fixed or correctly dismissed) | |
| - **NOT READY**: There are unresolved issues you couldn't fix, or you asked for clarification | |
| You MUST run this command at the very end: | |
| ```bash | |
| # If ALL issues are resolved: | |
| echo "READY_TO_MERGE" > /tmp/merge-status | |
| # If there are unresolved issues: | |
| echo "NOT_READY" > /tmp/merge-status | |
| ``` | |
| Then write a short summary of what you did. If not ready, explain what remains. | |
| env: | |
| SENTRY_API_TOKEN: ${{ secrets.SENTRY_API_TOKEN }} | |
| FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} | |
| - name: Stop conversation streamer | |
| if: always() | |
| continue-on-error: true | |
| run: | | |
| if [ -f /tmp/streamer-pid ]; then | |
| kill $(cat /tmp/streamer-pid) 2>/dev/null || true | |
| fi | |
| - name: Check merge readiness | |
| id: merge_check | |
| if: steps.check_sentry.outputs.is_sentry == 'true' | |
| run: | | |
| if [ -f /tmp/merge-status ] && grep -q "READY_TO_MERGE" /tmp/merge-status; then | |
| echo "ready=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "ready=false" >> $GITHUB_OUTPUT | |
| echo "PR is not ready to merge. Skipping merge and deploy." | |
| fi | |
| - name: Check review confidence | |
| id: confidence_check | |
| if: steps.check_sentry.outputs.is_sentry == 'true' && steps.merge_check.outputs.ready == 'true' | |
| run: | | |
| # Fetch latest Greptile review comment | |
| GREPTILE_COMMENT=$(gh api repos/${{ github.repository }}/issues/${{ steps.pr_info.outputs.pr_number }}/comments \ | |
| --jq '[.[] | select(.user.login == "greptile-apps[bot]")] | last | .body // ""') | |
| # Greptile formats as "<h3>Confidence Score: X/5</h3>" — anchor to that | |
| SCORE=$(echo "$GREPTILE_COMMENT" | grep -oP '(?i)confidence\s+score[:\s]*\K\d(?=/5)' | head -1) | |
| # Fallback: last X/5 occurrence (summary score appears after per-file scores) | |
| if [ -z "$SCORE" ]; then | |
| SCORE=$(echo "$GREPTILE_COMMENT" | grep -oP '\d(?=/5)' | tail -1) | |
| fi | |
| if [ -z "$SCORE" ]; then | |
| echo "No confidence score found, proceeding" | |
| echo "sufficient=true" >> $GITHUB_OUTPUT | |
| elif [ "$SCORE" -lt 3 ]; then | |
| echo "Low confidence: $SCORE/5. Flagging for human review." | |
| gh pr edit ${{ steps.pr_info.outputs.pr_number }} --add-label "needs-human" --repo ${{ github.repository }} | |
| gh pr comment ${{ steps.pr_info.outputs.pr_number }} --body "Review confidence $SCORE/5 (threshold: 3/5). Flagging for human review." --repo ${{ github.repository }} | |
| echo "sufficient=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "Confidence: $SCORE/5 (sufficient)" | |
| echo "sufficient=true" >> $GITHUB_OUTPUT | |
| fi | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Wait for CI checks | |
| id: ci_check | |
| if: steps.check_sentry.outputs.is_sentry == 'true' && steps.merge_check.outputs.ready == 'true' && steps.confidence_check.outputs.sufficient == 'true' | |
| run: | | |
| echo "Waiting for CI checks to complete on PR #${{ steps.pr_info.outputs.pr_number }}..." | |
| for i in $(seq 1 60); do | |
| CHECKS_JSON=$(gh pr checks ${{ steps.pr_info.outputs.pr_number }} --repo ${{ github.repository }} --json name,state --jq '[.[] | select(.name != "process-review" and .name != "triage-and-fix" and .name != "Greptile Review")]') | |
| STATUS=$(echo "$CHECKS_JSON" | jq -r 'if length == 0 then "pass" elif [.[] | select(.state != "SUCCESS" and .state != "SKIPPED")] | length == 0 then "pass" elif [.[] | select(.state == "FAILURE")] | length > 0 then "fail" else "pending" end') | |
| echo "Check $i/60: $STATUS" | |
| if [ "$STATUS" = "pass" ]; then | |
| echo "passed=true" >> $GITHUB_OUTPUT | |
| exit 0 | |
| elif [ "$STATUS" = "fail" ]; then | |
| echo "CI checks failed. Skipping merge." | |
| echo "passed=false" >> $GITHUB_OUTPUT | |
| exit 0 | |
| fi | |
| sleep 30 | |
| done | |
| echo "CI checks timed out after 30 minutes." | |
| echo "passed=false" >> $GITHUB_OUTPUT | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_PAT }} | |
| - name: Merge PR | |
| id: merge_pr | |
| if: steps.check_sentry.outputs.is_sentry == 'true' && steps.merge_check.outputs.ready == 'true' && steps.confidence_check.outputs.sufficient == 'true' && steps.ci_check.outputs.passed == 'true' | |
| run: | | |
| gh pr merge ${{ steps.pr_info.outputs.pr_number }} \ | |
| --squash --admin --delete-branch \ | |
| --repo ${{ github.repository }} | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_PAT }} | |
| - name: Check hotfix label | |
| id: hotfix_check | |
| if: steps.merge_pr.conclusion == 'success' | |
| run: | | |
| ISSUE_NUMBER=$(gh pr view ${{ steps.pr_info.outputs.pr_number }} --repo ${{ github.repository }} --json body -q '.body' | grep -oP 'Fixes #\K\d+' | head -1) | |
| IS_HOTFIX=0 | |
| if [ -n "$ISSUE_NUMBER" ]; then | |
| IS_HOTFIX=$(gh issue view "$ISSUE_NUMBER" --repo ${{ github.repository }} --json labels -q '.labels[].name' | grep -c '^hotfix$' || true) | |
| fi | |
| echo "is_hotfix=${IS_HOTFIX}" >> "$GITHUB_OUTPUT" | |
| echo "issue_number=${ISSUE_NUMBER:-0}" >> "$GITHUB_OUTPUT" | |
| echo "Hotfix: ${IS_HOTFIX}, Issue: ${ISSUE_NUMBER:-none}" | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # --- Non-hotfix path: notify merged status and stop --- | |
| - name: Notify dashboard - merged (batch deploy) | |
| if: > | |
| steps.merge_pr.conclusion == 'success' && | |
| steps.hotfix_check.outputs.is_hotfix == '0' | |
| continue-on-error: true | |
| run: | | |
| curl -s -X POST "${{ secrets.APP_URL }}/webhooks/github-actions" \ | |
| -H "Authorization: Bearer ${{ secrets.ETHERNAL_WEBHOOK_SECRET }}" \ | |
| -H "Content-Type: application/json" \ | |
| -d "{ | |
| \"githubIssueNumber\": ${{ steps.hotfix_check.outputs.issue_number }}, | |
| \"githubPrNumber\": ${{ steps.pr_info.outputs.pr_number }}, | |
| \"status\": \"merged\", | |
| \"currentStep\": \"PR merged, awaiting batch deploy\" | |
| }" | |
| # --- Hotfix path: deploy inline (existing logic) --- | |
| - name: Notify dashboard - merging (hotfix) | |
| if: > | |
| steps.merge_pr.conclusion == 'success' && | |
| steps.hotfix_check.outputs.is_hotfix == '1' | |
| continue-on-error: true | |
| run: | | |
| curl -s -X POST "${{ secrets.APP_URL }}/webhooks/github-actions" \ | |
| -H "Authorization: Bearer ${{ secrets.ETHERNAL_WEBHOOK_SECRET }}" \ | |
| -H "Content-Type: application/json" \ | |
| -d "{ | |
| \"githubIssueNumber\": ${{ steps.hotfix_check.outputs.issue_number }}, | |
| \"githubPrNumber\": ${{ steps.pr_info.outputs.pr_number }}, | |
| \"status\": \"merging\", | |
| \"currentStep\": \"PR merged, deploying (hotfix)\" | |
| }" | |
| - name: Deploy (hotfix) | |
| id: deploy_hotfix | |
| if: > | |
| steps.merge_pr.conclusion == 'success' && | |
| steps.hotfix_check.outputs.is_hotfix == '1' | |
| env: | |
| GH_PAT: ${{ secrets.GH_PAT }} | |
| run: | | |
| git remote set-url origin "https://x-access-token:${GH_PAT}@github.com/${{ github.repository }}.git" | |
| git checkout develop | |
| git pull origin develop | |
| LAST_TAG=$(git describe --tags --abbrev=0) | |
| echo "Last tag: $LAST_TAG" | |
| echo "last_tag=$LAST_TAG" >> "$GITHUB_OUTPUT" | |
| # Get commits since last tag, filter out release machinery | |
| COMMITS=$(git log --oneline "$LAST_TAG..HEAD" | grep -viE '^[a-f0-9]+ (update changelog|[0-9]+\.[0-9]+\.[0-9]+|Merge (branch|develop|master))' || true) | |
| echo "Commits since $LAST_TAG:" | |
| echo "$COMMITS" | |
| if [ -z "$COMMITS" ]; then | |
| echo "No meaningful commits to deploy" | |
| exit 0 | |
| fi | |
| # Generate changelog entry | |
| VERSION_DATE=$(date +%Y-%m-%d) | |
| CURRENT_VERSION=$(node -p "require('./package.json').version") | |
| IFS='.' read -r MAJOR MINOR PATCH <<< "$CURRENT_VERSION" | |
| NEW_VERSION="$MAJOR.$MINOR.$((PATCH + 1))" | |
| CHANGELOG_ENTRY="## [$NEW_VERSION] - $VERSION_DATE | |
| ### Fixed | |
| $(echo "$COMMITS" | sed 's/^[a-f0-9]* /- /')" | |
| # Insert after header (line 5) | |
| head -5 CHANGELOG.md > CHANGELOG.tmp | |
| echo "" >> CHANGELOG.tmp | |
| echo "$CHANGELOG_ENTRY" >> CHANGELOG.tmp | |
| echo "" >> CHANGELOG.tmp | |
| tail -n +6 CHANGELOG.md >> CHANGELOG.tmp | |
| mv CHANGELOG.tmp CHANGELOG.md | |
| git add CHANGELOG.md | |
| git commit -m "update changelog" | |
| npm version patch --message '%s' | |
| git push origin develop | |
| git push origin "v$NEW_VERSION" | |
| # Sync master | |
| git checkout master | |
| git pull origin master | |
| git merge develop --no-ff -m "Merge develop into master for v$NEW_VERSION" | |
| git push origin master | |
| git checkout develop | |
| echo "Deployed v$NEW_VERSION" | |
| - name: Notify dashboard - deploying (hotfix) | |
| if: > | |
| steps.deploy_hotfix.conclusion == 'success' && | |
| steps.hotfix_check.outputs.is_hotfix == '1' | |
| continue-on-error: true | |
| run: | | |
| curl -s -X POST "${{ secrets.APP_URL }}/webhooks/github-actions" \ | |
| -H "Authorization: Bearer ${{ secrets.ETHERNAL_WEBHOOK_SECRET }}" \ | |
| -H "Content-Type: application/json" \ | |
| -d "{ | |
| \"githubIssueNumber\": ${{ steps.hotfix_check.outputs.issue_number }}, | |
| \"githubPrNumber\": ${{ steps.pr_info.outputs.pr_number }}, | |
| \"status\": \"deploying\", | |
| \"currentStep\": \"Deploying to production (hotfix)\" | |
| }" | |
| - name: Resolve Sentry issue (hotfix) | |
| if: > | |
| steps.deploy_hotfix.conclusion == 'success' && | |
| steps.hotfix_check.outputs.is_hotfix == '1' | |
| env: | |
| SENTRY_API_TOKEN: ${{ secrets.SENTRY_API_TOKEN }} | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| ISSUE_NUMBER=${{ steps.hotfix_check.outputs.issue_number }} | |
| if [ "$ISSUE_NUMBER" = "0" ] || [ -z "$ISSUE_NUMBER" ]; then | |
| echo "No linked issue found, skipping Sentry resolution" | |
| exit 0 | |
| fi | |
| # Extract Sentry issue ID from the GitHub issue body | |
| SENTRY_ISSUE_ID=$(gh issue view "$ISSUE_NUMBER" --repo ${{ github.repository }} --json body -q '.body' | grep -oP 'sentry(?:\.tryethernal\.com|\.io)/organizations/[^/]+/issues/\K\d+' | head -1) | |
| if [ -z "$SENTRY_ISSUE_ID" ]; then | |
| echo "No Sentry issue ID found in issue #$ISSUE_NUMBER body, skipping" | |
| exit 0 | |
| fi | |
| echo "Resolving Sentry issue $SENTRY_ISSUE_ID (from GitHub issue #$ISSUE_NUMBER)" | |
| curl -s -X PUT \ | |
| "https://sentry.io/api/0/issues/$SENTRY_ISSUE_ID/" \ | |
| -H "Authorization: Bearer $SENTRY_API_TOKEN" \ | |
| -H "Content-Type: application/json" \ | |
| -d '{"status": "resolved"}' | jq '{status, statusDetails}' | |
| - name: Notify dashboard - completed (hotfix) | |
| if: > | |
| steps.deploy_hotfix.conclusion == 'success' && | |
| steps.hotfix_check.outputs.is_hotfix == '1' | |
| continue-on-error: true | |
| run: | | |
| curl -s -X POST "${{ secrets.APP_URL }}/webhooks/github-actions" \ | |
| -H "Authorization: Bearer ${{ secrets.ETHERNAL_WEBHOOK_SECRET }}" \ | |
| -H "Content-Type: application/json" \ | |
| -d "{ | |
| \"githubIssueNumber\": ${{ steps.hotfix_check.outputs.issue_number }}, | |
| \"githubPrNumber\": ${{ steps.pr_info.outputs.pr_number }}, | |
| \"status\": \"completed\", | |
| \"currentStep\": \"Fix deployed and Sentry issue resolved (hotfix)\", | |
| \"completedAt\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" | |
| }" | |
| - name: Resolve swept-in non-hotfix Sentry issues | |
| if: > | |
| steps.deploy_hotfix.conclusion == 'success' && | |
| steps.hotfix_check.outputs.is_hotfix == '1' | |
| continue-on-error: true | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| SENTRY_API_TOKEN: ${{ secrets.SENTRY_API_TOKEN }} | |
| run: | | |
| # Use pre-deploy tag captured by the deploy step | |
| LAST_TAG=${{ steps.deploy_hotfix.outputs.last_tag }} | |
| HOTFIX_PR=${{ steps.pr_info.outputs.pr_number }} | |
| PR_NUMBERS=$(git log --oneline "$LAST_TAG..HEAD" | grep -viE '^[a-f0-9]+ (update changelog|[0-9]+\.[0-9]+\.[0-9]+|Merge (branch|develop|master))' | grep -oP '#\K\d+' | grep -v "^${HOTFIX_PR}$" || true) | |
| if [ -z "$PR_NUMBERS" ]; then | |
| echo "No other PRs swept into this hotfix release" | |
| exit 0 | |
| fi | |
| echo "Found swept-in PRs: $PR_NUMBERS" | |
| for PR_NUM in $PR_NUMBERS; do | |
| echo "--- Processing swept-in PR #$PR_NUM ---" | |
| ISSUE_NUMBER=$(gh pr view "$PR_NUM" --repo ${{ github.repository }} --json body -q '.body' 2>/dev/null | grep -oP 'Fixes #\K\d+' | head -1 || true) | |
| if [ -z "$ISSUE_NUMBER" ]; then | |
| echo "No linked issue in PR #$PR_NUM, skipping" | |
| continue | |
| fi | |
| IS_SENTRY=$(gh issue view "$ISSUE_NUMBER" --repo ${{ github.repository }} --json labels -q '.labels[].name' 2>/dev/null | grep -c '^sentry$' || true) | |
| if [ "$IS_SENTRY" = "0" ]; then | |
| echo "Issue #$ISSUE_NUMBER is not a sentry issue, skipping" | |
| continue | |
| fi | |
| SENTRY_ISSUE_ID=$(gh issue view "$ISSUE_NUMBER" --repo ${{ github.repository }} --json body -q '.body' | grep -oP 'sentry(?:\.tryethernal\.com|\.io)/organizations/[^/]+/issues/\K\d+' | head -1 || true) | |
| if [ -z "$SENTRY_ISSUE_ID" ]; then | |
| echo "No Sentry issue ID in issue #$ISSUE_NUMBER, skipping" | |
| continue | |
| fi | |
| echo "Resolving swept-in Sentry issue $SENTRY_ISSUE_ID (issue #$ISSUE_NUMBER, PR #$PR_NUM)" | |
| curl -s -X PUT \ | |
| "https://sentry.io/api/0/issues/$SENTRY_ISSUE_ID/" \ | |
| -H "Authorization: Bearer $SENTRY_API_TOKEN" \ | |
| -H "Content-Type: application/json" \ | |
| -d '{"status": "resolved"}' | jq '{status, statusDetails}' | |
| curl -s -X POST "${{ secrets.APP_URL }}/webhooks/github-actions" \ | |
| -H "Authorization: Bearer ${{ secrets.ETHERNAL_WEBHOOK_SECRET }}" \ | |
| -H "Content-Type: application/json" \ | |
| -d "{ | |
| \"githubIssueNumber\": ${ISSUE_NUMBER}, | |
| \"githubPrNumber\": ${PR_NUM}, | |
| \"status\": \"completed\", | |
| \"currentStep\": \"Fix deployed via hotfix release\", | |
| \"completedAt\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" | |
| }" | |
| done | |
| recover-stuck: | |
| if: github.event_name == 'schedule' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Find and merge stuck sentry PRs | |
| run: | | |
| # Find open sentry fix PRs not updated in the last hour | |
| STUCK_PRS=$(gh pr list --repo ${{ github.repository }} \ | |
| --search "head:fix/sentry- is:open" \ | |
| --json number,updatedAt \ | |
| --jq '[.[] | select((.updatedAt | fromdateiso8601) < (now - 3600))] | .[].number') | |
| if [ -z "$STUCK_PRS" ]; then | |
| echo "No stuck PRs found" | |
| exit 0 | |
| fi | |
| for PR_NUM in $STUCK_PRS; do | |
| echo "--- Checking PR #$PR_NUM ---" | |
| # Check mergeable status | |
| MERGEABLE=$(gh pr view "$PR_NUM" --repo ${{ github.repository }} --json mergeable -q '.mergeable') | |
| if [ "$MERGEABLE" != "MERGEABLE" ]; then | |
| echo "PR #$PR_NUM is $MERGEABLE, skipping" | |
| continue | |
| fi | |
| # Skip PRs flagged for human review | |
| HAS_NEEDS_HUMAN=$(gh pr view "$PR_NUM" --repo ${{ github.repository }} --json labels \ | |
| -q '[.labels[].name] | any(. == "needs-human")') | |
| if [ "$HAS_NEEDS_HUMAN" = "true" ]; then | |
| echo "PR #$PR_NUM has needs-human label, skipping" | |
| continue | |
| fi | |
| # Check CI status (exclude workflow-specific checks) | |
| FAILING=$(gh pr checks "$PR_NUM" --repo ${{ github.repository }} --json name,state \ | |
| --jq '[.[] | select(.name != "process-review" and .name != "triage-and-fix" and .name != "recover-stuck" and .name != "Greptile Review") | select(.state != "SUCCESS" and .state != "SKIPPED")] | length') | |
| if [ "$FAILING" != "0" ]; then | |
| echo "PR #$PR_NUM has failing CI, skipping" | |
| continue | |
| fi | |
| # Check if Greptile reviewed but process-review failed or never ran | |
| GREPTILE_ISSUE_COMMENTS=$(gh api repos/${{ github.repository }}/issues/$PR_NUM/comments \ | |
| --jq '[.[] | select(.user.login == "greptile-apps[bot]")] | length') | |
| GREPTILE_REVIEWS=$(gh api repos/${{ github.repository }}/pulls/$PR_NUM/reviews \ | |
| --jq '[.[] | select(.user.login == "greptile-apps[bot]")] | length') | |
| GREPTILE_COMMENTED=$(( GREPTILE_ISSUE_COMMENTS + GREPTILE_REVIEWS )) | |
| REVIEW_STATE=$(gh pr checks "$PR_NUM" --repo ${{ github.repository }} --json name,state \ | |
| --jq '[.[] | select(.name == "process-review")] | .[0].state // "NONE"') | |
| if [ "$GREPTILE_COMMENTED" != "0" ] && [ "$REVIEW_STATE" != "SUCCESS" ]; then | |
| # Check if we already retried (look for our retry comment) | |
| RETRY_COUNT=$(gh api repos/${{ github.repository }}/issues/$PR_NUM/comments \ | |
| --jq '[.[] | select(.body | contains("@claude Please process the Greptile review"))] | length') | |
| if [ "$RETRY_COUNT" -ge 2 ]; then | |
| echo "PR #$PR_NUM: review processing failed after $RETRY_COUNT retries. Flagging for human review." | |
| gh pr edit "$PR_NUM" --add-label "needs-human" --repo ${{ github.repository }} | |
| gh pr comment "$PR_NUM" --body "Automated review processing failed after multiple retries. Flagging for human review. @${{ vars.ONCALL_HANDLE }}" --repo ${{ github.repository }} | |
| continue | |
| fi | |
| echo "PR #$PR_NUM has unprocessed Greptile review (process-review: $REVIEW_STATE). Re-triggering review." | |
| gh pr comment "$PR_NUM" --body "@claude Please process the Greptile review comments on this PR." --repo ${{ github.repository }} | |
| continue | |
| fi | |
| echo "Merging stuck PR #$PR_NUM" | |
| gh pr merge "$PR_NUM" --squash --admin --delete-branch --repo ${{ github.repository }} || echo "Failed to merge #$PR_NUM" | |
| done | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_PAT }} |