|
| 1 | +#!/usr/bin/env bash |
| 2 | +# CI Failure Re-runner |
| 3 | +# Automatically detects failed CI jobs and re-runs them locally |
| 4 | +# Usage: |
| 5 | +# bin/ci-rerun-failures # Check current commit |
| 6 | +# bin/ci-rerun-failures --previous # Search recent commits for failures |
| 7 | +# bin/ci-rerun-failures 1964 # Check specific PR |
| 8 | + |
| 9 | +set -euo pipefail |
| 10 | + |
| 11 | +# Colors for output |
| 12 | +RED='\033[0;31m' |
| 13 | +GREEN='\033[0;32m' |
| 14 | +YELLOW='\033[1;33m' |
| 15 | +BLUE='\033[0;34m' |
| 16 | +NC='\033[0m' # No Color |
| 17 | + |
| 18 | +# Show help |
| 19 | +show_help() { |
| 20 | + cat << EOF |
| 21 | +${BLUE}CI Failure Re-runner${NC} |
| 22 | +
|
| 23 | +Automatically detects failed CI jobs from GitHub and re-runs them locally. |
| 24 | +
|
| 25 | +${YELLOW}Usage:${NC} |
| 26 | + bin/ci-rerun-failures [OPTIONS] [PR_NUMBER] |
| 27 | +
|
| 28 | +${YELLOW}Options:${NC} |
| 29 | + --previous, --prev, -p Search recent commits for failures (when current is clean) |
| 30 | + --help, -h Show this help message |
| 31 | +
|
| 32 | +${YELLOW}Examples:${NC} |
| 33 | + bin/ci-rerun-failures # Check current commit |
| 34 | + bin/ci-rerun-failures --previous # Find failures in recent commits |
| 35 | + bin/ci-rerun-failures 1964 # Check specific PR number |
| 36 | +
|
| 37 | +${YELLOW}Features:${NC} |
| 38 | + - Fetches actual CI failures from GitHub using gh CLI |
| 39 | + - Waits for in-progress CI jobs (polls every 30s) |
| 40 | + - Maps CI job names to local rake commands |
| 41 | + - Deduplicates commands |
| 42 | + - Shows what will run before executing |
| 43 | +
|
| 44 | +${YELLOW}Related Tools:${NC} |
| 45 | + bin/ci-run-failed-specs Run specific failing RSpec examples |
| 46 | + bin/ci-local Smart test detection based on code changes |
| 47 | +EOF |
| 48 | + exit 0 |
| 49 | +} |
| 50 | + |
| 51 | +# Parse arguments |
| 52 | +USE_PREVIOUS=false |
| 53 | +PR_NUMBER="" |
| 54 | + |
| 55 | +while [[ $# -gt 0 ]]; do |
| 56 | + case $1 in |
| 57 | + --help|-h) |
| 58 | + show_help |
| 59 | + ;; |
| 60 | + --previous|--prev|-p) |
| 61 | + USE_PREVIOUS=true |
| 62 | + shift |
| 63 | + ;; |
| 64 | + *) |
| 65 | + PR_NUMBER="$1" |
| 66 | + shift |
| 67 | + ;; |
| 68 | + esac |
| 69 | +done |
| 70 | + |
| 71 | +echo -e "${BLUE}=== CI Failure Re-runner ===${NC}" |
| 72 | +echo "" |
| 73 | + |
| 74 | +# Fetch PR info |
| 75 | +if [ -z "$PR_NUMBER" ]; then |
| 76 | + if ! gh pr view --json number,commits >/dev/null 2>&1; then |
| 77 | + echo -e "${RED}Error: Not on a PR branch or gh CLI not authenticated${NC}" |
| 78 | + echo "Usage: bin/ci-rerun-failures [--previous] [pr-number]" |
| 79 | + exit 1 |
| 80 | + fi |
| 81 | + PR_INFO=$(gh pr view --json number,commits) |
| 82 | + PR_NUMBER=$(echo "$PR_INFO" | jq -r '.number') |
| 83 | +else |
| 84 | + if ! gh pr view "$PR_NUMBER" --json number,commits >/dev/null 2>&1; then |
| 85 | + echo -e "${RED}Error: PR #$PR_NUMBER not found${NC}" |
| 86 | + exit 1 |
| 87 | + fi |
| 88 | + PR_INFO=$(gh pr view "$PR_NUMBER" --json number,commits) |
| 89 | +fi |
| 90 | + |
| 91 | +# Determine which commit to check |
| 92 | +if [ "$USE_PREVIOUS" = true ]; then |
| 93 | + # Show recent commits and let user see which had failures |
| 94 | + echo -e "${BLUE}Recent commits on this PR:${NC}" |
| 95 | + echo "$PR_INFO" | jq -r '.commits[-5:] | reverse | .[] | " \(.oid[0:8]) - \(.messageHeadline)"' |
| 96 | + echo "" |
| 97 | + |
| 98 | + # Check each commit from newest to oldest for failures |
| 99 | + FOUND_FAILURES=false |
| 100 | + for i in 2 3 4 5; do |
| 101 | + COMMIT_SHA=$(echo "$PR_INFO" | jq -r ".commits[-$i].oid // empty") |
| 102 | + if [ -z "$COMMIT_SHA" ]; then |
| 103 | + continue |
| 104 | + fi |
| 105 | + |
| 106 | + echo "Checking commit ${COMMIT_SHA:0:8}..." |
| 107 | + CHECK_RUNS=$(gh api "repos/{owner}/{repo}/commits/$COMMIT_SHA/check-runs" --jq '.check_runs' 2>/dev/null || echo "[]") |
| 108 | + FAILURE_COUNT=$(echo "$CHECK_RUNS" | jq '[.[] | select(.conclusion == "FAILURE")] | length') |
| 109 | + |
| 110 | + if [ "$FAILURE_COUNT" -gt 0 ]; then |
| 111 | + echo -e "${GREEN}Found $FAILURE_COUNT failed check(s) in commit ${COMMIT_SHA:0:8}${NC}" |
| 112 | + STATUS_JSON=$(echo "$CHECK_RUNS" | jq '{statusCheckRollup: [.[] | {name: .name, conclusion: .conclusion, status: .status}], number: '$PR_NUMBER'}') |
| 113 | + FOUND_FAILURES=true |
| 114 | + break |
| 115 | + fi |
| 116 | + done |
| 117 | + |
| 118 | + if [ "$FOUND_FAILURES" = false ]; then |
| 119 | + echo -e "${YELLOW}No failures found in recent commits. Using current commit.${NC}" |
| 120 | + STATUS_JSON=$(gh pr view "$PR_NUMBER" --json statusCheckRollup,number) |
| 121 | + fi |
| 122 | +else |
| 123 | + # Use current commit (HEAD of PR) |
| 124 | + echo "Fetching current PR status..." |
| 125 | + STATUS_JSON=$(gh pr view "$PR_NUMBER" --json statusCheckRollup,number) |
| 126 | +fi |
| 127 | + |
| 128 | +echo -e "${GREEN}Analyzing PR #$PR_NUMBER...${NC}" |
| 129 | +echo "" |
| 130 | + |
| 131 | +# Check for in-progress jobs |
| 132 | +IN_PROGRESS_COUNT=$(echo "$STATUS_JSON" | jq '[(.statusCheckRollup // [])[] | select(.status == "IN_PROGRESS")] | length') |
| 133 | +if [ "$IN_PROGRESS_COUNT" -gt 0 ]; then |
| 134 | + echo -e "${YELLOW}⏳ $IN_PROGRESS_COUNT CI jobs are still running...${NC}" |
| 135 | + echo "" |
| 136 | + read -p "Wait for CI to complete? [Y/n] " -n 1 -r |
| 137 | + echo |
| 138 | + if [[ $REPLY =~ ^[Yy]$ ]] || [[ -z $REPLY ]]; then |
| 139 | + echo "Waiting for CI to complete (checking every 30 seconds)..." |
| 140 | + while [ "$IN_PROGRESS_COUNT" -gt 0 ]; do |
| 141 | + sleep 30 |
| 142 | + STATUS_JSON=$(gh pr view "$PR_NUMBER" --json statusCheckRollup,number 2>/dev/null) |
| 143 | + IN_PROGRESS_COUNT=$(echo "$STATUS_JSON" | jq '[(.statusCheckRollup // [])[] | select(.status == "IN_PROGRESS")] | length') |
| 144 | + FAILED_COUNT=$(echo "$STATUS_JSON" | jq '[(.statusCheckRollup // [])[] | select(.conclusion == "FAILURE")] | length') |
| 145 | + echo -e " In progress: $IN_PROGRESS_COUNT | Failed: $FAILED_COUNT" |
| 146 | + done |
| 147 | + echo -e "${GREEN}✓ CI completed!${NC}" |
| 148 | + echo "" |
| 149 | + fi |
| 150 | +fi |
| 151 | + |
| 152 | +# Parse failed checks |
| 153 | +FAILED_CHECKS=$(echo "$STATUS_JSON" | jq -r '(.statusCheckRollup // [])[] | select(.conclusion == "FAILURE") | .name') |
| 154 | + |
| 155 | +if [ -z "$FAILED_CHECKS" ]; then |
| 156 | + echo -e "${GREEN}✓ No failed checks found! All CI jobs passed.${NC}" |
| 157 | + exit 0 |
| 158 | +fi |
| 159 | + |
| 160 | +echo -e "${YELLOW}Failed CI jobs:${NC}" |
| 161 | +echo "$FAILED_CHECKS" | while read -r check; do |
| 162 | + echo -e "${RED} ✗ $check${NC}" |
| 163 | +done |
| 164 | +echo "" |
| 165 | + |
| 166 | +# Map CI job names to local commands |
| 167 | +declare -A JOB_MAP |
| 168 | +JOB_MAP["lint-js-and-ruby"]="bundle exec rubocop && yarn run eslint --report-unused-disable-directives && yarn start format.listDifferent" |
| 169 | +JOB_MAP["rspec-package-tests"]="bundle exec rake run_rspec:gem" |
| 170 | +JOB_MAP["package-js-tests"]="yarn test" |
| 171 | +JOB_MAP["dummy-app-integration-tests (3.4, 22, latest)"]="bundle exec rake run_rspec:all_dummy" |
| 172 | +JOB_MAP["dummy-app-integration-tests (3.2, 20, minimum)"]="bundle exec rake run_rspec:all_dummy" |
| 173 | +JOB_MAP["examples"]="bundle exec rake run_rspec:shakapacker_examples" |
| 174 | + |
| 175 | +# Track what we'll run (deduplicated) |
| 176 | +declare -A COMMANDS_TO_RUN |
| 177 | + |
| 178 | +while IFS= read -r check; do |
| 179 | + for job_name in "${!JOB_MAP[@]}"; do |
| 180 | + if [[ "$check" == "$job_name"* ]]; then |
| 181 | + COMMANDS_TO_RUN["${JOB_MAP[$job_name]}"]="$job_name" |
| 182 | + break |
| 183 | + fi |
| 184 | + done |
| 185 | +done <<< "$FAILED_CHECKS" |
| 186 | + |
| 187 | +# Check if any commands were found (handle empty array with set -u) |
| 188 | +set +u |
| 189 | +NUM_COMMANDS=${#COMMANDS_TO_RUN[@]} |
| 190 | +set -u |
| 191 | + |
| 192 | +if [ "$NUM_COMMANDS" -eq 0 ]; then |
| 193 | + echo -e "${YELLOW}No local equivalents found for failed jobs.${NC}" |
| 194 | + echo "Failed jobs might be from Pro or other workflows." |
| 195 | + echo "" |
| 196 | + echo "You can still run common test suites:" |
| 197 | + echo " bundle exec rake run_rspec:all_dummy # Dummy app tests" |
| 198 | + echo " bundle exec rake run_rspec:gem # Gem tests" |
| 199 | + echo " yarn test # JS tests" |
| 200 | + exit 1 |
| 201 | +fi |
| 202 | + |
| 203 | +echo -e "${BLUE}Will run the following commands:${NC}" |
| 204 | +for cmd in "${!COMMANDS_TO_RUN[@]}"; do |
| 205 | + echo -e "${BLUE} • ${COMMANDS_TO_RUN[$cmd]}:${NC} $cmd" |
| 206 | +done |
| 207 | +echo "" |
| 208 | + |
| 209 | +# Offer to show GitHub Actions logs for more specific failures |
| 210 | +echo -e "${YELLOW}💡 Tip: For RSpec failures, you can run specific failing examples:${NC}" |
| 211 | +echo " 1. Go to the failed job on GitHub Actions" |
| 212 | +echo " 2. Copy the 'Failed examples:' section" |
| 213 | +echo " 3. Run: pbpaste | bin/ci-run-failed-specs" |
| 214 | +echo "" |
| 215 | + |
| 216 | +# Confirm before running |
| 217 | +read -p "Run these tests now? [Y/n] " -n 1 -r |
| 218 | +echo |
| 219 | +if [[ ! $REPLY =~ ^[Yy]$ ]] && [[ ! -z $REPLY ]]; then |
| 220 | + echo "Cancelled." |
| 221 | + exit 0 |
| 222 | +fi |
| 223 | + |
| 224 | +echo "" |
| 225 | + |
| 226 | +# Ensure dependencies |
| 227 | +if [ ! -d "node_modules" ] || [ ! -d "vendor/bundle" ]; then |
| 228 | + echo -e "${YELLOW}Installing dependencies...${NC}" |
| 229 | + bundle install && yarn install |
| 230 | + echo "" |
| 231 | +fi |
| 232 | + |
| 233 | +# Run commands |
| 234 | +FAILED_COMMANDS=() |
| 235 | + |
| 236 | +for cmd in "${!COMMANDS_TO_RUN[@]}"; do |
| 237 | + job_name="${COMMANDS_TO_RUN[$cmd]}" |
| 238 | + echo -e "${BLUE}▶ Running: $job_name${NC}" |
| 239 | + echo -e "${BLUE}Command: $cmd${NC}" |
| 240 | + echo "" |
| 241 | + |
| 242 | + if eval "$cmd"; then |
| 243 | + echo -e "${GREEN}✓ $job_name passed${NC}" |
| 244 | + echo "" |
| 245 | + else |
| 246 | + echo -e "${RED}✗ $job_name failed${NC}" |
| 247 | + echo "" |
| 248 | + FAILED_COMMANDS+=("$job_name") |
| 249 | + fi |
| 250 | +done |
| 251 | + |
| 252 | +# Summary |
| 253 | +echo "" |
| 254 | +echo -e "${BLUE}=== Summary ===${NC}" |
| 255 | + |
| 256 | +if [ ${#FAILED_COMMANDS[@]} -eq 0 ]; then |
| 257 | + echo -e "${GREEN}All local tests passed! ✓${NC}" |
| 258 | + echo "" |
| 259 | + echo "Push your changes and CI should pass." |
| 260 | + exit 0 |
| 261 | +else |
| 262 | + echo -e "${RED}Some tests still failing:${NC}" |
| 263 | + for cmd in "${FAILED_COMMANDS[@]}"; do |
| 264 | + echo -e "${RED} ✗ $cmd${NC}" |
| 265 | + done |
| 266 | + echo "" |
| 267 | + echo -e "${YELLOW}Fix these failures before pushing.${NC}" |
| 268 | + exit 1 |
| 269 | +fi |
0 commit comments