Fix /run-skipped-ci to detect and run only actually skipped checks

justin808 · claude · justin808 · commit e8f62c16d843 · 2025-11-12T13:05:35.000-10:00
The previous implementation tried to guess which tests to run based on a hardcoded list. This fix makes it smarter: 1. **Fetches actual skipped checks from the PR** - Uses GitHub API to find checks with conclusion='SKIPPED' - Maps workflow names to workflow files - Only triggers workflows that have skipped checks 2. **Added force_run input to ALL workflows** - main.yml, examples.yml, pro-integration-tests.yml, pro-package-tests.yml, pro-lint.yml - When force_run=true, detect-changes outputs true for all checks - Bypasses file change detection that causes skips 3. **Fixed matrix exclusion logic** - When force_run=true, runs BOTH latest and minimum matrices - Ensures comprehensive testing of skipped configurations 4. **Improved PR comments** - Lists all skipped checks that were detected - Shows which workflows were triggered - Clear explanation that force_run bypasses detect-changes Example comment: ``` 🚀 Skipped CI Checks - Trigger Results ✅ Successfully triggered skipped CI checks **Skipped checks detected:** - build-dummy-app-webpack-test-bundles (React on Rails Pro - Integration Tests) - lint-js-and-ruby (React on Rails Pro - Lint) **Triggered workflows:** - ✅ React on Rails Pro - Integration Tests - ✅ React on Rails Pro - Lint **Note:** These workflows will run with `force_run: true` to bypass detect-changes. ``` This fixes the issue where Pro tests were being skipped on PRs without Pro file changes, and ensures the command only runs what's actually needed. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml
@@ -13,8 +13,8 @@ on:
       - 'docs/**'
   workflow_dispatch:
     inputs:
-      run_minimum_tests:
-        description: 'Run minimum dependency matrix (Ruby 3.2)'
+      force_run:
+        description: 'Force run all jobs (bypass detect-changes)'
         required: false
         type: boolean
         default: false
@@ -38,6 +38,17 @@ jobs:
       - name: Detect relevant changes
         id: detect
         run: |
+          # If force_run is true, run everything
+          if [ "${{ inputs.force_run }}" = "true" ]; then
+            echo "run_lint=true" >> "$GITHUB_OUTPUT"
+            echo "run_js_tests=true" >> "$GITHUB_OUTPUT"
+            echo "run_ruby_tests=true" >> "$GITHUB_OUTPUT"
+            echo "run_dummy_tests=true" >> "$GITHUB_OUTPUT"
+            echo "run_generators=true" >> "$GITHUB_OUTPUT"
+            echo "docs_only=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
           BASE_REF="${{ github.event.pull_request.base.sha || github.event.before || 'origin/master' }}"
           script/ci-changes-detector "$BASE_REF"
         shell: bash
@@ -58,10 +69,9 @@ jobs:
           - ruby-version: '3.2'
             dependency-level: 'minimum'
         exclude:
-          # When run_minimum_tests is true, skip latest (run only minimum)
-          - ${{ inputs.run_minimum_tests && fromJSON('{"ruby-version": "3.4", "dependency-level": "latest"}') || fromJSON('{}') }}
-          # When run_minimum_tests is false, skip minimum on regular PRs (run only on master/workflow_dispatch)
-          - ${{ !inputs.run_minimum_tests && github.event_name == 'pull_request' && github.ref != 'refs/heads/master' && fromJSON('{"ruby-version": "3.2", "dependency-level": "minimum"}') || fromJSON('{}') }}
+          # Skip minimum dependency matrix on regular PRs (run only on master/workflow_dispatch/force_run)
+          - ruby-version: ${{ github.event_name == 'pull_request' && github.ref != 'refs/heads/master' && inputs.force_run != true && '3.2' || '' }}
+            dependency-level: ${{ github.event_name == 'pull_request' && github.ref != 'refs/heads/master' && inputs.force_run != true && 'minimum' || '' }}
     env:
       SKIP_YARN_COREPACK_CHECK: 0
       BUNDLE_FROZEN: ${{ matrix.dependency-level == 'minimum' && 'false' || 'true' }}
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -13,8 +13,8 @@ on:
       - 'docs/**'
   workflow_dispatch:
     inputs:
-      run_minimum_tests:
-        description: 'Run minimum dependency matrix (Ruby 3.2, Node 20)'
+      force_run:
+        description: 'Force run all jobs (bypass detect-changes)'
         required: false
         type: boolean
         default: false
@@ -38,6 +38,17 @@ jobs:
       - name: Detect relevant changes
         id: detect
         run: |
+          # If force_run is true, run everything
+          if [ "${{ inputs.force_run }}" = "true" ]; then
+            echo "run_lint=true" >> "$GITHUB_OUTPUT"
+            echo "run_js_tests=true" >> "$GITHUB_OUTPUT"
+            echo "run_ruby_tests=true" >> "$GITHUB_OUTPUT"
+            echo "run_dummy_tests=true" >> "$GITHUB_OUTPUT"
+            echo "run_generators=true" >> "$GITHUB_OUTPUT"
+            echo "docs_only=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
           BASE_REF="${{ github.event.pull_request.base.sha || github.event.before || 'origin/master' }}"
           script/ci-changes-detector "$BASE_REF"
         shell: bash
@@ -59,10 +70,10 @@ jobs:
             node-version: '20'
             dependency-level: 'minimum'
         exclude:
-          # When run_minimum_tests is true, skip latest (run only minimum)
-          - ${{ inputs.run_minimum_tests && fromJSON('{"ruby-version": "3.4", "node-version": "22", "dependency-level": "latest"}') || fromJSON('{}') }}
-          # When run_minimum_tests is false, skip minimum on regular PRs (run only on master/workflow_dispatch)
-          - ${{ !inputs.run_minimum_tests && github.event_name == 'pull_request' && github.ref != 'refs/heads/master' && fromJSON('{"ruby-version": "3.2", "node-version": "20", "dependency-level": "minimum"}') || fromJSON('{}') }}
+          # Skip minimum dependency matrix on regular PRs (run only on master/workflow_dispatch/force_run)
+          - ruby-version: ${{ github.event_name == 'pull_request' && github.ref != 'refs/heads/master' && inputs.force_run != true && '3.2' || '' }}
+            node-version: ${{ github.event_name == 'pull_request' && github.ref != 'refs/heads/master' && inputs.force_run != true && '20' || '' }}
+            dependency-level: ${{ github.event_name == 'pull_request' && github.ref != 'refs/heads/master' && inputs.force_run != true && 'minimum' || '' }}
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
@@ -149,10 +160,10 @@ jobs:
             node-version: '20'
             dependency-level: 'minimum'
         exclude:
-          # When run_minimum_tests is true, skip latest (run only minimum)
-          - ${{ inputs.run_minimum_tests && fromJSON('{"ruby-version": "3.4", "node-version": "22", "dependency-level": "latest"}') || fromJSON('{}') }}
-          # When run_minimum_tests is false, skip minimum on regular PRs (run only on master/workflow_dispatch)
-          - ${{ !inputs.run_minimum_tests && github.event_name == 'pull_request' && github.ref != 'refs/heads/master' && fromJSON('{"ruby-version": "3.2", "node-version": "20", "dependency-level": "minimum"}') || fromJSON('{}') }}
+          # Skip minimum dependency matrix on regular PRs (run only on master/workflow_dispatch/force_run)
+          - ruby-version: ${{ github.event_name == 'pull_request' && github.ref != 'refs/heads/master' && inputs.force_run != true && '3.2' || '' }}
+            node-version: ${{ github.event_name == 'pull_request' && github.ref != 'refs/heads/master' && inputs.force_run != true && '20' || '' }}
+            dependency-level: ${{ github.event_name == 'pull_request' && github.ref != 'refs/heads/master' && inputs.force_run != true && 'minimum' || '' }}
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
diff --git a/.github/workflows/pro-integration-tests.yml b/.github/workflows/pro-integration-tests.yml
@@ -6,6 +6,12 @@ on:
       - 'master'
   pull_request:
   workflow_dispatch:
+    inputs:
+      force_run:
+        description: 'Force run all jobs (bypass detect-changes)'
+        required: false
+        type: boolean
+        default: false
 
 defaults:
   run:
@@ -27,6 +33,14 @@ jobs:
         id: detect
         working-directory: .
         run: |
+          # If force_run is true, run everything
+          if [ "${{ inputs.force_run }}" = "true" ]; then
+            echo "run_pro_lint=true" >> "$GITHUB_OUTPUT"
+            echo "run_pro_tests=true" >> "$GITHUB_OUTPUT"
+            echo "docs_only=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
           BASE_REF="${{ github.event.pull_request.base.sha || github.event.before || 'origin/master' }}"
           script/ci-changes-detector "$BASE_REF"
         shell: bash
diff --git a/.github/workflows/pro-lint.yml b/.github/workflows/pro-lint.yml
@@ -6,6 +6,12 @@ on:
       - 'master'
   pull_request:
   workflow_dispatch:
+    inputs:
+      force_run:
+        description: 'Force run all jobs (bypass detect-changes)'
+        required: false
+        type: boolean
+        default: false
 
 defaults:
   run:
@@ -27,6 +33,14 @@ jobs:
         id: detect
         working-directory: .
         run: |
+          # If force_run is true, run everything
+          if [ "${{ inputs.force_run }}" = "true" ]; then
+            echo "run_pro_lint=true" >> "$GITHUB_OUTPUT"
+            echo "run_pro_tests=true" >> "$GITHUB_OUTPUT"
+            echo "docs_only=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
           BASE_REF="${{ github.event.pull_request.base.sha || github.event.before || 'origin/master' }}"
           script/ci-changes-detector "$BASE_REF"
         shell: bash
diff --git a/.github/workflows/pro-package-tests.yml b/.github/workflows/pro-package-tests.yml
@@ -6,6 +6,12 @@ on:
       - 'master'
   pull_request:
   workflow_dispatch:
+    inputs:
+      force_run:
+        description: 'Force run all jobs (bypass detect-changes)'
+        required: false
+        type: boolean
+        default: false
 
 defaults:
   run:
@@ -27,6 +33,14 @@ jobs:
         id: detect
         working-directory: .
         run: |
+          # If force_run is true, run everything
+          if [ "${{ inputs.force_run }}" = "true" ]; then
+            echo "run_pro_lint=true" >> "$GITHUB_OUTPUT"
+            echo "run_pro_tests=true" >> "$GITHUB_OUTPUT"
+            echo "docs_only=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
           BASE_REF="${{ github.event.pull_request.base.sha || github.event.before || 'origin/master' }}"
           script/ci-changes-detector "$BASE_REF"
         shell: bash
diff --git a/.github/workflows/run-skipped-ci.yml b/.github/workflows/run-skipped-ci.yml
@@ -81,53 +81,83 @@ jobs:
               sha: pr.data.head.sha
             };
 
-      - name: Trigger all workflows and collect results
+      - name: Get skipped checks and trigger workflows
         id: trigger_workflows
         uses: actions/github-script@v7
         with:
           script: |
             const prData = ${{ steps.pr.outputs.result }};
 
-            // Workflows that support minimum dependency testing
-            const workflowsWithMinimum = [
-              { id: 'main.yml', name: 'Main Tests' },
-              { id: 'examples.yml', name: 'Generator Tests' }
-            ];
+            // Fetch PR checks to find skipped ones
+            const { data: pr } = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number
+            });
 
-            // Pro workflows always run (no minimum matrix)
-            const proWorkflows = [
-              { id: 'pro-integration-tests.yml', name: 'Pro Integration Tests' },
-              { id: 'pro-package-tests.yml', name: 'Pro Package Tests' }
-            ];
+            const checks = await github.rest.checks.listForRef({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref: pr.head.sha,
+              per_page: 100
+            });
+
+            // Find all skipped checks
+            const skippedChecks = checks.data.check_runs.filter(check =>
+              check.conclusion === 'SKIPPED' && check.status === 'COMPLETED'
+            );
+
+            console.log(`Found ${skippedChecks.length} skipped checks:`);
+            skippedChecks.forEach(check => {
+              console.log(`  - ${check.name} (${check.workflow_name})`);
+            });
+
+            // Map workflow names to workflow files
+            const workflowMap = {
+              'Main test': 'main.yml',
+              'Generator tests': 'examples.yml',
+              'React on Rails Pro - Integration Tests': 'pro-integration-tests.yml',
+              'React on Rails Pro - Package Tests': 'pro-package-tests.yml',
+              'React on Rails Pro - Lint': 'pro-lint.yml'
+            };
+
+            // Get unique workflows that have skipped checks
+            const uniqueWorkflows = new Set();
+            skippedChecks.forEach(check => {
+              if (workflowMap[check.workflow_name]) {
+                uniqueWorkflows.add(check.workflow_name);
+              }
+            });
 
             const succeeded = [];
             const failed = [];
-            const skipped = [];
+            const notApplicable = [];
 
-            // Trigger workflows with minimum dependency testing
-            for (const workflow of workflowsWithMinimum) {
+            // Trigger each workflow that has skipped checks
+            for (const workflowName of uniqueWorkflows) {
+              const workflowFile = workflowMap[workflowName];
               try {
                 await github.rest.actions.createWorkflowDispatch({
                   owner: context.repo.owner,
                   repo: context.repo.repo,
-                  workflow_id: workflow.id,
+                  workflow_id: workflowFile,
                   ref: prData.ref,
                   inputs: {
-                    run_minimum_tests: 'true'
+                    force_run: 'true'
                   }
                 });
-                console.log(`✅ Triggered ${workflow.id} with run_minimum_tests=true`);
-                succeeded.push(workflow);
+                console.log(`✅ Triggered ${workflowFile} (${workflowName})`);
+                succeeded.push({ id: workflowFile, name: workflowName });
               } catch (error) {
-                console.error(`❌ Failed to trigger ${workflow.id}:`, error.message);
-                failed.push({ workflow: workflow.name, error: error.message });
+                console.error(`❌ Failed to trigger ${workflowFile}:`, error.message);
+                failed.push({ workflow: workflowName, error: error.message });
               }
             }
 
-            // Skip Pro workflows (they don't have minimum matrix, always run on PRs if needed)
-            for (const workflow of proWorkflows) {
-              console.log(`⏭️  Skipping ${workflow.id} (no minimum dependency matrix)`);
-              skipped.push(workflow);
+            // Note workflows with no skipped checks
+            if (uniqueWorkflows.size === 0) {
+              console.log('ℹ️  No skipped checks found - all tests already running on this PR');
+              notApplicable.push('No skipped checks to run');
             }
 
             // Wait a bit for workflows to queue
@@ -164,33 +194,39 @@ jobs:
             }
 
             // Build the comment body based on actual results
-            let status = '✅ **Successfully triggered skipped CI tests**';
-            if (failed.length > 0 && notFound.length > 0) {
+            let status;
+            if (notApplicable.length > 0) {
+              status = '✅ **All checks are already running - nothing to do!**';
+            } else if (failed.length > 0 && notFound.length > 0) {
               status = '❌ **Failed to trigger or verify workflows**';
             } else if (failed.length > 0) {
               status = '⚠️ **Some workflows failed to trigger**';
             } else if (notFound.length > 0) {
               status = '⚠️ **Workflows triggered but not yet verified**';
+            } else {
+              status = '✅ **Successfully triggered skipped CI checks**';
             }
 
-            const verifiedList = verified.length > 0 ? verified.map(w => `- ✅ ${w.name}`).join('\n') : '';
+            // List the skipped checks we found
+            const skippedChecksList = skippedChecks.length > 0
+              ? `\n**Skipped checks detected:**\n${skippedChecks.map(c => `- ${c.name} (${c.workflow_name})`).join('\n')}`
+              : '';
+
+            const verifiedList = verified.length > 0 ? `\n**Triggered workflows:**\n${verified.map(w => `- ✅ ${w.name}`).join('\n')}` : '';
             const notFoundList = notFound.length > 0 ? `\n\n**Triggered but not yet queued (may still start):**\n${notFound.map(w => `- ⏳ ${w.name}`).join('\n')}` : '';
             const failedList = failed.length > 0 ? `\n\n**Failed to trigger:**\n${failed.map(f => `- ❌ ${f.workflow}: ${f.error}`).join('\n')}` : '';
-            const skippedList = skipped.length > 0 ? `\n\n**Skipped (already run on PRs):**\n${skipped.map(w => `- ⏭️  ${w.name}`).join('\n')}` : '';
 
-            const body = `🚀 **Skipped CI Tests Triggered**
+            const body = `🚀 **Skipped CI Checks - Trigger Results**
 
             ${status}
+            ${skippedChecksList}
+            ${verifiedList}${notFoundList}${failedList}
 
-            ${verifiedList ? `**Running minimum dependency tests:**\n${verifiedList}` : ''}${notFoundList}${failedList}${skippedList}
-
-            ${verified.length > 0 ? `\n**What's running:**
-            - ✅ Minimum dependency versions (Ruby 3.2, Node 20)
-            - ✅ Generator tests with minimum dependencies
+            ${verified.length > 0 ? `\n**Note:** These workflows will run with \`force_run: true\` to bypass detect-changes logic that caused them to skip.
 
-            **Note:** Pro package tests and latest dependency tests are skipped because they already run on all PRs.
+            View progress in the [Actions tab](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions).` : ''}
 
-            View progress in the [Actions tab](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions).` : ''}`;
+            ${notApplicable.length > 0 ? `\nAll CI checks are already running on this PR. Use this command when you see skipped checks that you want to run.` : ''}`;
 
             // Post the comment
             await github.rest.issues.createComment({
