add skipping passed examples

Author: mhordynski

.github/docs/run-examples-caching.md

@@ -0,0 +1,31 @@
+# GitHub Actions Scripts
+
+## Example Caching System
+
+This directory contains scripts that implement a smart caching mechanism for running examples in CI:
+
+### How it works
+
+1. **`list_examples.sh`** - Discovers all runnable Python examples and generates a matrix that includes skip information
+2. **`example_cache.sh`** - Consolidated script with subcommands:
+   - `check <example_file>` - Checks if an example should be skipped based on previous successful run and file modifications
+   - `record <example_file>` - Records successful example runs with the current commit hash
+3. **`run_single_example.sh`** - Runs individual examples and records success when they complete
+
+### Cache behavior
+
+- Examples that passed on the current branch and haven't been modified since will be skipped
+- Cache is stored per branch using GitHub Actions cache
+- Cache keys include the branch name and commit hash
+- Fallback to main branch cache if current branch cache doesn't exist
+
+### Benefits
+
+- Dramatically reduces CI time when most examples are already passing
+- Only runs examples that are new, modified, or previously failed
+- Maintains full test coverage while optimizing for common scenarios
+
+### Cache storage
+
+Cache files are stored in `.example-cache/` with the format:
+- `{branch-name}-{example-path-normalized}.success` - Contains the commit hash of the last successful run

.github/scripts/example_cache.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if [ $# -lt 2 ]; then
+    echo "Usage: $0 <command> <example_file>"
+    echo "Commands:"
+    echo "  record <example_file>  - Record successful example run"
+    echo "  check <example_file>   - Check if example should be skipped"
+    exit 1
+fi
+
+COMMAND="$1"
+EXAMPLE_FILE="$2"
+CACHE_DIR=".example-cache"
+BRANCH_REF="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME:-main}}"
+
+# Normalize the branch name for cache key (replace / with -)
+CACHE_KEY=$(echo "$BRANCH_REF" | sed 's/\//-/g')
+SUCCESS_FILE="$CACHE_DIR/${CACHE_KEY}-$(echo "$EXAMPLE_FILE" | sed 's/\//-/g').success"
+
+mkdir -p "$CACHE_DIR"
+
+case "$COMMAND" in
+    "record")
+        # Record the current commit hash as successful
+        echo "$GITHUB_SHA" > "$SUCCESS_FILE"
+        echo "Recorded success for $EXAMPLE_FILE at commit $GITHUB_SHA"
+        ;;
+
+    "check")
+        # Check if we have a previous success record
+        if [ ! -f "$SUCCESS_FILE" ]; then
+            echo "false" # No previous success, must run
+            exit 0
+        fi
+
+        # Get the commit hash from the success file
+        LAST_SUCCESS_COMMIT=$(cat "$SUCCESS_FILE")
+
+        # Check if the example file has been modified since the last success
+        if git diff --quiet "$LAST_SUCCESS_COMMIT" HEAD -- "$EXAMPLE_FILE"; then
+            echo "true" # File unchanged since last success, can skip
+        else
+            echo "false" # File modified, must run
+        fi
+        ;;
+
+    *)
+        echo "Unknown command: $COMMAND"
+        echo "Available commands: record, check"
+        exit 1
+        ;;
+esac

.github/scripts/list_examples.sh

@@ -11,12 +11,19 @@ while IFS= read -r -d '' file; do
     fi
 done < <(find examples -name '*.py' -type f -print0)
 
-# Convert to JSON format for GitHub Actions matrix
-printf '{"example":['
+# Convert to JSON format for GitHub Actions matrix with skip information
+printf '{"include":['
 for i in "${!examples[@]}"; do
     if [ $i -gt 0 ]; then
         printf ','
     fi
-    printf '"%s"' "${examples[$i]}"
+
+    # Check if this example should be skipped
+    should_skip="false"
+    if [ -f ".github/scripts/example_cache.sh" ]; then
+        should_skip=$(bash .github/scripts/example_cache.sh check "${examples[$i]}" || echo "false")
+    fi
+
+    printf '{"example":"%s","skip":%s}' "${examples[$i]}" "$should_skip"
 done
 printf ']}'

.github/scripts/run_single_example.sh

@@ -83,4 +83,8 @@ elif [[ $exit_code -ne 0 ]]; then
   exit $exit_code
 else
   echo "Script completed successfully."
+  # Record success in cache
+  if [ -f ".github/scripts/example_cache.sh" ]; then
+    bash .github/scripts/example_cache.sh record "$EXAMPLE_FILE"
+  fi
 fi

.github/workflows/run-examples.yml

@@ -18,17 +18,32 @@ jobs:
     outputs:
       matrix: ${{ steps.generate.outputs.matrix }}
       examples-count: ${{ steps.generate.outputs.count }}
+      total-count: ${{ steps.generate.outputs.total-count }}
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Need full history for git diff
+
+      - name: Restore example cache
+        uses: actions/cache@v4
+        with:
+          path: .example-cache
+          key: examples-cache-${{ github.head_ref || github.ref_name }}-${{ github.sha }}
+          restore-keys: |
+            examples-cache-${{ github.head_ref || github.ref_name }}-
+            examples-cache-main-
 
       - name: Generate matrix
         id: generate
         run: |
           matrix=$(bash .github/scripts/list_examples.sh)
           echo "matrix=$matrix" >> $GITHUB_OUTPUT
-          count=$(echo "$matrix" | jq '.example | length')
-          echo "count=$count" >> $GITHUB_OUTPUT
-          echo "Found $count runnable examples"
+          total_count=$(echo "$matrix" | jq '.include | length')
+          skipped_count=$(echo "$matrix" | jq '[.include[] | select(.skip == true)] | length')
+          run_count=$((total_count - skipped_count))
+          echo "total-count=$total_count" >> $GITHUB_OUTPUT
+          echo "count=$run_count" >> $GITHUB_OUTPUT
+          echo "Found $total_count total examples ($run_count to run, $skipped_count cached)"
           echo "$matrix" | jq .
 
   # Second job: Run examples in parallel
@@ -89,6 +104,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Need full history for git diff
 
       - name: Install uv
         uses: astral-sh/setup-uv@v2
@@ -104,7 +121,27 @@ jobs:
           restore-keys: |
             uv-${{ runner.os }}-
 
+      - name: Restore example cache
+        uses: actions/cache@v4
+        with:
+          path: .example-cache
+          key: examples-cache-${{ github.head_ref || github.ref_name }}-${{ github.sha }}
+          restore-keys: |
+            examples-cache-${{ github.head_ref || github.ref_name }}-
+            examples-cache-main-
+
+      - name: Check if example should be skipped
+        id: check-skip
+        run: |
+          if [[ "${{ matrix.skip }}" == "true" ]]; then
+            echo "skip=true" >> $GITHUB_OUTPUT
+            echo "✅ Skipping ${{ matrix.example }} (cached from previous successful run)"
+          else
+            echo "skip=false" >> $GITHUB_OUTPUT
+          fi
+
       - name: Pre-install common dependencies
+        if: steps.check-skip.outputs.skip == 'false'
         run: |
           # Create a temporary environment with common ragbits packages to cache them
           uv venv --python 3.10 .temp-env
@@ -121,6 +158,7 @@ jobs:
           rm -rf .temp-env
 
       - name: Run example - ${{ matrix.example }}
+        if: steps.check-skip.outputs.skip == 'false'
         env:
           PR_BRANCH: ${{ github.head_ref }}
           GOOGLE_CLOUD_PROJECT: ${{ secrets.GCP_PROJECT_ID }}
@@ -135,6 +173,13 @@ jobs:
           chmod +x .github/scripts/run_single_example.sh
           ./.github/scripts/run_single_example.sh "${{ matrix.example }}"
 
+      - name: Save example cache
+        if: always()
+        uses: actions/cache/save@v4
+        with:
+          path: .example-cache
+          key: examples-cache-${{ github.head_ref || github.ref_name }}-${{ github.sha }}
+
   # Summary job: Report overall results
   examples-summary:
     name: Examples summary
@@ -143,10 +188,16 @@ jobs:
     if: always()
     steps:
       - name: Check results
+        env:
+          TOTAL_COUNT: ${{ needs.generate-matrix.outputs.total-count }}
+          RUN_COUNT: ${{ needs.generate-matrix.outputs.examples-count }}
         run: |
           echo "Examples matrix generation: ${{ needs.generate-matrix.result }}"
           echo "Examples execution: ${{ needs.examples.result }}"
-          echo "Total examples: ${{ needs.generate-matrix.outputs.examples-count }}"
+          echo "Total examples: $TOTAL_COUNT"
+          echo "Examples run: $RUN_COUNT"
+          cached_count=$((TOTAL_COUNT - RUN_COUNT))
+          echo "Examples cached: $cached_count"
 
           if [[ "${{ needs.generate-matrix.result }}" != "success" ]]; then
             echo "❌ Failed to generate examples matrix"
@@ -158,4 +209,4 @@ jobs:
             exit 1
           fi
 
-          echo "✅ All ${{ needs.generate-matrix.outputs.examples-count }} examples completed successfully"
+          echo "✅ All examples completed successfully ($RUN_COUNT run, $cached_count cached)"