bulk-generate.yml

name: "Bulk: Generate"
run-name: "Bulk: ${{ inputs.library }} for ${{ inputs.specification_id }}"

# Dispatches multiple implementation workflows
# Use for bulk operations like "update all matplotlib" or "generate all for spec"

on:
  workflow_dispatch:
    inputs:
      specification_id:
        description: "Specification ID (or 'all' for all specs)"
        required: true
        type: string
        default: 'all'
      library:
        description: "Library to generate (or 'all' for all libraries)"
        required: true
        type: choice
        default: 'all'
        options:
          - all
          - matplotlib
          - seaborn
          - plotly
          - bokeh
          - altair
          - plotnine
          - pygal
          - highcharts
          - letsplot
          - ggplot2
          - makie
          - chartjs
          - d3
          - echarts
          - muix
      dry_run:
        description: "List what would be generated without executing"
        type: boolean
        default: false
      pace_seconds:
        description: "Seconds to wait between dispatches (0 = fire all at once, default 120 = 2 min)"
        required: false
        default: '120'
      model:
        description: "Claude model to use across generate / review / repair (default sonnet)"
        required: false
        type: choice
        default: 'sonnet'
        options:
          - haiku
          - sonnet
          - opus
      change_requests:
        description: "JSON object {library: one-sentence-hint} from daily-regen similarity audit. Empty = no clusters."
        required: false
        type: string
        default: '{}'

env:
  ALL_LIBRARIES: "matplotlib seaborn plotly bokeh altair plotnine pygal highcharts letsplot ggplot2 makie chartjs d3 echarts muix"

# Serialise bulk-generate runs. Each run paces its own dispatches with
# `pace_seconds`; letting two runs overlap would interleave their
# dispatches and stack the Claude queue, which is exactly what the pacing
# exists to avoid. Second run waits for first to finish instead.
concurrency:
  group: bulk-generate
  cancel-in-progress: false

jobs:
  # ============================================================================
  # Build matrix of (spec, library) pairs to generate
  # ============================================================================
  build-matrix:
    runs-on: ubuntu-latest
    permissions:
      contents: read
    outputs:
      matrix: ${{ steps.build.outputs.matrix }}
      count: ${{ steps.build.outputs.count }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1

      - name: Build generation matrix
        id: build
        env:
          SPEC_INPUT: ${{ inputs.specification_id }}
          LIBRARY_INPUT: ${{ inputs.library }}
        run: |
          # Get list of specifications
          if [ "$SPEC_INPUT" == "all" ]; then
            SPECS=$(ls -d plots/*/ 2>/dev/null | xargs -I{} basename {} | tr '\n' ' ')
          else
            SPECS="$SPEC_INPUT"
          fi

          # Get list of libraries
          if [ "$LIBRARY_INPUT" == "all" ]; then
            LIBRARIES="$ALL_LIBRARIES"
          else
            LIBRARIES="$LIBRARY_INPUT"
          fi

          # Build matrix JSON
          MATRIX='{"include":['
          FIRST=true
          COUNT=0

          for SPEC in $SPECS; do
            # Skip if specification.md doesn't exist
            if [ ! -f "plots/${SPEC}/specification.md" ]; then
              echo "::warning::Skipping ${SPEC} - no specification.md found"
              continue
            fi

            for LIB in $LIBRARIES; do
              if [ "$FIRST" = true ]; then
                FIRST=false
              else
                MATRIX="${MATRIX},"
              fi
              MATRIX="${MATRIX}{\"specification_id\":\"${SPEC}\",\"library\":\"${LIB}\"}"
              COUNT=$((COUNT + 1))
            done
          done

          MATRIX="${MATRIX}]}"

          echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
          echo "count=$COUNT" >> $GITHUB_OUTPUT

          echo "Generated matrix with $COUNT items:"
          echo "$MATRIX" | jq .

  # ============================================================================
  # Display what will be generated (for review)
  # ============================================================================
  preview:
    needs: build-matrix
    runs-on: ubuntu-latest
    permissions:
      contents: read

    steps:
      - name: Preview generation plan
        env:
          MATRIX: ${{ needs.build-matrix.outputs.matrix }}
          COUNT: ${{ needs.build-matrix.outputs.count }}
          DRY_RUN: ${{ inputs.dry_run }}
        run: |
          echo "## Bulk Generation Plan"
          echo ""
          echo "**Total items:** $COUNT"
          echo "**Pacing:** ${{ inputs.pace_seconds || '120' }}s between dispatches (0 = fire all)"
          echo "**Dry run:** $DRY_RUN"
          echo ""
          echo "### Items to generate:"
          echo "$MATRIX" | jq -r '.include[] | "- \(.specification_id) / \(.library)"'

  # ============================================================================
  # Generate implementations — sequential, paced dispatch
  #
  # One runner loops through every (spec, library) pair and dispatches
  # impl-generate.yml with a configurable pause between each call. Default
  # pace is 120 s (2 min) to stay under the Claude Max concurrency budget on
  # Sonnet and avoid spinning up 9 heavy generation runs at once.
  # ============================================================================
  generate:
    needs: [build-matrix, preview]
    if: inputs.dry_run == false && needs.build-matrix.outputs.count != '0'
    runs-on: ubuntu-latest
    permissions:
      contents: read
      actions: write
    # Enough headroom: 9 libs × (120 s pace + ~30 s dispatch) ≈ 23 min;
    # bulk "all for all" could be very long — clamp at 6 h.
    timeout-minutes: 360

    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1

      - name: Dispatch impl-generate for each matrix item (paced)
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          MATRIX: ${{ needs.build-matrix.outputs.matrix }}
          PACE_SECONDS: ${{ inputs.pace_seconds || '120' }}
          MODEL: ${{ inputs.model || 'sonnet' }}
          CHANGE_REQUESTS: ${{ inputs.change_requests || '{}' }}
        run: |
          set -u

          pace="${PACE_SECONDS}"
          pairs=$(echo "$MATRIX" | jq -r '.include[] | "\(.specification_id) \(.library)"')
          total=$(echo "$pairs" | wc -l | tr -d ' ')

          # Validate change_requests is a JSON object early — bad JSON would
          # silently produce empty hints later and we'd never know.
          if ! echo "$CHANGE_REQUESTS" | jq -e 'type == "object"' >/dev/null 2>&1; then
            echo "::warning::change_requests input is not a valid JSON object; ignoring (got: ${CHANGE_REQUESTS})"
            CHANGE_REQUESTS='{}'
          fi
          flagged_count=$(echo "$CHANGE_REQUESTS" | jq 'length')
          echo "::notice::Dispatching $total item(s) with ${pace}s pacing between each (model=${MODEL}, change_requests for ${flagged_count} libs)"

          i=0
          failed=0
          while IFS=' ' read -r SPEC_ID LIBRARY; do
            i=$((i + 1))

            # Resolve issue number from specification.yaml (optional).
            SPEC_YAML="plots/${SPEC_ID}/specification.yaml"
            ISSUE=""
            if [ -f "$SPEC_YAML" ]; then
              ISSUE=$(yq '.issue' "$SPEC_YAML" 2>/dev/null || echo "")
              [ "$ISSUE" = "null" ] && ISSUE=""
            fi

            # Per-library divergence hint (empty if not flagged).
            HINT=$(echo "$CHANGE_REQUESTS" | jq -r --arg lib "$LIBRARY" '.[$lib] // ""')

            # Best-effort pending label so the issue shows the in-flight lib.
            if [ -n "$ISSUE" ]; then
              gh issue edit "$ISSUE" --add-label "impl:${LIBRARY}:pending" 2>/dev/null || true
            fi

            if [ -n "$HINT" ]; then
              echo "::notice::[$i/$total] $(date -u +%H:%M:%SZ)  dispatching impl-generate for ${SPEC_ID}/${LIBRARY} (issue: ${ISSUE:-none}, change_request: ${HINT})"
            else
              echo "::notice::[$i/$total] $(date -u +%H:%M:%SZ)  dispatching impl-generate for ${SPEC_ID}/${LIBRARY} (issue: ${ISSUE:-none})"
            fi

            # Retry dispatch up to 3× with linear backoff.
            dispatched=0
            for attempt in 1 2 3; do
              if [ -n "$ISSUE" ]; then
                gh workflow run impl-generate.yml --repo "${{ github.repository }}" \
                  -f specification_id="${SPEC_ID}" \
                  -f library="${LIBRARY}" \
                  -f issue_number="${ISSUE}" \
                  -f model="${MODEL}" \
                  -f change_request="${HINT}" && dispatched=1 && break
              else
                gh workflow run impl-generate.yml --repo "${{ github.repository }}" \
                  -f specification_id="${SPEC_ID}" \
                  -f library="${LIBRARY}" \
                  -f model="${MODEL}" \
                  -f change_request="${HINT}" && dispatched=1 && break
              fi
              echo "::warning::Dispatch attempt $attempt failed for ${SPEC_ID}/${LIBRARY}, retrying in 10s"
              sleep 10
            done

            if [ "$dispatched" = "0" ]; then
              echo "::error::Failed to dispatch ${SPEC_ID}/${LIBRARY} after 3 attempts — continuing"
              failed=$((failed + 1))
            fi

            # Pause before next dispatch, but not after the last item.
            if [ "$i" -lt "$total" ] && [ "$pace" -gt 0 ]; then
              echo "Sleeping ${pace}s before next dispatch…"
              sleep "$pace"
            fi
          done <<< "$pairs"

          echo "::notice::Done: $i dispatched, $failed failures"
          if [ "$failed" -gt 0 ]; then
            exit 1
          fi

  # ============================================================================
  # Summary
  # ============================================================================
  summary:
    needs: [build-matrix, generate]
    if: always()
    runs-on: ubuntu-latest
    permissions:
      contents: read

    steps:
      - name: Post summary
        env:
          COUNT: ${{ needs.build-matrix.outputs.count }}
          DRY_RUN: ${{ inputs.dry_run }}
          GENERATE_RESULT: ${{ needs.generate.result }}
        run: |
          echo "## Bulk Generation Complete"
          echo ""
          echo "**Total dispatched:** $COUNT"

          if [ "$DRY_RUN" == "true" ]; then
            echo "**Mode:** Dry run (no workflows triggered)"
          else
            echo "**Mode:** Live execution"
            echo "**Generate job result:** $GENERATE_RESULT"
          fi

          echo ""
          echo "Check the Actions tab for individual impl-generate workflow runs."