update-benchmarks.yml

name: Update Benchmarks

on:
  push:
    branches: [main]
  workflow_dispatch:

# Runs full benchmarks on main and opens a PR with updated results

permissions:
  contents: write
  pull-requests: write

jobs:
  # Run benchmarks for all validators (except jsonschema)
  benchmark:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        draft: [draft4, draft6, draft7, draft2019-09, draft2020-12]
        validator: [tjs, ajv, zod, joi, is-my-json-valid, z-schema, djv, jsen, schemasafe]

    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true

      - name: Use Node.js 22
        uses: actions/setup-node@v4
        with:
          node-version: 22
          cache: 'npm'

      - name: Install dependencies
        run: npm ci

      - name: Build
        run: npm run build

      - name: Run benchmark for ${{ matrix.validator }} ${{ matrix.draft }}
        run: |
          mkdir -p benchmarks/results
          npx tsx benchmarks/bench.ts -v ${{ matrix.validator }} ${{ matrix.draft }} \
            --json benchmarks/results/${{ matrix.validator }}-${{ matrix.draft }}.json

      - name: Upload benchmark results
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-${{ matrix.validator }}-${{ matrix.draft }}
          path: benchmarks/results/${{ matrix.validator }}-${{ matrix.draft }}.json

  # Merge results, update files, and open PR
  finalize:
    runs-on: ubuntu-latest
    needs: [benchmark]

    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true

      - name: Use Node.js 22
        uses: actions/setup-node@v4
        with:
          node-version: 22
          cache: 'npm'

      - name: Install dependencies
        run: npm ci

      - name: Build
        run: npm run build

      - name: Download all benchmark results
        uses: actions/download-artifact@v4
        with:
          path: benchmarks/results
          pattern: benchmark-*
          merge-multiple: true

      - name: Merge benchmark results
        run: |
          # Merge each validator's results from all drafts
          for v in $(ls benchmarks/results/*-draft4.json 2>/dev/null | xargs -n1 basename | sed 's/-draft4.json//'); do
            echo "Merging $v results..."
            npx tsx benchmarks/merge-isolated.ts "benchmarks/results/${v}.json" \
              "benchmarks/results/${v}-draft4.json" \
              "benchmarks/results/${v}-draft6.json" \
              "benchmarks/results/${v}-draft7.json" \
              "benchmarks/results/${v}-draft2019-09.json" \
              "benchmarks/results/${v}-draft2020-12.json"
          done

          # Clean up per-draft files
          rm -f benchmarks/results/*-draft*.json

      - name: Generate summary and reports
        run: |
          # Generate BENCHMARKS.md
          npx tsx benchmarks/generate-summary.ts

          # Update README.md (requires ajv.json)
          if [ -f benchmarks/results/ajv.json ]; then
            npx tsx benchmarks/update-readme.ts
          else
            echo "Skipping README update (ajv.json not found)"
          fi

          # Update SVG chart
          npx tsx benchmarks/update-svg.ts

          # Generate detailed markdown reports for each validator that has results
          for v in $(ls benchmarks/results/*.json 2>/dev/null | xargs -n1 basename | sed 's/\.json$//' | grep -v tjs); do
            UPPER_NAME=$(echo "$v" | tr '[:lower:]-' '[:upper:]_')
            npx tsx benchmarks/generate-report.ts "$v" -o "benchmarks/results/BENCHMARK_${UPPER_NAME}.md"
          done

      - name: Create PR with benchmark updates
        uses: peter-evans/create-pull-request@v7
        with:
          token: ${{ secrets.PAT_TOKEN }}
          commit-message: "chore: update benchmark results"
          title: "chore: update benchmark results"
          body: "Automated benchmark update from main branch."
          branch: chore/update-benchmarks
          delete-branch: true