Add Pytest Benchmarks Feature

.github/workflows/benchmark.yaml

@@ -0,0 +1,169 @@
+name: Benchmarks
+
+on:
+  pull_request:
+    types: [labeled, synchronize]
+  workflow_dispatch:
+    inputs:
+      mode:
+        description: 'Benchmark mode'
+        required: true
+        type: choice
+        options:
+          - run
+          - compare
+        default: 'run'
+      comparison_ref:
+        description: 'Git ref to compare against (only used in compare mode)'
+        required: false
+        default: 'main'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  FORCE_COLOR: "1"
+  MPLBACKEND: agg
+  UV_COMPILE_BYTECODE: "1"
+
+defaults:
+  run:
+    # to fail on error in multiline statements (-e), in pipes (-o pipefail), and on unset variables (-u).
+    shell: bash -euo pipefail {0}
+
+jobs:
+  benchmark:
+    # Run on 'pytest-benchmark' label or manual dispatch
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'pytest-benchmark'))
+    name: Run Benchmarks
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          filter: blob:none
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          cache-dependency-glob: pyproject.toml
+
+      - name: Install dependencies
+        run: uv pip install -e ".[test]" --system
+
+      - name: Run benchmarks
+        run: |
+          pytest tests/benchmarks/ \
+            --benchmark-enable \
+            --benchmark-only \
+            --benchmark-json=benchmark-results.json \
+            --benchmark-warmup=on \
+            --benchmark-disable-gc \
+            --benchmark-min-rounds=5 \
+            -v --color=yes
+
+      - name: Generate report (simple mode)
+        if: github.event.inputs.mode != 'compare'
+        run: |
+          python .scripts/ci/benchmark_report.py \
+            benchmark-results.json \
+            --pr-ref="${GITHUB_HEAD_REF:-$GITHUB_REF_NAME}" \
+            --output=benchmark-report.md \
+            --github-summary
+
+      - name: Run comparison benchmarks
+        if: github.event.inputs.mode == 'compare'
+        run: |
+          # Save current results
+          mv benchmark-results.json benchmark-pr.json
+
+          # Checkout base
+          git checkout ${{ github.event.inputs.comparison_ref || 'main' }}
+
+          # Reinstall in case dependencies changed
+          uv pip install -e ".[test]" --system
+
+          # Run benchmarks on base
+          pytest tests/benchmarks/ \
+            --benchmark-enable \
+            --benchmark-only \
+            --benchmark-json=benchmark-base.json \
+            --benchmark-warmup=on \
+            --benchmark-disable-gc \
+            --benchmark-min-rounds=5 \
+            -v --color=yes || true
+
+          # Checkout back
+          git checkout ${{ github.sha }}
+
+          # Rename for report
+          mv benchmark-pr.json benchmark-results.json
+
+      - name: Generate comparison report
+        if: github.event.inputs.mode == 'compare'
+        run: |
+          python .scripts/ci/benchmark_report.py \
+            benchmark-results.json \
+            --base-results=benchmark-base.json \
+            --pr-ref="${GITHUB_HEAD_REF:-$GITHUB_REF_NAME}" \
+            --base-ref="${{ github.event.inputs.comparison_ref || 'main' }}" \
+            --output=benchmark-report.md \
+            --github-summary
+
+      - name: Comment on PR
+        if: github.event_name == 'pull_request'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            const report = fs.readFileSync('benchmark-report.md', 'utf8');
+
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+
+            const botComment = comments.find(comment =>
+              comment.user.type === 'Bot' &&
+              comment.body.includes('## 📊 Benchmark Results')
+            );
+
+            if (botComment) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: botComment.id,
+                body: report
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body: report
+              });
+            }
+
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-results
+          path: |
+            benchmark-results.json
+            benchmark-base.json
+            benchmark-report.md
+          if-no-files-found: ignore

.gitignore

@@ -144,3 +144,6 @@ data
 # pixi
 .pixi
 pixi.lock
+
+# benchmarks
+benchmark*.json