Add checks parameter to config() for consistency with io()
#486
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Performance | |
| on: | |
| pull_request: | |
| branches: [main] | |
| concurrency: | |
| group: build-perf-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| jobs: | |
| build_perf: | |
| name: pcb build performance | |
| runs-on: ubicloud-standard-16-ubuntu-2404 | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Cache Cargo | |
| uses: Swatinem/rust-cache@v2 | |
| - name: Build pcb (head) | |
| run: cargo build -p pcb --release | |
| - name: Stash head binary | |
| run: | | |
| mkdir -p "$RUNNER_TEMP/pcb-head" | |
| cp "target/release/pcb" "$RUNNER_TEMP/pcb-head/pcb" | |
| - name: Create base worktree | |
| run: | | |
| git worktree add "$RUNNER_TEMP/pcb-base" "${{ github.event.pull_request.base.sha }}" | |
| - name: Build pcb (base) | |
| run: cargo build -p pcb --release --manifest-path "$RUNNER_TEMP/pcb-base/Cargo.toml" | |
| env: | |
| CARGO_TARGET_DIR: ${{ github.workspace }}/target | |
| - name: Stash base binary | |
| run: | | |
| mkdir -p "$RUNNER_TEMP/pcb-base-bin" | |
| cp "target/release/pcb" "$RUNNER_TEMP/pcb-base-bin/pcb" | |
| - name: Checkout demo workspace | |
| uses: actions/checkout@v5 | |
| with: | |
| repository: dioderobot/demo | |
| path: workspaces/demo | |
| - name: Checkout arduino workspace | |
| uses: actions/checkout@v5 | |
| with: | |
| repository: dioderobot/arduino | |
| path: workspaces/arduino | |
| - name: Install hyperfine | |
| run: | | |
| wget -q https://github.com/sharkdp/hyperfine/releases/download/v1.20.0/hyperfine_1.20.0_amd64.deb | |
| sudo dpkg -i hyperfine_1.20.0_amd64.deb | |
| - name: Run performance benchmarks | |
| run: | | |
| mkdir -p perf | |
| python3 bin/pcb-build-perf \ | |
| --pcb "$RUNNER_TEMP/pcb-base-bin/pcb" \ | |
| --workspace workspaces/demo \ | |
| --output perf/demo-base.json | |
| python3 bin/pcb-build-perf \ | |
| --pcb "$RUNNER_TEMP/pcb-base-bin/pcb" \ | |
| --workspace workspaces/arduino \ | |
| --output perf/arduino-base.json | |
| python3 bin/pcb-build-perf \ | |
| --pcb "$RUNNER_TEMP/pcb-head/pcb" \ | |
| --workspace workspaces/demo \ | |
| --output perf/demo-head.json | |
| python3 bin/pcb-build-perf \ | |
| --pcb "$RUNNER_TEMP/pcb-head/pcb" \ | |
| --workspace workspaces/arduino \ | |
| --output perf/arduino-head.json | |
| - name: Create PR comment body | |
| id: perf_comment | |
| run: | | |
| python3 - <<'PY' | |
| import json | |
| import math | |
| from pathlib import Path | |
| def load(path): | |
| with open(path, "r", encoding="utf-8") as handle: | |
| return json.load(handle) | |
| def fmt_ms(value, stddev=None): | |
| """Format time in ms with optional ± stddev.""" | |
| ms = value * 1000 | |
| if stddev is not None and stddev > 0: | |
| stddev_ms = stddev * 1000 | |
| # Use 1 decimal for small values, 0 for larger | |
| if stddev_ms < 1: | |
| return f"{ms:.0f}ms ±{stddev_ms:.1f}" | |
| return f"{ms:.0f}ms ±{stddev_ms:.0f}" | |
| return f"{ms:.0f}ms" | |
| def analyze_change(base, head): | |
| """Compare base vs head using median and check for significance.""" | |
| base_median = base["median_seconds"] | |
| head_median = head["median_seconds"] | |
| base_stddev = base["stddev_seconds"] | |
| head_stddev = head["stddev_seconds"] | |
| delta = head_median - base_median | |
| pct = (delta / base_median * 100) if base_median > 0 else None | |
| # Use pooled stddev for significance check | |
| # Change is significant if |delta| > combined noise floor | |
| noise = math.sqrt(base_stddev ** 2 + head_stddev ** 2) | |
| significant = False | |
| if pct is not None and abs(pct) >= 5: | |
| # Require delta to exceed 2x the noise floor | |
| if noise == 0 or abs(delta) > 2 * noise: | |
| significant = True | |
| return delta, pct, significant | |
| def compute_speedup(base, head): | |
| """Compute speedup ratio with uncertainty (like hyperfine comparison).""" | |
| base_mean = base["mean_seconds"] | |
| head_mean = head["mean_seconds"] | |
| base_stddev = base["stddev_seconds"] | |
| head_stddev = head["stddev_seconds"] | |
| if head_mean <= 0 or base_mean <= 0: | |
| return None, None | |
| ratio = base_mean / head_mean # >1 means head is faster | |
| # Propagate uncertainty: for ratio r = a/b, σ_r/r = sqrt((σ_a/a)² + (σ_b/b)²) | |
| rel_err_base = base_stddev / base_mean if base_mean > 0 else 0 | |
| rel_err_head = head_stddev / head_mean if head_mean > 0 else 0 | |
| rel_err = math.sqrt(rel_err_base ** 2 + rel_err_head ** 2) | |
| ratio_err = ratio * rel_err | |
| return ratio, ratio_err | |
| rows = [] | |
| has_significant = False | |
| for name in ["demo", "arduino"]: | |
| base = load(Path("perf") / f"{name}-base.json") | |
| head = load(Path("perf") / f"{name}-head.json") | |
| base_boards = {b["zen_path"]: b for b in base.get("boards", [])} | |
| head_boards = {b["zen_path"]: b for b in head.get("boards", [])} | |
| all_paths = sorted(set(base_boards) | set(head_boards)) | |
| for zen_path in all_paths: | |
| base_board = base_boards.get(zen_path) | |
| head_board = head_boards.get(zen_path) | |
| board_name = zen_path | |
| if head_board and head_board.get("name"): | |
| board_name = head_board["name"] | |
| elif base_board and base_board.get("name"): | |
| board_name = base_board["name"] | |
| label = f"{name}/{board_name}" | |
| if not base_board or not head_board: | |
| rows.append((label, "—", "—", "—", "missing")) | |
| continue | |
| delta, pct, significant = analyze_change(base_board, head_board) | |
| ratio, ratio_err = compute_speedup(base_board, head_board) | |
| base_str = fmt_ms(base_board["median_seconds"], base_board["stddev_seconds"]) | |
| head_str = fmt_ms(head_board["median_seconds"], head_board["stddev_seconds"]) | |
| if pct is None: | |
| change_text = "—" | |
| elif not significant: | |
| change_text = f"{pct:+.1f}%" | |
| else: | |
| has_significant = True | |
| if ratio is not None and ratio_err is not None: | |
| if pct < 0: | |
| change_text = f"**{ratio:.2f}× ±{ratio_err:.2f} faster**" | |
| else: | |
| change_text = f"**{1/ratio:.2f}× ±{ratio_err/ratio**2:.2f} slower**" | |
| else: | |
| if pct < 0: | |
| change_text = f"**{pct:+.1f}% faster**" | |
| else: | |
| change_text = f"**{pct:+.1f}% slower**" | |
| rows.append((label, base_str, head_str, change_text)) | |
| lines = ["### Build Performance", ""] | |
| lines.append("| Board | Base (median) | Head (median) | Change |") | |
| lines.append("|:------|-----:|-----:|:------|") | |
| for label, base, head, change in rows: | |
| lines.append(f"| {label} | {base} | {head} | {change} |") | |
| lines.append("") | |
| lines.append("<sub>Measured with [hyperfine](https://github.com/sharkdp/hyperfine). Times show median ±stddev.</sub>") | |
| lines.append("") | |
| lines.append("<!-- pcb-build-perf -->") | |
| body = "\n".join(lines) | |
| Path("perf/comment.txt").write_text(body, encoding="utf-8") | |
| # Always write to job summary | |
| import os | |
| with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: | |
| f.write(body + "\n") | |
| has_changes = "true" if has_significant else "false" | |
| with open(os.environ["GITHUB_OUTPUT"], "a") as f: | |
| f.write(f"has_changes={has_changes}\n") | |
| PY | |
| - name: Find existing comment | |
| id: find_comment | |
| uses: peter-evans/find-comment@v3 | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| comment-author: "github-actions[bot]" | |
| body-includes: "<!-- pcb-build-perf -->" | |
| - name: Create or update comment | |
| if: steps.perf_comment.outputs.has_changes == 'true' | |
| uses: peter-evans/create-or-update-comment@v4 | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| comment-id: ${{ steps.find_comment.outputs.comment-id }} | |
| body-path: perf/comment.txt | |
| edit-mode: replace | |
| - name: Delete stale comment | |
| if: steps.perf_comment.outputs.has_changes == 'false' && steps.find_comment.outputs.comment-id != '' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| await github.rest.issues.deleteComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: ${{ steps.find_comment.outputs.comment-id }} | |
| }) |