perf(table): cut point-read per-get overhead (#465) #1257
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Hardening policy: every `uses:` ref is pinned to a full commit | |
| # SHA; `actions/checkout` runs with `persist-credentials: false` | |
| # (this workflow does not push from the default identity — gh-pages | |
| # results use the App-token-scoped benchmark-action input); | |
| # `actions/create-github-app-token` lists explicit `permission-*` | |
| # scopes for the minimum the workflow actually exercises. | |
| name: Benchmark | |
| on: | |
| # Auto-triggered ONLY on merges/pushes to main; also runnable | |
| # manually from any branch via workflow_dispatch (see below). | |
| # `pull_request` is deliberately omitted: the self-hosted bench | |
| # host is shared with future cross-engine benches (issue #244) | |
| # and other perf workloads — every PR pushing a commit would | |
| # queue behind the host's serial concurrency group and hold up | |
| # every subsequent PR, even when the change can't affect the | |
| # bench (CI, docs, comment-only fixes). Restricting the | |
| # AUTO-trigger to main keeps the bench timeline a clean "one | |
| # entry per merged change" series, which is what the gh-pages | |
| # dashboard is meant to show. Manual runs through | |
| # workflow_dispatch don't update the dashboard's main-branch | |
| # series (auto-push gating below is push+main only), so they | |
| # can't contaminate the trend. | |
| push: | |
| branches: [main] | |
| # Manual dispatch from any branch when a contributor explicitly | |
| # wants to measure their PR against main before merging. | |
| # workflow_dispatch requires actions:write on the repo, so only | |
| # owner / maintainers can fire it — fork PR authors cannot. | |
| workflow_dispatch: | |
| # Strict serialisation across the entire repo: the self-hosted | |
| # bench host is shared with future cross-engine benches (RocksDB | |
| # vs lsm-tree per issue #244) and any other perf workload. Two | |
| # benches running simultaneously contaminate each other's CPU / | |
| # disk / cache state and the published numbers become unreadable. | |
| # `cancel-in-progress: false` makes queued runs wait their turn | |
| # instead of clobbering the in-flight bench — every measurement | |
| # completes on a quiet host. | |
| concurrency: | |
| group: bench-self-hosted | |
| cancel-in-progress: false | |
| jobs: | |
| benchmark: | |
| name: Performance regression check | |
| # Self-hosted bare-metal runner. Shared GitHub-hosted runners | |
| # have ±15-20% noise on every db_bench workload — large enough | |
| # to hide every win below ~20% and produce false-positive | |
| # regression alerts. The self-hosted host has stable thermal | |
| # state, no CPU contention from other jobs, and consistent | |
| # disk I/O — bench delta becomes interpretable per merged | |
| # commit on main instead of needing 20+ iteration medians to | |
| # escape the noise floor. (Per-PR comparison is available via | |
| # the `workflow_dispatch` manual trigger above; routine PR | |
| # pushes do not auto-queue against this runner.) Label | |
| # `gpu-private-systems` targets that specific host; other | |
| # self-hosted runners (if added later) without the label | |
| # won't pick the job up. | |
| runs-on: [self-hosted, gpu-private-systems] | |
| # 60 (was 30): absorbs the compare-rocksdb head-to-head pass added | |
| # below (bounded criterion settings, but the zstd22/10k write points | |
| # are seconds each) plus the first-run librocksdb-sys C++ compile. | |
| timeout-minutes: 60 | |
| steps: | |
| # v6 requires runner ≥2.327.1 (node24 runtime) — the self-hosted | |
| # runner has a current runner agent that satisfies this. | |
| # persist-credentials: false — git pushes from this workflow use the | |
| # explicit App token generated below, NOT the runner's GITHUB_TOKEN | |
| # written to .git/config. Disabling the default credential persistence | |
| # closes the credential-persistence attack surface (zizmor's | |
| # `artipacked` lint — ARTIfact-PACKED, the documented class of | |
| # token leakage where actions/upload-artifact captures the | |
| # whole working tree, including the .git/config that the | |
| # default checkout populated with GITHUB_TOKEN). | |
| - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 | |
| with: | |
| persist-credentials: false | |
| - name: Generate bot token | |
| id: bot-token | |
| # `push` and `workflow_dispatch` are both trusted contexts on | |
| # this repo (workflow_dispatch requires actions:write, which | |
| # only owner/maintainers hold — fork PRs can't trigger it). | |
| # The `pull_request` clause exists for paranoia: if the | |
| # trigger list is ever expanded to include `pull_request`, | |
| # only same-repo PRs get the token (fork PRs would still be | |
| # blocked). On `workflow_dispatch` the `github.event.pull_request` | |
| # context is absent, so the `pull_request` branch is gated by | |
| # `github.event_name == 'pull_request'` to avoid evaluating | |
| # a missing context. | |
| if: > | |
| github.event_name == 'push' || | |
| github.event_name == 'workflow_dispatch' || | |
| (github.event_name == 'pull_request' && | |
| github.event.pull_request.head.repo.full_name == github.repository) | |
| uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 | |
| with: | |
| app-id: ${{ secrets.RELEASER_APP_ID }} | |
| private-key: ${{ secrets.RELEASER_APP_PRIVATE_KEY }} | |
| # Least-privilege scope: benchmark-action/github-action-benchmark | |
| # needs contents:write (push results to gh-pages branch on main), | |
| # pull-requests:write (comment on PRs with comparison alerts). | |
| # No other scopes are exercised by this workflow. | |
| permission-contents: write | |
| permission-pull-requests: write | |
| - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 | |
| with: | |
| prefix-key: bench | |
| - name: Run benchmarks | |
| run: bash .github/scripts/run-benchmarks.sh 200000 | |
| - name: Store benchmark results | |
| if: steps.bot-token.outputs.token != '' | |
| uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # v1.22.1 | |
| with: | |
| name: "lsm-tree db_bench" | |
| tool: customBiggerIsBetter | |
| output-file-path: benchmark-results.json | |
| github-token: ${{ steps.bot-token.outputs.token }} | |
| # Push results to gh-pages only on main (not PRs) | |
| auto-push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} | |
| # On PRs, comment with comparison against baseline | |
| comment-on-alert: true | |
| alert-comment-cc-users: "@polaz" | |
| # 15% regression = alert, 25% = fail (shared runners have high variance) | |
| alert-threshold: "115%" | |
| fail-threshold: "125%" | |
| fail-on-alert: ${{ github.event_name == 'pull_request' }} | |
| # Store data for PRs without pushing (compare only) | |
| save-data-file: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} | |
| benchmark-data-dir-path: dev/bench | |
| # Head-to-head RocksDB comparison. Distinct from the db_bench | |
| # trend above (single-engine, value-over-commits): this runs the | |
| # standalone `tools/compare-rocksdb` criterion harness where every | |
| # scenario runs BOTH engines in the same process, so each criterion | |
| # group is an ours-vs-rocksdb overlay chart. Because the comparison | |
| # is a ratio measured on one host in one run, it stays meaningful | |
| # even if this runner's CPU changes between runs. | |
| # | |
| # Bounded criterion settings keep the heavy zstd22/10k write points | |
| # from dominating the timeout: a small sample count with a short | |
| # measurement window still yields a stable median for the dashboard | |
| # without the default 100-sample sweep (which on a multi-second-per | |
| # -iteration point would run for many minutes each). | |
| # | |
| # Runner requirement: librocksdb-sys builds from source via bindgen, | |
| # so the host must have a C/C++ toolchain + libclang. On Linux | |
| # bindgen finds the distro libclang.so without env-var help. | |
| # Pre-flight: librocksdb-sys builds C bindings via bindgen -> clang-sys, | |
| # which needs libclang + llvm-config on the host. When they are absent the | |
| # raw failure is a deep, cryptic clang-sys build-script panic. Fail early | |
| # with an actionable message naming the exact packages to install on the | |
| # self-hosted runner instead. | |
| - name: Verify libclang toolchain (compare-rocksdb prerequisite) | |
| run: | | |
| # Detection covers: llvm-config on PATH (Fedora/most), explicit | |
| # LIBCLANG_PATH, direct lib dirs (/usr/lib64 on Fedora), Debian/Ubuntu | |
| # multiarch (/usr/lib/x86_64-linux-gnu) + llvm-* dirs via the nested | |
| # glob, and finally ldconfig -p (the robust Linux probe that finds | |
| # libclang wherever the dynamic linker registered it). | |
| if ! command -v llvm-config >/dev/null 2>&1 \ | |
| && [ -z "${LLVM_CONFIG_PATH:-}" ] \ | |
| && [ -z "${LIBCLANG_PATH:-}" ] \ | |
| && ! ls /usr/lib*/libclang.so* /usr/lib*/*/libclang.so* >/dev/null 2>&1 \ | |
| && ! { command -v ldconfig >/dev/null 2>&1 && ldconfig -p | grep -q libclang; }; then | |
| echo "::error::libclang/llvm-config not found on the bench runner. compare-rocksdb (librocksdb-sys -> bindgen -> clang-sys) cannot build. Install LLVM dev tools on the runner host: Fedora 'dnf install -y clang clang-devel llvm llvm-devel', Debian/Ubuntu 'apt-get install -y clang libclang-dev llvm-dev', or set LIBCLANG_PATH/LLVM_CONFIG_PATH." | |
| exit 1 | |
| fi | |
| - name: Run compare-rocksdb head-to-head benches | |
| # Pin the published dashboard to the RocksDbParity preset (the apples-to- | |
| # apples comparison): every lsm-tree-only on-disk opt-in is OFF so the | |
| # numbers are not paying for protection RocksDB has no equivalent for. | |
| # Set explicitly rather than relying on the in-code default, so a future | |
| # default change cannot silently alter the public comparison. See | |
| # docs/BENCHMARKING.md. | |
| env: | |
| LSM_BENCH_PRESET: rocksdb-parity | |
| run: | | |
| cd tools/compare-rocksdb | |
| cargo bench --bench compare -- \ | |
| --sample-size 10 --warm-up-time 1 --measurement-time 5 | |
| # Publish the criterion HTML report tree (per-group overlay charts) | |
| # to gh-pages under dev/compare, alongside the db_bench dashboard at | |
| # dev/bench. Manual git push (rather than a third-party publish | |
| # action) keeps the workflow's "every `uses:` pinned to a SHA" | |
| # hardening policy intact and reuses the same App-scoped token the | |
| # benchmark-action step uses. Only publishes on main pushes; on PR / | |
| # workflow_dispatch runs the previous step still generates the report | |
| # tree in the workspace, but this step is skipped (no artifact upload, | |
| # no gh-pages mutation). | |
| - name: Publish compare-rocksdb overlay reports to gh-pages | |
| # Publish on main pushes AND on manual workflow_dispatch. The overlay is | |
| # a ratio SNAPSHOT (dev/compare is replaced wholesale each run), not the | |
| # per-commit trend that dev/bench is — so a manual dispatch refreshing it | |
| # can't contaminate any time series. This lets a maintainer populate the | |
| # overlays on demand (e.g. right after a runner fix) without waiting for | |
| # the next merge to main. The db_bench trend (benchmark-action auto-push / | |
| # save-data above) stays push+main only. | |
| if: > | |
| steps.bot-token.outputs.token != '' && | |
| ( | |
| (github.event_name == 'push' && github.ref == 'refs/heads/main') || | |
| github.event_name == 'workflow_dispatch' | |
| ) | |
| env: | |
| GH_TOKEN: ${{ steps.bot-token.outputs.token }} | |
| REPO: ${{ github.repository }} | |
| run: | | |
| set -euo pipefail | |
| src="tools/compare-rocksdb/target/criterion" | |
| if [ ! -d "$src/report" ]; then | |
| echo "::error::criterion report tree not found at $src/report" | |
| exit 1 | |
| fi | |
| # Explicit template (not bare `mktemp -d`, which is GNU-only and | |
| # fails on macOS/BSD with "too few X's") so the step survives a | |
| # future runner-OS change. | |
| work="$(mktemp -d "${TMPDIR:-/tmp}/compare-ghpages.XXXXXXXX")" | |
| # Remove the temp clone on every exit path (success, error, or | |
| # cancellation) so it doesn't leak disk on a long-lived | |
| # self-hosted runner. | |
| trap 'rm -rf "$work"' EXIT | |
| # Supply the token through a credential helper that reads it from | |
| # the environment at call time, so it appears neither in argv | |
| # (visible via `ps` on a shared self-hosted host) nor in a | |
| # persisted remote URL / .git/config. The clone + push both use a | |
| # plain, secret-free https URL; GIT_CONFIG_* injects the helper | |
| # into every git invocation in this step's environment. | |
| export GIT_CONFIG_COUNT=1 | |
| export GIT_CONFIG_KEY_0=credential.helper | |
| # SC2016: the single quotes are deliberate — `${GH_TOKEN}` must NOT | |
| # expand here. It is expanded later by the helper's own shell when | |
| # git invokes it, reading the token from the (exported) environment | |
| # so it never lands in argv or config. | |
| # shellcheck disable=SC2016 | |
| export GIT_CONFIG_VALUE_0='!f() { echo username=x-access-token; echo "password=${GH_TOKEN}"; }; f' | |
| git clone --depth 1 --branch gh-pages \ | |
| "https://github.com/${REPO}.git" "$work" | |
| # Replace the previous report tree wholesale so deleted / | |
| # renamed scenarios don't leave stale pages behind; dev/bench | |
| # and every other gh-pages path are untouched. | |
| rm -rf "$work/dev/compare" | |
| mkdir -p "$work/dev/compare" | |
| cp -R "$src/." "$work/dev/compare/" | |
| # Landing gallery: Criterion's own dev/compare/report/index.html is a | |
| # FLAT LIST of individual benchmarks with no overlay charts on it — a | |
| # visitor lands there and sees links, not the head-to-head. The actual | |
| # ours-vs-rocksdb overlays are Criterion's per-group `lines.svg` | |
| # (X=element count, one line per engine), buried under | |
| # <scenario>/report/. Generate a single landing page at | |
| # dev/compare/index.html that embeds every scenario's lines.svg inline, | |
| # so the head-to-head is the first thing shown. | |
| gallery="$work/dev/compare/index.html" | |
| { | |
| cat <<'HTML_HEAD' | |
| <!doctype html><html lang="en"><head><meta charset="utf-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1"> | |
| <title>lsm-tree vs RocksDB — head-to-head</title> | |
| <style> | |
| body{font-family:system-ui,-apple-system,sans-serif;max-width:1080px;margin:2rem auto;padding:0 1rem;background:#0d1117;color:#c9d1d9} | |
| h1{font-size:1.5rem} h2{font-size:1.1rem;margin:0 0 .5rem} | |
| a{color:#58a6ff;text-decoration:none} a:hover{text-decoration:underline} | |
| .intro{color:#8b949e;font-size:.95rem;line-height:1.5} | |
| .scenario{margin:1.5rem 0;padding-bottom:1rem;border-bottom:1px solid #30363d} | |
| .scenario img{max-width:100%;background:#fff;border-radius:6px;padding:6px} | |
| .links{font-size:.85rem;color:#8b949e} | |
| </style></head><body> | |
| <h1>lsm-tree vs RocksDB — head-to-head overlays</h1> | |
| <p class="intro">Each chart overlays <strong>ours</strong> against <strong>rocksdb</strong> on the | |
| same axes (X = element count, Y = average time; <strong>lower is faster</strong>). Generated by the | |
| <code>compare-rocksdb</code> harness — both engines run in the same process per scenario. | |
| · <a href="../bench/">single-engine db_bench trend →</a> | |
| · <a href="report/">raw criterion index →</a></p> | |
| HTML_HEAD | |
| for d in "$work"/dev/compare/*/; do | |
| name="$(basename "$d")" | |
| [ -f "$d/report/lines.svg" ] || continue | |
| printf '<div class="scenario"><h2>%s</h2><a href="%s/report/"><img src="%s/report/lines.svg" alt="%s overlay" loading="lazy"></a><div class="links"><a href="%s/report/">full report (violin, regression, per-size) →</a></div></div>\n' \ | |
| "$name" "$name" "$name" "$name" "$name" | |
| done | |
| echo '</body></html>' | |
| } > "$gallery" | |
| # Cross-link banner: the db_bench dashboard (dev/bench/index.html) is | |
| # regenerated by benchmark-action on every main push and has no link to | |
| # the overlay reports, so visitors never discover dev/compare. Inject a | |
| # banner pointing at the overlays right after <body>. Idempotent via the | |
| # marker id; awk (not sed) avoids `&`/`/` escaping pitfalls in the HTML. | |
| bench_index="$work/dev/bench/index.html" | |
| if [ -f "$bench_index" ] && ! grep -q "compare-overlay-banner" "$bench_index"; then | |
| banner='<div id="compare-overlay-banner" style="padding:8px 14px;background:#0d1117;color:#c9d1d9;font-family:system-ui,sans-serif;border-bottom:1px solid #30363d;font-size:14px">📊 <strong>lsm-tree benchmarks</strong> · you are viewing the single-engine db_bench trend · <a href="../compare/" style="color:#58a6ff;font-weight:600">RocksDB head-to-head overlays →</a></div>' | |
| awk -v b="$banner" 'BEGIN{done=0} {print} /<body/ && !done {print b; done=1}' "$bench_index" > "$bench_index.tmp" \ | |
| && mv "$bench_index.tmp" "$bench_index" | |
| fi | |
| cd "$work" | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git add dev/compare | |
| # Stage the banner edit too (file may be absent on a first-ever run). | |
| git add dev/bench/index.html 2>/dev/null || true | |
| if git diff --cached --quiet; then | |
| echo "::notice::compare-rocksdb reports unchanged — nothing to publish" | |
| else | |
| git commit -m "bench: refresh compare-rocksdb overlay reports (dev/compare)" | |
| git push origin gh-pages | |
| echo "::notice::published compare-rocksdb overlays to dev/compare/report/index.html" | |
| fi |