Skip to content

perf(table): cut point-read per-get overhead (#465) #1257

perf(table): cut point-read per-get overhead (#465)

perf(table): cut point-read per-get overhead (#465) #1257

Workflow file for this run

# Hardening policy: every `uses:` ref is pinned to a full commit
# SHA; `actions/checkout` runs with `persist-credentials: false`
# (this workflow does not push from the default identity — gh-pages
# results use the App-token-scoped benchmark-action input);
# `actions/create-github-app-token` lists explicit `permission-*`
# scopes for the minimum the workflow actually exercises.
name: Benchmark
on:
# Auto-triggered ONLY on merges/pushes to main; also runnable
# manually from any branch via workflow_dispatch (see below).
# `pull_request` is deliberately omitted: the self-hosted bench
# host is shared with future cross-engine benches (issue #244)
# and other perf workloads — every PR pushing a commit would
# queue behind the host's serial concurrency group and hold up
# every subsequent PR, even when the change can't affect the
# bench (CI, docs, comment-only fixes). Restricting the
# AUTO-trigger to main keeps the bench timeline a clean "one
# entry per merged change" series, which is what the gh-pages
# dashboard is meant to show. Manual runs through
# workflow_dispatch don't update the dashboard's main-branch
# series (auto-push gating below is push+main only), so they
# can't contaminate the trend.
push:
branches: [main]
# Manual dispatch from any branch when a contributor explicitly
# wants to measure their PR against main before merging.
# workflow_dispatch requires actions:write on the repo, so only
# owner / maintainers can fire it — fork PR authors cannot.
workflow_dispatch:
# Strict serialisation across the entire repo: the self-hosted
# bench host is shared with future cross-engine benches (RocksDB
# vs lsm-tree per issue #244) and any other perf workload. Two
# benches running simultaneously contaminate each other's CPU /
# disk / cache state and the published numbers become unreadable.
# `cancel-in-progress: false` makes queued runs wait their turn
# instead of clobbering the in-flight bench — every measurement
# completes on a quiet host.
concurrency:
group: bench-self-hosted
cancel-in-progress: false
jobs:
benchmark:
name: Performance regression check
# Self-hosted bare-metal runner. Shared GitHub-hosted runners
# have ±15-20% noise on every db_bench workload — large enough
# to hide every win below ~20% and produce false-positive
# regression alerts. The self-hosted host has stable thermal
# state, no CPU contention from other jobs, and consistent
# disk I/O — bench delta becomes interpretable per merged
# commit on main instead of needing 20+ iteration medians to
# escape the noise floor. (Per-PR comparison is available via
# the `workflow_dispatch` manual trigger above; routine PR
# pushes do not auto-queue against this runner.) Label
# `gpu-private-systems` targets that specific host; other
# self-hosted runners (if added later) without the label
# won't pick the job up.
runs-on: [self-hosted, gpu-private-systems]
# 60 (was 30): absorbs the compare-rocksdb head-to-head pass added
# below (bounded criterion settings, but the zstd22/10k write points
# are seconds each) plus the first-run librocksdb-sys C++ compile.
timeout-minutes: 60
steps:
# v6 requires runner ≥2.327.1 (node24 runtime) — the self-hosted
# runner has a current runner agent that satisfies this.
# persist-credentials: false — git pushes from this workflow use the
# explicit App token generated below, NOT the runner's GITHUB_TOKEN
# written to .git/config. Disabling the default credential persistence
# closes the credential-persistence attack surface (zizmor's
# `artipacked` lint — ARTIfact-PACKED, the documented class of
# token leakage where actions/upload-artifact captures the
# whole working tree, including the .git/config that the
# default checkout populated with GITHUB_TOKEN).
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
persist-credentials: false
- name: Generate bot token
id: bot-token
# `push` and `workflow_dispatch` are both trusted contexts on
# this repo (workflow_dispatch requires actions:write, which
# only owner/maintainers hold — fork PRs can't trigger it).
# The `pull_request` clause exists for paranoia: if the
# trigger list is ever expanded to include `pull_request`,
# only same-repo PRs get the token (fork PRs would still be
# blocked). On `workflow_dispatch` the `github.event.pull_request`
# context is absent, so the `pull_request` branch is gated by
# `github.event_name == 'pull_request'` to avoid evaluating
# a missing context.
if: >
github.event_name == 'push' ||
github.event_name == 'workflow_dispatch' ||
(github.event_name == 'pull_request' &&
github.event.pull_request.head.repo.full_name == github.repository)
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
with:
app-id: ${{ secrets.RELEASER_APP_ID }}
private-key: ${{ secrets.RELEASER_APP_PRIVATE_KEY }}
# Least-privilege scope: benchmark-action/github-action-benchmark
# needs contents:write (push results to gh-pages branch on main),
# pull-requests:write (comment on PRs with comparison alerts).
# No other scopes are exercised by this workflow.
permission-contents: write
permission-pull-requests: write
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
with:
toolchain: stable
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
with:
prefix-key: bench
- name: Run benchmarks
run: bash .github/scripts/run-benchmarks.sh 200000
- name: Store benchmark results
if: steps.bot-token.outputs.token != ''
uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # v1.22.1
with:
name: "lsm-tree db_bench"
tool: customBiggerIsBetter
output-file-path: benchmark-results.json
github-token: ${{ steps.bot-token.outputs.token }}
# Push results to gh-pages only on main (not PRs)
auto-push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
# On PRs, comment with comparison against baseline
comment-on-alert: true
alert-comment-cc-users: "@polaz"
# 15% regression = alert, 25% = fail (shared runners have high variance)
alert-threshold: "115%"
fail-threshold: "125%"
fail-on-alert: ${{ github.event_name == 'pull_request' }}
# Store data for PRs without pushing (compare only)
save-data-file: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
benchmark-data-dir-path: dev/bench
# Head-to-head RocksDB comparison. Distinct from the db_bench
# trend above (single-engine, value-over-commits): this runs the
# standalone `tools/compare-rocksdb` criterion harness where every
# scenario runs BOTH engines in the same process, so each criterion
# group is an ours-vs-rocksdb overlay chart. Because the comparison
# is a ratio measured on one host in one run, it stays meaningful
# even if this runner's CPU changes between runs.
#
# Bounded criterion settings keep the heavy zstd22/10k write points
# from dominating the timeout: a small sample count with a short
# measurement window still yields a stable median for the dashboard
# without the default 100-sample sweep (which on a multi-second-per
# -iteration point would run for many minutes each).
#
# Runner requirement: librocksdb-sys builds from source via bindgen,
# so the host must have a C/C++ toolchain + libclang. On Linux
# bindgen finds the distro libclang.so without env-var help.
# Pre-flight: librocksdb-sys builds C bindings via bindgen -> clang-sys,
# which needs libclang + llvm-config on the host. When they are absent the
# raw failure is a deep, cryptic clang-sys build-script panic. Fail early
# with an actionable message naming the exact packages to install on the
# self-hosted runner instead.
- name: Verify libclang toolchain (compare-rocksdb prerequisite)
run: |
# Detection covers: llvm-config on PATH (Fedora/most), explicit
# LIBCLANG_PATH, direct lib dirs (/usr/lib64 on Fedora), Debian/Ubuntu
# multiarch (/usr/lib/x86_64-linux-gnu) + llvm-* dirs via the nested
# glob, and finally ldconfig -p (the robust Linux probe that finds
# libclang wherever the dynamic linker registered it).
if ! command -v llvm-config >/dev/null 2>&1 \
&& [ -z "${LLVM_CONFIG_PATH:-}" ] \
&& [ -z "${LIBCLANG_PATH:-}" ] \
&& ! ls /usr/lib*/libclang.so* /usr/lib*/*/libclang.so* >/dev/null 2>&1 \
&& ! { command -v ldconfig >/dev/null 2>&1 && ldconfig -p | grep -q libclang; }; then
echo "::error::libclang/llvm-config not found on the bench runner. compare-rocksdb (librocksdb-sys -> bindgen -> clang-sys) cannot build. Install LLVM dev tools on the runner host: Fedora 'dnf install -y clang clang-devel llvm llvm-devel', Debian/Ubuntu 'apt-get install -y clang libclang-dev llvm-dev', or set LIBCLANG_PATH/LLVM_CONFIG_PATH."
exit 1
fi
- name: Run compare-rocksdb head-to-head benches
# Pin the published dashboard to the RocksDbParity preset (the apples-to-
# apples comparison): every lsm-tree-only on-disk opt-in is OFF so the
# numbers are not paying for protection RocksDB has no equivalent for.
# Set explicitly rather than relying on the in-code default, so a future
# default change cannot silently alter the public comparison. See
# docs/BENCHMARKING.md.
env:
LSM_BENCH_PRESET: rocksdb-parity
run: |
cd tools/compare-rocksdb
cargo bench --bench compare -- \
--sample-size 10 --warm-up-time 1 --measurement-time 5
# Publish the criterion HTML report tree (per-group overlay charts)
# to gh-pages under dev/compare, alongside the db_bench dashboard at
# dev/bench. Manual git push (rather than a third-party publish
# action) keeps the workflow's "every `uses:` pinned to a SHA"
# hardening policy intact and reuses the same App-scoped token the
# benchmark-action step uses. Only publishes on main pushes; on PR /
# workflow_dispatch runs the previous step still generates the report
# tree in the workspace, but this step is skipped (no artifact upload,
# no gh-pages mutation).
- name: Publish compare-rocksdb overlay reports to gh-pages
# Publish on main pushes AND on manual workflow_dispatch. The overlay is
# a ratio SNAPSHOT (dev/compare is replaced wholesale each run), not the
# per-commit trend that dev/bench is — so a manual dispatch refreshing it
# can't contaminate any time series. This lets a maintainer populate the
# overlays on demand (e.g. right after a runner fix) without waiting for
# the next merge to main. The db_bench trend (benchmark-action auto-push /
# save-data above) stays push+main only.
if: >
steps.bot-token.outputs.token != '' &&
(
(github.event_name == 'push' && github.ref == 'refs/heads/main') ||
github.event_name == 'workflow_dispatch'
)
env:
GH_TOKEN: ${{ steps.bot-token.outputs.token }}
REPO: ${{ github.repository }}
run: |
set -euo pipefail
src="tools/compare-rocksdb/target/criterion"
if [ ! -d "$src/report" ]; then
echo "::error::criterion report tree not found at $src/report"
exit 1
fi
# Explicit template (not bare `mktemp -d`, which is GNU-only and
# fails on macOS/BSD with "too few X's") so the step survives a
# future runner-OS change.
work="$(mktemp -d "${TMPDIR:-/tmp}/compare-ghpages.XXXXXXXX")"
# Remove the temp clone on every exit path (success, error, or
# cancellation) so it doesn't leak disk on a long-lived
# self-hosted runner.
trap 'rm -rf "$work"' EXIT
# Supply the token through a credential helper that reads it from
# the environment at call time, so it appears neither in argv
# (visible via `ps` on a shared self-hosted host) nor in a
# persisted remote URL / .git/config. The clone + push both use a
# plain, secret-free https URL; GIT_CONFIG_* injects the helper
# into every git invocation in this step's environment.
export GIT_CONFIG_COUNT=1
export GIT_CONFIG_KEY_0=credential.helper
# SC2016: the single quotes are deliberate — `${GH_TOKEN}` must NOT
# expand here. It is expanded later by the helper's own shell when
# git invokes it, reading the token from the (exported) environment
# so it never lands in argv or config.
# shellcheck disable=SC2016
export GIT_CONFIG_VALUE_0='!f() { echo username=x-access-token; echo "password=${GH_TOKEN}"; }; f'
git clone --depth 1 --branch gh-pages \
"https://github.com/${REPO}.git" "$work"
# Replace the previous report tree wholesale so deleted /
# renamed scenarios don't leave stale pages behind; dev/bench
# and every other gh-pages path are untouched.
rm -rf "$work/dev/compare"
mkdir -p "$work/dev/compare"
cp -R "$src/." "$work/dev/compare/"
# Landing gallery: Criterion's own dev/compare/report/index.html is a
# FLAT LIST of individual benchmarks with no overlay charts on it — a
# visitor lands there and sees links, not the head-to-head. The actual
# ours-vs-rocksdb overlays are Criterion's per-group `lines.svg`
# (X=element count, one line per engine), buried under
# <scenario>/report/. Generate a single landing page at
# dev/compare/index.html that embeds every scenario's lines.svg inline,
# so the head-to-head is the first thing shown.
gallery="$work/dev/compare/index.html"
{
cat <<'HTML_HEAD'
<!doctype html><html lang="en"><head><meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>lsm-tree vs RocksDB — head-to-head</title>
<style>
body{font-family:system-ui,-apple-system,sans-serif;max-width:1080px;margin:2rem auto;padding:0 1rem;background:#0d1117;color:#c9d1d9}
h1{font-size:1.5rem} h2{font-size:1.1rem;margin:0 0 .5rem}
a{color:#58a6ff;text-decoration:none} a:hover{text-decoration:underline}
.intro{color:#8b949e;font-size:.95rem;line-height:1.5}
.scenario{margin:1.5rem 0;padding-bottom:1rem;border-bottom:1px solid #30363d}
.scenario img{max-width:100%;background:#fff;border-radius:6px;padding:6px}
.links{font-size:.85rem;color:#8b949e}
</style></head><body>
<h1>lsm-tree vs RocksDB — head-to-head overlays</h1>
<p class="intro">Each chart overlays <strong>ours</strong> against <strong>rocksdb</strong> on the
same axes (X = element count, Y = average time; <strong>lower is faster</strong>). Generated by the
<code>compare-rocksdb</code> harness — both engines run in the same process per scenario.
&nbsp;&middot;&nbsp; <a href="../bench/">single-engine db_bench trend &rarr;</a>
&nbsp;&middot;&nbsp; <a href="report/">raw criterion index &rarr;</a></p>
HTML_HEAD
for d in "$work"/dev/compare/*/; do
name="$(basename "$d")"
[ -f "$d/report/lines.svg" ] || continue
printf '<div class="scenario"><h2>%s</h2><a href="%s/report/"><img src="%s/report/lines.svg" alt="%s overlay" loading="lazy"></a><div class="links"><a href="%s/report/">full report (violin, regression, per-size) &rarr;</a></div></div>\n' \
"$name" "$name" "$name" "$name" "$name"
done
echo '</body></html>'
} > "$gallery"
# Cross-link banner: the db_bench dashboard (dev/bench/index.html) is
# regenerated by benchmark-action on every main push and has no link to
# the overlay reports, so visitors never discover dev/compare. Inject a
# banner pointing at the overlays right after <body>. Idempotent via the
# marker id; awk (not sed) avoids `&`/`/` escaping pitfalls in the HTML.
bench_index="$work/dev/bench/index.html"
if [ -f "$bench_index" ] && ! grep -q "compare-overlay-banner" "$bench_index"; then
banner='<div id="compare-overlay-banner" style="padding:8px 14px;background:#0d1117;color:#c9d1d9;font-family:system-ui,sans-serif;border-bottom:1px solid #30363d;font-size:14px">&#128202; <strong>lsm-tree benchmarks</strong> &nbsp;&middot;&nbsp; you are viewing the single-engine db_bench trend &nbsp;&middot;&nbsp; <a href="../compare/" style="color:#58a6ff;font-weight:600">RocksDB head-to-head overlays &rarr;</a></div>'
awk -v b="$banner" 'BEGIN{done=0} {print} /<body/ && !done {print b; done=1}' "$bench_index" > "$bench_index.tmp" \
&& mv "$bench_index.tmp" "$bench_index"
fi
cd "$work"
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git add dev/compare
# Stage the banner edit too (file may be absent on a first-ever run).
git add dev/bench/index.html 2>/dev/null || true
if git diff --cached --quiet; then
echo "::notice::compare-rocksdb reports unchanged — nothing to publish"
else
git commit -m "bench: refresh compare-rocksdb overlay reports (dev/compare)"
git push origin gh-pages
echo "::notice::published compare-rocksdb overlays to dev/compare/report/index.html"
fi