test(shared-log): make getReceivedHeads native-aware to widen conformance leg #404
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks | |
| on: | |
| issue_comment: | |
| types: [created] | |
| workflow_dispatch: | |
| inputs: | |
| pr: | |
| description: Pull request number | |
| required: true | |
| type: number | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| jobs: | |
| bench: | |
| if: | | |
| (github.event_name == 'issue_comment' && | |
| github.event.issue.pull_request && | |
| startsWith(github.event.comment.body, '/bench') && | |
| contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association) | |
| ) || | |
| (github.event_name == 'workflow_dispatch') | |
| runs-on: ubuntu-22.04 | |
| concurrency: | |
| group: bench-${{ github.event.issue.number || inputs.pr }} | |
| cancel-in-progress: true | |
| steps: | |
| - name: Resolve PR details | |
| id: pr | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const prNumber = context.eventName === 'workflow_dispatch' | |
| ? Number(context.payload.inputs?.pr) | |
| : context.payload.issue.number; | |
| const { owner, repo } = context.repo; | |
| const pr = await github.rest.pulls.get({ owner, repo, pull_number: prNumber }); | |
| core.setOutput('number', String(prNumber)); | |
| core.setOutput('base_sha', pr.data.base.sha); | |
| core.setOutput('head_sha', pr.data.head.sha); | |
| core.setOutput('head_repo', pr.data.head.repo.full_name); | |
| const sameRepo = pr.data.head.repo.full_name === `${owner}/${repo}`; | |
| core.setOutput('same_repo', sameRepo ? 'true' : 'false'); | |
| const files = await github.paginate(github.rest.pulls.listFiles, { | |
| owner, | |
| repo, | |
| pull_number: prNumber, | |
| per_page: 100, | |
| }); | |
| const filenames = files.map((f) => f.filename); | |
| const commentBody = | |
| context.eventName === 'issue_comment' ? (context.payload.comment?.body || '') : ''; | |
| const argv = commentBody.trim().split(/\s+/).slice(1).filter(Boolean); | |
| const hasArgs = argv.length > 0; | |
| const wantAll = argv.includes('all'); | |
| const wantSharedLog = | |
| wantAll || argv.includes('shared-log') || argv.includes('sharedlog') || argv.includes('shared'); | |
| const wantTransport = | |
| wantAll || | |
| argv.includes('transport') || | |
| argv.includes('stream') || | |
| argv.includes('transfer') || | |
| argv.includes('ack'); | |
| const wantDocument = | |
| wantAll || | |
| argv.includes('document') || | |
| argv.includes('doc') || | |
| argv.includes('ingest') || | |
| argv.includes('file-ingest'); | |
| const touchedSharedLog = filenames.some( | |
| (p) => | |
| p.startsWith('packages/programs/data/shared-log/') || | |
| p.startsWith('packages/utils/rateless-iblt/'), | |
| ); | |
| const touchedIngestStack = filenames.some( | |
| (p) => | |
| p.startsWith('packages/programs/data/document/') || | |
| p.startsWith('packages/programs/data/shared-log/') || | |
| p.startsWith('packages/log/') || | |
| p.startsWith('packages/transport/stream/') || | |
| p.startsWith('packages/transport/stream-interface/') || | |
| p.startsWith('packages/transport/blocks/') || | |
| p.startsWith('packages/transport/blocks-interface/') || | |
| p.startsWith('packages/utils/indexer/') || | |
| p.startsWith('packages/utils/crypto/'), | |
| ); | |
| const touchedTransport = filenames.some( | |
| (p) => | |
| p.startsWith('packages/transport/stream/') || | |
| p.startsWith('packages/transport/stream-interface/') || | |
| p.startsWith('packages/utils/crypto/'), | |
| ); | |
| const runSharedLog = wantSharedLog || (!hasArgs && touchedSharedLog); | |
| const runTransport = wantTransport || (!hasArgs && touchedTransport); | |
| const runDocument = wantDocument || (!hasArgs && touchedIngestStack); | |
| core.setOutput('run_shared_log', runSharedLog ? 'true' : 'false'); | |
| core.setOutput('run_transport', runTransport ? 'true' : 'false'); | |
| core.setOutput('run_document', runDocument ? 'true' : 'false'); | |
| const selected = []; | |
| if (runSharedLog) selected.push('shared-log'); | |
| if (runTransport) selected.push('transport'); | |
| if (runDocument) selected.push('document'); | |
| core.setOutput('selected', selected.join(',') || 'none'); | |
| core.setOutput('mode', hasArgs ? 'manual' : 'auto'); | |
| core.setOutput('args', argv.join(' ')); | |
| const runAny = runSharedLog || runTransport || runDocument; | |
| core.setOutput('run_any', runAny ? 'true' : 'false'); | |
| - name: Skip forked PRs | |
| if: steps.pr.outputs.same_repo != 'true' | |
| run: | | |
| echo "Benchmarks are only supported for same-repo branches (forks are skipped for security)." | |
| - name: Checkout base | |
| if: steps.pr.outputs.same_repo == 'true' && steps.pr.outputs.run_any == 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ steps.pr.outputs.base_sha }} | |
| path: bench-base | |
| - name: Checkout head | |
| if: steps.pr.outputs.same_repo == 'true' && steps.pr.outputs.run_any == 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ steps.pr.outputs.head_sha }} | |
| path: bench-head | |
| - name: Setup Node | |
| if: steps.pr.outputs.same_repo == 'true' && steps.pr.outputs.run_any == 'true' | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 22.x | |
| - name: Setup pnpm | |
| if: steps.pr.outputs.same_repo == 'true' && steps.pr.outputs.run_any == 'true' | |
| uses: pnpm/action-setup@v4 | |
| with: | |
| version: 10.24.0 | |
| run_install: false | |
| - name: Install wasm-pack | |
| if: steps.pr.outputs.same_repo == 'true' && steps.pr.outputs.run_any == 'true' | |
| run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh | |
| - name: Install deps (base) | |
| if: steps.pr.outputs.same_repo == 'true' && steps.pr.outputs.run_any == 'true' | |
| working-directory: bench-base | |
| run: pnpm install --frozen-lockfile=false | |
| - name: Install deps (head) | |
| if: steps.pr.outputs.same_repo == 'true' && steps.pr.outputs.run_any == 'true' | |
| working-directory: bench-head | |
| run: pnpm install --frozen-lockfile=false | |
| - name: Build (base) | |
| if: steps.pr.outputs.same_repo == 'true' && steps.pr.outputs.run_any == 'true' | |
| working-directory: bench-base | |
| run: | | |
| if [ "${{ steps.pr.outputs.run_shared_log }}" = "true" ]; then | |
| pnpm --filter @peerbit/shared-log... run build | |
| fi | |
| if [ "${{ steps.pr.outputs.run_transport }}" = "true" ]; then | |
| pnpm --filter @peerbit/stream... run build | |
| fi | |
| if [ "${{ steps.pr.outputs.run_document }}" = "true" ]; then | |
| pnpm --filter @peerbit/document... run build | |
| fi | |
| - name: Build (head) | |
| if: steps.pr.outputs.same_repo == 'true' && steps.pr.outputs.run_any == 'true' | |
| working-directory: bench-head | |
| run: | | |
| if [ "${{ steps.pr.outputs.run_shared_log }}" = "true" ]; then | |
| pnpm --filter @peerbit/shared-log... run build | |
| fi | |
| if [ "${{ steps.pr.outputs.run_transport }}" = "true" ]; then | |
| pnpm --filter @peerbit/stream... run build | |
| fi | |
| if [ "${{ steps.pr.outputs.run_document }}" = "true" ]; then | |
| pnpm --filter @peerbit/document... run build | |
| fi | |
| - name: Run shared-log benchmarks (base) | |
| if: steps.pr.outputs.same_repo == 'true' && steps.pr.outputs.run_shared_log == 'true' | |
| working-directory: bench-base/packages/programs/data/shared-log | |
| env: | |
| NODE_OPTIONS: --no-warnings | |
| BENCH_JSON: "1" | |
| RIBLT_SIZES: "1000,10000,50000" | |
| RIBLT_WARMUP: "2" | |
| RIBLT_ITERATIONS: "10" | |
| SWEEP_BATCH_SIZES: "256,512,1024,4096,16384,65536" | |
| SWEEP_ENTRY_COUNT: "20000" | |
| SWEEP_WARMUP_RUNS: "0" | |
| SWEEP_RUNS: "1" | |
| SWEEP_TIMEOUT: "120000" | |
| run: | | |
| mkdir -p "$GITHUB_WORKSPACE/bench-results/base" | |
| node --loader ts-node/esm ./benchmark/rateless-iblt-startsync-cache.ts > "$GITHUB_WORKSPACE/bench-results/base/rateless-iblt-startsync-cache.json" | |
| node --loader ts-node/esm ./benchmark/rateless-iblt-sender-startsync.ts > "$GITHUB_WORKSPACE/bench-results/base/rateless-iblt-sender-startsync.json" | |
| node --loader ts-node/esm ./benchmark/sync-batch-sweep.ts > "$GITHUB_WORKSPACE/bench-results/base/sync-batch-sweep.json" | |
| node --loader ts-node/esm ./benchmark/pid-convergence.ts > "$GITHUB_WORKSPACE/bench-results/base/pid-convergence.json" | |
| - name: Run shared-log benchmarks (head) | |
| if: steps.pr.outputs.same_repo == 'true' && steps.pr.outputs.run_shared_log == 'true' | |
| working-directory: bench-head/packages/programs/data/shared-log | |
| env: | |
| NODE_OPTIONS: --no-warnings | |
| BENCH_JSON: "1" | |
| RIBLT_SIZES: "1000,10000,50000" | |
| RIBLT_WARMUP: "2" | |
| RIBLT_ITERATIONS: "10" | |
| SWEEP_BATCH_SIZES: "256,512,1024,4096,16384,65536" | |
| SWEEP_ENTRY_COUNT: "20000" | |
| SWEEP_WARMUP_RUNS: "0" | |
| SWEEP_RUNS: "1" | |
| SWEEP_TIMEOUT: "120000" | |
| run: | | |
| mkdir -p "$GITHUB_WORKSPACE/bench-results/head" | |
| node --loader ts-node/esm ./benchmark/rateless-iblt-startsync-cache.ts > "$GITHUB_WORKSPACE/bench-results/head/rateless-iblt-startsync-cache.json" | |
| node --loader ts-node/esm ./benchmark/rateless-iblt-sender-startsync.ts > "$GITHUB_WORKSPACE/bench-results/head/rateless-iblt-sender-startsync.json" | |
| node --loader ts-node/esm ./benchmark/sync-batch-sweep.ts > "$GITHUB_WORKSPACE/bench-results/head/sync-batch-sweep.json" | |
| node --loader ts-node/esm ./benchmark/pid-convergence.ts > "$GITHUB_WORKSPACE/bench-results/head/pid-convergence.json" | |
| - name: Run document benchmarks (base) | |
| if: steps.pr.outputs.same_repo == 'true' && steps.pr.outputs.run_document == 'true' | |
| working-directory: bench-base/packages/programs/data/document/document | |
| env: | |
| NODE_OPTIONS: --no-warnings | |
| BENCH_JSON: "1" | |
| DOC_WARMUP: "2" | |
| DOC_ITERATIONS: "10" | |
| FILE_INGEST_WARMUP: "1" | |
| FILE_INGEST_ITERATIONS: "3" | |
| FILE_INGEST_CHUNKS: "24" | |
| FILE_INGEST_CHUNK_BYTES: "262144" | |
| FILE_INGEST_READY_TIMEOUT_MS: "30000" | |
| run: | | |
| mkdir -p "$GITHUB_WORKSPACE/bench-results/base" | |
| if [ -f ./benchmark/document-put.ts ]; then | |
| node --loader ts-node/esm ./benchmark/document-put.ts > "$GITHUB_WORKSPACE/bench-results/base/document-put.json" | |
| else | |
| echo "document-put benchmark not present in base; skipping." | |
| fi | |
| if [ -f ./benchmark/file-ingest.ts ]; then | |
| node --loader ts-node/esm ./benchmark/file-ingest.ts > "$GITHUB_WORKSPACE/bench-results/base/file-ingest.json" | |
| else | |
| echo "file-ingest benchmark not present in base; skipping." | |
| fi | |
| - name: Run transport benchmarks (base) | |
| if: steps.pr.outputs.same_repo == 'true' && steps.pr.outputs.run_transport == 'true' | |
| working-directory: bench-base/packages/transport/stream | |
| env: | |
| NODE_OPTIONS: --no-warnings | |
| BENCH_JSON: "1" | |
| CHUNK_TRANSFER_WARMUP: "1" | |
| CHUNK_TRANSFER_ITERATIONS: "3" | |
| CHUNK_TRANSFER_CHUNKS: "24" | |
| CHUNK_TRANSFER_CHUNK_BYTES: "262144" | |
| CHUNK_TRANSFER_TIMEOUT_MS: "30000" | |
| run: | | |
| mkdir -p "$GITHUB_WORKSPACE/bench-results/base" | |
| if [ -f ./benchmark/chunk-transfer.ts ]; then | |
| node ./dist/benchmark/chunk-transfer.js > "$GITHUB_WORKSPACE/bench-results/base/chunk-transfer.json" | |
| else | |
| echo "chunk-transfer benchmark not present in base; skipping." | |
| fi | |
| - name: Run document benchmarks (head) | |
| if: steps.pr.outputs.same_repo == 'true' && steps.pr.outputs.run_document == 'true' | |
| working-directory: bench-head/packages/programs/data/document/document | |
| env: | |
| NODE_OPTIONS: --no-warnings | |
| BENCH_JSON: "1" | |
| DOC_WARMUP: "2" | |
| DOC_ITERATIONS: "10" | |
| FILE_INGEST_WARMUP: "1" | |
| FILE_INGEST_ITERATIONS: "3" | |
| FILE_INGEST_CHUNKS: "24" | |
| FILE_INGEST_CHUNK_BYTES: "262144" | |
| FILE_INGEST_READY_TIMEOUT_MS: "30000" | |
| run: | | |
| mkdir -p "$GITHUB_WORKSPACE/bench-results/head" | |
| if [ -f ./benchmark/document-put.ts ]; then | |
| node --loader ts-node/esm ./benchmark/document-put.ts > "$GITHUB_WORKSPACE/bench-results/head/document-put.json" | |
| else | |
| echo "document-put benchmark not present in head; skipping." | |
| fi | |
| if [ -f ./benchmark/file-ingest.ts ]; then | |
| node --loader ts-node/esm ./benchmark/file-ingest.ts > "$GITHUB_WORKSPACE/bench-results/head/file-ingest.json" | |
| else | |
| echo "file-ingest benchmark not present in head; skipping." | |
| fi | |
| - name: Run transport benchmarks (head) | |
| if: steps.pr.outputs.same_repo == 'true' && steps.pr.outputs.run_transport == 'true' | |
| working-directory: bench-head/packages/transport/stream | |
| env: | |
| NODE_OPTIONS: --no-warnings | |
| BENCH_JSON: "1" | |
| CHUNK_TRANSFER_WARMUP: "1" | |
| CHUNK_TRANSFER_ITERATIONS: "3" | |
| CHUNK_TRANSFER_CHUNKS: "24" | |
| CHUNK_TRANSFER_CHUNK_BYTES: "262144" | |
| CHUNK_TRANSFER_TIMEOUT_MS: "30000" | |
| run: | | |
| mkdir -p "$GITHUB_WORKSPACE/bench-results/head" | |
| if [ -f ./benchmark/chunk-transfer.ts ]; then | |
| node ./dist/benchmark/chunk-transfer.js > "$GITHUB_WORKSPACE/bench-results/head/chunk-transfer.json" | |
| else | |
| echo "chunk-transfer benchmark not present in head; skipping." | |
| fi | |
| - name: Comment results | |
| if: always() && steps.pr.outputs.same_repo == 'true' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('node:fs'); | |
| const { owner, repo } = context.repo; | |
| const prNumber = Number('${{ steps.pr.outputs.number }}'); | |
| const marker = '<!-- PEERBIT_BENCHMARKS -->'; | |
| const runUrl = `${process.env.GITHUB_SERVER_URL}/${owner}/${repo}/actions/runs/${process.env.GITHUB_RUN_ID}`; | |
| const triggerLabel = | |
| context.eventName === 'workflow_dispatch' ? '`workflow_dispatch`' : '`/bench`'; | |
| const selected = '${{ steps.pr.outputs.selected }}'; | |
| const mode = '${{ steps.pr.outputs.mode }}'; | |
| const args = '${{ steps.pr.outputs.args }}'; | |
| const suites = [ | |
| { | |
| title: 'shared-log: StartSync local decoder cache', | |
| base: 'bench-results/base/rateless-iblt-startsync-cache.json', | |
| head: 'bench-results/head/rateless-iblt-startsync-cache.json', | |
| }, | |
| { | |
| title: 'shared-log: sender StartSync setup (onMaybeMissingEntries)', | |
| base: 'bench-results/base/rateless-iblt-sender-startsync.json', | |
| head: 'bench-results/head/rateless-iblt-sender-startsync.json', | |
| }, | |
| { | |
| title: 'shared-log: sync catch-up batch sweep (including large-edge batch)', | |
| base: 'bench-results/base/sync-batch-sweep.json', | |
| head: 'bench-results/head/sync-batch-sweep.json', | |
| }, | |
| { | |
| title: 'shared-log: PID convergence (model)', | |
| base: 'bench-results/base/pid-convergence.json', | |
| head: 'bench-results/head/pid-convergence.json', | |
| }, | |
| { | |
| title: 'document: put', | |
| base: 'bench-results/base/document-put.json', | |
| head: 'bench-results/head/document-put.json', | |
| }, | |
| { | |
| title: 'transport: multi-hop chunk transfer', | |
| base: 'bench-results/base/chunk-transfer.json', | |
| head: 'bench-results/head/chunk-transfer.json', | |
| }, | |
| { | |
| title: 'document: chunked file-ingest', | |
| base: 'bench-results/base/file-ingest.json', | |
| head: 'bench-results/head/file-ingest.json', | |
| }, | |
| ]; | |
| const readJson = (file) => { | |
| try { | |
| return JSON.parse(fs.readFileSync(file, 'utf8')); | |
| } catch (error) { | |
| return { | |
| tasks: [], | |
| __parseError: error instanceof Error ? error.message : String(error), | |
| }; | |
| } | |
| }; | |
| const format = (value, digits = 3) => { | |
| if (value === null || value === undefined || Number.isNaN(value)) return '-'; | |
| return Number(value).toFixed(digits); | |
| }; | |
| const formatPercent = (value) => { | |
| if (value === null || value === undefined || Number.isNaN(value)) return '-'; | |
| const sign = value > 0 ? '+' : ''; | |
| return `${sign}${value.toFixed(2)}%`; | |
| }; | |
| const formatSuite = (baseJson, headJson) => { | |
| const baseMap = new Map((baseJson.tasks || []).map((t) => [t.name, t])); | |
| const headMap = new Map((headJson.tasks || []).map((t) => [t.name, t])); | |
| const taskNames = Array.from(new Set([...baseMap.keys(), ...headMap.keys()])); | |
| taskNames.sort(); | |
| const lines = []; | |
| lines.push('| Task | base mean (ms) | head mean (ms) | Δ mean | base ops/s | head ops/s | Δ ops/s |'); | |
| lines.push('|---|---:|---:|---:|---:|---:|---:|'); | |
| for (const name of taskNames) { | |
| const b = baseMap.get(name); | |
| const h = headMap.get(name); | |
| const baseMean = b?.mean_ms ?? null; | |
| const headMean = h?.mean_ms ?? null; | |
| const baseHz = b?.hz ?? null; | |
| const headHz = h?.hz ?? null; | |
| const meanDelta = | |
| baseMean != null && headMean != null ? ((headMean - baseMean) / baseMean) * 100 : null; | |
| const hzDelta = baseHz != null && headHz != null ? ((headHz - baseHz) / baseHz) * 100 : null; | |
| lines.push( | |
| `| ${name} | ${format(baseMean, 6)} | ${format(headMean, 6)} | ${formatPercent(meanDelta)} | ${format(baseHz, 0)} | ${format(headHz, 0)} | ${formatPercent(hzDelta)} |`, | |
| ); | |
| } | |
| return lines.join('\n'); | |
| }; | |
| let body = | |
| marker + | |
| '\n## Benchmarks\n\nTriggered by ' + | |
| triggerLabel + | |
| ' - Run: ' + | |
| runUrl + | |
| '\n\n' + | |
| 'Mode: ' + | |
| mode + | |
| (args ? ' (' + args + ')' : '') + | |
| ' • Selected: ' + | |
| selected + | |
| '\n\n'; | |
| let includedSuites = 0; | |
| for (const suite of suites) { | |
| const baseExists = fs.existsSync(suite.base); | |
| const headExists = fs.existsSync(suite.head); | |
| if (!baseExists && !headExists) continue; | |
| const baseJson = baseExists ? readJson(suite.base) : { tasks: [] }; | |
| const headJson = headExists ? readJson(suite.head) : { tasks: [] }; | |
| body += `### ${suite.title}\n\n`; | |
| body += formatSuite(baseJson, headJson); | |
| if (baseJson.__parseError || headJson.__parseError) { | |
| body += '\n\n'; | |
| body += [ | |
| baseJson.__parseError ? `base parse error: ${baseJson.__parseError}` : null, | |
| headJson.__parseError ? `head parse error: ${headJson.__parseError}` : null, | |
| ].filter(Boolean).join(' • '); | |
| } | |
| body += `\n\n`; | |
| includedSuites += 1; | |
| } | |
| if (includedSuites === 0) { | |
| body += '_No matching benchmark suites ran for this PR._\n'; | |
| } | |
| const comments = await github.paginate(github.rest.issues.listComments, { | |
| owner, | |
| repo, | |
| issue_number: prNumber, | |
| per_page: 100, | |
| }); | |
| const existing = comments.find((c) => typeof c.body === 'string' && c.body.includes(marker)); | |
| if (existing) { | |
| await github.rest.issues.updateComment({ | |
| owner, | |
| repo, | |
| comment_id: existing.id, | |
| body, | |
| }); | |
| } else { | |
| await github.rest.issues.createComment({ | |
| owner, | |
| repo, | |
| issue_number: prNumber, | |
| body, | |
| }); | |
| } |