|
| 1 | +#!/usr/bin/env node |
| 2 | + |
| 3 | +import { readFileSync, writeFileSync } from "node:fs"; |
| 4 | + |
| 5 | +function parseArgs(argv) { |
| 6 | + const args = { |
| 7 | + thresholdPct: 5, |
| 8 | + failOnRegression: true, |
| 9 | + }; |
| 10 | + |
| 11 | + for (let i = 0; i < argv.length; i += 1) { |
| 12 | + const arg = argv[i]; |
| 13 | + const next = argv[i + 1]; |
| 14 | + |
| 15 | + if (arg === "--base") { |
| 16 | + if (!next) throw new Error("Missing value for --base"); |
| 17 | + args.basePath = next; |
| 18 | + i += 1; |
| 19 | + continue; |
| 20 | + } |
| 21 | + if (arg === "--head") { |
| 22 | + if (!next) throw new Error("Missing value for --head"); |
| 23 | + args.headPath = next; |
| 24 | + i += 1; |
| 25 | + continue; |
| 26 | + } |
| 27 | + if (arg === "--threshold-pct") { |
| 28 | + if (!next) throw new Error("Missing value for --threshold-pct"); |
| 29 | + args.thresholdPct = Number(next); |
| 30 | + i += 1; |
| 31 | + continue; |
| 32 | + } |
| 33 | + if (arg === "--markdown") { |
| 34 | + if (!next) throw new Error("Missing value for --markdown"); |
| 35 | + args.markdownPath = next; |
| 36 | + i += 1; |
| 37 | + continue; |
| 38 | + } |
| 39 | + if (arg === "--json") { |
| 40 | + if (!next) throw new Error("Missing value for --json"); |
| 41 | + args.jsonPath = next; |
| 42 | + i += 1; |
| 43 | + continue; |
| 44 | + } |
| 45 | + if (arg === "--no-fail") { |
| 46 | + args.failOnRegression = false; |
| 47 | + continue; |
| 48 | + } |
| 49 | + |
| 50 | + throw new Error(`Unknown argument: ${arg}`); |
| 51 | + } |
| 52 | + |
| 53 | + if (!args.basePath || !args.headPath) { |
| 54 | + throw new Error("Usage: compare-bench.mjs --base <path> --head <path> [--threshold-pct <n>]"); |
| 55 | + } |
| 56 | + |
| 57 | + if (!Number.isFinite(args.thresholdPct) || args.thresholdPct < 0) { |
| 58 | + throw new Error("--threshold-pct must be a non-negative number"); |
| 59 | + } |
| 60 | + |
| 61 | + return args; |
| 62 | +} |
| 63 | + |
| 64 | +function formatNum(value) { |
| 65 | + return value.toLocaleString("en-US", { maximumFractionDigits: 2 }); |
| 66 | +} |
| 67 | + |
| 68 | +function formatPct(value) { |
| 69 | + const sign = value > 0 ? "+" : ""; |
| 70 | + return `${sign}${value.toFixed(2)}%`; |
| 71 | +} |
| 72 | + |
| 73 | +function loadBench(path) { |
| 74 | + return JSON.parse(readFileSync(path, "utf8")); |
| 75 | +} |
| 76 | + |
| 77 | +function compare(baseRun, headRun, thresholdPct) { |
| 78 | + const byName = (run) => new Map(run.results.map((result) => [result.name, result])); |
| 79 | + const baseMap = byName(baseRun); |
| 80 | + const headMap = byName(headRun); |
| 81 | + |
| 82 | + const names = [...baseMap.keys()].filter((name) => headMap.has(name)); |
| 83 | + names.sort(); |
| 84 | + |
| 85 | + const scenarios = names.map((name) => { |
| 86 | + const base = baseMap.get(name); |
| 87 | + const head = headMap.get(name); |
| 88 | + |
| 89 | + const deltaPct = ((head.mean - base.mean) / base.mean) * 100; |
| 90 | + const regression = deltaPct <= -thresholdPct; |
| 91 | + const improvement = deltaPct >= thresholdPct; |
| 92 | + |
| 93 | + return { |
| 94 | + name, |
| 95 | + baseMean: base.mean, |
| 96 | + headMean: head.mean, |
| 97 | + deltaPct, |
| 98 | + baseCiPct: base.ci95.marginPct, |
| 99 | + headCiPct: head.ci95.marginPct, |
| 100 | + baseErrors: base.errors, |
| 101 | + headErrors: head.errors, |
| 102 | + status: regression ? "REGRESSION" : improvement ? "IMPROVEMENT" : "OK", |
| 103 | + }; |
| 104 | + }); |
| 105 | + |
| 106 | + const regressions = scenarios.filter((item) => item.status === "REGRESSION"); |
| 107 | + |
| 108 | + return { |
| 109 | + generatedAt: new Date().toISOString(), |
| 110 | + thresholdPct, |
| 111 | + baseCommit: baseRun.git?.commit, |
| 112 | + headCommit: headRun.git?.commit, |
| 113 | + regressions: regressions.map((item) => item.name), |
| 114 | + pass: regressions.length === 0, |
| 115 | + scenarios, |
| 116 | + }; |
| 117 | +} |
| 118 | + |
| 119 | +function toMarkdown(report) { |
| 120 | + const lines = []; |
| 121 | + lines.push("# AWS Perf Compare"); |
| 122 | + lines.push(""); |
| 123 | + lines.push(`- Generated: ${report.generatedAt}`); |
| 124 | + lines.push(`- Threshold: ${report.thresholdPct}% throughput drop => regression`); |
| 125 | + if (report.baseCommit) lines.push(`- Base commit: ${report.baseCommit}`); |
| 126 | + if (report.headCommit) lines.push(`- Head commit: ${report.headCommit}`); |
| 127 | + lines.push(""); |
| 128 | + lines.push("| Scenario | Base req/s | Head req/s | Delta | Status | Base CI | Head CI |"); |
| 129 | + lines.push("|---|---:|---:|---:|---|---:|---:|"); |
| 130 | + |
| 131 | + for (const scenario of report.scenarios) { |
| 132 | + lines.push( |
| 133 | + `| ${scenario.name} | ${formatNum(scenario.baseMean)} | ${formatNum(scenario.headMean)} | ${formatPct(scenario.deltaPct)} | ${scenario.status} | ±${scenario.baseCiPct.toFixed(2)}% | ±${scenario.headCiPct.toFixed(2)}% |`, |
| 134 | + ); |
| 135 | + } |
| 136 | + |
| 137 | + lines.push(""); |
| 138 | + lines.push(`## Gate: ${report.pass ? "PASS" : "FAIL"}`); |
| 139 | + if (!report.pass) { |
| 140 | + lines.push(`Regressions: ${report.regressions.join(", ")}`); |
| 141 | + } |
| 142 | + |
| 143 | + return lines.join("\n"); |
| 144 | +} |
| 145 | + |
| 146 | +function main() { |
| 147 | + const args = parseArgs(process.argv.slice(2)); |
| 148 | + const baseRun = loadBench(args.basePath); |
| 149 | + const headRun = loadBench(args.headPath); |
| 150 | + const report = compare(baseRun, headRun, args.thresholdPct); |
| 151 | + const markdown = toMarkdown(report); |
| 152 | + |
| 153 | + if (args.markdownPath) { |
| 154 | + writeFileSync(args.markdownPath, markdown, "utf8"); |
| 155 | + } |
| 156 | + |
| 157 | + if (args.jsonPath) { |
| 158 | + writeFileSync(args.jsonPath, JSON.stringify(report, null, 2), "utf8"); |
| 159 | + } |
| 160 | + |
| 161 | + console.log(markdown); |
| 162 | + |
| 163 | + if (!report.pass && args.failOnRegression) { |
| 164 | + process.exit(2); |
| 165 | + } |
| 166 | +} |
| 167 | + |
| 168 | +main(); |
0 commit comments