perf: speed up bench script for CI (#15)

* perf: speed up bench script for CI

* fix: omit empty feature comparisons from report in quick mode

* fix: include key benchmarks table in markdown report

* refactor: move AB-only definitions inside quick mode guard

* ci: retrigger

Author: RostiMelk

tests/bench/run.ts

@@ -20,7 +20,7 @@ import {
   measureWithContentDetection,
 } from "../../src/utils/measure";
 import { createNormalizedLogo } from "../../src/utils/normalize";
-import { fmtNs, fmtP, type TTestResult, welchTTest } from "./welch";
+import { fmtNs, fmtP, welchTTest } from "./welch";
 
 const origCreateElement = document.createElement.bind(document);
 document.createElement = ((tag: string, options?: ElementCreationOptions) => {
@@ -64,14 +64,21 @@ const svgFiles = readdirSync(LOGOS_DIR)
   .filter((f) => f.endsWith(".svg"))
   .sort();
 
-console.log(`Loading ${svgFiles.length} real SVGs from static/logos/…`);
+const QUICK = !!(process.env.BENCH_QUICK || process.env.CI);
 
-const allLogos: RenderedLogo[] = [];
-for (const file of svgFiles) {
-  const buf = Buffer.from(await Bun.file(join(LOGOS_DIR, file)).arrayBuffer());
-  const { img, width, height } = await loadSvgAtWidth(buf, 400);
-  allLogos.push({ name: file.replace(/\.svg$/, ""), img, width, height });
-}
+console.log(
+  `Loading ${svgFiles.length} real SVGs from static/logos/…${QUICK ? " (quick mode)" : ""}`,
+);
+
+const allLogos: RenderedLogo[] = await Promise.all(
+  svgFiles.map(async (file) => {
+    const buf = Buffer.from(
+      await Bun.file(join(LOGOS_DIR, file)).arrayBuffer(),
+    );
+    const { img, width, height } = await loadSvgAtWidth(buf, 400);
+    return { name: file.replace(/\.svg$/, ""), img, width, height };
+  }),
+);
 
 const byPixels = [...allLogos].sort(
   (a, b) => a.width * a.height - b.width * b.height,
@@ -104,10 +111,10 @@ const normalized20 = Array.from({ length: 20 }, (_, i) => {
 
 const logos20 = allLogos.slice(0, 20);
 
-const TIME_BUDGET_MS = 2_000;
-const MIN_SAMPLES = 30;
+const TIME_BUDGET_MS = QUICK ? 800 : 2_000;
+const MIN_SAMPLES = QUICK ? 20 : 30;
 const MAX_SAMPLES = 2_000;
-const WARMUP_MS = 200;
+const WARMUP_MS = QUICK ? 80 : 200;
 
 function collectSamples(fn: () => void): number[] {
   const warmupEnd = Bun.nanoseconds() + WARMUP_MS * 1e6;
@@ -188,92 +195,13 @@ const benchMount20Baseline = () => {
   }
 };
 
-const benchReNormalize20 = () => {
-  for (let i = 0; i < 20; i++) {
-    const idx = i % allLogos.length;
-    const logo = createNormalizedLogo(
-      sources[idx]!,
-      realMeasurements[idx]!,
-      DEFAULT_BASE_SIZE,
-      DEFAULT_SCALE_FACTOR,
-      DEFAULT_DENSITY_FACTOR,
-    );
-    blackhole(getVisualCenterTransform(logo, DEFAULT_ALIGN_BY));
-  }
-};
-
 const keyBenchmarks: Record<string, () => void> = {
   "content detection (1 logo)": benchMeasure,
   "render pass (20 logos)": benchGetVCT20,
   "mount 20 logos (no detection)": benchMount20Baseline,
   "mount 20 logos (defaults)": benchMount20,
 };
 
-interface ABComparison {
-  name: string;
-  a: { label: string; fn: () => void };
-  b: { label: string; fn: () => void };
-}
-
-const medianMeasurement = realMeasurements[allLogos.indexOf(medianLogo)]!;
-
-const abComparisons: ABComparison[] = [
-  {
-    name: "densityAware: true vs false",
-    a: {
-      label: "true",
-      fn: benchMeasure,
-    },
-    b: {
-      label: "false",
-      fn: () =>
-        measureWithContentDetection(
-          medianLogo.img,
-          DEFAULT_CONTRAST_THRESHOLD,
-          false,
-        ),
-    },
-  },
-  {
-    name: "alignBy: visual-center-y vs bounds",
-    a: { label: "visual-center-y", fn: benchGetVCT20 },
-    b: {
-      label: "bounds",
-      fn: () => {
-        for (let i = 0; i < 20; i++)
-          getVisualCenterTransform(normalized20[i]!, "bounds");
-      },
-    },
-  },
-  {
-    name: "cropToContent: true vs false",
-    a: {
-      label: "true",
-      fn: () => {
-        if (medianMeasurement.contentBox)
-          blackhole(
-            cropToDataUrl(medianLogo.img, medianMeasurement.contentBox),
-          );
-      },
-    },
-    b: {
-      label: "false (noop)",
-      fn: () => {},
-    },
-  },
-  {
-    name: "layout update: full mount vs cached",
-    a: {
-      label: "full mount",
-      fn: benchMount20,
-    },
-    b: {
-      label: "cached re-normalize",
-      fn: benchReNormalize20,
-    },
-  },
-];
-
 console.log();
 console.log("─".repeat(72));
 console.log("  KEY BENCHMARKS");
@@ -293,77 +221,130 @@ for (const [name, fn] of Object.entries(keyBenchmarks)) {
   );
 }
 
-console.log();
-console.log("─".repeat(72));
-console.log("  FEATURE COMPARISONS (Welch's t-test, two-tailed)");
-console.log("  Legend: * p<0.05  ** p<0.01  *** p<0.001");
-console.log("─".repeat(72));
-
-interface ABResult {
-  name: string;
-  aLabel: string;
-  bLabel: string;
-  aMean: string;
-  bMean: string;
-  pctChange: number;
-  result: TTestResult;
-}
-
-const abResults: ABResult[] = [];
-
-for (const comp of abComparisons) {
-  const samplesA = collectSamples(comp.a.fn);
-  const samplesB = collectSamples(comp.b.fn);
-  const result = welchTTest(samplesA, samplesB);
-  const sA = stats(samplesA);
-  const sB = stats(samplesB);
-  const pctChange = sB.mean !== 0 ? ((sA.mean - sB.mean) / sB.mean) * 100 : 0;
-
-  const sig = result.significant ? `YES ${result.marker}` : "NO";
-  const sign = pctChange > 0 ? "+" : "";
-  console.log(`  > ${comp.name}`);
-  console.log(
-    `    ${comp.a.label}: ${fmtNs(sA.mean)}  vs  ${comp.b.label}: ${fmtNs(sB.mean)}  (${sign}${pctChange.toFixed(1)}%)`,
-  );
-  console.log(`    p=${fmtP(result.p)}  significant=${sig}`);
-  console.log();
-
-  abResults.push({
-    name: comp.name,
-    aLabel: comp.a.label,
-    bLabel: comp.b.label,
-    aMean: fmtNs(sA.mean),
-    bMean: fmtNs(sB.mean),
-    pctChange,
-    result,
-  });
-}
-
 const md: string[] = [
   "## react-logo-soup Benchmark Report",
   "",
   `Test fixtures: ${allLogos.length} real SVGs from static/logos/. ` +
     `${TIME_BUDGET_MS}ms budget per bench, ${MIN_SAMPLES}-${MAX_SAMPLES} samples.`,
   "",
-  "### Feature Comparisons (Welch's t-test)",
-  "",
-  "| Test | A | B | Delta | Sig |",
-  "|:-----|--:|--:|------:|:----|",
+  "| Benchmark | Mean | ± Stddev | Samples |",
+  "|:----------|-----:|---------:|--------:|",
 ];
 
-for (const r of abResults) {
-  const sig = r.result.significant ? `YES ${r.result.marker}` : "NO";
-  const sign = r.pctChange > 0 ? "+" : "";
+for (const [name, samples] of Object.entries(benchSamples)) {
+  const s = stats(samples);
   md.push(
-    `| ${r.name} | ${r.aMean} | ${r.bMean} | ${sign}${r.pctChange.toFixed(1)}% | ${fmtP(r.result.p)} ${sig} |`,
+    `| ${name} | ${fmtNs(s.mean)} | ${fmtNs(s.stddev)} | ${samples.length} |`,
   );
 }
 
 md.push("");
-md.push(
-  "A/B columns match the order in the test name. Sig: `*` p<0.05, `**` p<0.01, `***` p<0.001.",
-);
-md.push("");
+
+if (!QUICK) {
+  const medianMeasurement = realMeasurements[allLogos.indexOf(medianLogo)]!;
+
+  const benchReNormalize20 = () => {
+    for (let i = 0; i < 20; i++) {
+      const idx = i % allLogos.length;
+      const logo = createNormalizedLogo(
+        sources[idx]!,
+        realMeasurements[idx]!,
+        DEFAULT_BASE_SIZE,
+        DEFAULT_SCALE_FACTOR,
+        DEFAULT_DENSITY_FACTOR,
+      );
+      blackhole(getVisualCenterTransform(logo, DEFAULT_ALIGN_BY));
+    }
+  };
+
+  const abComparisons = [
+    {
+      name: "densityAware: true vs false",
+      a: { label: "true", fn: benchMeasure },
+      b: {
+        label: "false",
+        fn: () =>
+          measureWithContentDetection(
+            medianLogo.img,
+            DEFAULT_CONTRAST_THRESHOLD,
+            false,
+          ),
+      },
+    },
+    {
+      name: "alignBy: visual-center-y vs bounds",
+      a: { label: "visual-center-y", fn: benchGetVCT20 },
+      b: {
+        label: "bounds",
+        fn: () => {
+          for (let i = 0; i < 20; i++)
+            getVisualCenterTransform(normalized20[i]!, "bounds");
+        },
+      },
+    },
+    {
+      name: "cropToContent: true vs false",
+      a: {
+        label: "true",
+        fn: () => {
+          if (medianMeasurement.contentBox)
+            blackhole(
+              cropToDataUrl(medianLogo.img, medianMeasurement.contentBox),
+            );
+        },
+      },
+      b: { label: "false (noop)", fn: () => {} },
+    },
+    {
+      name: "layout update: full mount vs cached",
+      a: { label: "full mount", fn: benchMount20 },
+      b: { label: "cached re-normalize", fn: benchReNormalize20 },
+    },
+  ];
+
+  console.log();
+  console.log("─".repeat(72));
+  console.log("  FEATURE COMPARISONS (Welch's t-test, two-tailed)");
+  console.log("  Legend: * p<0.05  ** p<0.01  *** p<0.001");
+  console.log("─".repeat(72));
+
+  md.push(
+    "### Feature Comparisons (Welch's t-test)",
+    "",
+    "| Test | A | B | Delta | Sig |",
+    "|:-----|--:|--:|------:|:----|",
+  );
+
+  for (const comp of abComparisons) {
+    const samplesA = collectSamples(comp.a.fn);
+    const samplesB = collectSamples(comp.b.fn);
+    const result = welchTTest(samplesA, samplesB);
+    const sA = stats(samplesA);
+    const sB = stats(samplesB);
+    const pctChange = sB.mean !== 0 ? ((sA.mean - sB.mean) / sB.mean) * 100 : 0;
+
+    const sig = result.significant ? `YES ${result.marker}` : "NO";
+    const sign = pctChange > 0 ? "+" : "";
+    console.log(`  > ${comp.name}`);
+    console.log(
+      `    ${comp.a.label}: ${fmtNs(sA.mean)}  vs  ${comp.b.label}: ${fmtNs(sB.mean)}  (${sign}${pctChange.toFixed(1)}%)`,
+    );
+    console.log(`    p=${fmtP(result.p)}  significant=${sig}`);
+    console.log();
+
+    md.push(
+      `| ${comp.name} | ${fmtNs(sA.mean)} | ${fmtNs(sB.mean)} | ${sign}${pctChange.toFixed(1)}% | ${fmtP(result.p)} ${sig} |`,
+    );
+  }
+
+  md.push(
+    "",
+    "A/B columns match the order in the test name. Sig: `*` p<0.05, `**` p<0.01, `***` p<0.001.",
+    "",
+  );
+} else {
+  console.log("\n  Skipping AB comparisons in quick mode.");
+}
 
 const outDir = process.env.BENCH_OUT_DIR ?? "tmp";
 mkdirSync(outDir, { recursive: true });