feat(tinybench-plugin): bump tinybench and add walltime support

Author: GuillaumeLagrange

.github/workflows/codspeed.yml

@@ -7,8 +7,8 @@ on:
   workflow_dispatch:
 
 jobs:
-  codspeed:
-    name: Run CodSpeed
+  codspeed-instrumented:
+    name: Run CodSpeed instrumented
     runs-on: "ubuntu-latest"
     steps:
       - uses: "actions/checkout@v4"
@@ -35,3 +35,31 @@ jobs:
             pnpm --workspace-concurrency 1 -r bench-tinybench
             pnpm --workspace-concurrency 1 -r bench-benchmark-js
             pnpm --workspace-concurrency 1 -r bench-vitest
+
+  codspeed-walltime:
+    name: Run CodSpeed walltime
+    runs-on: "codspeed-macro"
+    steps:
+      - uses: "actions/checkout@v4"
+        with:
+          fetch-depth: 0
+      - name: Install valgrind
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y valgrind
+      - uses: pnpm/action-setup@v2
+      - uses: actions/setup-node@v3
+        with:
+          cache: pnpm
+          node-version-file: .nvmrc
+      - run: pnpm install --frozen-lockfile --prefer-offline
+      - run: pnpm moon run :build
+
+      - name: Run benchmarks
+        # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v2`
+        uses: CodSpeedHQ/action@main
+        with:
+          # Only tinybench supports walltime for now
+          run: |
+            pnpm moon run tinybench-plugin:bench
+            pnpm --workspace-concurrency 1 -r bench-tinybench

examples/with-javascript-cjs/package.json

@@ -9,6 +9,6 @@
     "@codspeed/benchmark.js-plugin": "workspace:*",
     "@codspeed/tinybench-plugin": "workspace:*",
     "benchmark": "^2.1.4",
-    "tinybench": "^2.5.0"
+    "tinybench": "^4.0.1"
   }
 }

examples/with-javascript-esm/package.json

@@ -10,6 +10,6 @@
     "@codspeed/benchmark.js-plugin": "workspace:*",
     "@codspeed/tinybench-plugin": "workspace:*",
     "benchmark": "^2.1.4",
-    "tinybench": "^2.5.0"
+    "tinybench": "^4.0.1"
   }
 }

examples/with-typescript-cjs/package.json

@@ -11,7 +11,7 @@
     "@types/benchmark": "^2.1.2",
     "benchmark": "^2.1.4",
     "esbuild-register": "^3.4.2",
-    "tinybench": "^2.5.0",
+    "tinybench": "^4.0.1",
     "typescript": "^5.1.3"
   }
 }

examples/with-typescript-esm/package.json

@@ -14,7 +14,7 @@
     "@types/benchmark": "^2.1.2",
     "benchmark": "^2.1.4",
     "esbuild-register": "^3.4.2",
-    "tinybench": "^2.5.0",
+    "tinybench": "^4.0.1",
     "typescript": "^5.1.3",
     "vitest": "^1.2.2"
   }

examples/with-typescript-simple-cjs/package.json

@@ -11,7 +11,7 @@
     "@types/benchmark": "^2.1.2",
     "benchmark": "^2.1.4",
     "esbuild-register": "^3.4.2",
-    "tinybench": "^2.5.0",
+    "tinybench": "^4.0.1",
     "typescript": "^5.1.3"
   }
 }

examples/with-typescript-simple-esm/package.json

@@ -12,7 +12,7 @@
     "@types/benchmark": "^2.1.2",
     "benchmark": "^2.1.4",
     "esbuild-register": "^3.4.2",
-    "tinybench": "^2.5.0",
+    "tinybench": "^4.0.1",
     "typescript": "^5.1.3"
   }
 }

packages/core/src/index.ts

@@ -10,6 +10,26 @@ export const isBound = native_core.isBound;
 
 export const mongoMeasurement = new MongoMeasurement();
 
+export enum MeasurementMode {
+  Instrumentation = "instrumentation",
+  WallTime = "walltime",
+}
+
+export function getMeasurementMode(): MeasurementMode {
+  const isCodSpeedEnabled = process.env.CODSPEED_ENV !== undefined;
+  if (isCodSpeedEnabled) {
+    // If CODSPEED_ENV is set, check CODSPEED_RUNNER_MODE
+    if (process.env.CODSPEED_RUNNER_MODE === "walltime") {
+      return MeasurementMode.WallTime;
+    } else {
+      return MeasurementMode.Instrumentation;
+    }
+  }
+
+  // Default to walltime mode when CODSPEED_ENV is not set
+  return MeasurementMode.WallTime;
+}
+
 export const setupCore = () => {
   native_core.Measurement.stopInstrumentation(
     `Metadata: codspeed-node ${__VERSION__}`
@@ -29,4 +49,5 @@ export type {
 export { getV8Flags, tryIntrospect } from "./introspection";
 export { optimizeFunction, optimizeFunctionSync } from "./optimization";
 export * from "./utils";
+export * from "./walltime";
 export const Measurement = native_core.Measurement;

packages/core/src/walltime.ts

@@ -0,0 +1,121 @@
+import fs from "fs";
+import path from "path";
+
+declare const __VERSION__: string;
+
+export interface BenchmarkStats {
+  min_ns: number;
+  max_ns: number;
+  mean_ns: number;
+  stdev_ns: number;
+  q1_ns: number;
+  median_ns: number;
+  q3_ns: number;
+  rounds: number;
+  total_time: number;
+  iqr_outlier_rounds: number;
+  stdev_outlier_rounds: number;
+  iter_per_round: number;
+  warmup_iters: number;
+}
+
+export interface BenchmarkConfig {
+  warmup_time_ns: number;
+  min_round_time_ns: number;
+  max_time_ns: number;
+  max_rounds: number | null;
+}
+
+export interface Benchmark {
+  name: string;
+  uri: string;
+  config: BenchmarkConfig;
+  stats: BenchmarkStats;
+}
+
+export interface InstrumentInfo {
+  type: string;
+  clock_info: {
+    implementation: string;
+    monotonic: boolean;
+    adjustable: boolean;
+    resolution: number;
+  };
+}
+
+export interface ResultData {
+  creator: {
+    name: string;
+    version: string;
+    pid: number;
+  };
+  instrument: InstrumentInfo;
+  benchmarks: Benchmark[];
+}
+
+export function getProfileFolder(): string | null {
+  return process.env.CODSPEED_PROFILE_FOLDER || null;
+}
+
+export function getCreatorMetadata() {
+  return {
+    creator: {
+      name: "@codspeed/core",
+      version: __VERSION__,
+      pid: process.pid,
+    },
+  };
+}
+
+export function getWalltimeInstrumentInfo(): InstrumentInfo {
+  return {
+    type: "walltime",
+    clock_info: {
+      implementation: "perf_counter",
+      monotonic: true,
+      adjustable: false,
+      resolution: 1e-9, // nanosecond resolution
+    },
+  };
+}
+
+export function writeWalltimeResults(
+  benchmarks: Benchmark[],
+  integrationName: string
+) {
+  const profileFolder = getProfileFolder();
+  let resultPath: string;
+
+  if (profileFolder) {
+    const resultsDir = path.join(profileFolder, "results");
+    fs.mkdirSync(resultsDir, { recursive: true });
+    resultPath = path.join(resultsDir, `${process.pid}.json`);
+  } else {
+    // Fallback: write to .codspeed in current working directory
+    const codspeedDir = path.join(process.cwd(), ".codspeed");
+    fs.mkdirSync(codspeedDir, { recursive: true });
+    resultPath = path.join(codspeedDir, `results_${Date.now()}.json`);
+  }
+
+  const data: ResultData = {
+    creator: {
+      name: integrationName,
+      version: __VERSION__,
+      pid: process.pid,
+    },
+    instrument: getWalltimeInstrumentInfo(),
+    benchmarks: benchmarks,
+  };
+
+  fs.writeFileSync(resultPath, JSON.stringify(data, null, 2));
+  console.log(`[CodSpeed] Results written to ${resultPath}`);
+}
+
+export function createDefaultBenchmarkConfig(): BenchmarkConfig {
+  return {
+    warmup_time_ns: 1_000_000_000, // 1 second default
+    min_round_time_ns: 1_000_000, // 1ms default
+    max_time_ns: 3_000_000_000, // 3 seconds default
+    max_rounds: null,
+  };
+}

packages/tinybench-plugin/benches/sample.ts

@@ -1,6 +1,7 @@
 import { Bench } from "tinybench";
 import { withCodSpeed } from "..";
 import parsePr from "./parsePr";
+import { registerTimingBenchmarks } from "./timing";
 
 const LONG_BODY =
   new Array(1_000)
@@ -33,6 +34,8 @@ bench
     parsePr({ body: LONG_BODY, title: "test", number: 124 });
   });
 
+registerTimingBenchmarks(bench);
+
 bench.run().then(() => {
   console.table(bench.table());
 });