feat(tinybench-plugin): bump tinybench and add walltime support

Author: GuillaumeLagrange

examples/with-javascript-cjs/package.json

@@ -9,6 +9,6 @@
     "@codspeed/benchmark.js-plugin": "workspace:*",
     "@codspeed/tinybench-plugin": "workspace:*",
     "benchmark": "^2.1.4",
-    "tinybench": "^2.5.0"
+    "tinybench": "^4.0.1"
   }
 }

examples/with-javascript-esm/package.json

@@ -10,6 +10,6 @@
     "@codspeed/benchmark.js-plugin": "workspace:*",
     "@codspeed/tinybench-plugin": "workspace:*",
     "benchmark": "^2.1.4",
-    "tinybench": "^2.5.0"
+    "tinybench": "^4.0.1"
   }
 }

examples/with-typescript-cjs/package.json

@@ -11,7 +11,7 @@
     "@types/benchmark": "^2.1.2",
     "benchmark": "^2.1.4",
     "esbuild-register": "^3.4.2",
-    "tinybench": "^2.5.0",
+    "tinybench": "^4.0.1",
     "typescript": "^5.1.3"
   }
 }

examples/with-typescript-esm/bench/tinybench/index.bench.ts

@@ -2,12 +2,14 @@ import { withCodSpeed } from "@codspeed/tinybench-plugin";
 import { Bench } from "tinybench";
 import { registerFiboBenchmarks } from "./fibo.bench";
 import { registerFoobarbazBenchmarks } from "./foobarbaz.bench";
+import { registerTimingBenchmarks } from "./timing.bench";
 
 export const bench = withCodSpeed(new Bench());
 
 (async () => {
   registerFiboBenchmarks(bench);
   registerFoobarbazBenchmarks(bench);
+  registerTimingBenchmarks(bench);
 
   await bench.run();
   console.table(bench.table());

examples/with-typescript-esm/bench/tinybench/timing.bench.ts

@@ -0,0 +1,19 @@
+import type { Bench } from "tinybench";
+
+const sleep = (ms: number): Promise<void> => {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+};
+
+export function registerTimingBenchmarks(bench: Bench) {
+  bench.add("wait 1ms", async () => {
+    await sleep(1);
+  });
+
+  bench.add("wait 500ms", async () => {
+    await sleep(500);
+  });
+
+  bench.add("wait 1sec", async () => {
+    await sleep(1000);
+  });
+}

examples/with-typescript-esm/package.json

@@ -14,7 +14,7 @@
     "@types/benchmark": "^2.1.2",
     "benchmark": "^2.1.4",
     "esbuild-register": "^3.4.2",
-    "tinybench": "^2.5.0",
+    "tinybench": "^4.0.1",
     "typescript": "^5.1.3",
     "vitest": "^1.2.2"
   }

examples/with-typescript-simple-cjs/package.json

@@ -11,7 +11,7 @@
     "@types/benchmark": "^2.1.2",
     "benchmark": "^2.1.4",
     "esbuild-register": "^3.4.2",
-    "tinybench": "^2.5.0",
+    "tinybench": "^4.0.1",
     "typescript": "^5.1.3"
   }
 }

examples/with-typescript-simple-esm/package.json

@@ -12,7 +12,7 @@
     "@types/benchmark": "^2.1.2",
     "benchmark": "^2.1.4",
     "esbuild-register": "^3.4.2",
-    "tinybench": "^2.5.0",
+    "tinybench": "^4.0.1",
     "typescript": "^5.1.3"
   }
 }

packages/core/src/index.ts

@@ -10,6 +10,26 @@ export const isBound = native_core.isBound;
 
 export const mongoMeasurement = new MongoMeasurement();
 
+export enum MeasurementMode {
+  Instrumentation = "instrumentation",
+  WallTime = "walltime",
+}
+
+export function getMeasurementMode(): MeasurementMode {
+  const isCodSpeedEnabled = process.env.CODSPEED_ENV !== undefined;
+  if (isCodSpeedEnabled) {
+    // If CODSPEED_ENV is set, check CODSPEED_RUNNER_MODE
+    if (process.env.CODSPEED_RUNNER_MODE === "walltime") {
+      return MeasurementMode.WallTime;
+    } else {
+      return MeasurementMode.Instrumentation;
+    }
+  }
+
+  // Default to walltime mode when CODSPEED_ENV is not set
+  return MeasurementMode.WallTime;
+}
+
 export const setupCore = () => {
   native_core.Measurement.stopInstrumentation(
     `Metadata: codspeed-node ${__VERSION__}`
@@ -29,4 +49,5 @@ export type {
 export { getV8Flags, tryIntrospect } from "./introspection";
 export { optimizeFunction, optimizeFunctionSync } from "./optimization";
 export * from "./utils";
+export * from "./walltime";
 export const Measurement = native_core.Measurement;

packages/core/src/walltime.ts

@@ -0,0 +1,121 @@
+import fs from "fs";
+import path from "path";
+
+declare const __VERSION__: string;
+
+export interface BenchmarkStats {
+  min_ns: number;
+  max_ns: number;
+  mean_ns: number;
+  stdev_ns: number;
+  q1_ns: number;
+  median_ns: number;
+  q3_ns: number;
+  rounds: number;
+  total_time: number;
+  iqr_outlier_rounds: number;
+  stdev_outlier_rounds: number;
+  iter_per_round: number;
+  warmup_iters: number;
+}
+
+export interface BenchmarkConfig {
+  warmup_time_ns: number;
+  min_round_time_ns: number;
+  max_time_ns: number;
+  max_rounds: number | null;
+}
+
+export interface Benchmark {
+  name: string;
+  uri: string;
+  config: BenchmarkConfig;
+  stats: BenchmarkStats;
+}
+
+export interface InstrumentInfo {
+  type: string;
+  clock_info: {
+    implementation: string;
+    monotonic: boolean;
+    adjustable: boolean;
+    resolution: number;
+  };
+}
+
+export interface ResultData {
+  creator: {
+    name: string;
+    version: string;
+    pid: number;
+  };
+  instrument: InstrumentInfo;
+  benchmarks: Benchmark[];
+}
+
+export function getProfileFolder(): string | null {
+  return process.env.CODSPEED_PROFILE_FOLDER || null;
+}
+
+export function getCreatorMetadata() {
+  return {
+    creator: {
+      name: "@codspeed/core",
+      version: __VERSION__,
+      pid: process.pid,
+    },
+  };
+}
+
+export function getWalltimeInstrumentInfo(): InstrumentInfo {
+  return {
+    type: "walltime",
+    clock_info: {
+      implementation: "perf_counter",
+      monotonic: true,
+      adjustable: false,
+      resolution: 1e-9, // nanosecond resolution
+    },
+  };
+}
+
+export function writeWalltimeResults(
+  benchmarks: Benchmark[],
+  integrationName: string
+) {
+  const profileFolder = getProfileFolder();
+  let resultPath: string;
+
+  if (profileFolder) {
+    const resultsDir = path.join(profileFolder, "results");
+    fs.mkdirSync(resultsDir, { recursive: true });
+    resultPath = path.join(resultsDir, `${process.pid}.json`);
+  } else {
+    // Fallback: write to .codspeed in current working directory
+    const codspeedDir = path.join(process.cwd(), ".codspeed");
+    fs.mkdirSync(codspeedDir, { recursive: true });
+    resultPath = path.join(codspeedDir, `results_${Date.now()}.json`);
+  }
+
+  const data: ResultData = {
+    creator: {
+      name: integrationName,
+      version: __VERSION__,
+      pid: process.pid,
+    },
+    instrument: getWalltimeInstrumentInfo(),
+    benchmarks: benchmarks,
+  };
+
+  fs.writeFileSync(resultPath, JSON.stringify(data, null, 2));
+  console.log(`[CodSpeed] Results written to ${resultPath}`);
+}
+
+export function createDefaultBenchmarkConfig(): BenchmarkConfig {
+  return {
+    warmup_time_ns: 1_000_000_000, // 1 second default
+    min_round_time_ns: 1_000_000, // 1ms default
+    max_time_ns: 3_000_000_000, // 3 seconds default
+    max_rounds: null,
+  };
+}