feat(tinybench-plugin): bump tinybench and add walltime support

Author: GuillaumeLagrange

.github/workflows/codspeed.yml

@@ -7,8 +7,8 @@ on:
   workflow_dispatch:
 
 jobs:
-  codspeed:
-    name: Run CodSpeed
+  codspeed-instrumented:
+    name: Run CodSpeed instrumented
     runs-on: "ubuntu-latest"
     steps:
       - uses: "actions/checkout@v4"
@@ -35,3 +35,31 @@ jobs:
             pnpm --workspace-concurrency 1 -r bench-tinybench
             pnpm --workspace-concurrency 1 -r bench-benchmark-js
             pnpm --workspace-concurrency 1 -r bench-vitest
+
+  codspeed-walltime:
+    name: Run CodSpeed walltime
+    runs-on: "codspeed-macro"
+    steps:
+      - uses: "actions/checkout@v4"
+        with:
+          fetch-depth: 0
+      - name: Install valgrind
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y valgrind
+      - uses: pnpm/action-setup@v2
+      - uses: actions/setup-node@v3
+        with:
+          cache: pnpm
+          node-version-file: .nvmrc
+      - run: pnpm install --frozen-lockfile --prefer-offline
+      - run: pnpm moon run :build
+
+      - name: Run benchmarks
+        # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v2`
+        uses: CodSpeedHQ/action@main
+        with:
+          # Only tinybench supports walltime for now
+          run: |
+            pnpm moon run tinybench-plugin:bench
+            pnpm --workspace-concurrency 1 -r bench-tinybench

examples/with-javascript-cjs/package.json

@@ -10,6 +10,6 @@
     "@codspeed/benchmark.js-plugin": "workspace:*",
     "@codspeed/tinybench-plugin": "workspace:*",
     "benchmark": "^2.1.4",
-    "tinybench": "^2.5.0"
+    "tinybench": "^4.0.1"
   }
 }

examples/with-javascript-esm/package.json

@@ -11,7 +11,7 @@
     "@types/benchmark": "^2.1.2",
     "benchmark": "^2.1.4",
     "esbuild-register": "^3.4.2",
-    "tinybench": "^2.5.0",
+    "tinybench": "^4.0.1",
     "typescript": "^5.1.3"
   }
 }

examples/with-typescript-cjs/package.json

@@ -14,7 +14,7 @@
     "@types/benchmark": "^2.1.2",
     "benchmark": "^2.1.4",
     "esbuild-register": "^3.4.2",
-    "tinybench": "^2.5.0",
+    "tinybench": "^4.0.1",
     "typescript": "^5.1.3",
     "vitest": "^1.2.2"
   }

examples/with-typescript-esm/package.json

@@ -11,7 +11,7 @@
     "@types/benchmark": "^2.1.2",
     "benchmark": "^2.1.4",
     "esbuild-register": "^3.4.2",
-    "tinybench": "^2.5.0",
+    "tinybench": "^4.0.1",
     "typescript": "^5.1.3"
   }
 }

examples/with-typescript-simple-cjs/package.json

@@ -12,7 +12,7 @@
     "@types/benchmark": "^2.1.2",
     "benchmark": "^2.1.4",
     "esbuild-register": "^3.4.2",
-    "tinybench": "^2.5.0",
+    "tinybench": "^4.0.1",
     "typescript": "^5.1.3"
   }
 }

examples/with-typescript-simple-esm/package.json

@@ -10,6 +10,26 @@ export const isBound = native_core.isBound;
 
 export const mongoMeasurement = new MongoMeasurement();
 
+export enum MeasurementMode {
+  Instrumentation = "instrumentation",
+  WallTime = "walltime",
+}
+
+export function getMeasurementMode(): MeasurementMode {
+  const isCodSpeedEnabled = process.env.CODSPEED_ENV !== undefined;
+  if (isCodSpeedEnabled) {
+    // If CODSPEED_ENV is set, check CODSPEED_RUNNER_MODE
+    if (process.env.CODSPEED_RUNNER_MODE === "walltime") {
+      return MeasurementMode.WallTime;
+    } else {
+      return MeasurementMode.Instrumentation;
+    }
+  }
+
+  // Default to walltime mode when CODSPEED_ENV is not set
+  return MeasurementMode.WallTime;
+}
+
 export const setupCore = () => {
   native_core.Measurement.stopInstrumentation(
     `Metadata: codspeed-node ${__VERSION__}`
@@ -29,4 +49,5 @@ export type {
 export { getV8Flags, tryIntrospect } from "./introspection";
 export { optimizeFunction, optimizeFunctionSync } from "./optimization";
 export * from "./utils";
+export * from "./walltime";
 export const Measurement = native_core.Measurement;

packages/core/src/index.ts

@@ -0,0 +1,41 @@
+import fs from "fs";
+import path from "path";
+import { Benchmark, ResultData } from "./interfaces";
+
+declare const __VERSION__: string;
+
+export function getProfileFolder(): string | null {
+  return process.env.CODSPEED_PROFILE_FOLDER || null;
+}
+
+export function writeWalltimeResults(benchmarks: Benchmark[]) {
+  const profileFolder = getProfileFolder();
+  let resultPath: string;
+
+  if (profileFolder) {
+    const resultsDir = path.join(profileFolder, "results");
+    fs.mkdirSync(resultsDir, { recursive: true });
+    resultPath = path.join(resultsDir, `${process.pid}.json`);
+  } else {
+    // Fallback: write to .codspeed in current working directory
+    const codspeedDir = path.join(process.cwd(), ".codspeed");
+    fs.mkdirSync(codspeedDir, { recursive: true });
+    resultPath = path.join(codspeedDir, `results_${Date.now()}.json`);
+  }
+
+  const data: ResultData = {
+    creator: {
+      name: "codspeed-node",
+      version: __VERSION__,
+      pid: process.pid,
+    },
+    instrument: { type: "walltime" },
+    benchmarks: benchmarks,
+  };
+
+  fs.writeFileSync(resultPath, JSON.stringify(data, null, 2));
+  console.log(`[CodSpeed] Results written to ${resultPath}`);
+}
+
+export * from "./interfaces";
+export * from "./quantiles";

packages/core/src/walltime/index.ts

@@ -0,0 +1,43 @@
+export interface BenchmarkStats {
+  min_ns: number;
+  max_ns: number;
+  mean_ns: number;
+  stdev_ns: number;
+  q1_ns: number;
+  median_ns: number;
+  q3_ns: number;
+  rounds: number;
+  total_time: number;
+  iqr_outlier_rounds: number;
+  stdev_outlier_rounds: number;
+  iter_per_round: number;
+  warmup_iters: number;
+}
+
+export interface BenchmarkConfig {
+  warmup_time_ns: number | null;
+  min_round_time_ns: number | null;
+  max_time_ns?: number | null;
+  max_rounds?: number | null;
+}
+
+export interface Benchmark {
+  name: string;
+  uri: string;
+  config: BenchmarkConfig;
+  stats: BenchmarkStats;
+}
+
+export interface InstrumentInfo {
+  type: string;
+}
+
+export interface ResultData {
+  creator: {
+    name: string;
+    version: string;
+    pid: number;
+  };
+  instrument: { type: "walltime" };
+  benchmarks: Benchmark[];
+}