--wip-- [skip ci]

Author: GuillaumeLagrange

examples/with-typescript-esm/bench/tinybench/index.bench.ts

@@ -2,12 +2,14 @@ import { withCodSpeed } from "@codspeed/tinybench-plugin";
 import { Bench } from "tinybench";
 import { registerFiboBenchmarks } from "./fibo.bench";
 import { registerFoobarbazBenchmarks } from "./foobarbaz.bench";
+import { registerTimingBenchmarks } from "./timing.bench";
 
 export const bench = withCodSpeed(new Bench());
 
 (async () => {
   registerFiboBenchmarks(bench);
   registerFoobarbazBenchmarks(bench);
+  registerTimingBenchmarks(bench);
 
   await bench.run();
   console.table(bench.table());

examples/with-typescript-esm/bench/tinybench/timing.bench.ts

@@ -0,0 +1,19 @@
+import type { Bench } from "tinybench";
+
+const sleep = (ms: number): Promise<void> => {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+};
+
+export function registerTimingBenchmarks(bench: Bench) {
+  bench.add("wait 1ms", async () => {
+    await sleep(1);
+  });
+
+  bench.add("wait 500ms", async () => {
+    await sleep(500);
+  });
+
+  bench.add("wait 1sec", async () => {
+    await sleep(1000);
+  });
+}

packages/core/src/walltime.ts

@@ -0,0 +1,121 @@
+import fs from "fs";
+import path from "path";
+
+declare const __VERSION__: string;
+
+export interface BenchmarkStats {
+  min_ns: number;
+  max_ns: number;
+  mean_ns: number;
+  stdev_ns: number;
+  q1_ns: number;
+  median_ns: number;
+  q3_ns: number;
+  rounds: number;
+  total_time: number;
+  iqr_outlier_rounds: number;
+  stdev_outlier_rounds: number;
+  iter_per_round: number;
+  warmup_iters: number;
+}
+
+export interface BenchmarkConfig {
+  warmup_time_ns: number;
+  min_round_time_ns: number;
+  max_time_ns: number;
+  max_rounds: number | null;
+}
+
+export interface Benchmark {
+  name: string;
+  uri: string;
+  config: BenchmarkConfig;
+  stats: BenchmarkStats;
+}
+
+export interface InstrumentInfo {
+  type: string;
+  clock_info: {
+    implementation: string;
+    monotonic: boolean;
+    adjustable: boolean;
+    resolution: number;
+  };
+}
+
+export interface ResultData {
+  creator: {
+    name: string;
+    version: string;
+    pid: number;
+  };
+  instrument: InstrumentInfo;
+  benchmarks: Benchmark[];
+}
+
+export function getProfileFolder(): string | null {
+  return process.env.CODSPEED_PROFILE_FOLDER || null;
+}
+
+export function getCreatorMetadata() {
+  return {
+    creator: {
+      name: "@codspeed/core",
+      version: __VERSION__,
+      pid: process.pid,
+    },
+  };
+}
+
+export function getWalltimeInstrumentInfo(): InstrumentInfo {
+  return {
+    type: "walltime",
+    clock_info: {
+      implementation: "perf_counter",
+      monotonic: true,
+      adjustable: false,
+      resolution: 1e-9, // nanosecond resolution
+    },
+  };
+}
+
+export function writeWalltimeResults(
+  benchmarks: Benchmark[],
+  integrationName: string
+) {
+  const profileFolder = getProfileFolder();
+  let resultPath: string;
+
+  if (profileFolder) {
+    const resultsDir = path.join(profileFolder, "results");
+    fs.mkdirSync(resultsDir, { recursive: true });
+    resultPath = path.join(resultsDir, `${process.pid}.json`);
+  } else {
+    // Fallback: write to .codspeed in current working directory
+    const codspeedDir = path.join(process.cwd(), ".codspeed");
+    fs.mkdirSync(codspeedDir, { recursive: true });
+    resultPath = path.join(codspeedDir, `results_${Date.now()}.json`);
+  }
+
+  const data: ResultData = {
+    creator: {
+      name: integrationName,
+      version: __VERSION__,
+      pid: process.pid,
+    },
+    instrument: getWalltimeInstrumentInfo(),
+    benchmarks: benchmarks,
+  };
+
+  fs.writeFileSync(resultPath, JSON.stringify(data, null, 2));
+  console.log(`[CodSpeed] Results written to ${resultPath}`);
+}
+
+export function createDefaultBenchmarkConfig(): BenchmarkConfig {
+  return {
+    warmup_time_ns: 1_000_000_000, // 1 second default
+    min_round_time_ns: 1_000_000, // 1ms default
+    max_time_ns: 3_000_000_000, // 3 seconds default
+    max_rounds: null,
+  };
+}

packages/tinybench-plugin/src/index.ts

@@ -1,8 +1,6 @@
 import {
   getGitDir,
-  getMeasurementMode,
   Measurement,
-  MeasurementMode,
   mongoMeasurement,
   SetupInstrumentsRequestBody,
   SetupInstrumentsResponse,
@@ -21,7 +19,12 @@ tryIntrospect();
 export const taskUriMap = new WeakMap<Bench, Map<string, string>>();
 
 export function withCodSpeed(bench: Bench): Bench {
-  const measurementMode = getMeasurementMode();
+  // Check if CODSPEED_ENV is defined
+  if (!process.env.CODSPEED_ENV) {
+    // If CODSPEED_ENV is not defined, return bench unchanged (run as normal tinybench)
+    return bench;
+  }
+
   const rootCallingFile = getCallingFile();
 
   // Initialize URI mapping for this bench instance
@@ -40,20 +43,18 @@ export function withCodSpeed(bench: Bench): Bench {
     return rawAdd.bind(bench)(name, fn, opts);
   };
 
-  // Apply the appropriate measurement strategy based on mode and instrumentation
-  if (
-    measurementMode === MeasurementMode.Instrumentation &&
-    Measurement.isInstrumented()
-  ) {
+  // Apply the appropriate measurement strategy based on CODSPEED_RUNNER_MODE
+  const runnerMode = process.env.CODSPEED_RUNNER_MODE;
+
+  if (runnerMode === "instrumentation") {
     runInstrumentedBench(bench, rootCallingFile);
-  } else if (measurementMode === MeasurementMode.WallTime) {
+  } else if (runnerMode === "walltime") {
     runWalltimeBench(bench, rootCallingFile);
   } else {
-    // Fallback: instrumentation requested but not available
     const rawRun = bench.run;
     bench.run = async () => {
       console.warn(
-        `[CodSpeed] ${bench.tasks.length} benches detected but no instrumentation found, falling back to tinybench`
+        `[CodSpeed] CODSPEED_ENV is defined but CODSPEED_RUNNER_MODE="${runnerMode}" is not recognized, falling back to tinybench`
       );
       return await rawRun.bind(bench)();
     };

packages/tinybench-plugin/src/index.unit.test.ts

@@ -0,0 +1,60 @@
+import { Bench } from "tinybench";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { withCodSpeed } from ".";
+
+const mockInstrumented = vi.hoisted(() => ({
+  runInstrumentedBench: vi.fn(),
+}));
+
+vi.mock("./instrumented", () => ({
+  ...mockInstrumented,
+}));
+
+const mockWalltime = vi.hoisted(() => ({
+  runWalltimeBench: vi.fn(),
+}));
+
+vi.mock("./walltime", () => ({
+  ...mockWalltime,
+}));
+
+describe("withCodSpeed behavior without different codspeed modes", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    // Ensure no CODSPEED env vars are set
+    delete process.env.CODSPEED_ENV;
+    delete process.env.CODSPEED_RUNNER_MODE;
+  });
+
+  it("should return the same bench instance unchanged when no CODSPEED_ENV", async () => {
+    const originalBench = new Bench({ iterations: 10, time: 10 });
+    const wrappedBench = withCodSpeed(originalBench);
+    const shouldBeCalled = vi.fn();
+    wrappedBench.add("test task", shouldBeCalled);
+    await wrappedBench.run();
+
+    // Should return the exact same instance
+    expect(wrappedBench).toBe(originalBench);
+    expect(shouldBeCalled.mock.calls.length).toBeGreaterThan(1000);
+  });
+
+  it("should run in instrumented mode when CODSPEED_RUNNER_MODE=instrumentation", async () => {
+    process.env.CODSPEED_ENV = "true";
+    process.env.CODSPEED_RUNNER_MODE = "instrumentation";
+
+    withCodSpeed(new Bench());
+
+    expect(mockInstrumented.runInstrumentedBench).toHaveBeenCalled();
+    expect(mockWalltime.runWalltimeBench).not.toHaveBeenCalled();
+  });
+
+  it("should run in walltime mode when CODSPEED_RUNNER_MODE=walltime", async () => {
+    process.env.CODSPEED_ENV = "true";
+    process.env.CODSPEED_RUNNER_MODE = "walltime";
+
+    withCodSpeed(new Bench());
+
+    expect(mockInstrumented.runInstrumentedBench).not.toHaveBeenCalled();
+    expect(mockWalltime.runWalltimeBench).toHaveBeenCalled();
+  });
+});

packages/tinybench-plugin/src/instrumented.ts

@@ -0,0 +1,74 @@
+import {
+  Measurement,
+  mongoMeasurement,
+  optimizeFunction,
+  setupCore,
+  teardownCore,
+} from "@codspeed/core";
+import { Bench } from "tinybench";
+import { taskUriMap } from "./index";
+
+declare const __VERSION__: string;
+
+function getTaskUri(
+  bench: Bench,
+  taskName: string,
+  rootCallingFile: string
+): string {
+  const uriMap = taskUriMap.get(bench);
+  return uriMap?.get(taskName) || `${rootCallingFile}::${taskName}`;
+}
+
+export function runInstrumentedBench(
+  bench: Bench,
+  rootCallingFile: string
+): void {
+  bench.run = async () => {
+    console.log(
+      `[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (instrumented mode)`
+    );
+    setupCore();
+
+    for (const task of bench.tasks) {
+      const uri = getTaskUri(bench, task.name, rootCallingFile);
+
+      // Access private fnOpts to get hooks
+      const fnOpts = (task as any).fnOpts;
+
+      // Call beforeAll hook if it exists
+      await fnOpts?.beforeAll?.call(task, "run");
+
+      // run optimizations
+      await optimizeFunction(async () => {
+        await fnOpts?.beforeEach?.call(task, "run");
+        await (task as any).fn(); // Access private fn property
+        await fnOpts?.afterEach?.call(task, "run");
+      });
+
+      // run instrumented benchmark
+      await fnOpts?.beforeEach?.call(task, "run");
+
+      await mongoMeasurement.start(uri);
+      global.gc?.();
+      await (async function __codspeed_root_frame__() {
+        Measurement.startInstrumentation();
+        await (task as any).fn(); // Access private fn property
+        Measurement.stopInstrumentation(uri);
+      })();
+      await mongoMeasurement.stop(uri);
+
+      await fnOpts?.afterEach?.call(task, "run");
+
+      await fnOpts?.afterAll?.call(task, "run");
+
+      // print results
+      console.log(
+        `    ✔ ${Measurement.isInstrumented() ? "Measured" : "Checked"} ${uri}`
+      );
+    }
+
+    teardownCore();
+    console.log(`[CodSpeed] Done running ${bench.tasks.length} benches.`);
+    return bench.tasks;
+  };
+}