feat(vitest-plugin): add walltime support

Author: GuillaumeLagrange

packages/vitest-plugin/benches/timing.bench.ts

@@ -0,0 +1,19 @@
+import { bench, describe } from "vitest";
+
+const sleep = (ms: number): Promise<void> => {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+};
+
+describe("timing tests", () => {
+  bench("wait 1ms", async () => {
+    await sleep(1);
+  });
+
+  bench("wait 500ms", async () => {
+    await sleep(1);
+  });
+
+  bench("wait 1sec", async () => {
+    await sleep(1);
+  });
+});

packages/vitest-plugin/moon.yml

@@ -6,6 +6,8 @@ tasks:
     local: true
     options:
       cache: false
+    deps:
+      - build
 
   test:
     command: vitest --run

packages/vitest-plugin/rollup.config.ts

@@ -21,8 +21,14 @@ export default defineConfig([
     external: ["@codspeed/core", /^vitest/],
   },
   {
-    input: "src/runner.ts",
-    output: { file: "dist/runner.mjs", format: "es" },
+    input: "src/instrumented.ts",
+    output: { file: "dist/instrumented.mjs", format: "es" },
+    plugins: jsPlugins(pkg.version),
+    external: ["@codspeed/core", /^vitest/],
+  },
+  {
+    input: "src/walltime.ts",
+    output: { file: "dist/walltime.mjs", format: "es" },
     plugins: jsPlugins(pkg.version),
     external: ["@codspeed/core", /^vitest/],
   },

packages/vitest-plugin/src/__tests__/index.test.ts

@@ -1,5 +1,5 @@
 import { fromPartial } from "@total-typescript/shoehorn";
-import { describe, expect, it, vi } from "vitest";
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
 import codspeedPlugin from "../index";
 
 const coreMocks = vi.hoisted(() => {
@@ -25,6 +25,18 @@ vi.mock("@codspeed/core", async (importOriginal) => {
 console.warn = vi.fn();
 
 describe("codSpeedPlugin", () => {
+  beforeAll(() => {
+    // Set environment variables to trigger instrumented mode
+    process.env.CODSPEED_ENV = "1";
+    process.env.CODSPEED_RUNNER_MODE = "instrumentation";
+  });
+
+  afterAll(() => {
+    // Clean up environment variables
+    delete process.env.CODSPEED_ENV;
+    delete process.env.CODSPEED_RUNNER_MODE;
+  });
+
   it("should have a name", async () => {
     expect(resolvedCodSpeedPlugin.name).toBe("codspeed:vitest");
   });
@@ -96,7 +108,9 @@ describe("codSpeedPlugin", () => {
             ],
           },
         },
-        runner: expect.stringContaining("packages/vitest-plugin/src/runner.ts"),
+        runner: expect.stringContaining(
+          "packages/vitest-plugin/src/instrumented.ts"
+        ),
       },
     });
   });

packages/vitest-plugin/src/__tests__/runner.test.ts renamed to packages/vitest-plugin/src/__tests__/instrumented.test.ts

@@ -1,7 +1,7 @@
 import { fromPartial } from "@total-typescript/shoehorn";
 import { describe, expect, it, Suite, vi } from "vitest";
 import { getBenchFn } from "vitest/suite";
-import CodSpeedRunner from "../runner";
+import { InstrumentedRunner as CodSpeedRunner } from "../instrumented";
 
 const coreMocks = vi.hoisted(() => {
   return {
@@ -53,24 +53,24 @@ describe("CodSpeedRunner", () => {
     // setup
     expect(coreMocks.setupCore).toHaveBeenCalledTimes(1);
     expect(console.log).toHaveBeenCalledWith(
-      "[CodSpeed] running suite packages/vitest-plugin/src/__tests__/runner.test.ts"
+      "[CodSpeed] running suite packages/vitest-plugin/src/__tests__/instrumented.test.ts"
     );
 
     // run
     expect(coreMocks.mongoMeasurement.start).toHaveBeenCalledWith(
-      "packages/vitest-plugin/src/__tests__/runner.test.ts::test bench"
+      "packages/vitest-plugin/src/__tests__/instrumented.test.ts::test bench"
     );
     expect(coreMocks.Measurement.startInstrumentation).toHaveBeenCalledTimes(1);
     expect(benchFn).toHaveBeenCalledTimes(8);
     expect(coreMocks.Measurement.stopInstrumentation).toHaveBeenCalledTimes(1);
     expect(coreMocks.mongoMeasurement.stop).toHaveBeenCalledTimes(1);
     expect(console.log).toHaveBeenCalledWith(
-      "[CodSpeed] packages/vitest-plugin/src/__tests__/runner.test.ts::test bench done"
+      "[CodSpeed] packages/vitest-plugin/src/__tests__/instrumented.test.ts::test bench done"
     );
 
     // teardown
     expect(console.log).toHaveBeenCalledWith(
-      "[CodSpeed] running suite packages/vitest-plugin/src/__tests__/runner.test.ts done"
+      "[CodSpeed] running suite packages/vitest-plugin/src/__tests__/instrumented.test.ts done"
     );
     expect(coreMocks.teardownCore).toHaveBeenCalledTimes(1);
   });
@@ -107,24 +107,24 @@ describe("CodSpeedRunner", () => {
     // setup
     expect(coreMocks.setupCore).toHaveBeenCalledTimes(1);
     expect(console.log).toHaveBeenCalledWith(
-      "[CodSpeed] running suite packages/vitest-plugin/src/__tests__/runner.test.ts"
+      "[CodSpeed] running suite packages/vitest-plugin/src/__tests__/instrumented.test.ts"
     );
 
     // run
     expect(coreMocks.mongoMeasurement.start).toHaveBeenCalledWith(
-      "packages/vitest-plugin/src/__tests__/runner.test.ts::nested suite::test bench"
+      "packages/vitest-plugin/src/__tests__/instrumented.test.ts::nested suite::test bench"
     );
     expect(coreMocks.Measurement.startInstrumentation).toHaveBeenCalledTimes(1);
     expect(benchFn).toHaveBeenCalledTimes(8);
     expect(coreMocks.Measurement.stopInstrumentation).toHaveBeenCalledTimes(1);
     expect(coreMocks.mongoMeasurement.stop).toHaveBeenCalledTimes(1);
     expect(console.log).toHaveBeenCalledWith(
-      "[CodSpeed] packages/vitest-plugin/src/__tests__/runner.test.ts::nested suite::test bench done"
+      "[CodSpeed] packages/vitest-plugin/src/__tests__/instrumented.test.ts::nested suite::test bench done"
     );
 
     // teardown
     expect(console.log).toHaveBeenCalledWith(
-      "[CodSpeed] running suite packages/vitest-plugin/src/__tests__/runner.test.ts done"
+      "[CodSpeed] running suite packages/vitest-plugin/src/__tests__/instrumented.test.ts done"
     );
     expect(coreMocks.teardownCore).toHaveBeenCalledTimes(1);
   });

packages/vitest-plugin/src/common.ts

@@ -36,4 +36,4 @@ export function patchRootSuiteWithFullFilePath(suite: Suite) {
     throw new Error("Could not find a git repository");
   }
   suite.name = path.relative(gitDir, suite.filepath);
-}
+}

packages/vitest-plugin/src/index.ts

@@ -1,4 +1,5 @@
 import {
+  getCodspeedRunnerMode,
   getV8Flags,
   Measurement,
   mongoMeasurement,
@@ -19,32 +20,51 @@ function getCodSpeedFileFromName(name: string) {
   return join(__dirname, `${name}.${fileExtension}`);
 }
 
+function getRunnerFile(): string | undefined {
+  const codspeedRunnerMode = getCodspeedRunnerMode();
+  if (codspeedRunnerMode === "disabled") {
+    return undefined;
+  }
+
+  return getCodSpeedFileFromName(codspeedRunnerMode);
+}
+
 export default function codspeedPlugin(): Plugin {
   return {
     name: "codspeed:vitest",
     apply(_, { mode }) {
       if (mode !== "benchmark") {
         return false;
       }
-      if (!Measurement.isInstrumented()) {
+      if (
+        getCodspeedRunnerMode() == "instrumented" &&
+        !Measurement.isInstrumented()
+      ) {
         console.warn("[CodSpeed] bench detected but no instrumentation found");
       }
       return true;
     },
     enforce: "post",
     config(): UserConfig {
-      return {
+      const runnerFile = getRunnerFile();
+      const config: UserConfig = {
         test: {
           pool: "forks",
           poolOptions: {
             forks: {
               execArgv: getV8Flags(),
             },
           },
-          runner: getCodSpeedFileFromName("runner"),
           globalSetup: [getCodSpeedFileFromName("globalSetup")],
         },
       };
+
+      // Only set custom runner when CODSPEED_ENV is set
+      if (runnerFile) {
+        config.test!.runner = runnerFile;
+      }
+
+      return config;
     },
   };
 }

packages/vitest-plugin/src/instrumented.ts

@@ -106,3 +106,4 @@ export class InstrumentedRunner extends NodeBenchmarkRunner {
   }
 }
 
+export default InstrumentedRunner;