--wip-- [skip ci]

Author: GuillaumeLagrange

packages/vitest-plugin/src/index.ts

@@ -65,12 +65,6 @@ export default function codspeedPlugin(): Plugin {
       // Only set custom runner when CODSPEED_ENV is set
       if (runnerFile) {
         config.test!.runner = runnerFile;
-        
-        // Add CodSpeedReporter when in walltime mode
-        const runnerMode = process.env.CODSPEED_RUNNER_MODE;
-        if (runnerMode !== "instrumentation") {
-          config.test!.reporters = [getCodSpeedFileFromName("reporter")];
-        }
       }
 
       return config;

packages/vitest-plugin/src/walltime.ts

@@ -1,16 +1,48 @@
-import { logDebug, setupCore, teardownCore } from "@codspeed/core";
-import { Suite } from "vitest";
+import {
+  calculateQuantiles,
+  logDebug,
+  mongoMeasurement,
+  setupCore,
+  teardownCore,
+  writeWalltimeResults,
+  type Benchmark as CoreBenchmark,
+  type BenchmarkStats,
+} from "@codspeed/core";
+import { Suite, Task } from "vitest";
 import { NodeBenchmarkRunner } from "vitest/runners";
 import { patchRootSuiteWithFullFilePath } from "./common";
 
+declare const __VERSION__: string;
+
 const currentFileName =
   typeof __filename === "string"
     ? __filename
     : new URL("walltime.mjs", import.meta.url).pathname;
 
+interface VitestBenchmarkResult {
+  name: string;
+  options?: {
+    iterations?: number;
+    time?: number;
+    warmupIterations?: number;
+    warmupTime?: number;
+  };
+  result?: {
+    error?: unknown;
+    duration?: number;
+    mean?: number;
+    min?: number;
+    max?: number;
+    stdDev?: number;
+    samples?: number[];
+    variance?: number;
+    rme?: number;
+  };
+}
+
 /**
  * WalltimeRunner uses Vitest's default benchmark execution
- * and relies on the CodSpeedReporter to capture and process results
+ * and extracts results from the suite after completion
  */
 export class WalltimeRunner extends NodeBenchmarkRunner {
   async runSuite(suite: Suite): Promise<void> {
@@ -19,12 +51,149 @@ export class WalltimeRunner extends NodeBenchmarkRunner {
 
     patchRootSuiteWithFullFilePath(suite);
 
+    console.log(
+      `[CodSpeed] running with @codspeed/vitest-plugin v${__VERSION__} (walltime mode)`
+    );
+
     // Let Vitest's default benchmark runner handle execution
-    // Results will be captured by CodSpeedReporter via onTaskUpdate
     await super.runSuite(suite);
 
+    // Extract benchmark results from the completed suite
+    const benchmarks = await this.extractBenchmarkResults(suite);
+
+    // Write results to JSON file using core function
+    if (benchmarks.length > 0) {
+      writeWalltimeResults(benchmarks);
+      console.log(
+        `[CodSpeed] Done collecting walltime data for ${benchmarks.length} benches.`
+      );
+    }
+
     teardownCore();
   }
+
+  private async extractBenchmarkResults(
+    suite: Suite,
+    parentPath = ""
+  ): Promise<CoreBenchmark[]> {
+    const benchmarks: CoreBenchmark[] = [];
+    const currentPath = parentPath
+      ? `${parentPath}::${suite.name}`
+      : suite.name;
+
+    for (const task of suite.tasks) {
+      if (task.result?.state === "pass") {
+        const benchmark = await this.processBenchmarkTask(task, currentPath);
+        if (benchmark) {
+          benchmarks.push(benchmark);
+        }
+      } else if (task.type === "suite") {
+        const nestedBenchmarks = await this.extractBenchmarkResults(
+          task,
+          currentPath
+        );
+        benchmarks.push(...nestedBenchmarks);
+      }
+    }
+
+    return benchmarks;
+  }
+
+  private async processBenchmarkTask(
+    task: Task,
+    suitePath: string
+  ): Promise<CoreBenchmark | null> {
+    const uri = `${suitePath}::${task.name}`;
+
+    // Notify mongoMeasurement about the benchmark
+    await mongoMeasurement.start(uri);
+    await mongoMeasurement.stop(uri);
+
+    const benchmarkResult = task.meta?.benchmark as
+      | VitestBenchmarkResult
+      | undefined;
+
+    if (benchmarkResult?.result) {
+      const stats = this.convertVitestResultToBenchmarkStats(
+        benchmarkResult.result
+      );
+
+      const coreBenchmark: CoreBenchmark = {
+        name: task.name,
+        uri,
+        config: {
+          warmup_time_ns: benchmarkResult.options?.warmupTime
+            ? benchmarkResult.options.warmupTime * 1_000_000
+            : null,
+          min_round_time_ns: benchmarkResult.options?.time
+            ? benchmarkResult.options.time * 1_000_000
+            : null,
+        },
+        stats,
+      };
+
+      console.log(`    ✔ Collected walltime data for ${uri}`);
+      return coreBenchmark;
+    } else {
+      console.warn(`    ⚠ No result data available for ${uri}`);
+      return null;
+    }
+  }
+
+  private convertVitestResultToBenchmarkStats(
+    result: NonNullable<VitestBenchmarkResult["result"]>
+  ): BenchmarkStats {
+    // Convert milliseconds to nanoseconds
+    const ms_to_ns = (ms: number) => ms * 1_000_000;
+
+    // Use samples if available, otherwise create from basic stats
+    let sortedTimesNs: number[];
+    let meanNs: number;
+    let stdevNs: number;
+
+    if (result.samples && result.samples.length > 0) {
+      sortedTimesNs = result.samples.map(ms_to_ns).sort((a, b) => a - b);
+      meanNs = result.mean
+        ? ms_to_ns(result.mean)
+        : sortedTimesNs.reduce((a, b) => a + b, 0) / sortedTimesNs.length;
+      stdevNs = result.stdDev
+        ? ms_to_ns(result.stdDev)
+        : ms_to_ns(Math.sqrt(result.variance || 0));
+    } else {
+      // Fallback to basic stats if no samples
+      meanNs = result.mean
+        ? ms_to_ns(result.mean)
+        : ms_to_ns(result.duration || 0);
+      stdevNs = result.stdDev ? ms_to_ns(result.stdDev) : 0;
+      sortedTimesNs = [meanNs]; // Single sample
+    }
+
+    const {
+      q1_ns,
+      q3_ns,
+      median_ns,
+      iqr_outlier_rounds,
+      stdev_outlier_rounds,
+    } = calculateQuantiles({ meanNs, stdevNs, sortedTimesNs });
+
+    return {
+      min_ns: result.min ? ms_to_ns(result.min) : sortedTimesNs[0] || meanNs,
+      max_ns: result.max
+        ? ms_to_ns(result.max)
+        : sortedTimesNs[sortedTimesNs.length - 1] || meanNs,
+      mean_ns: meanNs,
+      stdev_ns: stdevNs,
+      q1_ns,
+      median_ns,
+      q3_ns,
+      total_time: (result.duration || meanNs / 1_000_000) / 1000, // convert to seconds
+      iter_per_round: sortedTimesNs.length,
+      rounds: 1, // Vitest typically runs one round
+      iqr_outlier_rounds,
+      stdev_outlier_rounds,
+      warmup_iters: 0, // Will be filled from options if available
+    };
+  }
 }
 
-export default WalltimeRunner;
+export default WalltimeRunner;