Skip to content

Commit acf0046

Browse files
GuillaumeLagrangeGuillaumeLagrange
committed
--wip-- [skip ci]
1 parent 3e49eb1 commit acf0046

File tree

3 files changed

+174
-175
lines changed

3 files changed

+174
-175
lines changed

‎packages/vitest-plugin/src/index.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,12 @@ export default function codspeedPlugin(): Plugin {
6565
// Only set custom runner when CODSPEED_ENV is set
6666
if (runnerFile) {
6767
config.test!.runner = runnerFile;
68+
69+
// Add CodSpeedReporter when in walltime mode
70+
const runnerMode = process.env.CODSPEED_RUNNER_MODE;
71+
if (runnerMode !== "instrumentation") {
72+
config.test!.reporters = [getCodSpeedFileFromName("reporter")];
73+
}
6874
}
6975

7076
return config;

‎packages/vitest-plugin/src/reporter.ts

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
import {
2+
calculateQuantiles,
3+
mongoMeasurement,
4+
writeWalltimeResults,
5+
type Benchmark as CoreBenchmark,
6+
type BenchmarkStats,
7+
} from "@codspeed/core";
8+
import type { Reporter, Task, TaskResultPack } from "vitest";
9+
10+
declare const __VERSION__: string;
11+
12+
interface VitestBenchmarkResult {
13+
name: string;
14+
options?: {
15+
iterations?: number;
16+
time?: number;
17+
warmupIterations?: number;
18+
warmupTime?: number;
19+
};
20+
result?: {
21+
error?: unknown;
22+
duration?: number;
23+
mean?: number;
24+
min?: number;
25+
max?: number;
26+
stdDev?: number;
27+
samples?: number[];
28+
variance?: number;
29+
rme?: number;
30+
};
31+
}
32+
33+
class CodSpeedReporter implements Reporter {
34+
private benchmarks: CoreBenchmark[] = [];
35+
private suiteNames = new Map<string, string>();
36+
37+
onInit() {
38+
console.log(`[CodSpeed] running with @codspeed/vitest-plugin v${__VERSION__} (walltime mode)`);
39+
}
40+
41+
onTaskUpdate(packs: TaskResultPack[]) {
42+
for (const pack of packs) {
43+
for (const result of pack) {
44+
this.processTask(result[1]);
45+
}
46+
}
47+
}
48+
49+
private processTask(task: Task) {
50+
// Build suite name path
51+
const suitePath = this.buildSuitePath(task);
52+
53+
if (task.type === "benchmark" && task.result?.state === "pass") {
54+
this.processBenchmarkTask(task, suitePath);
55+
}
56+
}
57+
58+
private buildSuitePath(task: Task): string {
59+
const parts: string[] = [];
60+
let current = task.suite;
61+
62+
while (current) {
63+
if (current.name && current.name !== task.file?.name) {
64+
parts.unshift(current.name);
65+
}
66+
current = current.suite;
67+
}
68+
69+
// Add the file path
70+
if (task.file?.name) {
71+
parts.unshift(task.file.name);
72+
}
73+
74+
return parts.join("::");
75+
}
76+
77+
private async processBenchmarkTask(task: Task, suitePath: string) {
78+
const uri = `${suitePath}::${task.name}`;
79+
80+
// Notify mongoMeasurement about the benchmark
81+
await mongoMeasurement.start(uri);
82+
await mongoMeasurement.stop(uri);
83+
84+
const benchmarkResult = task.meta?.benchmark as VitestBenchmarkResult | undefined;
85+
86+
if (benchmarkResult?.result) {
87+
const stats = this.convertVitestResultToBenchmarkStats(benchmarkResult.result);
88+
89+
const coreBenchmark: CoreBenchmark = {
90+
name: task.name,
91+
uri,
92+
config: {
93+
warmup_time_ns: benchmarkResult.options?.warmupTime
94+
? benchmarkResult.options.warmupTime * 1_000_000
95+
: null,
96+
min_round_time_ns: benchmarkResult.options?.time
97+
? benchmarkResult.options.time * 1_000_000
98+
: null,
99+
},
100+
stats,
101+
};
102+
103+
this.benchmarks.push(coreBenchmark);
104+
console.log(` ✔ Collected walltime data for ${uri}`);
105+
} else {
106+
console.warn(` ⚠ No result data available for ${uri}`);
107+
}
108+
}
109+
110+
private convertVitestResultToBenchmarkStats(result: NonNullable<VitestBenchmarkResult["result"]>): BenchmarkStats {
111+
// Convert milliseconds to nanoseconds
112+
const ms_to_ns = (ms: number) => ms * 1_000_000;
113+
114+
// Use samples if available, otherwise create from basic stats
115+
let sortedTimesNs: number[];
116+
let meanNs: number;
117+
let stdevNs: number;
118+
119+
if (result.samples && result.samples.length > 0) {
120+
sortedTimesNs = result.samples.map(ms_to_ns).sort((a, b) => a - b);
121+
meanNs = result.mean ? ms_to_ns(result.mean) : sortedTimesNs.reduce((a, b) => a + b, 0) / sortedTimesNs.length;
122+
stdevNs = result.stdDev ? ms_to_ns(result.stdDev) : ms_to_ns(Math.sqrt(result.variance || 0));
123+
} else {
124+
// Fallback to basic stats if no samples
125+
meanNs = result.mean ? ms_to_ns(result.mean) : ms_to_ns(result.duration || 0);
126+
stdevNs = result.stdDev ? ms_to_ns(result.stdDev) : 0;
127+
sortedTimesNs = [meanNs]; // Single sample
128+
}
129+
130+
const { q1_ns, q3_ns, median_ns, iqr_outlier_rounds, stdev_outlier_rounds } =
131+
calculateQuantiles({ meanNs, stdevNs, sortedTimesNs });
132+
133+
return {
134+
min_ns: result.min ? ms_to_ns(result.min) : sortedTimesNs[0] || meanNs,
135+
max_ns: result.max ? ms_to_ns(result.max) : sortedTimesNs[sortedTimesNs.length - 1] || meanNs,
136+
mean_ns: meanNs,
137+
stdev_ns: stdevNs,
138+
q1_ns,
139+
median_ns,
140+
q3_ns,
141+
total_time: (result.duration || meanNs / 1_000_000) / 1000, // convert to seconds
142+
iter_per_round: sortedTimesNs.length,
143+
rounds: 1, // Vitest typically runs one round
144+
iqr_outlier_rounds,
145+
stdev_outlier_rounds,
146+
warmup_iters: 0, // Will be filled from options if available
147+
};
148+
}
149+
150+
onFinished() {
151+
// Write results to JSON file using core function
152+
if (this.benchmarks.length > 0) {
153+
writeWalltimeResults(this.benchmarks);
154+
console.log(`[CodSpeed] Done collecting walltime data for ${this.benchmarks.length} benches.`);
155+
}
156+
}
157+
}
158+
159+
export default CodSpeedReporter;

‎packages/vitest-plugin/src/walltime.ts

Lines changed: 9 additions & 175 deletions
Original file line numberDiff line numberDiff line change
@@ -1,196 +1,30 @@
1-
import {
2-
calculateQuantiles,
3-
logDebug,
4-
mongoMeasurement,
5-
setupCore,
6-
teardownCore,
7-
writeWalltimeResults,
8-
type Benchmark as CoreBenchmark,
9-
type BenchmarkStats,
10-
} from "@codspeed/core";
11-
import { Benchmark, Suite } from "vitest";
1+
import { logDebug, setupCore, teardownCore } from "@codspeed/core";
2+
import { Suite } from "vitest";
123
import { NodeBenchmarkRunner } from "vitest/runners";
13-
import { getBenchFn } from "vitest/suite";
14-
import { callSuiteHook, patchRootSuiteWithFullFilePath } from "./common";
15-
16-
declare const __VERSION__: string;
4+
import { patchRootSuiteWithFullFilePath } from "./common";
175

186
const currentFileName =
197
typeof __filename === "string"
208
? __filename
219
: new URL("walltime.mjs", import.meta.url).pathname;
2210

2311
/**
24-
* @deprecated
25-
* TODO: try to use something like `updateTask` from `@vitest/runner` instead to use the output
26-
* of vitest instead console.log but at the moment, `updateTask` is not exposed
12+
* WalltimeRunner uses Vitest's default benchmark execution
13+
* and relies on the CodSpeedReporter to capture and process results
2714
*/
28-
function logCodSpeed(message: string) {
29-
console.log(`[CodSpeed] ${message}`);
30-
}
31-
32-
async function runWalltimeBench(
33-
benchmark: Benchmark,
34-
currentSuiteName: string
35-
): Promise<CoreBenchmark | null> {
36-
const uri = `${currentSuiteName}::${benchmark.name}`;
37-
const fn = getBenchFn(benchmark);
38-
39-
// Run the benchmark function multiple times to collect timing data
40-
const samples: number[] = [];
41-
const iterations = 100; // Similar to tinybench default
42-
43-
// Warmup phase
44-
await callSuiteHook(benchmark.suite, benchmark, "beforeEach");
45-
for (let i = 0; i < 10; i++) {
46-
await fn();
47-
}
48-
await callSuiteHook(benchmark.suite, benchmark, "afterEach");
49-
50-
// Measurement phase
51-
await mongoMeasurement.start(uri);
52-
53-
for (let i = 0; i < iterations; i++) {
54-
await callSuiteHook(benchmark.suite, benchmark, "beforeEach");
55-
56-
const startTime = process.hrtime.bigint();
57-
await fn();
58-
const endTime = process.hrtime.bigint();
59-
60-
await callSuiteHook(benchmark.suite, benchmark, "afterEach");
61-
62-
// Convert nanoseconds to milliseconds for consistency with tinybench
63-
const durationMs = Number(endTime - startTime) / 1_000_000;
64-
samples.push(durationMs);
65-
}
66-
67-
await mongoMeasurement.stop(uri);
68-
69-
if (samples.length === 0) {
70-
console.warn(` ⚠ No timing data available for ${uri}`);
71-
return null;
72-
}
73-
74-
// Convert to BenchmarkStats format
75-
const stats = convertSamplesToBenchmarkStats(samples);
76-
77-
const coreBenchmark: CoreBenchmark = {
78-
name: benchmark.name,
79-
uri,
80-
config: {
81-
warmup_time_ns: null, // Not configurable in Vitest
82-
min_round_time_ns: null, // Not configurable in Vitest
83-
},
84-
stats,
85-
};
86-
87-
console.log(` ✔ Collected walltime data for ${uri}`);
88-
return coreBenchmark;
89-
}
90-
91-
function convertSamplesToBenchmarkStats(samples: number[]): BenchmarkStats {
92-
// All samples are in milliseconds, convert to nanoseconds
93-
const ms_to_ns = (ms: number) => ms * 1_000_000;
94-
95-
const sortedTimesNs = samples.map(ms_to_ns).sort((a, b) => a - b);
96-
const meanMs = samples.reduce((a, b) => a + b, 0) / samples.length;
97-
const meanNs = ms_to_ns(meanMs);
98-
99-
// Calculate standard deviation
100-
const variance =
101-
samples.reduce((acc, val) => acc + Math.pow(val - meanMs, 2), 0) /
102-
samples.length;
103-
const stdevMs = Math.sqrt(variance);
104-
const stdevNs = ms_to_ns(stdevMs);
105-
106-
const { q1_ns, q3_ns, median_ns, iqr_outlier_rounds, stdev_outlier_rounds } =
107-
calculateQuantiles({ meanNs, stdevNs, sortedTimesNs });
108-
109-
return {
110-
min_ns: sortedTimesNs[0],
111-
max_ns: sortedTimesNs[sortedTimesNs.length - 1],
112-
mean_ns: meanNs,
113-
stdev_ns: stdevNs,
114-
q1_ns,
115-
median_ns,
116-
q3_ns,
117-
total_time: samples.reduce((a, b) => a + b, 0) / 1000, // convert from ms to seconds
118-
iter_per_round: sortedTimesNs.length,
119-
rounds: 1, // Single round like tinybench
120-
iqr_outlier_rounds,
121-
stdev_outlier_rounds,
122-
warmup_iters: 10, // Fixed warmup iterations
123-
};
124-
}
125-
126-
async function runWalltimeBenchmarkSuite(
127-
suite: Suite,
128-
parentSuiteName?: string
129-
): Promise<CoreBenchmark[]> {
130-
const currentSuiteName = parentSuiteName
131-
? parentSuiteName + "::" + suite.name
132-
: suite.name;
133-
134-
const benchmarks: CoreBenchmark[] = [];
135-
136-
// do not call `beforeAll` if we are in the root suite, since it is already called by vitest
137-
if (parentSuiteName !== undefined) {
138-
await callSuiteHook(suite, suite, "beforeAll");
139-
}
140-
141-
for (const task of suite.tasks) {
142-
if (task.mode !== "run") continue;
143-
144-
if (task.meta?.benchmark) {
145-
const benchmark = await runWalltimeBench(
146-
task as Benchmark,
147-
currentSuiteName
148-
);
149-
if (benchmark) {
150-
benchmarks.push(benchmark);
151-
}
152-
} else if (task.type === "suite") {
153-
const nestedBenchmarks = await runWalltimeBenchmarkSuite(
154-
task,
155-
currentSuiteName
156-
);
157-
benchmarks.push(...nestedBenchmarks);
158-
}
159-
}
160-
161-
// do not call `afterAll` if we are in the root suite, since it is already called by vitest
162-
if (parentSuiteName !== undefined) {
163-
await callSuiteHook(suite, suite, "afterAll");
164-
}
165-
166-
return benchmarks;
167-
}
168-
16915
export class WalltimeRunner extends NodeBenchmarkRunner {
17016
async runSuite(suite: Suite): Promise<void> {
17117
logDebug(`PROCESS PID: ${process.pid} in ${currentFileName}`);
17218
setupCore();
17319

17420
patchRootSuiteWithFullFilePath(suite);
17521

176-
console.log(
177-
`[CodSpeed] running with @codspeed/vitest-plugin v${__VERSION__} (walltime mode)`
178-
);
179-
logCodSpeed(`running suite ${suite.name}`);
180-
181-
const benchmarks = await runWalltimeBenchmarkSuite(suite);
182-
183-
// Write results to JSON file using core function
184-
if (benchmarks.length > 0) {
185-
writeWalltimeResults(benchmarks);
186-
}
187-
188-
console.log(
189-
`[CodSpeed] Done collecting walltime data for ${benchmarks.length} benches.`
190-
);
191-
logCodSpeed(`running suite ${suite.name} done`);
22+
// Let Vitest's default benchmark runner handle execution
23+
// Results will be captured by CodSpeedReporter via onTaskUpdate
24+
await super.runSuite(suite);
19225

19326
teardownCore();
19427
}
19528
}
19629

30+
export default WalltimeRunner;

0 commit comments

Comments
 (0)