Skip to content

Commit 52b8163

Browse files
committed
fixup! feat(vitest-plugin): add walltime support
1 parent 4409831 commit 52b8163

File tree

2 files changed

+65
-29
lines changed

2 files changed

+65
-29
lines changed
Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,36 @@
1-
import { bench, describe } from "vitest";
1+
import { bench, describe, type BenchOptions } from "vitest";
22

33
const sleep = (ms: number): Promise<void> => {
44
return new Promise((resolve) => setTimeout(resolve, ms));
55
};
66

7+
const timingBenchOptions: BenchOptions = {
8+
iterations: 5,
9+
warmupIterations: 0,
10+
};
11+
712
describe("timing tests", () => {
8-
bench("wait 1ms", async () => {
9-
await sleep(1);
10-
});
13+
bench(
14+
"wait 1ms",
15+
async () => {
16+
await sleep(1);
17+
},
18+
timingBenchOptions
19+
);
1120

12-
bench("wait 500ms", async () => {
13-
await sleep(500);
14-
});
21+
bench(
22+
"wait 500ms",
23+
async () => {
24+
await sleep(500);
25+
},
26+
timingBenchOptions
27+
);
1528

16-
bench("wait 1sec", async () => {
17-
await sleep(1_000);
18-
});
29+
bench(
30+
"wait 1sec",
31+
async () => {
32+
await sleep(1_000);
33+
},
34+
timingBenchOptions
35+
);
1936
});

packages/vitest-plugin/src/walltime.ts

Lines changed: 38 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
import {
22
calculateQuantiles,
3+
msToNs,
4+
msToS,
35
writeWalltimeResults,
46
type Benchmark,
57
type BenchmarkStats,
68
} from "@codspeed/core";
7-
import { BenchTaskResult, Suite, Task } from "vitest";
9+
import { BenchTaskResult, Suite, type Custom } from "vitest";
810
import { NodeBenchmarkRunner } from "vitest/runners";
11+
import { getBenchOptions } from "vitest/suite";
912
import { patchRootSuiteWithFullFilePath } from "./common";
1013

1114
declare const __VERSION__: string;
@@ -58,7 +61,11 @@ export class WalltimeRunner extends NodeBenchmarkRunner {
5861
: suite.name;
5962

6063
for (const task of suite.tasks) {
61-
if (task.meta?.benchmark && task.result?.state === "pass") {
64+
if (
65+
task.meta.benchmark &&
66+
task.type === "custom" &&
67+
task.result?.state === "pass"
68+
) {
6269
const benchmark = await this.processBenchmarkTask(task, currentPath);
6370
if (benchmark) {
6471
benchmarks.push(benchmark);
@@ -76,7 +83,7 @@ export class WalltimeRunner extends NodeBenchmarkRunner {
7683
}
7784

7885
private async processBenchmarkTask(
79-
task: Task,
86+
task: Custom,
8087
suitePath: string
8188
): Promise<Benchmark | null> {
8289
const uri = `${suitePath}::${task.name}`;
@@ -88,8 +95,12 @@ export class WalltimeRunner extends NodeBenchmarkRunner {
8895
}
8996

9097
try {
98+
// Get tinybench configuration options from vitest
99+
const benchOptions = getBenchOptions(task);
100+
91101
const stats = this.convertVitestResultToBenchmarkStats(
92-
result as VitestTaskResult
102+
result as VitestTaskResult,
103+
benchOptions
93104
);
94105

95106
if (stats === null) {
@@ -101,8 +112,13 @@ export class WalltimeRunner extends NodeBenchmarkRunner {
101112
name: task.name,
102113
uri,
103114
config: {
104-
warmup_time_ns: null, // Vitest doesn't expose this in task.result
105-
min_round_time_ns: null, // Vitest doesn't expose this in task.result
115+
max_rounds: benchOptions.iterations ?? null,
116+
max_time_ns: benchOptions.time ? msToNs(benchOptions.time) : null,
117+
min_round_time_ns: null, // tinybench does not have an option for this
118+
warmup_time_ns:
119+
benchOptions.warmupIterations !== 0 && benchOptions.warmupTime
120+
? msToNs(benchOptions.warmupTime)
121+
: null,
106122
},
107123
stats,
108124
};
@@ -119,23 +135,26 @@ export class WalltimeRunner extends NodeBenchmarkRunner {
119135
}
120136

121137
private convertVitestResultToBenchmarkStats(
122-
result: VitestTaskResult
138+
result: VitestTaskResult,
139+
benchOptions: {
140+
time?: number;
141+
warmupTime?: number;
142+
warmupIterations?: number;
143+
iterations?: number;
144+
}
123145
): BenchmarkStats | null {
124146
const benchmark = result.benchmark;
125147

126148
if (!benchmark) {
127149
throw new Error("No benchmark data available in result");
128150
}
129151

130-
// All tinybench times are in milliseconds, convert to nanoseconds
131-
const ms_to_ns = (ms: number) => ms * 1_000_000;
132-
133152
const { totalTime, min, max, mean, sd, samples } = benchmark;
134153

135154
// Get individual sample times in nanoseconds and sort them
136-
const sortedTimesNs = samples.map(ms_to_ns).sort((a, b) => a - b);
137-
const meanNs = ms_to_ns(mean);
138-
const stdevNs = ms_to_ns(sd);
155+
const sortedTimesNs = samples.map(msToNs).sort((a, b) => a - b);
156+
const meanNs = msToNs(mean);
157+
const stdevNs = msToNs(sd);
139158

140159
if (sortedTimesNs.length == 0) {
141160
// Sometimes the benchmarks can be completely optimized out and not even run, but its beforeEach and afterEach hooks are still executed, and the task is still considered a success.
@@ -152,19 +171,19 @@ export class WalltimeRunner extends NodeBenchmarkRunner {
152171
} = calculateQuantiles({ meanNs, stdevNs, sortedTimesNs });
153172

154173
return {
155-
min_ns: ms_to_ns(min),
156-
max_ns: ms_to_ns(max),
174+
min_ns: msToNs(min),
175+
max_ns: msToNs(max),
157176
mean_ns: meanNs,
158177
stdev_ns: stdevNs,
159178
q1_ns,
160179
median_ns,
161180
q3_ns,
162-
total_time: totalTime / 1_000, // convert from ms to seconds
163-
iter_per_round: sortedTimesNs.length,
164-
rounds: 1, // Tinybench only runs one round
181+
total_time: msToS(totalTime),
182+
iter_per_round: 1, // as there is only one round in tinybench, we define that there were n rounds of 1 iteration
183+
rounds: sortedTimesNs.length,
165184
iqr_outlier_rounds,
166185
stdev_outlier_rounds,
167-
warmup_iters: 0, // TODO: get warmup iters here
186+
warmup_iters: benchOptions.warmupIterations ?? 0,
168187
};
169188
}
170189
}

0 commit comments

Comments
 (0)