1
1
import {
2
2
calculateQuantiles ,
3
+ msToNs ,
4
+ msToS ,
3
5
writeWalltimeResults ,
4
6
type Benchmark ,
5
7
type BenchmarkStats ,
6
8
} from "@codspeed/core" ;
7
- import { BenchTaskResult , Suite , Task } from "vitest" ;
9
+ import { BenchTaskResult , Suite , type Custom } from "vitest" ;
8
10
import { NodeBenchmarkRunner } from "vitest/runners" ;
11
+ import { getBenchOptions } from "vitest/suite" ;
9
12
import { patchRootSuiteWithFullFilePath } from "./common" ;
10
13
11
14
declare const __VERSION__ : string ;
@@ -58,7 +61,11 @@ export class WalltimeRunner extends NodeBenchmarkRunner {
58
61
: suite . name ;
59
62
60
63
for ( const task of suite . tasks ) {
61
- if ( task . meta ?. benchmark && task . result ?. state === "pass" ) {
64
+ if (
65
+ task . meta . benchmark &&
66
+ task . type === "custom" &&
67
+ task . result ?. state === "pass"
68
+ ) {
62
69
const benchmark = await this . processBenchmarkTask ( task , currentPath ) ;
63
70
if ( benchmark ) {
64
71
benchmarks . push ( benchmark ) ;
@@ -76,7 +83,7 @@ export class WalltimeRunner extends NodeBenchmarkRunner {
76
83
}
77
84
78
85
private async processBenchmarkTask (
79
- task : Task ,
86
+ task : Custom ,
80
87
suitePath : string
81
88
) : Promise < Benchmark | null > {
82
89
const uri = `${ suitePath } ::${ task . name } ` ;
@@ -88,8 +95,12 @@ export class WalltimeRunner extends NodeBenchmarkRunner {
88
95
}
89
96
90
97
try {
98
+ // Get tinybench configuration options from vitest
99
+ const benchOptions = getBenchOptions ( task ) ;
100
+
91
101
const stats = this . convertVitestResultToBenchmarkStats (
92
- result as VitestTaskResult
102
+ result as VitestTaskResult ,
103
+ benchOptions
93
104
) ;
94
105
95
106
if ( stats === null ) {
@@ -101,8 +112,13 @@ export class WalltimeRunner extends NodeBenchmarkRunner {
101
112
name : task . name ,
102
113
uri,
103
114
config : {
104
- warmup_time_ns : null , // Vitest doesn't expose this in task.result
105
- min_round_time_ns : null , // Vitest doesn't expose this in task.result
115
+ max_rounds : benchOptions . iterations ?? null ,
116
+ max_time_ns : benchOptions . time ? msToNs ( benchOptions . time ) : null ,
117
+ min_round_time_ns : null , // tinybench does not have an option for this
118
+ warmup_time_ns :
119
+ benchOptions . warmupIterations !== 0 && benchOptions . warmupTime
120
+ ? msToNs ( benchOptions . warmupTime )
121
+ : null ,
106
122
} ,
107
123
stats,
108
124
} ;
@@ -119,23 +135,26 @@ export class WalltimeRunner extends NodeBenchmarkRunner {
119
135
}
120
136
121
137
private convertVitestResultToBenchmarkStats (
122
- result : VitestTaskResult
138
+ result : VitestTaskResult ,
139
+ benchOptions : {
140
+ time ?: number ;
141
+ warmupTime ?: number ;
142
+ warmupIterations ?: number ;
143
+ iterations ?: number ;
144
+ }
123
145
) : BenchmarkStats | null {
124
146
const benchmark = result . benchmark ;
125
147
126
148
if ( ! benchmark ) {
127
149
throw new Error ( "No benchmark data available in result" ) ;
128
150
}
129
151
130
- // All tinybench times are in milliseconds, convert to nanoseconds
131
- const ms_to_ns = ( ms : number ) => ms * 1_000_000 ;
132
-
133
152
const { totalTime, min, max, mean, sd, samples } = benchmark ;
134
153
135
154
// Get individual sample times in nanoseconds and sort them
136
- const sortedTimesNs = samples . map ( ms_to_ns ) . sort ( ( a , b ) => a - b ) ;
137
- const meanNs = ms_to_ns ( mean ) ;
138
- const stdevNs = ms_to_ns ( sd ) ;
155
+ const sortedTimesNs = samples . map ( msToNs ) . sort ( ( a , b ) => a - b ) ;
156
+ const meanNs = msToNs ( mean ) ;
157
+ const stdevNs = msToNs ( sd ) ;
139
158
140
159
if ( sortedTimesNs . length == 0 ) {
141
160
// Sometimes the benchmarks can be completely optimized out and not even run, but its beforeEach and afterEach hooks are still executed, and the task is still considered a success.
@@ -152,19 +171,19 @@ export class WalltimeRunner extends NodeBenchmarkRunner {
152
171
} = calculateQuantiles ( { meanNs, stdevNs, sortedTimesNs } ) ;
153
172
154
173
return {
155
- min_ns : ms_to_ns ( min ) ,
156
- max_ns : ms_to_ns ( max ) ,
174
+ min_ns : msToNs ( min ) ,
175
+ max_ns : msToNs ( max ) ,
157
176
mean_ns : meanNs ,
158
177
stdev_ns : stdevNs ,
159
178
q1_ns,
160
179
median_ns,
161
180
q3_ns,
162
- total_time : totalTime / 1_000 , // convert from ms to seconds
163
- iter_per_round : sortedTimesNs . length ,
164
- rounds : 1 , // Tinybench only runs one round
181
+ total_time : msToS ( totalTime ) ,
182
+ iter_per_round : 1 , // as there is only one round in tinybench, we define that there were n rounds of 1 iteration
183
+ rounds : sortedTimesNs . length ,
165
184
iqr_outlier_rounds,
166
185
stdev_outlier_rounds,
167
- warmup_iters : 0 , // TODO: get warmup iters here
186
+ warmup_iters : benchOptions . warmupIterations ?? 0 ,
168
187
} ;
169
188
}
170
189
}
0 commit comments