Skip to content

Commit 7569b4d

Browse files
GuillaumeLagrangeGuillaumeLagrange
committed
--wip-- [skip ci]
1 parent bc1c260 commit 7569b4d

File tree

5 files changed

+418
-66
lines changed

5 files changed

+418
-66
lines changed

‎packages/core/src/index.ts

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,28 @@ export const isBound = native_core.isBound;
1010

1111
export const mongoMeasurement = new MongoMeasurement();
1212

13+
export enum MeasurementMode {
14+
Instrumentation = "instrumentation",
15+
WallTime = "walltime",
16+
}
17+
18+
export function getMeasurementMode(): MeasurementMode {
19+
// Check if CodSpeed is enabled
20+
const isCodSpeedEnabled = process.env.CODSPEED_ENV !== undefined;
21+
22+
if (isCodSpeedEnabled) {
23+
// If CODSPEED_ENV is set, check CODSPEED_RUNNER_MODE
24+
if (process.env.CODSPEED_RUNNER_MODE === "walltime") {
25+
return MeasurementMode.WallTime;
26+
} else {
27+
return MeasurementMode.Instrumentation;
28+
}
29+
} else {
30+
// Default to walltime mode when CODSPEED_ENV is not set
31+
return MeasurementMode.WallTime;
32+
}
33+
}
34+
1335
export const setupCore = () => {
1436
native_core.Measurement.stopInstrumentation(
1537
`Metadata: codspeed-node ${__VERSION__}`
@@ -29,4 +51,5 @@ export type {
2951
export { getV8Flags, tryIntrospect } from "./introspection";
3052
export { optimizeFunction, optimizeFunctionSync } from "./optimization";
3153
export * from "./utils";
54+
export * from "./walltime";
3255
export const Measurement = native_core.Measurement;

‎packages/core/src/walltime.ts

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
import fs from "fs";
2+
import path from "path";
3+
4+
declare const __VERSION__: string;
5+
6+
export interface BenchmarkStats {
7+
min_ns: number;
8+
max_ns: number;
9+
mean_ns: number;
10+
stdev_ns: number;
11+
q1_ns: number;
12+
median_ns: number;
13+
q3_ns: number;
14+
rounds: number;
15+
total_time: number;
16+
iqr_outlier_rounds: number;
17+
stdev_outlier_rounds: number;
18+
iter_per_round: number;
19+
warmup_iters: number;
20+
}
21+
22+
export interface BenchmarkConfig {
23+
warmup_time_ns: number;
24+
min_round_time_ns: number;
25+
max_time_ns: number;
26+
max_rounds: number | null;
27+
}
28+
29+
export interface Benchmark {
30+
name: string;
31+
uri: string;
32+
config: BenchmarkConfig;
33+
stats: BenchmarkStats;
34+
}
35+
36+
export interface InstrumentInfo {
37+
type: string;
38+
clock_info: {
39+
implementation: string;
40+
monotonic: boolean;
41+
adjustable: boolean;
42+
resolution: number;
43+
};
44+
}
45+
46+
export interface ResultData {
47+
creator: {
48+
name: string;
49+
version: string;
50+
pid: number;
51+
};
52+
instrument: InstrumentInfo;
53+
benchmarks: Benchmark[];
54+
}
55+
56+
export function getProfileFolder(): string | null {
57+
return process.env.CODSPEED_PROFILE_FOLDER || null;
58+
}
59+
60+
export function getCreatorMetadata() {
61+
return {
62+
creator: {
63+
name: "@codspeed/core",
64+
version: __VERSION__,
65+
pid: process.pid,
66+
},
67+
};
68+
}
69+
70+
export function getWalltimeInstrumentInfo(): InstrumentInfo {
71+
return {
72+
type: "walltime",
73+
clock_info: {
74+
implementation: "perf_counter",
75+
monotonic: true,
76+
adjustable: false,
77+
resolution: 1e-9, // nanosecond resolution
78+
},
79+
};
80+
}
81+
82+
export function writeWalltimeResults(
83+
benchmarks: Benchmark[],
84+
integrationName: string
85+
) {
86+
const profileFolder = getProfileFolder();
87+
let resultPath: string;
88+
89+
if (profileFolder) {
90+
const resultsDir = path.join(profileFolder, "results");
91+
fs.mkdirSync(resultsDir, { recursive: true });
92+
resultPath = path.join(resultsDir, `${process.pid}.json`);
93+
} else {
94+
// Fallback: write to .codspeed in current working directory
95+
const codspeedDir = path.join(process.cwd(), ".codspeed");
96+
fs.mkdirSync(codspeedDir, { recursive: true });
97+
resultPath = path.join(codspeedDir, `results_${Date.now()}.json`);
98+
}
99+
100+
const data: ResultData = {
101+
creator: {
102+
name: integrationName,
103+
version: __VERSION__,
104+
pid: process.pid,
105+
},
106+
instrument: getWalltimeInstrumentInfo(),
107+
benchmarks: benchmarks,
108+
};
109+
110+
fs.writeFileSync(resultPath, JSON.stringify(data, null, 2));
111+
console.log(`[CodSpeed] Results written to ${resultPath}`);
112+
}
113+
114+
export function createDefaultBenchmarkConfig(): BenchmarkConfig {
115+
return {
116+
warmup_time_ns: 1_000_000_000, // 1 second default
117+
min_round_time_ns: 1_000_000, // 1ms default
118+
max_time_ns: 3_000_000_000, // 3 seconds default
119+
max_rounds: null,
120+
};
121+
}

‎packages/tinybench-plugin/src/index.ts

Lines changed: 36 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -1,94 +1,64 @@
11
import {
22
getGitDir,
3+
getMeasurementMode,
34
Measurement,
5+
MeasurementMode,
46
mongoMeasurement,
5-
optimizeFunction,
6-
setupCore,
77
SetupInstrumentsRequestBody,
88
SetupInstrumentsResponse,
9-
teardownCore,
109
tryIntrospect,
1110
} from "@codspeed/core";
1211
import path from "path";
1312
import { get as getStackTrace } from "stack-trace";
14-
import { Bench, Task } from "tinybench";
13+
import { Bench } from "tinybench";
1514
import { fileURLToPath } from "url";
16-
17-
declare const __VERSION__: string;
15+
import { runInstrumentedBench } from "./instrumented";
16+
import { runWalltimeBench } from "./walltime";
1817

1918
tryIntrospect();
2019

21-
type CodSpeedBenchOptions = Task["opts"] & {
22-
uri: string;
23-
};
24-
25-
function isCodSpeedBenchOptions(
26-
options: Task["opts"]
27-
): options is CodSpeedBenchOptions {
28-
return "uri" in options;
29-
}
20+
// Store URI mapping externally since fnOpts is private
21+
export const taskUriMap = new WeakMap<Bench, Map<string, string>>();
3022

3123
export function withCodSpeed(bench: Bench): Bench {
32-
if (!Measurement.isInstrumented()) {
24+
const measurementMode = getMeasurementMode();
25+
const rootCallingFile = getCallingFile();
26+
27+
// Initialize URI mapping for this bench instance
28+
if (!taskUriMap.has(bench)) {
29+
taskUriMap.set(bench, new Map());
30+
}
31+
const uriMap = taskUriMap.get(bench)!;
32+
33+
// Setup URI generation for tasks
34+
const rawAdd = bench.add;
35+
bench.add = (name, fn, opts?) => {
36+
const callingFile = getCallingFile();
37+
const uri = `${callingFile}::${name}`;
38+
// Store URI mapping
39+
uriMap.set(name, uri);
40+
return rawAdd.bind(bench)(name, fn, opts);
41+
};
42+
43+
// Apply the appropriate measurement strategy based on mode and instrumentation
44+
if (
45+
measurementMode === MeasurementMode.Instrumentation &&
46+
Measurement.isInstrumented()
47+
) {
48+
runInstrumentedBench(bench, rootCallingFile);
49+
} else if (measurementMode === MeasurementMode.WallTime) {
50+
runWalltimeBench(bench, rootCallingFile);
51+
} else {
52+
// Fallback: instrumentation requested but not available
3353
const rawRun = bench.run;
3454
bench.run = async () => {
3555
console.warn(
3656
`[CodSpeed] ${bench.tasks.length} benches detected but no instrumentation found, falling back to tinybench`
3757
);
3858
return await rawRun.bind(bench)();
3959
};
40-
return bench;
4160
}
4261

43-
const rawAdd = bench.add;
44-
bench.add = (name, fn, opts: CodSpeedBenchOptions) => {
45-
const callingFile = getCallingFile();
46-
const uri = `${callingFile}::${name}`;
47-
const options = Object.assign({}, opts ?? {}, { uri });
48-
return rawAdd.bind(bench)(name, fn, options);
49-
};
50-
const rootCallingFile = getCallingFile();
51-
52-
bench.run = async () => {
53-
console.log(`[CodSpeed] running with @codspeed/tinybench v${__VERSION__}`);
54-
setupCore();
55-
for (const task of bench.tasks) {
56-
const uri = isCodSpeedBenchOptions(task.opts)
57-
? task.opts.uri
58-
: `${rootCallingFile}::${task.name}`;
59-
60-
await task.opts.beforeAll?.call(task);
61-
62-
// run optimizations
63-
await optimizeFunction(async () => {
64-
await task.opts.beforeEach?.call(task);
65-
await task.fn();
66-
await task.opts.afterEach?.call(task);
67-
});
68-
69-
// run instrumented benchmark
70-
await task.opts.beforeEach?.call(task);
71-
72-
await mongoMeasurement.start(uri);
73-
global.gc?.();
74-
await (async function __codspeed_root_frame__() {
75-
Measurement.startInstrumentation();
76-
await task.fn();
77-
Measurement.stopInstrumentation(uri);
78-
})();
79-
await mongoMeasurement.stop(uri);
80-
81-
await task.opts.afterEach?.call(task);
82-
83-
await task.opts.afterAll?.call(task);
84-
85-
// print results
86-
console.log(` ✔ Measured ${uri}`);
87-
}
88-
teardownCore();
89-
console.log(`[CodSpeed] Done running ${bench.tasks.length} benches.`);
90-
return bench.tasks;
91-
};
9262
return bench;
9363
}
9464

‎packages/tinybench-plugin/src/instrumented.ts

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
import {
2+
mongoMeasurement,
3+
optimizeFunction,
4+
setupCore,
5+
teardownCore,
6+
} from "@codspeed/core";
7+
import { Bench } from "tinybench";
8+
import { taskUriMap } from "./index";
9+
10+
declare const __VERSION__: string;
11+
12+
function getTaskUri(
13+
bench: Bench,
14+
taskName: string,
15+
rootCallingFile: string
16+
): string {
17+
const uriMap = taskUriMap.get(bench);
18+
return uriMap?.get(taskName) || `${rootCallingFile}::${taskName}`;
19+
}
20+
21+
export function runInstrumentedBench(
22+
bench: Bench,
23+
rootCallingFile: string
24+
): void {
25+
bench.run = async () => {
26+
console.log(
27+
`[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (instrumented mode)`
28+
);
29+
setupCore();
30+
31+
for (const task of bench.tasks) {
32+
const uri = getTaskUri(bench, task.name, rootCallingFile);
33+
34+
// Access private fnOpts to get hooks
35+
const fnOpts = (task as any).fnOpts;
36+
37+
// Call beforeAll hook if it exists
38+
await fnOpts?.beforeAll?.call(task, "run");
39+
40+
// run optimizations
41+
await optimizeFunction(async () => {
42+
await fnOpts?.beforeEach?.call(task, "run");
43+
await (task as any).fn(); // Access private fn property
44+
await fnOpts?.afterEach?.call(task, "run");
45+
});
46+
47+
// run instrumented benchmark
48+
await fnOpts?.beforeEach?.call(task, "run");
49+
50+
await mongoMeasurement.start(uri);
51+
global.gc?.();
52+
await (async function __codspeed_root_frame__() {
53+
Measurement.startInstrumentation();
54+
await task.fn();
55+
Measurement.stopInstrumentation(uri);
56+
})();
57+
await mongoMeasurement.stop(uri);
58+
59+
await fnOpts?.afterEach?.call(task, "run");
60+
61+
await fnOpts?.afterAll?.call(task, "run");
62+
63+
// print results
64+
console.log(` ✔ Measured ${uri}`);
65+
}
66+
67+
teardownCore();
68+
console.log(`[CodSpeed] Done running ${bench.tasks.length} benches.`);
69+
return bench.tasks;
70+
};
71+
}

0 commit comments

Comments
 (0)