Skip to content

Commit c567084

Browse files
GuillaumeLagrangeGuillaumeLagrange
committed
--wip-- [skip ci]
1 parent bc1c260 commit c567084

File tree

5 files changed

+409
-64
lines changed

5 files changed

+409
-64
lines changed

‎packages/core/src/index.ts

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,28 @@ export const isBound = native_core.isBound;
1010

1111
export const mongoMeasurement = new MongoMeasurement();
1212

13+
export enum MeasurementMode {
14+
Instrumentation = "instrumentation",
15+
WallTime = "walltime",
16+
}
17+
18+
export function getMeasurementMode(): MeasurementMode {
19+
// Check if CodSpeed is enabled
20+
const isCodSpeedEnabled = process.env.CODSPEED_ENV !== undefined;
21+
22+
if (isCodSpeedEnabled) {
23+
// If CODSPEED_ENV is set, check CODSPEED_RUNNER_MODE
24+
if (process.env.CODSPEED_RUNNER_MODE === "walltime") {
25+
return MeasurementMode.WallTime;
26+
} else {
27+
return MeasurementMode.Instrumentation;
28+
}
29+
} else {
30+
// Default to walltime mode when CODSPEED_ENV is not set
31+
return MeasurementMode.WallTime;
32+
}
33+
}
34+
1335
export const setupCore = () => {
1436
native_core.Measurement.stopInstrumentation(
1537
`Metadata: codspeed-node ${__VERSION__}`
@@ -29,4 +51,5 @@ export type {
2951
export { getV8Flags, tryIntrospect } from "./introspection";
3052
export { optimizeFunction, optimizeFunctionSync } from "./optimization";
3153
export * from "./utils";
54+
export * from "./walltime";
3255
export const Measurement = native_core.Measurement;

‎packages/core/src/walltime.ts

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
import fs from "fs";
2+
import path from "path";
3+
4+
declare const __VERSION__: string;
5+
6+
export interface BenchmarkStats {
7+
min_ns: number;
8+
max_ns: number;
9+
mean_ns: number;
10+
stdev_ns: number;
11+
q1_ns: number;
12+
median_ns: number;
13+
q3_ns: number;
14+
rounds: number;
15+
total_time: number;
16+
iqr_outlier_rounds: number;
17+
stdev_outlier_rounds: number;
18+
iter_per_round: number;
19+
warmup_iters: number;
20+
}
21+
22+
export interface BenchmarkConfig {
23+
warmup_time_ns: number;
24+
min_round_time_ns: number;
25+
max_time_ns: number;
26+
max_rounds: number | null;
27+
}
28+
29+
export interface Benchmark {
30+
name: string;
31+
uri: string;
32+
config: BenchmarkConfig;
33+
stats: BenchmarkStats;
34+
}
35+
36+
export interface InstrumentInfo {
37+
type: string;
38+
clock_info: {
39+
implementation: string;
40+
monotonic: boolean;
41+
adjustable: boolean;
42+
resolution: number;
43+
};
44+
}
45+
46+
export interface ResultData {
47+
creator: {
48+
name: string;
49+
version: string;
50+
pid: number;
51+
};
52+
instrument: InstrumentInfo;
53+
benchmarks: Benchmark[];
54+
}
55+
56+
export function getProfileFolder(): string | null {
57+
return process.env.CODSPEED_PROFILE_FOLDER || null;
58+
}
59+
60+
export function getCreatorMetadata() {
61+
return {
62+
creator: {
63+
name: "@codspeed/core",
64+
version: __VERSION__,
65+
pid: process.pid,
66+
},
67+
};
68+
}
69+
70+
export function getWalltimeInstrumentInfo(): InstrumentInfo {
71+
return {
72+
type: "walltime",
73+
clock_info: {
74+
implementation: "perf_counter",
75+
monotonic: true,
76+
adjustable: false,
77+
resolution: 1e-9, // nanosecond resolution
78+
},
79+
};
80+
}
81+
82+
export function writeWalltimeResults(
83+
benchmarks: Benchmark[],
84+
integrationName: string
85+
) {
86+
const profileFolder = getProfileFolder();
87+
let resultPath: string;
88+
89+
if (profileFolder) {
90+
const resultsDir = path.join(profileFolder, "results");
91+
fs.mkdirSync(resultsDir, { recursive: true });
92+
resultPath = path.join(resultsDir, `${process.pid}.json`);
93+
} else {
94+
// Fallback: write to .codspeed in current working directory
95+
const codspeedDir = path.join(process.cwd(), ".codspeed");
96+
fs.mkdirSync(codspeedDir, { recursive: true });
97+
resultPath = path.join(codspeedDir, `results_${Date.now()}.json`);
98+
}
99+
100+
const data: ResultData = {
101+
creator: {
102+
name: integrationName,
103+
version: __VERSION__,
104+
pid: process.pid,
105+
},
106+
instrument: getWalltimeInstrumentInfo(),
107+
benchmarks: benchmarks,
108+
};
109+
110+
fs.writeFileSync(resultPath, JSON.stringify(data, null, 2));
111+
console.log(`[CodSpeed] Results written to ${resultPath}`);
112+
}
113+
114+
export function createDefaultBenchmarkConfig(): BenchmarkConfig {
115+
return {
116+
warmup_time_ns: 1_000_000_000, // 1 second default
117+
min_round_time_ns: 1_000_000, // 1ms default
118+
max_time_ns: 3_000_000_000, // 3 seconds default
119+
max_rounds: null,
120+
};
121+
}

‎packages/tinybench-plugin/src/index.ts

Lines changed: 33 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1,94 +1,63 @@
11
import {
22
getGitDir,
3-
Measurement,
43
mongoMeasurement,
5-
optimizeFunction,
6-
setupCore,
74
SetupInstrumentsRequestBody,
85
SetupInstrumentsResponse,
9-
teardownCore,
106
tryIntrospect,
117
} from "@codspeed/core";
128
import path from "path";
139
import { get as getStackTrace } from "stack-trace";
1410
import { Bench, Task } from "tinybench";
1511
import { fileURLToPath } from "url";
12+
import { runInstrumentedBench } from "./instrumented";
13+
import { runWalltimeBench } from "./walltime";
1614

1715
declare const __VERSION__: string;
1816

1917
tryIntrospect();
2018

21-
type CodSpeedBenchOptions = Task["opts"] & {
22-
uri: string;
23-
};
24-
25-
function isCodSpeedBenchOptions(
26-
options: Task["opts"]
27-
): options is CodSpeedBenchOptions {
28-
return "uri" in options;
29-
}
19+
// Store URI mapping externally since fnOpts is private
20+
export const taskUriMap = new WeakMap<Bench, Map<string, string>>();
3021

3122
export function withCodSpeed(bench: Bench): Bench {
32-
if (!Measurement.isInstrumented()) {
23+
const measurementMode = getMeasurementMode();
24+
const rootCallingFile = getCallingFile();
25+
26+
// Initialize URI mapping for this bench instance
27+
if (!taskUriMap.has(bench)) {
28+
taskUriMap.set(bench, new Map());
29+
}
30+
const uriMap = taskUriMap.get(bench)!;
31+
32+
// Setup URI generation for tasks
33+
const rawAdd = bench.add;
34+
bench.add = (name, fn, opts?) => {
35+
const callingFile = getCallingFile();
36+
const uri = `${callingFile}::${name}`;
37+
// Store URI mapping
38+
uriMap.set(name, uri);
39+
return rawAdd.bind(bench)(name, fn, opts);
40+
};
41+
42+
// Apply the appropriate measurement strategy based on mode and instrumentation
43+
if (
44+
measurementMode === MeasurementMode.Instrumentation &&
45+
InstrumentHooks.isInstrumented()
46+
) {
47+
runInstrumentedBench(bench, rootCallingFile);
48+
} else if (measurementMode === MeasurementMode.WallTime) {
49+
runWalltimeBench(bench, rootCallingFile);
50+
} else {
51+
// Fallback: instrumentation requested but not available
3352
const rawRun = bench.run;
3453
bench.run = async () => {
3554
console.warn(
3655
`[CodSpeed] ${bench.tasks.length} benches detected but no instrumentation found, falling back to tinybench`
3756
);
3857
return await rawRun.bind(bench)();
3958
};
40-
return bench;
4159
}
4260

43-
const rawAdd = bench.add;
44-
bench.add = (name, fn, opts: CodSpeedBenchOptions) => {
45-
const callingFile = getCallingFile();
46-
const uri = `${callingFile}::${name}`;
47-
const options = Object.assign({}, opts ?? {}, { uri });
48-
return rawAdd.bind(bench)(name, fn, options);
49-
};
50-
const rootCallingFile = getCallingFile();
51-
52-
bench.run = async () => {
53-
console.log(`[CodSpeed] running with @codspeed/tinybench v${__VERSION__}`);
54-
setupCore();
55-
for (const task of bench.tasks) {
56-
const uri = isCodSpeedBenchOptions(task.opts)
57-
? task.opts.uri
58-
: `${rootCallingFile}::${task.name}`;
59-
60-
await task.opts.beforeAll?.call(task);
61-
62-
// run optimizations
63-
await optimizeFunction(async () => {
64-
await task.opts.beforeEach?.call(task);
65-
await task.fn();
66-
await task.opts.afterEach?.call(task);
67-
});
68-
69-
// run instrumented benchmark
70-
await task.opts.beforeEach?.call(task);
71-
72-
await mongoMeasurement.start(uri);
73-
global.gc?.();
74-
await (async function __codspeed_root_frame__() {
75-
Measurement.startInstrumentation();
76-
await task.fn();
77-
Measurement.stopInstrumentation(uri);
78-
})();
79-
await mongoMeasurement.stop(uri);
80-
81-
await task.opts.afterEach?.call(task);
82-
83-
await task.opts.afterAll?.call(task);
84-
85-
// print results
86-
console.log(` ✔ Measured ${uri}`);
87-
}
88-
teardownCore();
89-
console.log(`[CodSpeed] Done running ${bench.tasks.length} benches.`);
90-
return bench.tasks;
91-
};
9261
return bench;
9362
}
9463

‎packages/tinybench-plugin/src/instrumented.ts

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
import {
2+
InstrumentHooks,
3+
mongoMeasurement,
4+
optimizeFunction,
5+
setupCore,
6+
teardownCore,
7+
} from "@codspeed/core";
8+
import { Bench, Task } from "tinybench";
9+
import { taskUriMap } from "./index";
10+
11+
declare const __VERSION__: string;
12+
13+
function getTaskUri(bench: Bench, taskName: string, rootCallingFile: string): string {
14+
const uriMap = taskUriMap.get(bench);
15+
return uriMap?.get(taskName) || `${rootCallingFile}::${taskName}`;
16+
}
17+
18+
export function runInstrumentedBench(
19+
bench: Bench,
20+
rootCallingFile: string
21+
): void {
22+
bench.run = async () => {
23+
console.log(
24+
`[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (instrumented mode)`
25+
);
26+
setupCore();
27+
28+
for (const task of bench.tasks) {
29+
const uri = getTaskUri(bench, task.name, rootCallingFile);
30+
31+
// Access private fnOpts to get hooks
32+
const fnOpts = (task as any).fnOpts;
33+
34+
// Call beforeAll hook if it exists
35+
await fnOpts?.beforeAll?.call(task, "run");
36+
37+
// run optimizations
38+
await optimizeFunction(async () => {
39+
await fnOpts?.beforeEach?.call(task, "run");
40+
await (task as any).fn(); // Access private fn property
41+
await fnOpts?.afterEach?.call(task, "run");
42+
});
43+
44+
// run instrumented benchmark
45+
await fnOpts?.beforeEach?.call(task, "run");
46+
47+
await mongoMeasurement.start(uri);
48+
global.gc?.();
49+
await (async function __codspeed_root_frame__() {
50+
const benchmarkId = InstrumentHooks.startBenchmark();
51+
await (task as any).fn(); // Access private fn property
52+
InstrumentHooks.stopBenchmark();
53+
InstrumentHooks.setExecutedBenchmark(benchmarkId, uri);
54+
})();
55+
await mongoMeasurement.stop(uri);
56+
57+
await fnOpts?.afterEach?.call(task, "run");
58+
59+
await fnOpts?.afterAll?.call(task, "run");
60+
61+
// print results
62+
console.log(` ✔ Measured ${uri}`);
63+
}
64+
65+
teardownCore();
66+
console.log(`[CodSpeed] Done running ${bench.tasks.length} benches.`);
67+
return bench.tasks;
68+
};
69+
}

0 commit comments

Comments
 (0)