diff --git a/test/benchmarks/driver_bench/src/driver.mts b/test/benchmarks/driver_bench/src/driver.mts index 434c5b9d51b..d21048cf8f7 100644 --- a/test/benchmarks/driver_bench/src/driver.mts +++ b/test/benchmarks/driver_bench/src/driver.mts @@ -8,17 +8,18 @@ const __dirname = import.meta.dirname; const require = module.createRequire(__dirname); export const TAG = { - // Special tag that marks a benchmark as a spec-required benchmark + /** Special tag that marks a benchmark as a spec-required benchmark */ spec: 'spec-benchmark', - // Special tag that enables our perf monitoring tooling to create alerts when regressions in this - // benchmark's performance are detected + /** Special tag that enables our perf monitoring tooling to create alerts when regressions in this benchmark's performance are detected */ alert: 'alerting-benchmark', - // Tag marking a benchmark as being related to cursor performance + /** Tag marking a benchmark as being related to cursor performance */ cursor: 'cursor-benchmark', - // Tag marking a benchmark as being related to read performance + /** Tag marking a benchmark as being related to read performance */ read: 'read-benchmark', - // Tag marking a benchmark as being related to write performance - write: 'write-benchmark' + /** Tag marking a benchmark as being related to write performance */ + write: 'write-benchmark', + /** A tag for the cpu baseline task */ + reference: 'reference' }; /** @@ -135,7 +136,7 @@ export type Metric = { name: 'megabytes_per_second' | 'normalized_throughput'; value: number; metadata: { - tags?: string[]; + tags: ReadonlyArray; improvement_direction: 'up' | 'down'; }; }; @@ -150,7 +151,11 @@ export type MetricInfo = { metrics: Metric[]; }; -export function metrics(test_name: string, result: number, tags?: string[]): MetricInfo { +export function metrics( + test_name: string, + result: number, + tags: ReadonlyArray +): MetricInfo { return { info: { test_name, diff --git a/test/benchmarks/driver_bench/src/main.mts b/test/benchmarks/driver_bench/src/main.mts index 90e58e584c7..251ae42b4b3 100644 --- a/test/benchmarks/driver_bench/src/main.mts +++ b/test/benchmarks/driver_bench/src/main.mts @@ -18,7 +18,8 @@ import { MONGODB_DRIVER_PATH, MONGODB_DRIVER_REVISION, MONGODB_DRIVER_VERSION, - snakeToCamel + snakeToCamel, + TAG } from './driver.mjs'; const __dirname = import.meta.dirname; @@ -176,7 +177,7 @@ function calculateCompositeBenchmarks(results: MetricInfo[]) { if (compositeName === 'readBench') readBenchResult = compositeAverage; if (compositeName === 'writeBench') writeBenchResult = compositeAverage; - compositeResults.push(metrics(compositeName, compositeAverage)); + compositeResults.push(metrics(compositeName, compositeAverage, [TAG.spec])); console.log('avg:', compositeAverage, 'mb/s'); @@ -192,7 +193,7 @@ function calculateCompositeBenchmarks(results: MetricInfo[]) { console.log('avg:', driverBench, 'mb/s'); console.groupEnd(); - compositeResults.push(metrics('driverBench', driverBench)); + compositeResults.push(metrics('driverBench', driverBench, [TAG.spec])); console.groupEnd(); return [...results, ...compositeResults]; diff --git a/test/benchmarks/driver_bench/src/runner.mts b/test/benchmarks/driver_bench/src/runner.mts index b68bc9cfe49..58490de2e03 100644 --- a/test/benchmarks/driver_bench/src/runner.mts +++ b/test/benchmarks/driver_bench/src/runner.mts @@ -9,12 +9,13 @@ const [, , benchmarkFile] = process.argv; type BenchmarkModule = { taskSize: number; + tags: ReadonlyArray; + before?: () => Promise; beforeEach?: () => Promise; run: () => Promise; afterEach?: () => Promise; after?: () => Promise; - tags?: string[]; }; const benchmarkName = snakeToCamel(path.basename(benchmarkFile, '.mjs')); @@ -22,6 +23,10 @@ const benchmark: BenchmarkModule = await import(`./${benchmarkFile}`); if (typeof benchmark.taskSize !== 'number') throw new Error('missing taskSize'); if (typeof benchmark.run !== 'function') throw new Error('missing run'); +if (!Array.isArray(benchmark.tags)) throw new Error('tags must be an array'); +if (benchmark.tags.length === 0 || !benchmark.tags.every(t => typeof t === 'string')) { + throw new Error('must have more than one tag and all tags must be strings'); +} /** CRITICAL SECTION: time task took in seconds */ async function timeTask() { @@ -81,14 +86,6 @@ function percentileIndex(percentile: number, count: number) { const medianExecution = durations[percentileIndex(50, count)]; const megabytesPerSecond = benchmark.taskSize / medianExecution; -const tags = benchmark.tags; -if ( - tags && - (!Array.isArray(tags) || (tags.length > 0 && !tags.every(t => typeof t === 'string'))) -) { - throw new Error('If tags is specified, it MUST be an array of strings'); -} - console.log( ' '.repeat(3), ...['total time:', totalDuration, 'sec,'], @@ -100,6 +97,6 @@ console.log( await fs.writeFile( `results_${path.basename(benchmarkFile, '.mjs')}.json`, - JSON.stringify(metrics(benchmarkName, megabytesPerSecond, tags), undefined, 2) + '\n', + JSON.stringify(metrics(benchmarkName, megabytesPerSecond, benchmark.tags), undefined, 2) + '\n', 'utf8' ); diff --git a/test/benchmarks/driver_bench/src/suites/node_specific/cpu_baseline.mts b/test/benchmarks/driver_bench/src/suites/node_specific/cpu_baseline.mts index 9760f90921b..3b4300519a8 100644 --- a/test/benchmarks/driver_bench/src/suites/node_specific/cpu_baseline.mts +++ b/test/benchmarks/driver_bench/src/suites/node_specific/cpu_baseline.mts @@ -1,5 +1,7 @@ import assert from 'node:assert/strict'; +import { TAG } from '../../driver.mjs'; + const findPrimesBelow = 1_000_000; const expectedPrimes = 78_498; @@ -9,6 +11,9 @@ const expectedPrimes = 78_498; const stableRegionMean = 42.82; export const taskSize = 3.1401000000000003 / stableRegionMean; // ~3MB worth of work scaled down by the mean of the current stable region in CI to bring this value to roughly 1 + +export const tags = [TAG.reference]; + /** @see https://en.wikipedia.org/wiki/Sieve_of_Eratosthenes */ export function sieveOfEratosthenes(n: number) { // Create a boolean array "prime[0..n]" and initialize