diff --git a/test/benchmarks/driver_bench/src/driver.mts b/test/benchmarks/driver_bench/src/driver.mts index 358b10af67e..01674fbc0b5 100644 --- a/test/benchmarks/driver_bench/src/driver.mts +++ b/test/benchmarks/driver_bench/src/driver.mts @@ -112,7 +112,20 @@ const COLLECTION_NAME = 'corpus'; const SPEC_DIRECTORY = path.resolve(__dirname, '..', '..', 'driverBench', 'spec'); -export function metrics(test_name: string, result: number, count: number) { +export type Metric = { + name: 'megabytes_per_second'; + value: number; +}; + +export type MetricInfo = { + info: { + test_name: string; + args: Record; + }; + metrics: Metric[]; +}; + +export function metrics(test_name: string, result: number): MetricInfo { return { info: { test_name, @@ -125,11 +138,7 @@ export function metrics(test_name: string, result: number, count: number) { ]) ) }, - metrics: [ - { name: 'megabytes_per_second', value: result }, - // Reporting the count so we can verify programmatically or in UI how many iterations we reached - { name: 'count', value: count } - ] + metrics: [{ name: 'megabytes_per_second', value: result }] } as const; } diff --git a/test/benchmarks/driver_bench/src/main.mts b/test/benchmarks/driver_bench/src/main.mts index 2801ff421af..8de6a6a8f89 100644 --- a/test/benchmarks/driver_bench/src/main.mts +++ b/test/benchmarks/driver_bench/src/main.mts @@ -1,4 +1,5 @@ /* eslint-disable no-console */ +import assert from 'node:assert/strict'; import child_process from 'node:child_process'; import events from 'node:events'; import fs from 'node:fs/promises'; @@ -7,6 +8,9 @@ import path from 'node:path'; import util from 'node:util'; import { + type Metric, + type MetricInfo, + metrics, MONGODB_BSON_PATH, MONGODB_BSON_REVISION, MONGODB_BSON_VERSION, @@ -81,7 +85,7 @@ console.log(systemInfo()); const runnerPath = path.join(__dirname, 'runner.mjs'); -const results = []; +const results: MetricInfo[] = []; for (const [suite, benchmarks] of Object.entries(tests)) { console.group(snakeToCamel(suite)); @@ -106,4 +110,94 @@ for (const [suite, benchmarks] of Object.entries(tests)) { console.groupEnd(); } -await fs.writeFile('results.json', JSON.stringify(results, undefined, 2), 'utf8'); +function calculateCompositeBenchmarks(results: MetricInfo[]) { + const composites = { + singleBench: ['findOne', 'smallDocInsertOne', 'largeDocInsertOne'], + multiBench: [ + 'findManyAndEmptyCursor', + 'gridFsDownload', + 'gridFsUpload', + 'largeDocBulkInsert', + 'smallDocBulkInsert' + ], + // parallelBench: [ + // 'ldjsonMultiFileUpload', + // 'ldjsonMultiFileExport', + // 'gridfsMultiFileUpload', + // 'gridfsMultiFileDownload' + // ], + readBench: [ + 'findOne', + 'findManyAndEmptyCursor', + 'gridFsDownload' + // 'gridfsMultiFileDownload', + // 'ldjsonMultiFileExport' + ], + writeBench: [ + 'smallDocInsertOne', + 'largeDocInsertOne', + 'smallDocBulkInsert', + 'largeDocBulkInsert', + 'gridFsUpload' + // 'ldjsonMultiFileUpload', + // 'gridfsMultiFileUpload' + ] + }; + + const aMetricInfo = + (testName: string) => + ({ info: { test_name } }: MetricInfo) => + test_name === testName; + + const anMBsMetric = ({ name }: Metric) => name === 'megabytes_per_second'; + + let readBenchResult; + let writeBenchResult; + + console.group('composite scores'); + + const compositeResults: MetricInfo[] = []; + for (const [compositeName, compositeTests] of Object.entries(composites)) { + console.group(`${compositeName}: ${compositeTests.join(', ')}`); + + let sum = 0; + for (const testName of compositeTests) { + const testScore = results.find(aMetricInfo(testName)); + assert.ok(testScore, `${compositeName} suite requires ${testName} for composite score`); + + const metric = testScore.metrics.find(anMBsMetric); + assert.ok(metric, `${testName} is missing a megabytes_per_second metric`); + + sum += metric.value; + } + + const compositeAverage = sum / compositeTests.length; + + if (compositeName === 'readBench') readBenchResult = compositeAverage; + if (compositeName === 'writeBench') writeBenchResult = compositeAverage; + + compositeResults.push(metrics(compositeName, compositeAverage)); + + console.log('avg:', compositeAverage, 'mb/s'); + + console.groupEnd(); + } + + assert.ok(typeof readBenchResult === 'number'); + assert.ok(typeof writeBenchResult === 'number'); + + const driverBench = (readBenchResult + writeBenchResult) / 2; + + console.group('driverBench: readBench, writeBench'); + console.log('avg:', driverBench, 'mb/s'); + console.groupEnd(); + + compositeResults.push(metrics('driverBench', driverBench)); + + console.groupEnd(); + return [...results, ...compositeResults]; +} + +const finalResults = calculateCompositeBenchmarks(results); + +await fs.writeFile('results.json', JSON.stringify(finalResults, undefined, 2), 'utf8'); diff --git a/test/benchmarks/driver_bench/src/runner.mts b/test/benchmarks/driver_bench/src/runner.mts index 43580ec219a..287838e407b 100644 --- a/test/benchmarks/driver_bench/src/runner.mts +++ b/test/benchmarks/driver_bench/src/runner.mts @@ -101,6 +101,6 @@ console.log( await fs.writeFile( `results_${path.basename(benchmarkFile, '.mjs')}.json`, - JSON.stringify(metrics(benchmarkName, megabytesPerSecond, count), undefined, 2) + '\n', + JSON.stringify(metrics(benchmarkName, megabytesPerSecond), undefined, 2) + '\n', 'utf8' );