From b3fde6eda00ce84c16d560ab412ff5b90e73fc3b Mon Sep 17 00:00:00 2001 From: Warren James Date: Thu, 13 Feb 2025 15:50:05 -0500 Subject: [PATCH 01/20] add tags to benchmarks --- test/benchmarks/driver_bench/src/driver.mts | 6 ++++++ .../src/suites/multi_bench/find_many_and_empty_cursor.mts | 4 +++- .../src/suites/multi_bench/grid_fs_download.mts | 4 +++- .../driver_bench/src/suites/multi_bench/grid_fs_upload.mts | 3 ++- .../src/suites/multi_bench/large_doc_bulk_insert.mts | 3 ++- .../src/suites/multi_bench/small_doc_bulk_insert.mts | 3 ++- .../aggregate_a_million_documents_and_to_array.mts | 3 ++- .../aggregate_a_million_tweets_and_to_array.mts | 3 ++- .../src/suites/node_specific/find_many_and_to_array.mts | 4 +++- .../driver_bench/src/suites/node_specific/ping.mts | 3 ++- .../suites/parallel_bench/gridfs_multi_file_download.mts | 3 ++- .../src/suites/parallel_bench/gridfs_multi_file_upload.mts | 3 ++- .../src/suites/parallel_bench/ldjson_multi_file_export.mts | 3 ++- .../src/suites/parallel_bench/ldjson_multi_file_upload.mts | 3 ++- .../driver_bench/src/suites/single_bench/find_one.mts | 3 ++- .../src/suites/single_bench/large_doc_insert_one.mts | 3 ++- .../driver_bench/src/suites/single_bench/run_command.mts | 3 ++- .../src/suites/single_bench/small_doc_insert_one.mts | 3 ++- 18 files changed, 43 insertions(+), 17 deletions(-) diff --git a/test/benchmarks/driver_bench/src/driver.mts b/test/benchmarks/driver_bench/src/driver.mts index ab172fe7bd6..004e647ad55 100644 --- a/test/benchmarks/driver_bench/src/driver.mts +++ b/test/benchmarks/driver_bench/src/driver.mts @@ -7,6 +7,12 @@ import process from 'node:process'; const __dirname = import.meta.dirname; const require = module.createRequire(__dirname); +export const SPEC_TAG = 'spec-benchmark'; +export const ALERT_TAG = 'alerting-benchmark'; +export const CURSOR_TAG = 'cursor-benchmark'; +export const READ_TAG = 'read-benchmark'; +export const WRITE_TAG = 'write-benchmark'; + /** * The path to the MongoDB Node.js driver. * This MUST be set to the directory the driver is installed in diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts index 09c4cadd97a..e99c627f9fb 100644 --- a/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts @@ -1,8 +1,10 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ -import { driver, type mongodb } from '../../driver.mjs'; +import { driver, ALERT_TAG, SPEC_TAG, type mongodb, CURSOR_TAG, READ_TAG } from '../../driver.mjs'; export const taskSize = 16.22; +export const tags = [ALERT_TAG, SPEC_TAG, CURSOR_TAG, READ_TAG]; + let collection: mongodb.Collection; export async function before() { diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_download.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_download.mts index 31b85447858..7ad484e1137 100644 --- a/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_download.mts +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_download.mts @@ -1,10 +1,12 @@ import { Readable, Writable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; -import { driver, type mongodb } from '../../driver.mjs'; +import { driver, SPEC_TAG, ALERT_TAG, type mongodb, CURSOR_TAG, READ_TAG } from '../../driver.mjs'; export const taskSize = 52.43; +export const tags = [ALERT_TAG, SPEC_TAG, CURSOR_TAG, READ_TAG]; + let bucket: mongodb.GridFSBucket; let bin: Uint8Array; let _id: mongodb.ObjectId; diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_upload.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_upload.mts index 86361639d3c..d51eb8ebbb3 100644 --- a/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_upload.mts +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_upload.mts @@ -1,9 +1,10 @@ import { Readable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; -import { driver, type mongodb } from '../../driver.mjs'; +import { ALERT_TAG, driver, SPEC_TAG, WRITE_TAG, type mongodb } from '../../driver.mjs'; export const taskSize = 52.43; +export const tags = [ALERT_TAG, SPEC_TAG, WRITE_TAG]; let bucket: mongodb.GridFSBucket; let uploadStream: mongodb.GridFSBucketWriteStream; diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/large_doc_bulk_insert.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/large_doc_bulk_insert.mts index 5b913aefe6c..05a5ce01032 100644 --- a/test/benchmarks/driver_bench/src/suites/multi_bench/large_doc_bulk_insert.mts +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/large_doc_bulk_insert.mts @@ -1,6 +1,7 @@ -import { driver, type mongodb } from '../../driver.mjs'; +import { ALERT_TAG, driver, SPEC_TAG, WRITE_TAG, type mongodb } from '../../driver.mjs'; export const taskSize = 27.31; +export const tags = [SPEC_TAG, ALERT_TAG, WRITE_TAG]; let collection: mongodb.Collection; let documents: any[]; diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts index 922002f49b4..80c58d66e30 100644 --- a/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts @@ -1,6 +1,7 @@ -import { driver, type mongodb } from '../../driver.mjs'; +import { ALERT_TAG, driver, SPEC_TAG, WRITE_TAG, type mongodb } from '../../driver.mjs'; export const taskSize = 2.75; +export const tags = [SPEC_TAG, ALERT_TAG, WRITE_TAG]; let collection: mongodb.Collection; let documents: any[]; diff --git a/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_documents_and_to_array.mts b/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_documents_and_to_array.mts index 9a9e3f92fdd..0bdaac3ada8 100644 --- a/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_documents_and_to_array.mts +++ b/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_documents_and_to_array.mts @@ -1,6 +1,7 @@ -import { driver, type mongodb } from '../../driver.mjs'; +import { ALERT_TAG, driver, READ_TAG, type mongodb } from '../../driver.mjs'; export const taskSize = 16; +export const tags = [ALERT_TAG, READ_TAG]; let db: mongodb.Db; diff --git a/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_tweets_and_to_array.mts b/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_tweets_and_to_array.mts index da2ba5cc416..6cf2a1665e1 100644 --- a/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_tweets_and_to_array.mts +++ b/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_tweets_and_to_array.mts @@ -1,6 +1,7 @@ -import { driver, type mongodb } from '../../driver.mjs'; +import { ALERT_TAG, driver, READ_TAG, type mongodb } from '../../driver.mjs'; export const taskSize = 1500; +export const tags = [ALERT_TAG, READ_TAG]; let db: mongodb.Db; let tweet: Record; diff --git a/test/benchmarks/driver_bench/src/suites/node_specific/find_many_and_to_array.mts b/test/benchmarks/driver_bench/src/suites/node_specific/find_many_and_to_array.mts index 778d44c3a3d..9156a2e19ea 100644 --- a/test/benchmarks/driver_bench/src/suites/node_specific/find_many_and_to_array.mts +++ b/test/benchmarks/driver_bench/src/suites/node_specific/find_many_and_to_array.mts @@ -1,7 +1,9 @@ -import { driver, type mongodb } from '../../driver.mjs'; +import { ALERT_TAG, driver, READ_TAG, SPEC_TAG, type mongodb } from '../../driver.mjs'; export const taskSize = 16.22; +export const tags = [SPEC_TAG, ALERT_TAG,READ_TAG] + let collection: mongodb.Collection; export async function before() { diff --git a/test/benchmarks/driver_bench/src/suites/node_specific/ping.mts b/test/benchmarks/driver_bench/src/suites/node_specific/ping.mts index ac8e462381a..6d60e8b39cd 100644 --- a/test/benchmarks/driver_bench/src/suites/node_specific/ping.mts +++ b/test/benchmarks/driver_bench/src/suites/node_specific/ping.mts @@ -1,7 +1,8 @@ -import { driver, type mongodb } from '../../driver.mjs'; +import { ALERT_TAG, driver, WRITE_TAG, type mongodb } from '../../driver.mjs'; // { ping: 1 } is 15 bytes of BSON x 10,000 iterations export const taskSize = 0.15; +export const tags = [ALERT_TAG]; let db: mongodb.Db; diff --git a/test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_download.mts b/test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_download.mts index 9345826a604..af79b03a36b 100644 --- a/test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_download.mts +++ b/test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_download.mts @@ -2,9 +2,10 @@ import { createReadStream, createWriteStream, promises as fs } from 'node:fs'; import path from 'node:path'; import { pipeline } from 'node:stream/promises'; -import { driver, type mongodb, PARALLEL_DIRECTORY, TEMP_DIRECTORY } from '../../driver.mjs'; +import { ALERT_TAG, CURSOR_TAG, driver, type mongodb, PARALLEL_DIRECTORY, READ_TAG, SPEC_TAG, TEMP_DIRECTORY } from '../../driver.mjs'; export const taskSize = 262.144; +export const tags = [SPEC_TAG, ALERT_TAG, READ_TAG, CURSOR_TAG]; let bucket: mongodb.GridFSBucket; diff --git a/test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_upload.mts b/test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_upload.mts index 6c29c18ea15..c880f63b102 100644 --- a/test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_upload.mts +++ b/test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_upload.mts @@ -3,9 +3,10 @@ import path from 'node:path'; import { Readable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; -import { driver, type mongodb, PARALLEL_DIRECTORY } from '../../driver.mjs'; +import { ALERT_TAG, driver, type mongodb, PARALLEL_DIRECTORY, SPEC_TAG, WRITE_TAG } from '../../driver.mjs'; export const taskSize = 262.144; +export const tags = [SPEC_TAG, ALERT_TAG, WRITE_TAG]; let bucket: mongodb.GridFSBucket; diff --git a/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_export.mts b/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_export.mts index 81d05dcb526..2ff18383f6d 100644 --- a/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_export.mts +++ b/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_export.mts @@ -3,9 +3,10 @@ import path from 'node:path'; import readline from 'node:readline/promises'; import stream from 'node:stream/promises'; -import { driver, EJSON, type mongodb, PARALLEL_DIRECTORY, TEMP_DIRECTORY } from '../../driver.mjs'; +import { ALERT_TAG, driver, EJSON, type mongodb, PARALLEL_DIRECTORY, SPEC_TAG, TEMP_DIRECTORY, WRITE_TAG } from '../../driver.mjs'; export const taskSize = 565; +export const tags = [SPEC_TAG, ALERT_TAG, WRITE_TAG]; let collection: mongodb.Collection; diff --git a/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_upload.mts b/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_upload.mts index 5f5dab24f90..7ddc2f36064 100644 --- a/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_upload.mts +++ b/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_upload.mts @@ -2,9 +2,10 @@ import { createReadStream, promises as fs } from 'node:fs'; import path from 'node:path'; import readline from 'node:readline/promises'; -import { driver, type mongodb, PARALLEL_DIRECTORY } from '../../driver.mjs'; +import { ALERT_TAG, driver, type mongodb, PARALLEL_DIRECTORY, SPEC_TAG, WRITE_TAG } from '../../driver.mjs'; export const taskSize = 565; +export const tags = [SPEC_TAG, ALERT_TAG, WRITE_TAG]; const directory = path.resolve(PARALLEL_DIRECTORY, 'ldjson_multi'); let collection: mongodb.Collection; diff --git a/test/benchmarks/driver_bench/src/suites/single_bench/find_one.mts b/test/benchmarks/driver_bench/src/suites/single_bench/find_one.mts index d99ac5f9bfe..7707a133091 100644 --- a/test/benchmarks/driver_bench/src/suites/single_bench/find_one.mts +++ b/test/benchmarks/driver_bench/src/suites/single_bench/find_one.mts @@ -1,6 +1,7 @@ -import { driver, type mongodb } from '../../driver.mjs'; +import { ALERT_TAG, CURSOR_TAG, driver, READ_TAG, SPEC_TAG, type mongodb } from '../../driver.mjs'; export const taskSize = 16.22; +export const tags = [SPEC_TAG, ALERT_TAG, CURSOR_TAG, READ_TAG]; let collection: mongodb.Collection<{ _id: number }>; diff --git a/test/benchmarks/driver_bench/src/suites/single_bench/large_doc_insert_one.mts b/test/benchmarks/driver_bench/src/suites/single_bench/large_doc_insert_one.mts index 4dcae894903..965d87e95f9 100644 --- a/test/benchmarks/driver_bench/src/suites/single_bench/large_doc_insert_one.mts +++ b/test/benchmarks/driver_bench/src/suites/single_bench/large_doc_insert_one.mts @@ -1,6 +1,7 @@ -import { driver, type mongodb } from '../../driver.mjs'; +import { ALERT_TAG, driver, SPEC_TAG, WRITE_TAG, type mongodb } from '../../driver.mjs'; export const taskSize = 27.31; +export const tags = [SPEC_TAG, ALERT_TAG, WRITE_TAG]; let collection: mongodb.Collection; let documents: Record[]; diff --git a/test/benchmarks/driver_bench/src/suites/single_bench/run_command.mts b/test/benchmarks/driver_bench/src/suites/single_bench/run_command.mts index a2884ebb343..e758470e990 100644 --- a/test/benchmarks/driver_bench/src/suites/single_bench/run_command.mts +++ b/test/benchmarks/driver_bench/src/suites/single_bench/run_command.mts @@ -1,7 +1,8 @@ -import { driver, type mongodb } from '../../driver.mjs'; +import { ALERT_TAG, driver, SPEC_TAG, type mongodb } from '../../driver.mjs'; // { hello: true } is 13 bytes of BSON x 10,000 iterations export const taskSize = 0.13; +export const tags = [SPEC_TAG, ALERT_TAG]; let db: mongodb.Db; diff --git a/test/benchmarks/driver_bench/src/suites/single_bench/small_doc_insert_one.mts b/test/benchmarks/driver_bench/src/suites/single_bench/small_doc_insert_one.mts index 4995a9b2539..59e81a2a900 100644 --- a/test/benchmarks/driver_bench/src/suites/single_bench/small_doc_insert_one.mts +++ b/test/benchmarks/driver_bench/src/suites/single_bench/small_doc_insert_one.mts @@ -1,6 +1,7 @@ -import { driver, type mongodb } from '../../driver.mjs'; +import { ALERT_TAG, driver, SPEC_TAG, WRITE_TAG, type mongodb } from '../../driver.mjs'; export const taskSize = 2.75; +export const tags = [SPEC_TAG, ALERT_TAG, WRITE_TAG]; let collection: mongodb.Collection; let documents: Record[]; From 48e9df41cf5c8b2f0129867217718630b41a42bb Mon Sep 17 00:00:00 2001 From: Warren James Date: Thu, 13 Feb 2025 16:13:58 -0500 Subject: [PATCH 02/20] add normalized results --- test/benchmarks/driver_bench/src/driver.mts | 2 +- test/benchmarks/driver_bench/src/main.mts | 36 ++++++++++++++++++--- test/benchmarks/driver_bench/src/runner.mts | 1 + 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/test/benchmarks/driver_bench/src/driver.mts b/test/benchmarks/driver_bench/src/driver.mts index 004e647ad55..f14f0995f86 100644 --- a/test/benchmarks/driver_bench/src/driver.mts +++ b/test/benchmarks/driver_bench/src/driver.mts @@ -124,7 +124,7 @@ export const PARALLEL_DIRECTORY = path.resolve(SPEC_DIRECTORY, 'parallel'); export const TEMP_DIRECTORY = path.resolve(SPEC_DIRECTORY, 'tmp'); export type Metric = { - name: 'megabytes_per_second'; + name: 'megabytes_per_second' | 'normalized_throughput'; value: number; }; diff --git a/test/benchmarks/driver_bench/src/main.mts b/test/benchmarks/driver_bench/src/main.mts index 67309b7d49a..405ae0e010e 100644 --- a/test/benchmarks/driver_bench/src/main.mts +++ b/test/benchmarks/driver_bench/src/main.mts @@ -85,7 +85,7 @@ console.log(systemInfo()); const runnerPath = path.join(__dirname, 'runner.mjs'); -const results: MetricInfo[] = []; +let results: MetricInfo[] = []; for (const [suite, benchmarks] of Object.entries(tests)) { console.group(snakeToCamel(suite)); @@ -146,8 +146,8 @@ function calculateCompositeBenchmarks(results: MetricInfo[]) { const aMetricInfo = (testName: string) => - ({ info: { test_name } }: MetricInfo) => - test_name === testName; + ({ info: { test_name } }: MetricInfo) => + test_name === testName; const anMBsMetric = ({ name }: Metric) => name === 'megabytes_per_second'; @@ -198,6 +198,32 @@ function calculateCompositeBenchmarks(results: MetricInfo[]) { return [...results, ...compositeResults]; } -const finalResults = calculateCompositeBenchmarks(results); +function calculateNormalizedResults(results: MetricInfo[]): MetricInfo[] { + const primesBench = results.find(r => r.info.test_name === 'primes'); + const pingBench = results.find(r => r.info.test_name === 'ping'); + let pingMegabytesPerSecond: number; -await fs.writeFile('results.json', JSON.stringify(finalResults, undefined, 2), 'utf8'); + if (pingBench) { + const pingThroughput = pingBench.metrics[0].value; + for (const bench of results) { + if (bench.info.test_name === 'primes' || bench.info.test_name === 'ping') continue; + const normalizedThroughput = bench.metrics[0].value / pingThroughput; + bench.metrics.push({name: 'normalized_throughput', value: normalizedThroughput}); + + } + + if (primesBench) { + const normalizedThroughput = pingBench.metrics[0].value / primesBench.metrics[0].value; + pingMegabytesPerSecond = pingBench.metrics[0].value; + pingBench.metrics.push({ name: 'normalized_throughput', value: normalizedThroughput }); + } + } + + return results; +} + +results = calculateNormalizedResults(results); +results = calculateCompositeBenchmarks(results); + + +await fs.writeFile('results.json', JSON.stringify(results, undefined, 2), 'utf8'); diff --git a/test/benchmarks/driver_bench/src/runner.mts b/test/benchmarks/driver_bench/src/runner.mts index f360b2ef0ac..7d57fde8410 100644 --- a/test/benchmarks/driver_bench/src/runner.mts +++ b/test/benchmarks/driver_bench/src/runner.mts @@ -14,6 +14,7 @@ type BenchmarkModule = { run: () => Promise; afterEach?: () => Promise; after?: () => Promise; + tags?: string[]; }; const benchmarkName = snakeToCamel(path.basename(benchmarkFile, '.mjs')); From 1f00dcaf6459036d908c8dcbd42559eba77a9f5f Mon Sep 17 00:00:00 2001 From: Warren James Date: Fri, 14 Feb 2025 13:49:16 -0500 Subject: [PATCH 03/20] add handling for tags --- test/benchmarks/driver_bench/src/driver.mts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/benchmarks/driver_bench/src/driver.mts b/test/benchmarks/driver_bench/src/driver.mts index f14f0995f86..1bec27111e0 100644 --- a/test/benchmarks/driver_bench/src/driver.mts +++ b/test/benchmarks/driver_bench/src/driver.mts @@ -132,11 +132,12 @@ export type MetricInfo = { info: { test_name: string; args: Record; + tags?: string[] }; metrics: Metric[]; }; -export function metrics(test_name: string, result: number): MetricInfo { +export function metrics(test_name: string, result: number, tags?: string[]): MetricInfo { return { info: { test_name, @@ -147,7 +148,8 @@ export function metrics(test_name: string, result: number): MetricInfo { key, typeof value === 'number' ? value : value ? 1 : 0 ]) - ) + ), + tags }, metrics: [{ name: 'megabytes_per_second', value: result }] } as const; From 3285105f0b642a9555ce93713b40d9141c6ef126 Mon Sep 17 00:00:00 2001 From: Warren James Date: Fri, 14 Feb 2025 13:55:26 -0500 Subject: [PATCH 04/20] do not mutate array --- test/benchmarks/driver_bench/src/main.mts | 31 ++++++++++++++------- test/benchmarks/driver_bench/src/runner.mts | 4 ++- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/test/benchmarks/driver_bench/src/main.mts b/test/benchmarks/driver_bench/src/main.mts index 405ae0e010e..e13a08fed92 100644 --- a/test/benchmarks/driver_bench/src/main.mts +++ b/test/benchmarks/driver_bench/src/main.mts @@ -201,22 +201,33 @@ function calculateCompositeBenchmarks(results: MetricInfo[]) { function calculateNormalizedResults(results: MetricInfo[]): MetricInfo[] { const primesBench = results.find(r => r.info.test_name === 'primes'); const pingBench = results.find(r => r.info.test_name === 'ping'); - let pingMegabytesPerSecond: number; if (pingBench) { + const newMetrics: MetricInfo[] = []; const pingThroughput = pingBench.metrics[0].value; for (const bench of results) { - if (bench.info.test_name === 'primes' || bench.info.test_name === 'ping') continue; - const normalizedThroughput = bench.metrics[0].value / pingThroughput; - bench.metrics.push({name: 'normalized_throughput', value: normalizedThroughput}); - + if (bench.info.test_name === 'primes') { + newMetrics.push({ ...bench }); + } + else if (bench.info.test_name === 'ping') { + // Compute ping's normalized_throughput against the primes bench if present + if (primesBench) { + const primesThroughput = primesBench.metrics[0].value; + const normalizedThroughput: Metric = { 'name': 'normalized_throughput', value: bench.metrics[0].value / primesThroughput }; + const newInfo: MetricInfo = { info: { ...bench.info }, metrics: [...bench.metrics, normalizedThroughput] }; + newMetrics.push(newInfo); + } else { + newMetrics.push({ ...bench }); + } + } else { + // Compute normalized_throughput of benchmarks against ping bench + const normalizedThroughput: Metric = { 'name': 'normalized_throughput', value: bench.metrics[0].value / pingThroughput }; + const newInfo: MetricInfo = { info: { ...bench.info }, metrics: [...bench.metrics, normalizedThroughput] } + newMetrics.push(newInfo); + } } - if (primesBench) { - const normalizedThroughput = pingBench.metrics[0].value / primesBench.metrics[0].value; - pingMegabytesPerSecond = pingBench.metrics[0].value; - pingBench.metrics.push({ name: 'normalized_throughput', value: normalizedThroughput }); - } + return newMetrics; } return results; diff --git a/test/benchmarks/driver_bench/src/runner.mts b/test/benchmarks/driver_bench/src/runner.mts index 7d57fde8410..09ae2226783 100644 --- a/test/benchmarks/driver_bench/src/runner.mts +++ b/test/benchmarks/driver_bench/src/runner.mts @@ -17,6 +17,7 @@ type BenchmarkModule = { tags?: string[]; }; + const benchmarkName = snakeToCamel(path.basename(benchmarkFile, '.mjs')); const benchmark: BenchmarkModule = await import(`./${benchmarkFile}`); @@ -80,6 +81,7 @@ function percentileIndex(percentile: number, count: number) { const medianExecution = durations[percentileIndex(50, count)]; const megabytesPerSecond = benchmark.taskSize / medianExecution; +const tags = benchmark.tags; console.log( ' '.repeat(3), @@ -92,6 +94,6 @@ console.log( await fs.writeFile( `results_${path.basename(benchmarkFile, '.mjs')}.json`, - JSON.stringify(metrics(benchmarkName, megabytesPerSecond), undefined, 2) + '\n', + JSON.stringify(metrics(benchmarkName, megabytesPerSecond, tags), undefined, 2) + '\n', 'utf8' ); From 88294b3eed17d338e64b7b29880cf072ee4ecafe Mon Sep 17 00:00:00 2001 From: Warren James Date: Tue, 18 Feb 2025 17:16:42 -0500 Subject: [PATCH 05/20] Add scaling constant for ping normalized throughput --- test/benchmarks/driver_bench/src/driver.mts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/benchmarks/driver_bench/src/driver.mts b/test/benchmarks/driver_bench/src/driver.mts index 1bec27111e0..d17802082e1 100644 --- a/test/benchmarks/driver_bench/src/driver.mts +++ b/test/benchmarks/driver_bench/src/driver.mts @@ -13,6 +13,8 @@ export const CURSOR_TAG = 'cursor-benchmark'; export const READ_TAG = 'read-benchmark'; export const WRITE_TAG = 'write-benchmark'; +export const NORMALIZED_PING_SCALING_CONST = 1000; + /** * The path to the MongoDB Node.js driver. * This MUST be set to the directory the driver is installed in From 2aa2f3899801320bcf13d5558256c94b7c70caa9 Mon Sep 17 00:00:00 2001 From: Warren James Date: Tue, 18 Feb 2025 17:17:26 -0500 Subject: [PATCH 06/20] cleanup --- test/benchmarks/driver_bench/src/main.mts | 21 ++++++------------- .../src/suites/node_specific/ping.mts | 2 +- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/test/benchmarks/driver_bench/src/main.mts b/test/benchmarks/driver_bench/src/main.mts index e13a08fed92..24a7394665b 100644 --- a/test/benchmarks/driver_bench/src/main.mts +++ b/test/benchmarks/driver_bench/src/main.mts @@ -18,6 +18,7 @@ import { MONGODB_DRIVER_PATH, MONGODB_DRIVER_REVISION, MONGODB_DRIVER_VERSION, + NORMALIZED_PING_SCALING_CONST, snakeToCamel } from './driver.mjs'; @@ -203,38 +204,28 @@ function calculateNormalizedResults(results: MetricInfo[]): MetricInfo[] { const pingBench = results.find(r => r.info.test_name === 'ping'); if (pingBench) { - const newMetrics: MetricInfo[] = []; const pingThroughput = pingBench.metrics[0].value; for (const bench of results) { - if (bench.info.test_name === 'primes') { - newMetrics.push({ ...bench }); - } - else if (bench.info.test_name === 'ping') { + if (bench.info.test_name === 'ping') { // Compute ping's normalized_throughput against the primes bench if present if (primesBench) { const primesThroughput = primesBench.metrics[0].value; - const normalizedThroughput: Metric = { 'name': 'normalized_throughput', value: bench.metrics[0].value / primesThroughput }; - const newInfo: MetricInfo = { info: { ...bench.info }, metrics: [...bench.metrics, normalizedThroughput] }; - newMetrics.push(newInfo); - } else { - newMetrics.push({ ...bench }); + const normalizedThroughput: Metric = { 'name': 'normalized_throughput', value: NORMALIZED_PING_SCALING_CONST * bench.metrics[0].value / primesThroughput }; + bench.metrics.push(normalizedThroughput); } } else { // Compute normalized_throughput of benchmarks against ping bench const normalizedThroughput: Metric = { 'name': 'normalized_throughput', value: bench.metrics[0].value / pingThroughput }; - const newInfo: MetricInfo = { info: { ...bench.info }, metrics: [...bench.metrics, normalizedThroughput] } - newMetrics.push(newInfo); + bench.metrics.push(normalizedThroughput); } } - - return newMetrics; } return results; } -results = calculateNormalizedResults(results); results = calculateCompositeBenchmarks(results); +results = calculateNormalizedResults(results); await fs.writeFile('results.json', JSON.stringify(results, undefined, 2), 'utf8'); diff --git a/test/benchmarks/driver_bench/src/suites/node_specific/ping.mts b/test/benchmarks/driver_bench/src/suites/node_specific/ping.mts index 6d60e8b39cd..40230ebfed4 100644 --- a/test/benchmarks/driver_bench/src/suites/node_specific/ping.mts +++ b/test/benchmarks/driver_bench/src/suites/node_specific/ping.mts @@ -1,4 +1,4 @@ -import { ALERT_TAG, driver, WRITE_TAG, type mongodb } from '../../driver.mjs'; +import { ALERT_TAG, driver, type mongodb } from '../../driver.mjs'; // { ping: 1 } is 15 bytes of BSON x 10,000 iterations export const taskSize = 0.15; From ee0636948ce38b4dcb4e9b81a6f5dbe03ac683a2 Mon Sep 17 00:00:00 2001 From: Warren James Date: Tue, 18 Feb 2025 17:34:41 -0500 Subject: [PATCH 07/20] document tags --- test/benchmarks/driver_bench/readme.md | 17 +++++++++++++++++ test/benchmarks/driver_bench/src/driver.mts | 6 ++++++ 2 files changed, 23 insertions(+) diff --git a/test/benchmarks/driver_bench/readme.md b/test/benchmarks/driver_bench/readme.md index 81f8b859745..52e9fe9926a 100644 --- a/test/benchmarks/driver_bench/readme.md +++ b/test/benchmarks/driver_bench/readme.md @@ -51,6 +51,8 @@ type BenchmarkModule = { run: () => Promise; afterEach?: () => Promise; after?: () => Promise; + + tags?: string[]; }; ``` @@ -58,6 +60,21 @@ Just like mocha we have once before and once after as well as before each and af The `driver.mts` module is intended to hold various helpers for setup and teardown and help abstract some of the driver API. +## Benchmark tags +The `tags` property of `BenchmarkModule` is where a benchmark's tags should be added to facilitate +performance alerting and filter of results via our internal tools. + +Tags are defined in `driver.mts` and should end with a _TAG suffix. +Whenever a new tag is defined it should be documented in the table below . + +| tag variable name | tag string value | purpose | +|-------------------|------------------------|--------------------------------------------------------------------------------------------------------------------------------------| +| `SPEC_TAG` | `'spec-benchmark'` | Special tag that marks a benchmark as a spec-required benchmark | +| `ALERT_TAG` | `'alerting-benchmark'` | Special tag that enables our perf monitoring tooling to create alerts when regressions in this benchmark's performance are detected | +| `CURSOR_TAG` | `'cursor-benchmark'` | Tag marking a benchmark as being related to cursor performance | +| `READ_TAG` | `'read-benchmark'` | Tag marking a benchmark as being related to read performance | +| `WRITE_TAG` | `'write-benchmark'` | Tag marking a benchmark as being related to write performance | + ## Wishlist - Make it so runner can handle: `./lib/suites/multi_bench/grid_fs_upload.mjs` as an argument so shell path autocomplete makes it easier to pick a benchmark diff --git a/test/benchmarks/driver_bench/src/driver.mts b/test/benchmarks/driver_bench/src/driver.mts index d17802082e1..fc69ce7acaa 100644 --- a/test/benchmarks/driver_bench/src/driver.mts +++ b/test/benchmarks/driver_bench/src/driver.mts @@ -7,10 +7,16 @@ import process from 'node:process'; const __dirname = import.meta.dirname; const require = module.createRequire(__dirname); +// Special tag that marks a benchmark as a spec-required benchmark export const SPEC_TAG = 'spec-benchmark'; +// Special tag that enables our perf monitoring tooling to create alerts when regressions in this +// benchmark's performance are detected export const ALERT_TAG = 'alerting-benchmark'; +// Tag marking a benchmark as being related to cursor performance export const CURSOR_TAG = 'cursor-benchmark'; +// Tag marking a benchmark as being related to read performance export const READ_TAG = 'read-benchmark'; +// Tag marking a benchmark as being related to write performance export const WRITE_TAG = 'write-benchmark'; export const NORMALIZED_PING_SCALING_CONST = 1000; From 6f3293ddecbd25795673325395768e10d87e6e1d Mon Sep 17 00:00:00 2001 From: Warren James Date: Tue, 18 Feb 2025 17:45:25 -0500 Subject: [PATCH 08/20] add tags --- .../aggregate_a_million_documents_and_to_array.mts | 4 ++-- .../node_specific/aggregate_a_million_tweets_and_to_array.mts | 2 +- .../src/suites/node_specific/find_many_and_to_array.mts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_documents_and_to_array.mts b/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_documents_and_to_array.mts index 0bdaac3ada8..5518393d86d 100644 --- a/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_documents_and_to_array.mts +++ b/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_documents_and_to_array.mts @@ -1,7 +1,7 @@ -import { ALERT_TAG, driver, READ_TAG, type mongodb } from '../../driver.mjs'; +import { ALERT_TAG, CURSOR_TAG, driver, READ_TAG, type mongodb } from '../../driver.mjs'; export const taskSize = 16; -export const tags = [ALERT_TAG, READ_TAG]; +export const tags = [ALERT_TAG, CURSOR_TAG, READ_TAG]; let db: mongodb.Db; diff --git a/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_tweets_and_to_array.mts b/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_tweets_and_to_array.mts index 6cf2a1665e1..bf8aeb900d3 100644 --- a/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_tweets_and_to_array.mts +++ b/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_tweets_and_to_array.mts @@ -1,7 +1,7 @@ import { ALERT_TAG, driver, READ_TAG, type mongodb } from '../../driver.mjs'; export const taskSize = 1500; -export const tags = [ALERT_TAG, READ_TAG]; +export const tags = [ALERT_TAG, CURSOR_TAG, READ_TAG]; let db: mongodb.Db; let tweet: Record; diff --git a/test/benchmarks/driver_bench/src/suites/node_specific/find_many_and_to_array.mts b/test/benchmarks/driver_bench/src/suites/node_specific/find_many_and_to_array.mts index 9156a2e19ea..801cc6b810d 100644 --- a/test/benchmarks/driver_bench/src/suites/node_specific/find_many_and_to_array.mts +++ b/test/benchmarks/driver_bench/src/suites/node_specific/find_many_and_to_array.mts @@ -1,8 +1,8 @@ -import { ALERT_TAG, driver, READ_TAG, SPEC_TAG, type mongodb } from '../../driver.mjs'; +import { ALERT_TAG, CURSOR_TAG, driver, READ_TAG, SPEC_TAG, type mongodb } from '../../driver.mjs'; export const taskSize = 16.22; -export const tags = [SPEC_TAG, ALERT_TAG,READ_TAG] +export const tags = [SPEC_TAG, ALERT_TAG,READ_TAG, CURSOR_TAG] let collection: mongodb.Collection; From e2525366123117dfcd3c54e5078051cc79fc4b89 Mon Sep 17 00:00:00 2001 From: Warren James Date: Wed, 19 Feb 2025 15:01:13 -0500 Subject: [PATCH 09/20] address review comments --- test/benchmarks/driver_bench/src/driver.mts | 26 +++++++++++-------- test/benchmarks/driver_bench/src/main.mts | 26 +++++++------------ test/benchmarks/driver_bench/src/runner.mts | 6 +++++ .../find_many_and_empty_cursor.mts | 4 +-- .../suites/multi_bench/grid_fs_download.mts | 4 +-- .../src/suites/multi_bench/grid_fs_upload.mts | 4 +-- .../multi_bench/large_doc_bulk_insert.mts | 4 +-- .../multi_bench/small_doc_bulk_insert.mts | 4 +-- ...egate_a_million_documents_and_to_array.mts | 4 +-- ...ggregate_a_million_tweets_and_to_array.mts | 4 +-- .../node_specific/find_many_and_to_array.mts | 4 +-- .../src/suites/node_specific/ping.mts | 4 +-- .../gridfs_multi_file_download.mts | 4 +-- .../gridfs_multi_file_upload.mts | 4 +-- .../ldjson_multi_file_export.mts | 4 +-- .../ldjson_multi_file_upload.mts | 4 +-- .../src/suites/single_bench/find_one.mts | 4 +-- .../single_bench/large_doc_insert_one.mts | 4 +-- .../src/suites/single_bench/run_command.mts | 4 +-- .../single_bench/small_doc_insert_one.mts | 5 ++-- 20 files changed, 66 insertions(+), 61 deletions(-) diff --git a/test/benchmarks/driver_bench/src/driver.mts b/test/benchmarks/driver_bench/src/driver.mts index fc69ce7acaa..aa031892f59 100644 --- a/test/benchmarks/driver_bench/src/driver.mts +++ b/test/benchmarks/driver_bench/src/driver.mts @@ -7,18 +7,22 @@ import process from 'node:process'; const __dirname = import.meta.dirname; const require = module.createRequire(__dirname); -// Special tag that marks a benchmark as a spec-required benchmark -export const SPEC_TAG = 'spec-benchmark'; -// Special tag that enables our perf monitoring tooling to create alerts when regressions in this -// benchmark's performance are detected -export const ALERT_TAG = 'alerting-benchmark'; -// Tag marking a benchmark as being related to cursor performance -export const CURSOR_TAG = 'cursor-benchmark'; -// Tag marking a benchmark as being related to read performance -export const READ_TAG = 'read-benchmark'; -// Tag marking a benchmark as being related to write performance -export const WRITE_TAG = 'write-benchmark'; +export const TAG = { + // Special tag that marks a benchmark as a spec-required benchmark + spec: 'spec-benchmark', + // Special tag that enables our perf monitoring tooling to create alerts when regressions in this + // benchmark's performance are detected + alert: 'alerting-benchmark', + // Tag marking a benchmark as being related to cursor performance + cursor: 'cursor-benchmark', + // Tag marking a benchmark as being related to read performance + read: 'read-benchmark', + // Tag marking a benchmark as being related to write performance + write: 'write-benchmark' +}; +// Constant used to scale normalized_throughput results for ping benchmark to be roughly in the +// range 0 - 9 to make it easier to work with export const NORMALIZED_PING_SCALING_CONST = 1000; /** diff --git a/test/benchmarks/driver_bench/src/main.mts b/test/benchmarks/driver_bench/src/main.mts index 24a7394665b..457e23fc5ac 100644 --- a/test/benchmarks/driver_bench/src/main.mts +++ b/test/benchmarks/driver_bench/src/main.mts @@ -203,22 +203,16 @@ function calculateNormalizedResults(results: MetricInfo[]): MetricInfo[] { const primesBench = results.find(r => r.info.test_name === 'primes'); const pingBench = results.find(r => r.info.test_name === 'ping'); - if (pingBench) { - const pingThroughput = pingBench.metrics[0].value; - for (const bench of results) { - if (bench.info.test_name === 'ping') { - // Compute ping's normalized_throughput against the primes bench if present - if (primesBench) { - const primesThroughput = primesBench.metrics[0].value; - const normalizedThroughput: Metric = { 'name': 'normalized_throughput', value: NORMALIZED_PING_SCALING_CONST * bench.metrics[0].value / primesThroughput }; - bench.metrics.push(normalizedThroughput); - } - } else { - // Compute normalized_throughput of benchmarks against ping bench - const normalizedThroughput: Metric = { 'name': 'normalized_throughput', value: bench.metrics[0].value / pingThroughput }; - bench.metrics.push(normalizedThroughput); - } - } + assert.ok(pingBench); + assert.ok(primesBench); + const pingThroughput = pingBench.metrics[0].value; + const primesThroughput = primesBench.metrics[0].value; + primesBench.metrics.push({ 'name': 'normalized_throughput', value: NORMALIZED_PING_SCALING_CONST * pingThroughput / primesThroughput }); + + for (const bench of results) { + if (bench.info.test_name === 'ping' || bench.info.test_name === 'primes') continue; + // Compute normalized_throughput of benchmarks against ping bench + bench.metrics.push({ 'name': 'normalized_throughput', value: bench.metrics[0].value / pingThroughput }); } return results; diff --git a/test/benchmarks/driver_bench/src/runner.mts b/test/benchmarks/driver_bench/src/runner.mts index 09ae2226783..93e149e262a 100644 --- a/test/benchmarks/driver_bench/src/runner.mts +++ b/test/benchmarks/driver_bench/src/runner.mts @@ -81,7 +81,13 @@ function percentileIndex(percentile: number, count: number) { const medianExecution = durations[percentileIndex(50, count)]; const megabytesPerSecond = benchmark.taskSize / medianExecution; + const tags = benchmark.tags; +if (tags && + (!Array.isArray(tags) || (tags.length > 0 && !tags.every(t => typeof t === 'string')))) { + + throw new Error('If tags is specified, it MUST be an array of strings'); +} console.log( ' '.repeat(3), diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts index e99c627f9fb..2150e8b340e 100644 --- a/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts @@ -1,9 +1,9 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ -import { driver, ALERT_TAG, SPEC_TAG, type mongodb, CURSOR_TAG, READ_TAG } from '../../driver.mjs'; +import { driver, TAG, type mongodb, CURSOR_TAG, READ_TAG } from '../../driver.mjs'; export const taskSize = 16.22; -export const tags = [ALERT_TAG, SPEC_TAG, CURSOR_TAG, READ_TAG]; +export const tags = [TAG.alert, TAG.spec, TAG.cursor, TAG.read]; let collection: mongodb.Collection; diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_download.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_download.mts index 7ad484e1137..67b1c802517 100644 --- a/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_download.mts +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_download.mts @@ -1,11 +1,11 @@ import { Readable, Writable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; -import { driver, SPEC_TAG, ALERT_TAG, type mongodb, CURSOR_TAG, READ_TAG } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; export const taskSize = 52.43; -export const tags = [ALERT_TAG, SPEC_TAG, CURSOR_TAG, READ_TAG]; +export const tags = [TAG.alert, TAG.spec, TAG.cursor, TAG.read]; let bucket: mongodb.GridFSBucket; let bin: Uint8Array; diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_upload.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_upload.mts index d51eb8ebbb3..dc3438c1015 100644 --- a/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_upload.mts +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_upload.mts @@ -1,10 +1,10 @@ import { Readable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; -import { ALERT_TAG, driver, SPEC_TAG, WRITE_TAG, type mongodb } from '../../driver.mjs'; +import { driver, TAG, type mongodb } from '../../driver.mjs'; export const taskSize = 52.43; -export const tags = [ALERT_TAG, SPEC_TAG, WRITE_TAG]; +export const tags = [TAG.alert, TAG.spec, TAG.write]; let bucket: mongodb.GridFSBucket; let uploadStream: mongodb.GridFSBucketWriteStream; diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/large_doc_bulk_insert.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/large_doc_bulk_insert.mts index 05a5ce01032..2fe2e8fcd19 100644 --- a/test/benchmarks/driver_bench/src/suites/multi_bench/large_doc_bulk_insert.mts +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/large_doc_bulk_insert.mts @@ -1,7 +1,7 @@ -import { ALERT_TAG, driver, SPEC_TAG, WRITE_TAG, type mongodb } from '../../driver.mjs'; +import { driver, TAG, type mongodb } from '../../driver.mjs'; export const taskSize = 27.31; -export const tags = [SPEC_TAG, ALERT_TAG, WRITE_TAG]; +export const tags = [TAG.alert, TAG.spec, TAG.write]; let collection: mongodb.Collection; let documents: any[]; diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts index 80c58d66e30..aa0dc0df81b 100644 --- a/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts @@ -1,7 +1,7 @@ -import { ALERT_TAG, driver, SPEC_TAG, WRITE_TAG, type mongodb } from '../../driver.mjs'; +import { TAG, driver, type mongodb } from '../../driver.mjs'; export const taskSize = 2.75; -export const tags = [SPEC_TAG, ALERT_TAG, WRITE_TAG]; +export const tags = [TAG.alert, TAG.spec, TAG.write]; let collection: mongodb.Collection; let documents: any[]; diff --git a/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_documents_and_to_array.mts b/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_documents_and_to_array.mts index 5518393d86d..5a2ea49463a 100644 --- a/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_documents_and_to_array.mts +++ b/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_documents_and_to_array.mts @@ -1,7 +1,7 @@ -import { ALERT_TAG, CURSOR_TAG, driver, READ_TAG, type mongodb } from '../../driver.mjs'; +import { driver, TAG, type mongodb } from '../../driver.mjs'; export const taskSize = 16; -export const tags = [ALERT_TAG, CURSOR_TAG, READ_TAG]; +export const tags = [TAG.alert, TAG.cursor, TAG.read]; let db: mongodb.Db; diff --git a/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_tweets_and_to_array.mts b/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_tweets_and_to_array.mts index bf8aeb900d3..7967419d2f3 100644 --- a/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_tweets_and_to_array.mts +++ b/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_tweets_and_to_array.mts @@ -1,7 +1,7 @@ -import { ALERT_TAG, driver, READ_TAG, type mongodb } from '../../driver.mjs'; +import {driver, TAG, type mongodb } from '../../driver.mjs'; export const taskSize = 1500; -export const tags = [ALERT_TAG, CURSOR_TAG, READ_TAG]; +export const tags = [TAG.alert, TAG.cursor, TAG.read]; let db: mongodb.Db; let tweet: Record; diff --git a/test/benchmarks/driver_bench/src/suites/node_specific/find_many_and_to_array.mts b/test/benchmarks/driver_bench/src/suites/node_specific/find_many_and_to_array.mts index 801cc6b810d..3419418942a 100644 --- a/test/benchmarks/driver_bench/src/suites/node_specific/find_many_and_to_array.mts +++ b/test/benchmarks/driver_bench/src/suites/node_specific/find_many_and_to_array.mts @@ -1,8 +1,8 @@ -import { ALERT_TAG, CURSOR_TAG, driver, READ_TAG, SPEC_TAG, type mongodb } from '../../driver.mjs'; +import { driver, TAG, type mongodb } from '../../driver.mjs'; export const taskSize = 16.22; -export const tags = [SPEC_TAG, ALERT_TAG,READ_TAG, CURSOR_TAG] +export const tags = [TAG.alert, TAG.spec, TAG.cursor, TAG.read]; let collection: mongodb.Collection; diff --git a/test/benchmarks/driver_bench/src/suites/node_specific/ping.mts b/test/benchmarks/driver_bench/src/suites/node_specific/ping.mts index 40230ebfed4..429e70a1576 100644 --- a/test/benchmarks/driver_bench/src/suites/node_specific/ping.mts +++ b/test/benchmarks/driver_bench/src/suites/node_specific/ping.mts @@ -1,8 +1,8 @@ -import { ALERT_TAG, driver, type mongodb } from '../../driver.mjs'; +import { TAG, driver, type mongodb } from '../../driver.mjs'; // { ping: 1 } is 15 bytes of BSON x 10,000 iterations export const taskSize = 0.15; -export const tags = [ALERT_TAG]; +export const tags = [TAG.alert]; let db: mongodb.Db; diff --git a/test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_download.mts b/test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_download.mts index af79b03a36b..656919ed9b2 100644 --- a/test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_download.mts +++ b/test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_download.mts @@ -2,10 +2,10 @@ import { createReadStream, createWriteStream, promises as fs } from 'node:fs'; import path from 'node:path'; import { pipeline } from 'node:stream/promises'; -import { ALERT_TAG, CURSOR_TAG, driver, type mongodb, PARALLEL_DIRECTORY, READ_TAG, SPEC_TAG, TEMP_DIRECTORY } from '../../driver.mjs'; +import { driver, type mongodb, PARALLEL_DIRECTORY, TAG, TEMP_DIRECTORY } from '../../driver.mjs'; export const taskSize = 262.144; -export const tags = [SPEC_TAG, ALERT_TAG, READ_TAG, CURSOR_TAG]; +export const tags = [TAG.spec, TAG.alert, TAG.read, TAG.cursor]; let bucket: mongodb.GridFSBucket; diff --git a/test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_upload.mts b/test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_upload.mts index c880f63b102..0942f6bc414 100644 --- a/test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_upload.mts +++ b/test/benchmarks/driver_bench/src/suites/parallel_bench/gridfs_multi_file_upload.mts @@ -3,10 +3,10 @@ import path from 'node:path'; import { Readable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; -import { ALERT_TAG, driver, type mongodb, PARALLEL_DIRECTORY, SPEC_TAG, WRITE_TAG } from '../../driver.mjs'; +import { driver, type mongodb, PARALLEL_DIRECTORY, TAG } from '../../driver.mjs'; export const taskSize = 262.144; -export const tags = [SPEC_TAG, ALERT_TAG, WRITE_TAG]; +export const tags = [TAG.spec, TAG.alert, TAG.write]; let bucket: mongodb.GridFSBucket; diff --git a/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_export.mts b/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_export.mts index 2ff18383f6d..48b32d294f3 100644 --- a/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_export.mts +++ b/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_export.mts @@ -3,10 +3,10 @@ import path from 'node:path'; import readline from 'node:readline/promises'; import stream from 'node:stream/promises'; -import { ALERT_TAG, driver, EJSON, type mongodb, PARALLEL_DIRECTORY, SPEC_TAG, TEMP_DIRECTORY, WRITE_TAG } from '../../driver.mjs'; +import { driver, EJSON, type mongodb, PARALLEL_DIRECTORY, TAG, TEMP_DIRECTORY } from '../../driver.mjs'; export const taskSize = 565; -export const tags = [SPEC_TAG, ALERT_TAG, WRITE_TAG]; +export const tags = [TAG.spec, TAG.alert, TAG.write]; let collection: mongodb.Collection; diff --git a/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_upload.mts b/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_upload.mts index 7ddc2f36064..46a53823924 100644 --- a/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_upload.mts +++ b/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_upload.mts @@ -2,10 +2,10 @@ import { createReadStream, promises as fs } from 'node:fs'; import path from 'node:path'; import readline from 'node:readline/promises'; -import { ALERT_TAG, driver, type mongodb, PARALLEL_DIRECTORY, SPEC_TAG, WRITE_TAG } from '../../driver.mjs'; +import { driver, type mongodb, PARALLEL_DIRECTORY, TAG } from '../../driver.mjs'; export const taskSize = 565; -export const tags = [SPEC_TAG, ALERT_TAG, WRITE_TAG]; +export const tags = [TAG.spec, TAG.alert, TAG.write]; const directory = path.resolve(PARALLEL_DIRECTORY, 'ldjson_multi'); let collection: mongodb.Collection; diff --git a/test/benchmarks/driver_bench/src/suites/single_bench/find_one.mts b/test/benchmarks/driver_bench/src/suites/single_bench/find_one.mts index 7707a133091..09eef87c840 100644 --- a/test/benchmarks/driver_bench/src/suites/single_bench/find_one.mts +++ b/test/benchmarks/driver_bench/src/suites/single_bench/find_one.mts @@ -1,7 +1,7 @@ -import { ALERT_TAG, CURSOR_TAG, driver, READ_TAG, SPEC_TAG, type mongodb } from '../../driver.mjs'; +import { driver, TAG, type mongodb } from '../../driver.mjs'; export const taskSize = 16.22; -export const tags = [SPEC_TAG, ALERT_TAG, CURSOR_TAG, READ_TAG]; +export const tags = [TAG.spec, TAG.alert, TAG.cursor, TAG.read]; let collection: mongodb.Collection<{ _id: number }>; diff --git a/test/benchmarks/driver_bench/src/suites/single_bench/large_doc_insert_one.mts b/test/benchmarks/driver_bench/src/suites/single_bench/large_doc_insert_one.mts index 965d87e95f9..ffbf12b8141 100644 --- a/test/benchmarks/driver_bench/src/suites/single_bench/large_doc_insert_one.mts +++ b/test/benchmarks/driver_bench/src/suites/single_bench/large_doc_insert_one.mts @@ -1,7 +1,7 @@ -import { ALERT_TAG, driver, SPEC_TAG, WRITE_TAG, type mongodb } from '../../driver.mjs'; +import { driver, TAG, type mongodb } from '../../driver.mjs'; export const taskSize = 27.31; -export const tags = [SPEC_TAG, ALERT_TAG, WRITE_TAG]; +export const tags = [TAG.spec, TAG.alert, TAG.write]; let collection: mongodb.Collection; let documents: Record[]; diff --git a/test/benchmarks/driver_bench/src/suites/single_bench/run_command.mts b/test/benchmarks/driver_bench/src/suites/single_bench/run_command.mts index e758470e990..cdcc7819328 100644 --- a/test/benchmarks/driver_bench/src/suites/single_bench/run_command.mts +++ b/test/benchmarks/driver_bench/src/suites/single_bench/run_command.mts @@ -1,8 +1,8 @@ -import { ALERT_TAG, driver, SPEC_TAG, type mongodb } from '../../driver.mjs'; +import { driver, TAG, type mongodb } from '../../driver.mjs'; // { hello: true } is 13 bytes of BSON x 10,000 iterations export const taskSize = 0.13; -export const tags = [SPEC_TAG, ALERT_TAG]; +export const tags = [TAG.spec, TAG.alert]; let db: mongodb.Db; diff --git a/test/benchmarks/driver_bench/src/suites/single_bench/small_doc_insert_one.mts b/test/benchmarks/driver_bench/src/suites/single_bench/small_doc_insert_one.mts index 59e81a2a900..120870226cc 100644 --- a/test/benchmarks/driver_bench/src/suites/single_bench/small_doc_insert_one.mts +++ b/test/benchmarks/driver_bench/src/suites/single_bench/small_doc_insert_one.mts @@ -1,7 +1,8 @@ -import { ALERT_TAG, driver, SPEC_TAG, WRITE_TAG, type mongodb } from '../../driver.mjs'; +import { driver, TAG, type mongodb } from '../../driver.mjs'; export const taskSize = 2.75; -export const tags = [SPEC_TAG, ALERT_TAG, WRITE_TAG]; +export const tags = [TAG.spec, TAG.alert, TAG.write]; + let collection: mongodb.Collection; let documents: Record[]; From 4d3ec5dd6fd8d82b8382956b200be30affe333af Mon Sep 17 00:00:00 2001 From: Warren James Date: Wed, 19 Feb 2025 15:02:40 -0500 Subject: [PATCH 10/20] add error message --- test/benchmarks/driver_bench/src/main.mts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/benchmarks/driver_bench/src/main.mts b/test/benchmarks/driver_bench/src/main.mts index 457e23fc5ac..b540b718aa3 100644 --- a/test/benchmarks/driver_bench/src/main.mts +++ b/test/benchmarks/driver_bench/src/main.mts @@ -203,8 +203,8 @@ function calculateNormalizedResults(results: MetricInfo[]): MetricInfo[] { const primesBench = results.find(r => r.info.test_name === 'primes'); const pingBench = results.find(r => r.info.test_name === 'ping'); - assert.ok(pingBench); - assert.ok(primesBench); + assert.ok(pingBench, 'ping bench results not found!'); + assert.ok(primesBench, 'primes bench results not found!'); const pingThroughput = pingBench.metrics[0].value; const primesThroughput = primesBench.metrics[0].value; primesBench.metrics.push({ 'name': 'normalized_throughput', value: NORMALIZED_PING_SCALING_CONST * pingThroughput / primesThroughput }); From 6897f8c5e74b4fd0603959dbc6a45f217a9e5628 Mon Sep 17 00:00:00 2001 From: Warren James Date: Thu, 20 Feb 2025 16:49:01 -0500 Subject: [PATCH 11/20] update readme --- test/benchmarks/driver_bench/readme.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/benchmarks/driver_bench/readme.md b/test/benchmarks/driver_bench/readme.md index 52e9fe9926a..d660a997672 100644 --- a/test/benchmarks/driver_bench/readme.md +++ b/test/benchmarks/driver_bench/readme.md @@ -64,16 +64,16 @@ The `driver.mts` module is intended to hold various helpers for setup and teardo The `tags` property of `BenchmarkModule` is where a benchmark's tags should be added to facilitate performance alerting and filter of results via our internal tools. -Tags are defined in `driver.mts` and should end with a _TAG suffix. +Tags are defined in `driver.mts` under the `TAG` enum. Whenever a new tag is defined it should be documented in the table below . | tag variable name | tag string value | purpose | |-------------------|------------------------|--------------------------------------------------------------------------------------------------------------------------------------| -| `SPEC_TAG` | `'spec-benchmark'` | Special tag that marks a benchmark as a spec-required benchmark | -| `ALERT_TAG` | `'alerting-benchmark'` | Special tag that enables our perf monitoring tooling to create alerts when regressions in this benchmark's performance are detected | -| `CURSOR_TAG` | `'cursor-benchmark'` | Tag marking a benchmark as being related to cursor performance | -| `READ_TAG` | `'read-benchmark'` | Tag marking a benchmark as being related to read performance | -| `WRITE_TAG` | `'write-benchmark'` | Tag marking a benchmark as being related to write performance | +| `TAG.spec` | `'spec-benchmark'` | Special tag that marks a benchmark as a spec-required benchmark | +| `TAG.alert` | `'alerting-benchmark'` | Special tag that enables our perf monitoring tooling to create alerts when regressions in this benchmark's performance are detected | +| `TAG.cursor` | `'cursor-benchmark'` | Tag marking a benchmark as being related to cursor performance | +| `TAG.read` | `'read-benchmark'` | Tag marking a benchmark as being related to read performance | +| `TAG.write` | `'write-benchmark'` | Tag marking a benchmark as being related to write performance | ## Wishlist From 81c7978de7bea0d28f0dff4d16739bfbdf566bf9 Mon Sep 17 00:00:00 2001 From: Warren James Date: Fri, 21 Feb 2025 11:49:02 -0500 Subject: [PATCH 12/20] rename primes to cpu_baseline --- test/benchmarks/driver_bench/src/main.mts | 16 ++++++++++------ .../{primes.mts => cpu_baseline.mts} | 4 +--- 2 files changed, 11 insertions(+), 9 deletions(-) rename test/benchmarks/driver_bench/src/suites/node_specific/{primes.mts => cpu_baseline.mts} (85%) diff --git a/test/benchmarks/driver_bench/src/main.mts b/test/benchmarks/driver_bench/src/main.mts index b540b718aa3..234938f5038 100644 --- a/test/benchmarks/driver_bench/src/main.mts +++ b/test/benchmarks/driver_bench/src/main.mts @@ -200,19 +200,23 @@ function calculateCompositeBenchmarks(results: MetricInfo[]) { } function calculateNormalizedResults(results: MetricInfo[]): MetricInfo[] { - const primesBench = results.find(r => r.info.test_name === 'primes'); + const baselineBench = results.find(r => r.info.test_name === 'cpuBaseline'); const pingBench = results.find(r => r.info.test_name === 'ping'); assert.ok(pingBench, 'ping bench results not found!'); - assert.ok(primesBench, 'primes bench results not found!'); + assert.ok(baselineBench, 'baseline results not found!'); const pingThroughput = pingBench.metrics[0].value; - const primesThroughput = primesBench.metrics[0].value; - primesBench.metrics.push({ 'name': 'normalized_throughput', value: NORMALIZED_PING_SCALING_CONST * pingThroughput / primesThroughput }); + const cpuBaseline = baselineBench.metrics[0].value; for (const bench of results) { - if (bench.info.test_name === 'ping' || bench.info.test_name === 'primes') continue; + if (bench.info.test_name === 'cpuBaseline') continue; + if (bench.info.test_name === 'ping') { + bench.metrics.push({ 'name': 'normalized_throughput', value: bench.metrics[0].value / cpuBaseline }); + } // Compute normalized_throughput of benchmarks against ping bench - bench.metrics.push({ 'name': 'normalized_throughput', value: bench.metrics[0].value / pingThroughput }); + else { + bench.metrics.push({ 'name': 'normalized_throughput', value: bench.metrics[0].value / pingThroughput }); + } } return results; diff --git a/test/benchmarks/driver_bench/src/suites/node_specific/primes.mts b/test/benchmarks/driver_bench/src/suites/node_specific/cpu_baseline.mts similarity index 85% rename from test/benchmarks/driver_bench/src/suites/node_specific/primes.mts rename to test/benchmarks/driver_bench/src/suites/node_specific/cpu_baseline.mts index 6b5906408cc..f953ac2ed93 100644 --- a/test/benchmarks/driver_bench/src/suites/node_specific/primes.mts +++ b/test/benchmarks/driver_bench/src/suites/node_specific/cpu_baseline.mts @@ -6,11 +6,9 @@ const expectedPrimes = 78_498; // byteLength of // BSON.serialize({ primes: Buffer.from(new Int32Array(sieveOfEratosthenes(findPrimesBelow)).buffer) }).byteLength) // a bin data of int32s -const byteLength = 314_010; -export const taskSize = 3.1401000000000003; // ~3MB worth of work +export const taskSize = .0031401000000000003; // ~3MB worth of work, scaled down 1000 times to make the relative ping measurement be on the order of 10^0 -assert.equal(taskSize, byteLength * 10e-6); // taskSize should stay hardcoded, checking here the math is done right. /** @see https://en.wikipedia.org/wiki/Sieve_of_Eratosthenes */ export function sieveOfEratosthenes(n: number) { From eb8c690b8702ec4b9972e56d277f9d80f8e07879 Mon Sep 17 00:00:00 2001 From: Warren James Date: Fri, 21 Feb 2025 13:43:33 -0500 Subject: [PATCH 13/20] remove unneeded imports --- .../src/suites/multi_bench/find_many_and_empty_cursor.mts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts index 2150e8b340e..248d5c10461 100644 --- a/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ -import { driver, TAG, type mongodb, CURSOR_TAG, READ_TAG } from '../../driver.mjs'; +import { driver, TAG, type mongodb } from '../../driver.mjs'; export const taskSize = 16.22; From 1c7e1f6d65cc00ba989e6a4fd46fa9e8cba9f8c0 Mon Sep 17 00:00:00 2001 From: Warren James Date: Fri, 21 Feb 2025 13:44:13 -0500 Subject: [PATCH 14/20] update scaling for cpu_scaling --- .../driver_bench/src/suites/node_specific/cpu_baseline.mts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/benchmarks/driver_bench/src/suites/node_specific/cpu_baseline.mts b/test/benchmarks/driver_bench/src/suites/node_specific/cpu_baseline.mts index f953ac2ed93..4b4f593b0a9 100644 --- a/test/benchmarks/driver_bench/src/suites/node_specific/cpu_baseline.mts +++ b/test/benchmarks/driver_bench/src/suites/node_specific/cpu_baseline.mts @@ -7,9 +7,8 @@ const expectedPrimes = 78_498; // BSON.serialize({ primes: Buffer.from(new Int32Array(sieveOfEratosthenes(findPrimesBelow)).buffer) }).byteLength) // a bin data of int32s -export const taskSize = .0031401000000000003; // ~3MB worth of work, scaled down 1000 times to make the relative ping measurement be on the order of 10^0 - - +const stableRegionMean = 42.82; +export const taskSize = 3.1401000000000003/stableRegionMean; // ~3MB worth of work scaled down by the mean of the current stable region in CI to bring this value to roughly 1 /** @see https://en.wikipedia.org/wiki/Sieve_of_Eratosthenes */ export function sieveOfEratosthenes(n: number) { // Create a boolean array "prime[0..n]" and initialize From 7eab24fdffd5166a9d0030ef09a414b5eb283f7a Mon Sep 17 00:00:00 2001 From: Warren James Date: Fri, 21 Feb 2025 16:11:00 -0500 Subject: [PATCH 15/20] lint --- test/benchmarks/driver_bench/src/driver.mts | 2 +- test/benchmarks/driver_bench/src/main.mts | 16 ++++++++++------ test/benchmarks/driver_bench/src/runner.mts | 8 ++++---- .../multi_bench/find_many_and_empty_cursor.mts | 2 +- .../src/suites/multi_bench/grid_fs_upload.mts | 2 +- .../suites/multi_bench/large_doc_bulk_insert.mts | 2 +- .../suites/multi_bench/small_doc_bulk_insert.mts | 2 +- ...ggregate_a_million_documents_and_to_array.mts | 2 +- .../aggregate_a_million_tweets_and_to_array.mts | 2 +- .../src/suites/node_specific/cpu_baseline.mts | 2 +- .../node_specific/find_many_and_to_array.mts | 2 +- .../src/suites/node_specific/ping.mts | 2 +- .../parallel_bench/ldjson_multi_file_export.mts | 9 ++++++++- .../src/suites/single_bench/find_one.mts | 2 +- .../suites/single_bench/large_doc_insert_one.mts | 2 +- .../src/suites/single_bench/run_command.mts | 2 +- .../suites/single_bench/small_doc_insert_one.mts | 3 +-- 17 files changed, 36 insertions(+), 26 deletions(-) diff --git a/test/benchmarks/driver_bench/src/driver.mts b/test/benchmarks/driver_bench/src/driver.mts index aa031892f59..4d14ee29ae9 100644 --- a/test/benchmarks/driver_bench/src/driver.mts +++ b/test/benchmarks/driver_bench/src/driver.mts @@ -144,7 +144,7 @@ export type MetricInfo = { info: { test_name: string; args: Record; - tags?: string[] + tags?: string[]; }; metrics: Metric[]; }; diff --git a/test/benchmarks/driver_bench/src/main.mts b/test/benchmarks/driver_bench/src/main.mts index 234938f5038..667ab96cc52 100644 --- a/test/benchmarks/driver_bench/src/main.mts +++ b/test/benchmarks/driver_bench/src/main.mts @@ -18,7 +18,6 @@ import { MONGODB_DRIVER_PATH, MONGODB_DRIVER_REVISION, MONGODB_DRIVER_VERSION, - NORMALIZED_PING_SCALING_CONST, snakeToCamel } from './driver.mjs'; @@ -147,8 +146,8 @@ function calculateCompositeBenchmarks(results: MetricInfo[]) { const aMetricInfo = (testName: string) => - ({ info: { test_name } }: MetricInfo) => - test_name === testName; + ({ info: { test_name } }: MetricInfo) => + test_name === testName; const anMBsMetric = ({ name }: Metric) => name === 'megabytes_per_second'; @@ -211,11 +210,17 @@ function calculateNormalizedResults(results: MetricInfo[]): MetricInfo[] { for (const bench of results) { if (bench.info.test_name === 'cpuBaseline') continue; if (bench.info.test_name === 'ping') { - bench.metrics.push({ 'name': 'normalized_throughput', value: bench.metrics[0].value / cpuBaseline }); + bench.metrics.push({ + name: 'normalized_throughput', + value: bench.metrics[0].value / cpuBaseline + }); } // Compute normalized_throughput of benchmarks against ping bench else { - bench.metrics.push({ 'name': 'normalized_throughput', value: bench.metrics[0].value / pingThroughput }); + bench.metrics.push({ + name: 'normalized_throughput', + value: bench.metrics[0].value / pingThroughput + }); } } @@ -225,5 +230,4 @@ function calculateNormalizedResults(results: MetricInfo[]): MetricInfo[] { results = calculateCompositeBenchmarks(results); results = calculateNormalizedResults(results); - await fs.writeFile('results.json', JSON.stringify(results, undefined, 2), 'utf8'); diff --git a/test/benchmarks/driver_bench/src/runner.mts b/test/benchmarks/driver_bench/src/runner.mts index 93e149e262a..b68bc9cfe49 100644 --- a/test/benchmarks/driver_bench/src/runner.mts +++ b/test/benchmarks/driver_bench/src/runner.mts @@ -17,7 +17,6 @@ type BenchmarkModule = { tags?: string[]; }; - const benchmarkName = snakeToCamel(path.basename(benchmarkFile, '.mjs')); const benchmark: BenchmarkModule = await import(`./${benchmarkFile}`); @@ -83,9 +82,10 @@ const medianExecution = durations[percentileIndex(50, count)]; const megabytesPerSecond = benchmark.taskSize / medianExecution; const tags = benchmark.tags; -if (tags && - (!Array.isArray(tags) || (tags.length > 0 && !tags.every(t => typeof t === 'string')))) { - +if ( + tags && + (!Array.isArray(tags) || (tags.length > 0 && !tags.every(t => typeof t === 'string'))) +) { throw new Error('If tags is specified, it MUST be an array of strings'); } diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts index 248d5c10461..deb745ea646 100644 --- a/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ -import { driver, TAG, type mongodb } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; export const taskSize = 16.22; diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_upload.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_upload.mts index dc3438c1015..002c530ec01 100644 --- a/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_upload.mts +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_upload.mts @@ -1,7 +1,7 @@ import { Readable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; -import { driver, TAG, type mongodb } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; export const taskSize = 52.43; export const tags = [TAG.alert, TAG.spec, TAG.write]; diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/large_doc_bulk_insert.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/large_doc_bulk_insert.mts index 2fe2e8fcd19..6ca9edaaf64 100644 --- a/test/benchmarks/driver_bench/src/suites/multi_bench/large_doc_bulk_insert.mts +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/large_doc_bulk_insert.mts @@ -1,4 +1,4 @@ -import { driver, TAG, type mongodb } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; export const taskSize = 27.31; export const tags = [TAG.alert, TAG.spec, TAG.write]; diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts index aa0dc0df81b..0047731fe1b 100644 --- a/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts @@ -1,4 +1,4 @@ -import { TAG, driver, type mongodb } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; export const taskSize = 2.75; export const tags = [TAG.alert, TAG.spec, TAG.write]; diff --git a/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_documents_and_to_array.mts b/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_documents_and_to_array.mts index 5a2ea49463a..87f24ef3fee 100644 --- a/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_documents_and_to_array.mts +++ b/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_documents_and_to_array.mts @@ -1,4 +1,4 @@ -import { driver, TAG, type mongodb } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; export const taskSize = 16; export const tags = [TAG.alert, TAG.cursor, TAG.read]; diff --git a/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_tweets_and_to_array.mts b/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_tweets_and_to_array.mts index 7967419d2f3..4432f40a4ff 100644 --- a/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_tweets_and_to_array.mts +++ b/test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_tweets_and_to_array.mts @@ -1,4 +1,4 @@ -import {driver, TAG, type mongodb } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; export const taskSize = 1500; export const tags = [TAG.alert, TAG.cursor, TAG.read]; diff --git a/test/benchmarks/driver_bench/src/suites/node_specific/cpu_baseline.mts b/test/benchmarks/driver_bench/src/suites/node_specific/cpu_baseline.mts index 4b4f593b0a9..9760f90921b 100644 --- a/test/benchmarks/driver_bench/src/suites/node_specific/cpu_baseline.mts +++ b/test/benchmarks/driver_bench/src/suites/node_specific/cpu_baseline.mts @@ -8,7 +8,7 @@ const expectedPrimes = 78_498; // a bin data of int32s const stableRegionMean = 42.82; -export const taskSize = 3.1401000000000003/stableRegionMean; // ~3MB worth of work scaled down by the mean of the current stable region in CI to bring this value to roughly 1 +export const taskSize = 3.1401000000000003 / stableRegionMean; // ~3MB worth of work scaled down by the mean of the current stable region in CI to bring this value to roughly 1 /** @see https://en.wikipedia.org/wiki/Sieve_of_Eratosthenes */ export function sieveOfEratosthenes(n: number) { // Create a boolean array "prime[0..n]" and initialize diff --git a/test/benchmarks/driver_bench/src/suites/node_specific/find_many_and_to_array.mts b/test/benchmarks/driver_bench/src/suites/node_specific/find_many_and_to_array.mts index 3419418942a..f435995ea98 100644 --- a/test/benchmarks/driver_bench/src/suites/node_specific/find_many_and_to_array.mts +++ b/test/benchmarks/driver_bench/src/suites/node_specific/find_many_and_to_array.mts @@ -1,4 +1,4 @@ -import { driver, TAG, type mongodb } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; export const taskSize = 16.22; diff --git a/test/benchmarks/driver_bench/src/suites/node_specific/ping.mts b/test/benchmarks/driver_bench/src/suites/node_specific/ping.mts index 429e70a1576..2f95eea344d 100644 --- a/test/benchmarks/driver_bench/src/suites/node_specific/ping.mts +++ b/test/benchmarks/driver_bench/src/suites/node_specific/ping.mts @@ -1,4 +1,4 @@ -import { TAG, driver, type mongodb } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; // { ping: 1 } is 15 bytes of BSON x 10,000 iterations export const taskSize = 0.15; diff --git a/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_export.mts b/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_export.mts index 48b32d294f3..62fef2aa600 100644 --- a/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_export.mts +++ b/test/benchmarks/driver_bench/src/suites/parallel_bench/ldjson_multi_file_export.mts @@ -3,7 +3,14 @@ import path from 'node:path'; import readline from 'node:readline/promises'; import stream from 'node:stream/promises'; -import { driver, EJSON, type mongodb, PARALLEL_DIRECTORY, TAG, TEMP_DIRECTORY } from '../../driver.mjs'; +import { + driver, + EJSON, + type mongodb, + PARALLEL_DIRECTORY, + TAG, + TEMP_DIRECTORY +} from '../../driver.mjs'; export const taskSize = 565; export const tags = [TAG.spec, TAG.alert, TAG.write]; diff --git a/test/benchmarks/driver_bench/src/suites/single_bench/find_one.mts b/test/benchmarks/driver_bench/src/suites/single_bench/find_one.mts index 09eef87c840..e079c723c3e 100644 --- a/test/benchmarks/driver_bench/src/suites/single_bench/find_one.mts +++ b/test/benchmarks/driver_bench/src/suites/single_bench/find_one.mts @@ -1,4 +1,4 @@ -import { driver, TAG, type mongodb } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; export const taskSize = 16.22; export const tags = [TAG.spec, TAG.alert, TAG.cursor, TAG.read]; diff --git a/test/benchmarks/driver_bench/src/suites/single_bench/large_doc_insert_one.mts b/test/benchmarks/driver_bench/src/suites/single_bench/large_doc_insert_one.mts index ffbf12b8141..70cf758888b 100644 --- a/test/benchmarks/driver_bench/src/suites/single_bench/large_doc_insert_one.mts +++ b/test/benchmarks/driver_bench/src/suites/single_bench/large_doc_insert_one.mts @@ -1,4 +1,4 @@ -import { driver, TAG, type mongodb } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; export const taskSize = 27.31; export const tags = [TAG.spec, TAG.alert, TAG.write]; diff --git a/test/benchmarks/driver_bench/src/suites/single_bench/run_command.mts b/test/benchmarks/driver_bench/src/suites/single_bench/run_command.mts index cdcc7819328..0a55c4fae37 100644 --- a/test/benchmarks/driver_bench/src/suites/single_bench/run_command.mts +++ b/test/benchmarks/driver_bench/src/suites/single_bench/run_command.mts @@ -1,4 +1,4 @@ -import { driver, TAG, type mongodb } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; // { hello: true } is 13 bytes of BSON x 10,000 iterations export const taskSize = 0.13; diff --git a/test/benchmarks/driver_bench/src/suites/single_bench/small_doc_insert_one.mts b/test/benchmarks/driver_bench/src/suites/single_bench/small_doc_insert_one.mts index 120870226cc..070b0fff7e5 100644 --- a/test/benchmarks/driver_bench/src/suites/single_bench/small_doc_insert_one.mts +++ b/test/benchmarks/driver_bench/src/suites/single_bench/small_doc_insert_one.mts @@ -1,9 +1,8 @@ -import { driver, TAG, type mongodb } from '../../driver.mjs'; +import { driver, type mongodb, TAG } from '../../driver.mjs'; export const taskSize = 2.75; export const tags = [TAG.spec, TAG.alert, TAG.write]; - let collection: mongodb.Collection; let documents: Record[]; let smallDoc: Record; From 929e61dfa4d02f5b793b3997007471b1035dcef6 Mon Sep 17 00:00:00 2001 From: Warren James Date: Fri, 21 Feb 2025 16:17:21 -0500 Subject: [PATCH 16/20] add improvement_direction --- test/benchmarks/driver_bench/src/driver.mts | 6 +++++- test/benchmarks/driver_bench/src/main.mts | 10 ++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/test/benchmarks/driver_bench/src/driver.mts b/test/benchmarks/driver_bench/src/driver.mts index 4d14ee29ae9..e925e916a14 100644 --- a/test/benchmarks/driver_bench/src/driver.mts +++ b/test/benchmarks/driver_bench/src/driver.mts @@ -138,6 +138,9 @@ export const TEMP_DIRECTORY = path.resolve(SPEC_DIRECTORY, 'tmp'); export type Metric = { name: 'megabytes_per_second' | 'normalized_throughput'; value: number; + metadata?: { + improvement_direction: 'up' | 'down' + }; }; export type MetricInfo = { @@ -163,7 +166,8 @@ export function metrics(test_name: string, result: number, tags?: string[]): Met ), tags }, - metrics: [{ name: 'megabytes_per_second', value: result }] + // FIXME: For now all of our metrics are of throughput so their improvement_direction is up, + metrics: [{ name: 'megabytes_per_second', value: result, metadata: {improvement_direction: 'up'} }] } as const; } diff --git a/test/benchmarks/driver_bench/src/main.mts b/test/benchmarks/driver_bench/src/main.mts index 667ab96cc52..04e573b7734 100644 --- a/test/benchmarks/driver_bench/src/main.mts +++ b/test/benchmarks/driver_bench/src/main.mts @@ -212,14 +212,20 @@ function calculateNormalizedResults(results: MetricInfo[]): MetricInfo[] { if (bench.info.test_name === 'ping') { bench.metrics.push({ name: 'normalized_throughput', - value: bench.metrics[0].value / cpuBaseline + value: bench.metrics[0].value / cpuBaseline, + metadata: { + improvement_direction: 'up' + } }); } // Compute normalized_throughput of benchmarks against ping bench else { bench.metrics.push({ name: 'normalized_throughput', - value: bench.metrics[0].value / pingThroughput + value: bench.metrics[0].value / pingThroughput, + metadata: { + improvement_direction: 'up' + } }); } } From 0efd13e90b92ce285276b25a9af94ae552846a01 Mon Sep 17 00:00:00 2001 From: Warren James Date: Fri, 21 Feb 2025 16:19:37 -0500 Subject: [PATCH 17/20] lint --- test/benchmarks/driver_bench/src/driver.mts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/benchmarks/driver_bench/src/driver.mts b/test/benchmarks/driver_bench/src/driver.mts index e925e916a14..ada62784f29 100644 --- a/test/benchmarks/driver_bench/src/driver.mts +++ b/test/benchmarks/driver_bench/src/driver.mts @@ -139,7 +139,7 @@ export type Metric = { name: 'megabytes_per_second' | 'normalized_throughput'; value: number; metadata?: { - improvement_direction: 'up' | 'down' + improvement_direction: 'up' | 'down'; }; }; @@ -167,7 +167,9 @@ export function metrics(test_name: string, result: number, tags?: string[]): Met tags }, // FIXME: For now all of our metrics are of throughput so their improvement_direction is up, - metrics: [{ name: 'megabytes_per_second', value: result, metadata: {improvement_direction: 'up'} }] + metrics: [ + { name: 'megabytes_per_second', value: result, metadata: { improvement_direction: 'up' } } + ] } as const; } From 362ebb167a66d2401bea6b02e9e5f35d74fe8277 Mon Sep 17 00:00:00 2001 From: Warren James Date: Fri, 21 Feb 2025 16:24:46 -0500 Subject: [PATCH 18/20] require improvement direction --- test/benchmarks/driver_bench/src/driver.mts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/benchmarks/driver_bench/src/driver.mts b/test/benchmarks/driver_bench/src/driver.mts index ada62784f29..794b7809a45 100644 --- a/test/benchmarks/driver_bench/src/driver.mts +++ b/test/benchmarks/driver_bench/src/driver.mts @@ -138,7 +138,7 @@ export const TEMP_DIRECTORY = path.resolve(SPEC_DIRECTORY, 'tmp'); export type Metric = { name: 'megabytes_per_second' | 'normalized_throughput'; value: number; - metadata?: { + metadata: { improvement_direction: 'up' | 'down'; }; }; From 9191fa31b06ddd3cb5c28ad4819d94c05429c298 Mon Sep 17 00:00:00 2001 From: Warren James Date: Fri, 21 Feb 2025 16:42:15 -0500 Subject: [PATCH 19/20] add ticket to fixme comment --- test/benchmarks/driver_bench/src/driver.mts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/benchmarks/driver_bench/src/driver.mts b/test/benchmarks/driver_bench/src/driver.mts index 794b7809a45..9f79a70311c 100644 --- a/test/benchmarks/driver_bench/src/driver.mts +++ b/test/benchmarks/driver_bench/src/driver.mts @@ -166,7 +166,7 @@ export function metrics(test_name: string, result: number, tags?: string[]): Met ), tags }, - // FIXME: For now all of our metrics are of throughput so their improvement_direction is up, + // FIXME(NODE-6781): For now all of our metrics are of throughput so their improvement_direction is up, metrics: [ { name: 'megabytes_per_second', value: result, metadata: { improvement_direction: 'up' } } ] From 072378302d014213357193f76827ad92f75ec137 Mon Sep 17 00:00:00 2001 From: Warren James Date: Fri, 21 Feb 2025 17:12:22 -0500 Subject: [PATCH 20/20] remove dead code --- test/benchmarks/driver_bench/src/driver.mts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/benchmarks/driver_bench/src/driver.mts b/test/benchmarks/driver_bench/src/driver.mts index 9f79a70311c..dffe48408d6 100644 --- a/test/benchmarks/driver_bench/src/driver.mts +++ b/test/benchmarks/driver_bench/src/driver.mts @@ -21,10 +21,6 @@ export const TAG = { write: 'write-benchmark' }; -// Constant used to scale normalized_throughput results for ping benchmark to be roughly in the -// range 0 - 9 to make it easier to work with -export const NORMALIZED_PING_SCALING_CONST = 1000; - /** * The path to the MongoDB Node.js driver. * This MUST be set to the directory the driver is installed in