From 6c3e6d81f1b6fd12f76e7cb197acc46221f8fa60 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Tue, 4 Feb 2025 16:44:21 -0500 Subject: [PATCH 01/12] test(NODE-6705): isolate and warmup benchmarks --- test/benchmarks/driver_bench/.gitignore | 3 + test/benchmarks/driver_bench/package.json | 11 ++ test/benchmarks/driver_bench/src/driver.mts | 181 ++++++++++++++++++ test/benchmarks/driver_bench/src/main.mts | 89 +++++++++ test/benchmarks/driver_bench/src/runner.mts | 111 +++++++++++ .../find_many_and_empty_cursor.mts | 24 +++ .../multi_bench/small_doc_bulk_insert.mts | 25 +++ test/benchmarks/driver_bench/tsconfig.json | 34 ++++ 8 files changed, 478 insertions(+) create mode 100644 test/benchmarks/driver_bench/.gitignore create mode 100644 test/benchmarks/driver_bench/package.json create mode 100644 test/benchmarks/driver_bench/src/driver.mts create mode 100644 test/benchmarks/driver_bench/src/main.mts create mode 100644 test/benchmarks/driver_bench/src/runner.mts create mode 100644 test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts create mode 100644 test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts create mode 100644 test/benchmarks/driver_bench/tsconfig.json diff --git a/test/benchmarks/driver_bench/.gitignore b/test/benchmarks/driver_bench/.gitignore new file mode 100644 index 00000000000..ce54f56f966 --- /dev/null +++ b/test/benchmarks/driver_bench/.gitignore @@ -0,0 +1,3 @@ +results.json +results_*.json +package-lock.json diff --git a/test/benchmarks/driver_bench/package.json b/test/benchmarks/driver_bench/package.json new file mode 100644 index 00000000000..897b3d51e08 --- /dev/null +++ b/test/benchmarks/driver_bench/package.json @@ -0,0 +1,11 @@ +{ + "name": "driver_bench", + "version": "1.0.0", + "scripts": { + "prestart": "tsc", + "start": "node lib/main.js" + }, + "devDependencies": { + "@types/node": "^22.13.0" + } +} diff --git a/test/benchmarks/driver_bench/src/driver.mts b/test/benchmarks/driver_bench/src/driver.mts new file mode 100644 index 00000000000..06fba33a3ff --- /dev/null +++ b/test/benchmarks/driver_bench/src/driver.mts @@ -0,0 +1,181 @@ +import child_process from 'node:child_process'; +import fs from 'node:fs/promises'; +import module from 'node:module'; +import path from 'node:path'; +import process from 'node:process'; + +const __dirname = import.meta.dirname; +const require = module.createRequire(__dirname); + +/** + * The path to the MongoDB Node.js driver. + * This MUST be set to the directory the driver is installed in + * NOT the file "lib/index.js" that is the driver's export. + */ +export const MONGODB_DRIVER_PATH = (() => { + let driverPath = process.env.MONGODB_DRIVER_PATH; + if (!driverPath?.length) { + driverPath = path.resolve(__dirname, '../../../..'); + } + return driverPath; +})(); + +/** Grab the version from the package.json */ +export const { version: MONGODB_DRIVER_VERSION } = require( + path.join(MONGODB_DRIVER_PATH, 'package.json') +); + +/** + * Use git to optionally determine the git revision, + * but the benchmarks could be run against an npm installed version so this should be allowed to fail + */ +export const MONGODB_DRIVER_REVISION = (() => { + try { + return child_process + .execSync('git rev-parse --short HEAD', { + cwd: MONGODB_DRIVER_PATH, + encoding: 'utf8' + }) + .trim(); + } catch { + return 'unknown revision'; + } +})(); + +/** + * Find the BSON dependency inside the driver PATH given and grab the version from the package.json. + */ +export const MONGODB_BSON_PATH = path.join(MONGODB_DRIVER_PATH, 'node_modules', 'bson'); +export const { version: MONGODB_BSON_VERSION } = require( + path.join(MONGODB_BSON_PATH, 'package.json') +); + +/** + * If you need to test BSON changes, you should clone, checkout and build BSON. + * run: `npm link` with no arguments to register the link. + * Then in the driver you are testing run `npm link bson` to use your local build. + * + * This will symlink the BSON into the driver's node_modules directory. So here + * we can find the revision of the BSON we are testing against if .git exists. + */ +export const MONGODB_BSON_REVISION = await (async () => { + const bsonGitExists = await fs.access(path.join(MONGODB_BSON_PATH, '.git')).then( + () => true, + () => false + ); + if (!bsonGitExists) { + return 'installed from npm'; + } + try { + return child_process + .execSync('git rev-parse --short HEAD', { + cwd: path.join(MONGODB_BSON_PATH), + encoding: 'utf8' + }) + .trim(); + } catch { + return 'unknown revision'; + } +})(); + +export const MONGODB_CLIENT_OPTIONS = (() => { + const optionsString = process.env.MONGODB_CLIENT_OPTIONS; + let options = undefined; + if (optionsString?.length) { + options = JSON.parse(optionsString); + } + return { ...options }; +})(); + +export const MONGODB_URI = (() => { + if (process.env.MONGODB_URI?.length) return process.env.MONGODB_URI; + return 'mongodb://127.0.0.1:27017'; +})(); + +export function snakeToCamel(name: string) { + return name + .split('_') + .map((s, i) => (i !== 0 ? s[0].toUpperCase() + s.slice(1) : s)) + .join(''); +} + +import type mongodb from '../../../../mongodb.js'; +export type { mongodb }; + +const { MongoClient /* GridFSBucket */ } = require(path.join(MONGODB_DRIVER_PATH)); + +const DB_NAME = 'perftest'; +const COLLECTION_NAME = 'corpus'; + +const SPEC_DIRECTORY = path.resolve(__dirname, '..', '..', 'driverBench', 'spec'); + +export function metrics(test_name: string, result: number, count: number) { + return { + info: { + test_name, + // Args can only be a map of string -> int32. So if its a number leave it be, + // if it is anything else test for truthiness and set to 1 or 0. + args: Object.fromEntries( + Object.entries(MONGODB_CLIENT_OPTIONS).map(([key, value]) => [ + key, + typeof value === 'number' ? value : value ? 1 : 0 + ]) + ) + }, + metrics: [ + { name: 'megabytes_per_second', value: result }, + { name: 'count', value: count } + ] + } as const; +} + +/** + * This class exists to abstract some of the driver API so we can gloss over version differences. + * For use in setup/teardown mostly. + */ +export class DriverTester { + private utilClient: mongodb.MongoClient; + private client: mongodb.MongoClient; + constructor() { + this.utilClient = new MongoClient(MONGODB_URI, MONGODB_CLIENT_OPTIONS); + this.client = new MongoClient(MONGODB_URI, MONGODB_CLIENT_OPTIONS); + } + + get ns() { + return this.utilClient.db(DB_NAME).collection(COLLECTION_NAME); + } + + async drop() { + await this.ns.drop().catch(() => null); + await this.utilClient + .db(DB_NAME) + .dropDatabase() + .catch(() => null); + } + + async create() { + return await this.utilClient.db(DB_NAME).createCollection(COLLECTION_NAME); + } + + async load(filePath: string, type: 'json' | 'string' | 'buffer'): Promise { + const content = await fs.readFile(path.join(SPEC_DIRECTORY, filePath)); + if (type === 'buffer') return content; + const string = content.toString('utf8'); + if (type === 'string') return string; + if (type === 'json') return JSON.parse(string); + throw new Error('unknown type: ' + type); + } + + async insertManyOf(document: Record, length: number, addId = false) { + await this.ns.insertMany( + Array.from({ length }, (_, _id) => ({ ...(addId ? { _id } : {}), ...document })) as any[] + ); + } + + async close() { + await this.client.close(); + await this.utilClient.close(); + } +} + +export const driver = new DriverTester(); diff --git a/test/benchmarks/driver_bench/src/main.mts b/test/benchmarks/driver_bench/src/main.mts new file mode 100644 index 00000000000..9e7601e0ab6 --- /dev/null +++ b/test/benchmarks/driver_bench/src/main.mts @@ -0,0 +1,89 @@ +/* eslint-disable no-console */ +import child_process from 'node:child_process'; +import events from 'node:events'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import util from 'node:util'; + +import { + MONGODB_BSON_PATH, + MONGODB_BSON_REVISION, + MONGODB_BSON_VERSION, + MONGODB_CLIENT_OPTIONS, + MONGODB_DRIVER_PATH, + MONGODB_DRIVER_REVISION, + MONGODB_DRIVER_VERSION, + snakeToCamel +} from './driver.mjs'; + +const __dirname = import.meta.dirname; + +/** Find every mjs file in the suites folder */ +async function getBenchmarks(): Promise< + Record>> +> { + const tests: Record< + string, + Record> + > = Object.create(null); + const suites = await fs.readdir(path.join(__dirname, 'suites')); + for (const suite of suites) { + const benchmarks = await fs.readdir(path.join(__dirname, 'suites', suite)); + for (const benchmark of benchmarks) { + if (!benchmark.endsWith('.mjs')) continue; + tests[suite] ??= Object.create(null); + tests[suite][benchmark] = { benchFile: path.join('suites', suite, benchmark) }; + } + } + return tests; +} + +const hw = os.cpus(); +const ram = os.totalmem() / 1024 ** 3; +const platform = { name: hw[0].model, cores: hw.length, ram: `${ram}GB` }; + +const systemInfo = () => + [ + `\n- cpu: ${platform.name}`, + `- cores: ${platform.cores}`, + `- arch: ${os.arch()}`, + `- os: ${process.platform} (${os.release()})`, + `- ram: ${platform.ram}`, + `- node: ${process.version}`, + `- driver: ${MONGODB_DRIVER_VERSION} (${MONGODB_DRIVER_REVISION}): ${MONGODB_DRIVER_PATH}`, + ` - options ${util.inspect(MONGODB_CLIENT_OPTIONS)}`, + `- bson: ${MONGODB_BSON_VERSION} (${MONGODB_BSON_REVISION}): (${MONGODB_BSON_PATH})\n` + ].join('\n'); + +console.log(systemInfo()); + +const tests = await getBenchmarks(); +const runnerPath = path.join(__dirname, 'runner.mjs'); + +const results = []; + +for (const [suite, benchmarks] of Object.entries(tests)) { + console.group(snakeToCamel(suite)); + + for (const [benchmark, { benchFile }] of Object.entries(benchmarks)) { + console.log(snakeToCamel(path.basename(benchmark, '.mjs'))); + + const runner = child_process.fork(runnerPath, [benchFile], { stdio: 'inherit' }); + + const [exitCode] = await events.once(runner, 'close'); + if (exitCode !== 0) { + throw new Error(`Benchmark exited with failure: ${exitCode}`); + } + + const result = JSON.parse( + await fs.readFile(`results_${path.basename(benchmark, '.mjs')}.json`, 'utf8') + ); + + results.push(result); + } + + console.groupEnd(); +} + +await fs.writeFile('results.json', JSON.stringify(results, undefined, 2), 'utf8'); diff --git a/test/benchmarks/driver_bench/src/runner.mts b/test/benchmarks/driver_bench/src/runner.mts new file mode 100644 index 00000000000..0865cff42a5 --- /dev/null +++ b/test/benchmarks/driver_bench/src/runner.mts @@ -0,0 +1,111 @@ +/* eslint-disable no-console */ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import process from 'node:process'; + +import { metrics, snakeToCamel } from './driver.mjs'; + +const [, , benchmarkFile] = process.argv; + +type BenchmarkModule = { + taskSize: number; + before?: () => Promise; + beforeEach?: () => Promise; + run: () => Promise; + afterEach?: () => Promise; + after?: () => Promise; +}; + +const benchmarkName = snakeToCamel(path.basename(benchmarkFile, '.mjs')); +const benchmark: BenchmarkModule = await import(`./${benchmarkFile}`); + +if (typeof benchmark.taskSize !== 'number') throw new Error('missing taskSize'); +if (typeof benchmark.run !== 'function') throw new Error('missing run'); + +/** CRITICAL SECTION: time task took in seconds */ +async function timeTask() { + const start = performance.now(); + await benchmark.run(); + const end = performance.now(); + return (end - start) / 1000; +} + +/** 1 min in seconds */ +const ONE_MIN = 1 * 60; +/** 5 min in seconds */ +const FIVE_MIN = 5 * 60; +/** Don't run more than 100 iterations */ +const MAX_COUNT = 100; + +await benchmark.before?.(); + +// for 1/10th the max iterations +const warmupIterations = (MAX_COUNT / 10) | 0; + +// Warm Up. +for (let i = 0; i < warmupIterations; i++) { + await benchmark.beforeEach?.(); + await timeTask(); + await benchmark.afterEach?.(); +} + +// Allocate an obscene amount of space +const data = new Float64Array(10_000_000); + +// Test. +let totalDuration = 0; +let count = 0; +do { + await benchmark.beforeEach?.(); + + data[count] = await timeTask(); + + await benchmark.afterEach?.(); + + totalDuration += data[count]; // time moves up by benchmark exec time not wall clock + count += 1; + + // must run for at least one minute + if (totalDuration < ONE_MIN) continue; + + // 100 runs OR five minutes + if (count > 100 || totalDuration > FIVE_MIN) break; + + // count exceeds data space, we never intend to have more than a million data points let alone 10M + if (count === data.length) break; + + // else: more than one min, less than 100 iterations, less than 5min + + // eslint-disable-next-line no-constant-condition +} while (true); + +await benchmark.after?.(); + +const durations = data.subarray(0, count).toSorted((a, b) => a - b); + +function percentileIndex(percentile: number, count: number) { + return Math.max(Math.floor((count * percentile) / 100 - 1), 0); +} + +const medianExecution = durations[percentileIndex(50, count)]; + +console.log( + ' ', + benchmarkName, + 'finished in', + totalDuration, + 'sec and ran', + count, + 'iterations.', + 'median exec time', + medianExecution, + 'sec', + benchmark.taskSize / medianExecution, + 'mb/sec' +); + +await fs.writeFile( + `results_${path.basename(benchmarkFile, '.mjs')}.json`, + JSON.stringify(metrics(benchmarkName, medianExecution, count), undefined, 2) + '\n', + 'utf8' +); diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts new file mode 100644 index 00000000000..9048a90b771 --- /dev/null +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts @@ -0,0 +1,24 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { driver, type mongodb } from '../../driver.mjs'; + +export const taskSize = 16.22; + +let collection: mongodb.Collection; + +export async function before() { + await driver.drop(); + collection = await driver.create(); + const tweet = await driver.load('single_and_multi_document/tweet.json', 'json'); + await driver.insertManyOf(tweet, 10000); +} + +export async function run() { + for await (const doc of collection.find({})) { + // empty + } +} + +export async function after() { + await driver.drop(); + await driver.close(); +} diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts new file mode 100644 index 00000000000..10dca1d4ea9 --- /dev/null +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts @@ -0,0 +1,25 @@ +import { driver, type mongodb } from '../../driver.mjs'; + +export const taskSize = 2.75; + +let collection: mongodb.Collection; +let documents: any[]; + +export async function before() { + const smallDoc = await driver.load('single_and_multi_document/small_doc.json', 'json'); + documents = Array.from({ length: 10000 }, () => ({ ...smallDoc })) as any[]; +} + +export async function beforeEach() { + await driver.drop(); + collection = await driver.create(); +} + +export async function run() { + await collection.insertMany(documents, { ordered: true }); +} + +export async function after() { + await driver.drop(); + await driver.close(); +} diff --git a/test/benchmarks/driver_bench/tsconfig.json b/test/benchmarks/driver_bench/tsconfig.json new file mode 100644 index 00000000000..d7eb0f5c29e --- /dev/null +++ b/test/benchmarks/driver_bench/tsconfig.json @@ -0,0 +1,34 @@ +{ + "compilerOptions": { + "allowJs": true, + "checkJs": false, + "strict": true, + "alwaysStrict": true, + "target": "ESNext", + "verbatimModuleSyntax": true, + "module": "NodeNext", + "moduleResolution": "NodeNext", + "skipLibCheck": true, + // We don't make use of tslib helpers, all syntax used is supported by target engine + "importHelpers": false, + "noEmitHelpers": true, + // Never emit error filled code + "noEmitOnError": true, + "outDir": "lib", + // We want the sourcemaps in a separate file + "inlineSourceMap": false, + "sourceMap": true, + // we include sources in the release + "inlineSources": false, + // Prevents web types from being suggested by vscode. + "types": [ + "node" + ], + "forceConsistentCasingInFileNames": true, + "noImplicitOverride": true, + "noImplicitReturns": true, + }, + "include": [ + "src/**/*" + ] +} From 5ebb72db701b5e8a2fe641e628b07844d58fc613 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Tue, 4 Feb 2025 17:12:19 -0500 Subject: [PATCH 02/12] test: run new bench in CI --- .evergreen/config.in.yml | 25 ++++++++++++++--- .evergreen/config.yml | 33 +++++++++++++++++++++++ .evergreen/generate_evergreen_tasks.js | 31 ++++++++++++++++++--- .evergreen/run-benchmarks.sh | 6 +++++ test/benchmarks/driver_bench/package.json | 2 +- 5 files changed, 88 insertions(+), 9 deletions(-) diff --git a/.evergreen/config.in.yml b/.evergreen/config.in.yml index a2a0532a934..171942cbdab 100644 --- a/.evergreen/config.in.yml +++ b/.evergreen/config.in.yml @@ -142,7 +142,7 @@ functions: working_dir: "src" timeout_secs: 300 binary: bash - args: + args: - .evergreen/run-tests.sh "run serverless tests": @@ -158,11 +158,11 @@ functions: timeout_secs: 300 working_dir: src binary: bash - env: + env: AUTH: 'auth' SSL: 'ssl' add_expansions_to_env: true - args: + args: - .evergreen/run-serverless-tests.sh "start-load-balancer": @@ -882,6 +882,23 @@ functions: binary: bash args: - ${PROJECT_DIRECTORY}/.evergreen/run-benchmarks.sh + + # TEMP! + "run new spec driver benchmarks": + - command: subprocess.exec + type: test + params: + working_dir: "src" + env: + PROJECT_DIRECTORY: ${PROJECT_DIRECTORY} + MONGODB_URI: ${MONGODB_URI} + DRIVERS_TOOLS: ${DRIVERS_TOOLS} + MONGODB_CLIENT_OPTIONS: ${MONGODB_CLIENT_OPTIONS} + NEW_BENCH: "true" + binary: bash + args: + - ${PROJECT_DIRECTORY}/.evergreen/run-benchmarks.sh + "run x509 auth tests": - command: subprocess.exec type: test @@ -1191,7 +1208,7 @@ task_groups: binary: bash args: - ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh - + tasks: - ".serverless" diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 5fe7bf2efdb..2c5f6efac92 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -825,6 +825,20 @@ functions: binary: bash args: - ${PROJECT_DIRECTORY}/.evergreen/run-benchmarks.sh + run new spec driver benchmarks: + - command: subprocess.exec + type: test + params: + working_dir: src + env: + PROJECT_DIRECTORY: ${PROJECT_DIRECTORY} + MONGODB_URI: ${MONGODB_URI} + DRIVERS_TOOLS: ${DRIVERS_TOOLS} + MONGODB_CLIENT_OPTIONS: ${MONGODB_CLIENT_OPTIONS} + NEW_BENCH: 'true' + binary: bash + args: + - ${PROJECT_DIRECTORY}/.evergreen/run-benchmarks.sh run x509 auth tests: - command: subprocess.exec type: test @@ -3305,6 +3319,24 @@ tasks: - command: perf.send params: file: src/results.json + - name: run-spec-benchmark-tests-node-server-new + tags: + - run-spec-benchmark-tests + - performance + exec_timeout_secs: 3600 + commands: + - command: expansions.update + type: setup + params: + updates: + - {key: NODE_LTS_VERSION, value: v22.11.0} + - {key: VERSION, value: v6.0-perf} + - {key: TOPOLOGY, value: server} + - {key: AUTH, value: noauth} + - {key: MONGODB_CLIENT_OPTIONS, value: '{}'} + - func: install dependencies + - func: bootstrap mongo-orchestration + - func: run new spec driver benchmarks - name: run-unit-tests-node-16 tags: - unit-tests @@ -5122,6 +5154,7 @@ buildvariants: - run-spec-benchmark-tests-node-server-timeoutMS-0 - run-spec-benchmark-tests-node-server-monitorCommands-true - run-spec-benchmark-tests-node-server-logging + - run-spec-benchmark-tests-node-server-new - name: rhel8-custom-dependency-tests display_name: Custom Dependency Version Test run_on: rhel80-large diff --git a/.evergreen/generate_evergreen_tasks.js b/.evergreen/generate_evergreen_tasks.js index b263dc053cb..0e6b735fb8e 100644 --- a/.evergreen/generate_evergreen_tasks.js +++ b/.evergreen/generate_evergreen_tasks.js @@ -212,7 +212,7 @@ TASKS.push( { func: 'run socks5 tests' } ] } - ] + ] ); TASKS.push({ @@ -432,8 +432,8 @@ for (const { const expansions = { NODE_LTS_VERSION, NPM_VERSION }; const taskNames = tasks.map(({ name }) => name); - expansions.CLIENT_ENCRYPTION = String(!!clientEncryption) - expansions.TEST_CSFLE = expansions.CLIENT_ENCRYPTION + expansions.CLIENT_ENCRYPTION = String(!!clientEncryption); + expansions.TEST_CSFLE = expansions.CLIENT_ENCRYPTION; BUILD_VARIANTS.push({ name, display_name, run_on, expansions, tasks: taskNames }); } @@ -752,6 +752,28 @@ function addPerformanceTasks() { ] }); + // temp + const makePerfTaskNEW = (name, MONGODB_CLIENT_OPTIONS) => ({ + name, + tags: ['run-spec-benchmark-tests', 'performance'], + exec_timeout_secs: 3600, + commands: [ + updateExpansions({ + NODE_LTS_VERSION: 'v22.11.0', + VERSION: 'v6.0-perf', + TOPOLOGY: 'server', + AUTH: 'noauth', + MONGODB_CLIENT_OPTIONS: JSON.stringify(MONGODB_CLIENT_OPTIONS) + }), + ...[ + 'install dependencies', + 'bootstrap mongo-orchestration', + 'run new spec driver benchmarks' + ].map(func => ({ func })) + // No perf send! just testing + ] + }); + const tasks = [ makePerfTask('run-spec-benchmark-tests-node-server', {}), makePerfTask('run-spec-benchmark-tests-node-server-timeoutMS-120000', { timeoutMS: 120000 }), @@ -762,7 +784,8 @@ function addPerformanceTasks() { makePerfTask('run-spec-benchmark-tests-node-server-logging', { mongodbLogPath: 'stderr', mongodbLogComponentSeverities: { default: 'trace' } - }) + }), + makePerfTaskNEW('run-spec-benchmark-tests-node-server-new', {}) ]; TASKS.push(...tasks); diff --git a/.evergreen/run-benchmarks.sh b/.evergreen/run-benchmarks.sh index c81d6924716..9593c0db41c 100644 --- a/.evergreen/run-benchmarks.sh +++ b/.evergreen/run-benchmarks.sh @@ -11,6 +11,12 @@ export MONGODB_CLIENT_OPTIONS=$MONGODB_CLIENT_OPTIONS npm run build:ts +if [[ "${NEW_BENCH:-}" == "true" ]]; then + pushd test/benchmarks/driver_bench + npm start + popd + exit 0 +fi # If MONGODB_CLIENT_OPTIONS contains mongodbLogComponentSeverities redirect stderr to a file if [[ $MONGODB_CLIENT_OPTIONS == *"mongodbLogComponentSeverities"* ]]; then diff --git a/test/benchmarks/driver_bench/package.json b/test/benchmarks/driver_bench/package.json index 897b3d51e08..489d77dc609 100644 --- a/test/benchmarks/driver_bench/package.json +++ b/test/benchmarks/driver_bench/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "scripts": { "prestart": "tsc", - "start": "node lib/main.js" + "start": "node lib/main.mjs" }, "devDependencies": { "@types/node": "^22.13.0" From c1c919360c85d7ec1a713dd3e9a9ec95cc79e422 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Tue, 4 Feb 2025 19:08:47 -0500 Subject: [PATCH 03/12] comments --- test/benchmarks/driver_bench/src/driver.mts | 21 +++++++++++++++---- .../find_many_and_empty_cursor.mts | 5 ++++- .../multi_bench/small_doc_bulk_insert.mts | 11 +++++++--- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/test/benchmarks/driver_bench/src/driver.mts b/test/benchmarks/driver_bench/src/driver.mts index 06fba33a3ff..c911e490dec 100644 --- a/test/benchmarks/driver_bench/src/driver.mts +++ b/test/benchmarks/driver_bench/src/driver.mts @@ -102,7 +102,7 @@ export function snakeToCamel(name: string) { import type mongodb from '../../../../mongodb.js'; export type { mongodb }; -const { MongoClient /* GridFSBucket */ } = require(path.join(MONGODB_DRIVER_PATH)); +const { MongoClient, GridFSBucket } = require(path.join(MONGODB_DRIVER_PATH)); const DB_NAME = 'perftest'; const COLLECTION_NAME = 'corpus'; @@ -124,6 +124,7 @@ export function metrics(test_name: string, result: number, count: number) { }, metrics: [ { name: 'megabytes_per_second', value: result }, + // Reporting the count so we can verify programmatically or in UI how many iterations we reached { name: 'count', value: count } ] } as const; @@ -135,16 +136,28 @@ export function metrics(test_name: string, result: number, count: number) { */ export class DriverTester { private utilClient: mongodb.MongoClient; - private client: mongodb.MongoClient; + public client: mongodb.MongoClient; constructor() { this.utilClient = new MongoClient(MONGODB_URI, MONGODB_CLIENT_OPTIONS); this.client = new MongoClient(MONGODB_URI, MONGODB_CLIENT_OPTIONS); } - get ns() { + private get ns() { return this.utilClient.db(DB_NAME).collection(COLLECTION_NAME); } + public get db() { + return this.client.db(DB_NAME); + } + + public get collection() { + return this.client.db(DB_NAME).collection(COLLECTION_NAME); + } + + public get bucket(): mongodb.GridFSBucket { + return new GridFSBucket(this.db); + } + async drop() { await this.ns.drop().catch(() => null); await this.utilClient @@ -154,7 +167,7 @@ export class DriverTester { } async create() { - return await this.utilClient.db(DB_NAME).createCollection(COLLECTION_NAME); + await this.utilClient.db(DB_NAME).createCollection(COLLECTION_NAME); } async load(filePath: string, type: 'json' | 'string' | 'buffer'): Promise { diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts index 9048a90b771..f404c1f057c 100644 --- a/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts @@ -7,9 +7,12 @@ let collection: mongodb.Collection; export async function before() { await driver.drop(); - collection = await driver.create(); + await driver.create(); + const tweet = await driver.load('single_and_multi_document/tweet.json', 'json'); await driver.insertManyOf(tweet, 10000); + + collection = driver.collection; } export async function run() { diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts index 10dca1d4ea9..36396ec549a 100644 --- a/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts @@ -4,15 +4,20 @@ export const taskSize = 2.75; let collection: mongodb.Collection; let documents: any[]; +let smallDoc: any; export async function before() { - const smallDoc = await driver.load('single_and_multi_document/small_doc.json', 'json'); - documents = Array.from({ length: 10000 }, () => ({ ...smallDoc })) as any[]; + smallDoc = await driver.load('single_and_multi_document/small_doc.json', 'json'); } export async function beforeEach() { await driver.drop(); - collection = await driver.create(); + await driver.create(); + + // Make new "documents" so the _id field is not carried over from the last run + documents = Array.from({ length: 10000 }, () => ({ ...smallDoc })) as any[]; + + collection = driver.collection; } export async function run() { From 908647b29bcec61f03cf8814f495dfa448abbd7d Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Wed, 5 Feb 2025 10:22:06 -0500 Subject: [PATCH 04/12] fixes --- test/benchmarks/driver_bench/src/main.mts | 5 +++++ test/benchmarks/driver_bench/src/runner.mts | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/test/benchmarks/driver_bench/src/main.mts b/test/benchmarks/driver_bench/src/main.mts index 9e7601e0ab6..516f1a440d0 100644 --- a/test/benchmarks/driver_bench/src/main.mts +++ b/test/benchmarks/driver_bench/src/main.mts @@ -18,6 +18,7 @@ import { } from './driver.mjs'; const __dirname = import.meta.dirname; +const alphabetically = (a: string, b: string) => String.prototype.localeCompare.call(a, b); /** Find every mjs file in the suites folder */ async function getBenchmarks(): Promise< @@ -28,8 +29,12 @@ async function getBenchmarks(): Promise< Record> > = Object.create(null); const suites = await fs.readdir(path.join(__dirname, 'suites')); + suites.sort(alphabetically); + for (const suite of suites) { const benchmarks = await fs.readdir(path.join(__dirname, 'suites', suite)); + benchmarks.sort(alphabetically); + for (const benchmark of benchmarks) { if (!benchmark.endsWith('.mjs')) continue; tests[suite] ??= Object.create(null); diff --git a/test/benchmarks/driver_bench/src/runner.mts b/test/benchmarks/driver_bench/src/runner.mts index 0865cff42a5..dd1b25ed3f1 100644 --- a/test/benchmarks/driver_bench/src/runner.mts +++ b/test/benchmarks/driver_bench/src/runner.mts @@ -88,6 +88,7 @@ function percentileIndex(percentile: number, count: number) { } const medianExecution = durations[percentileIndex(50, count)]; +const megabytesPerSecond = benchmark.taskSize / medianExecution; console.log( ' ', @@ -100,12 +101,12 @@ console.log( 'median exec time', medianExecution, 'sec', - benchmark.taskSize / medianExecution, + megabytesPerSecond, 'mb/sec' ); await fs.writeFile( `results_${path.basename(benchmarkFile, '.mjs')}.json`, - JSON.stringify(metrics(benchmarkName, medianExecution, count), undefined, 2) + '\n', + JSON.stringify(metrics(benchmarkName, megabytesPerSecond, count), undefined, 2) + '\n', 'utf8' ); From 409625691d85a185161fbbbbbe825aa2d7c82420 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Wed, 5 Feb 2025 14:34:54 -0500 Subject: [PATCH 05/12] chore: improve final print, sort, and predict finish time --- test/benchmarks/driver_bench/src/main.mts | 29 ++++++++++++++++----- test/benchmarks/driver_bench/src/runner.mts | 19 +++++--------- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/test/benchmarks/driver_bench/src/main.mts b/test/benchmarks/driver_bench/src/main.mts index 516f1a440d0..008a00f1368 100644 --- a/test/benchmarks/driver_bench/src/main.mts +++ b/test/benchmarks/driver_bench/src/main.mts @@ -18,12 +18,22 @@ import { } from './driver.mjs'; const __dirname = import.meta.dirname; -const alphabetically = (a: string, b: string) => String.prototype.localeCompare.call(a, b); + +export const alphabetically = (a: unknown, b: unknown) => { + const res = `${a}`.localeCompare(`${b}`, 'en-US', { + usage: 'sort', + numeric: true, + ignorePunctuation: false + }); + return res < 0 ? -1 : res > 0 ? 1 : 0; +}; /** Find every mjs file in the suites folder */ -async function getBenchmarks(): Promise< - Record>> -> { +async function getBenchmarks(): Promise<{ + tests: Record>>; + total: number; +}> { + let total = 0; const tests: Record< string, Record> @@ -39,15 +49,21 @@ async function getBenchmarks(): Promise< if (!benchmark.endsWith('.mjs')) continue; tests[suite] ??= Object.create(null); tests[suite][benchmark] = { benchFile: path.join('suites', suite, benchmark) }; + total += 1; } } - return tests; + return { tests, total }; } const hw = os.cpus(); const ram = os.totalmem() / 1024 ** 3; const platform = { name: hw[0].model, cores: hw.length, ram: `${ram}GB` }; +const { tests, total } = await getBenchmarks(); + +const earliest = new Date(Date.now() + total * 60 * 1000); // plus one min per bench +const latest = new Date(Date.now() + total * 6 * 60 * 1000); // plus six min per bench (if we overshoot the 5 min limit) + const systemInfo = () => [ `\n- cpu: ${platform.name}`, @@ -56,6 +72,8 @@ const systemInfo = () => `- os: ${process.platform} (${os.release()})`, `- ram: ${platform.ram}`, `- node: ${process.version}`, + `- running ${total} benchmarks`, + ` - finishes soonest: ${earliest.toLocaleTimeString('en-US', { timeZoneName: 'short' })} latest ${latest.toLocaleTimeString('en-US', { timeZoneName: 'short' })}`, `- driver: ${MONGODB_DRIVER_VERSION} (${MONGODB_DRIVER_REVISION}): ${MONGODB_DRIVER_PATH}`, ` - options ${util.inspect(MONGODB_CLIENT_OPTIONS)}`, `- bson: ${MONGODB_BSON_VERSION} (${MONGODB_BSON_REVISION}): (${MONGODB_BSON_PATH})\n` @@ -63,7 +81,6 @@ const systemInfo = () => console.log(systemInfo()); -const tests = await getBenchmarks(); const runnerPath = path.join(__dirname, 'runner.mjs'); const results = []; diff --git a/test/benchmarks/driver_bench/src/runner.mts b/test/benchmarks/driver_bench/src/runner.mts index dd1b25ed3f1..8ba70e17266 100644 --- a/test/benchmarks/driver_bench/src/runner.mts +++ b/test/benchmarks/driver_bench/src/runner.mts @@ -69,7 +69,7 @@ do { if (totalDuration < ONE_MIN) continue; // 100 runs OR five minutes - if (count > 100 || totalDuration > FIVE_MIN) break; + if (count >= 100 || totalDuration >= FIVE_MIN) break; // count exceeds data space, we never intend to have more than a million data points let alone 10M if (count === data.length) break; @@ -91,18 +91,11 @@ const medianExecution = durations[percentileIndex(50, count)]; const megabytesPerSecond = benchmark.taskSize / medianExecution; console.log( - ' ', - benchmarkName, - 'finished in', - totalDuration, - 'sec and ran', - count, - 'iterations.', - 'median exec time', - medianExecution, - 'sec', - megabytesPerSecond, - 'mb/sec' + ' '.repeat(3), + ...['total time:', totalDuration, 'sec,'], + ...['ran:', count, 'times,'], + ...['time per run:', medianExecution, 'sec,'], + ...['throughput:', megabytesPerSecond, 'mb/sec'] ); await fs.writeFile( From 0709f1011be47783f3608cf539d5a196b8a55281 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Wed, 5 Feb 2025 15:07:16 -0500 Subject: [PATCH 06/12] chore: prints --- test/benchmarks/driver_bench/src/main.mts | 3 ++- test/benchmarks/driver_bench/src/runner.mts | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/benchmarks/driver_bench/src/main.mts b/test/benchmarks/driver_bench/src/main.mts index 008a00f1368..0521b675169 100644 --- a/test/benchmarks/driver_bench/src/main.mts +++ b/test/benchmarks/driver_bench/src/main.mts @@ -73,7 +73,8 @@ const systemInfo = () => `- ram: ${platform.ram}`, `- node: ${process.version}`, `- running ${total} benchmarks`, - ` - finishes soonest: ${earliest.toLocaleTimeString('en-US', { timeZoneName: 'short' })} latest ${latest.toLocaleTimeString('en-US', { timeZoneName: 'short' })}`, + ` - finishes soonest: ${earliest.toLocaleTimeString('en-US', { timeZoneName: 'short' })}`, + ` latest: ${latest.toLocaleTimeString('en-US', { timeZoneName: 'short' })}`, `- driver: ${MONGODB_DRIVER_VERSION} (${MONGODB_DRIVER_REVISION}): ${MONGODB_DRIVER_PATH}`, ` - options ${util.inspect(MONGODB_CLIENT_OPTIONS)}`, `- bson: ${MONGODB_BSON_VERSION} (${MONGODB_BSON_REVISION}): (${MONGODB_BSON_PATH})\n` diff --git a/test/benchmarks/driver_bench/src/runner.mts b/test/benchmarks/driver_bench/src/runner.mts index 8ba70e17266..031be53c966 100644 --- a/test/benchmarks/driver_bench/src/runner.mts +++ b/test/benchmarks/driver_bench/src/runner.mts @@ -94,7 +94,7 @@ console.log( ' '.repeat(3), ...['total time:', totalDuration, 'sec,'], ...['ran:', count, 'times,'], - ...['time per run:', medianExecution, 'sec,'], + ...['median time per run:', medianExecution, 'sec,'], ...['throughput:', megabytesPerSecond, 'mb/sec'] ); From a1a617b27a26d938efb7d3c95fbea502e958d201 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Wed, 5 Feb 2025 16:38:10 -0500 Subject: [PATCH 07/12] chore: todos --- .evergreen/config.in.yml | 2 +- .evergreen/generate_evergreen_tasks.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.in.yml b/.evergreen/config.in.yml index 171942cbdab..9041647e934 100644 --- a/.evergreen/config.in.yml +++ b/.evergreen/config.in.yml @@ -883,7 +883,7 @@ functions: args: - ${PROJECT_DIRECTORY}/.evergreen/run-benchmarks.sh - # TEMP! + # TODO(NODE-6729): Remove this task when the original tasks are using the new runner "run new spec driver benchmarks": - command: subprocess.exec type: test diff --git a/.evergreen/generate_evergreen_tasks.js b/.evergreen/generate_evergreen_tasks.js index 0e6b735fb8e..943796c76d3 100644 --- a/.evergreen/generate_evergreen_tasks.js +++ b/.evergreen/generate_evergreen_tasks.js @@ -752,7 +752,7 @@ function addPerformanceTasks() { ] }); - // temp + // TODO(NODE-6729): Remove this task when the original tasks are using the new runner const makePerfTaskNEW = (name, MONGODB_CLIENT_OPTIONS) => ({ name, tags: ['run-spec-benchmark-tests', 'performance'], From bacb2037bc1ce89f9827936c31645d8215a32f8d Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Wed, 5 Feb 2025 16:38:18 -0500 Subject: [PATCH 08/12] chore: no pkg --- test/benchmarks/driver_bench/package.json | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/benchmarks/driver_bench/package.json b/test/benchmarks/driver_bench/package.json index 489d77dc609..1f39320e570 100644 --- a/test/benchmarks/driver_bench/package.json +++ b/test/benchmarks/driver_bench/package.json @@ -1,11 +1,9 @@ { "name": "driver_bench", - "version": "1.0.0", + "version": "0.0.0", + "private": true, "scripts": { "prestart": "tsc", "start": "node lib/main.mjs" - }, - "devDependencies": { - "@types/node": "^22.13.0" } } From bbfbaaf55d3f1bdc80876c6ce708a98edc5f444d Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Wed, 5 Feb 2025 16:38:40 -0500 Subject: [PATCH 09/12] chore: no ns getter and util client closed --- test/benchmarks/driver_bench/src/driver.mts | 38 ++++++++++++--------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/test/benchmarks/driver_bench/src/driver.mts b/test/benchmarks/driver_bench/src/driver.mts index c911e490dec..adbd5198acd 100644 --- a/test/benchmarks/driver_bench/src/driver.mts +++ b/test/benchmarks/driver_bench/src/driver.mts @@ -135,17 +135,11 @@ export function metrics(test_name: string, result: number, count: number) { * For use in setup/teardown mostly. */ export class DriverTester { - private utilClient: mongodb.MongoClient; public client: mongodb.MongoClient; constructor() { - this.utilClient = new MongoClient(MONGODB_URI, MONGODB_CLIENT_OPTIONS); this.client = new MongoClient(MONGODB_URI, MONGODB_CLIENT_OPTIONS); } - private get ns() { - return this.utilClient.db(DB_NAME).collection(COLLECTION_NAME); - } - public get db() { return this.client.db(DB_NAME); } @@ -159,15 +153,21 @@ export class DriverTester { } async drop() { - await this.ns.drop().catch(() => null); - await this.utilClient - .db(DB_NAME) - .dropDatabase() - .catch(() => null); + const utilClient = new MongoClient(MONGODB_URI, MONGODB_CLIENT_OPTIONS); + const db = utilClient.db(DB_NAME); + const collection = db.collection(COLLECTION_NAME); + await collection.drop().catch(() => null); + await db.dropDatabase().catch(() => null); + utilClient.close(); } async create() { - await this.utilClient.db(DB_NAME).createCollection(COLLECTION_NAME); + const utilClient = new MongoClient(MONGODB_URI, MONGODB_CLIENT_OPTIONS); + try { + await utilClient.db(DB_NAME).createCollection(COLLECTION_NAME); + } finally { + utilClient.close(); + } } async load(filePath: string, type: 'json' | 'string' | 'buffer'): Promise { @@ -180,14 +180,20 @@ export class DriverTester { } async insertManyOf(document: Record, length: number, addId = false) { - await this.ns.insertMany( - Array.from({ length }, (_, _id) => ({ ...(addId ? { _id } : {}), ...document })) as any[] - ); + const utilClient = new MongoClient(MONGODB_URI, MONGODB_CLIENT_OPTIONS); + const db = utilClient.db(DB_NAME); + const collection = db.collection(COLLECTION_NAME); + try { + await collection.insertMany( + Array.from({ length }, (_, _id) => ({ ...(addId ? { _id } : {}), ...document })) as any[] + ); + } finally { + utilClient.close(); + } } async close() { await this.client.close(); - await this.utilClient.close(); } } From 50964d28fb62c0eb35adb07c4bbf13bbf098d454 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Wed, 5 Feb 2025 16:38:52 -0500 Subject: [PATCH 10/12] chore: simpler TS --- test/benchmarks/driver_bench/src/main.mts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/test/benchmarks/driver_bench/src/main.mts b/test/benchmarks/driver_bench/src/main.mts index 0521b675169..2801ff421af 100644 --- a/test/benchmarks/driver_bench/src/main.mts +++ b/test/benchmarks/driver_bench/src/main.mts @@ -30,14 +30,11 @@ export const alphabetically = (a: unknown, b: unknown) => { /** Find every mjs file in the suites folder */ async function getBenchmarks(): Promise<{ - tests: Record>>; + tests: Record>; total: number; }> { let total = 0; - const tests: Record< - string, - Record> - > = Object.create(null); + const tests: Record> = Object.create(null); const suites = await fs.readdir(path.join(__dirname, 'suites')); suites.sort(alphabetically); @@ -48,7 +45,7 @@ async function getBenchmarks(): Promise<{ for (const benchmark of benchmarks) { if (!benchmark.endsWith('.mjs')) continue; tests[suite] ??= Object.create(null); - tests[suite][benchmark] = { benchFile: path.join('suites', suite, benchmark) }; + tests[suite][benchmark] = path.join('suites', suite, benchmark); total += 1; } } @@ -89,7 +86,7 @@ const results = []; for (const [suite, benchmarks] of Object.entries(tests)) { console.group(snakeToCamel(suite)); - for (const [benchmark, { benchFile }] of Object.entries(benchmarks)) { + for (const [benchmark, benchFile] of Object.entries(benchmarks)) { console.log(snakeToCamel(path.basename(benchmark, '.mjs'))); const runner = child_process.fork(runnerPath, [benchFile], { stdio: 'inherit' }); From c0e96ee67094c8421f13042ee1bc4e7421bbdc0f Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Wed, 5 Feb 2025 17:33:57 -0500 Subject: [PATCH 11/12] chore: await close --- test/benchmarks/driver_bench/src/driver.mts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/benchmarks/driver_bench/src/driver.mts b/test/benchmarks/driver_bench/src/driver.mts index adbd5198acd..79a4851e677 100644 --- a/test/benchmarks/driver_bench/src/driver.mts +++ b/test/benchmarks/driver_bench/src/driver.mts @@ -166,7 +166,7 @@ export class DriverTester { try { await utilClient.db(DB_NAME).createCollection(COLLECTION_NAME); } finally { - utilClient.close(); + await utilClient.close(); } } @@ -188,7 +188,7 @@ export class DriverTester { Array.from({ length }, (_, _id) => ({ ...(addId ? { _id } : {}), ...document })) as any[] ); } finally { - utilClient.close(); + await utilClient.close(); } } From 50e008cdfeaf26f385929608684c78d5980164eb Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Wed, 5 Feb 2025 17:36:26 -0500 Subject: [PATCH 12/12] chore: go to jail --- test/benchmarks/driver_bench/src/driver.mts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/benchmarks/driver_bench/src/driver.mts b/test/benchmarks/driver_bench/src/driver.mts index 79a4851e677..6c399ffa162 100644 --- a/test/benchmarks/driver_bench/src/driver.mts +++ b/test/benchmarks/driver_bench/src/driver.mts @@ -158,7 +158,7 @@ export class DriverTester { const collection = db.collection(COLLECTION_NAME); await collection.drop().catch(() => null); await db.dropDatabase().catch(() => null); - utilClient.close(); + await utilClient.close(); } async create() {