diff --git a/.evergreen/config.in.yml b/.evergreen/config.in.yml index a2a0532a934..9041647e934 100644 --- a/.evergreen/config.in.yml +++ b/.evergreen/config.in.yml @@ -142,7 +142,7 @@ functions: working_dir: "src" timeout_secs: 300 binary: bash - args: + args: - .evergreen/run-tests.sh "run serverless tests": @@ -158,11 +158,11 @@ functions: timeout_secs: 300 working_dir: src binary: bash - env: + env: AUTH: 'auth' SSL: 'ssl' add_expansions_to_env: true - args: + args: - .evergreen/run-serverless-tests.sh "start-load-balancer": @@ -882,6 +882,23 @@ functions: binary: bash args: - ${PROJECT_DIRECTORY}/.evergreen/run-benchmarks.sh + + # TODO(NODE-6729): Remove this task when the original tasks are using the new runner + "run new spec driver benchmarks": + - command: subprocess.exec + type: test + params: + working_dir: "src" + env: + PROJECT_DIRECTORY: ${PROJECT_DIRECTORY} + MONGODB_URI: ${MONGODB_URI} + DRIVERS_TOOLS: ${DRIVERS_TOOLS} + MONGODB_CLIENT_OPTIONS: ${MONGODB_CLIENT_OPTIONS} + NEW_BENCH: "true" + binary: bash + args: + - ${PROJECT_DIRECTORY}/.evergreen/run-benchmarks.sh + "run x509 auth tests": - command: subprocess.exec type: test @@ -1191,7 +1208,7 @@ task_groups: binary: bash args: - ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh - + tasks: - ".serverless" diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 5fe7bf2efdb..2c5f6efac92 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -825,6 +825,20 @@ functions: binary: bash args: - ${PROJECT_DIRECTORY}/.evergreen/run-benchmarks.sh + run new spec driver benchmarks: + - command: subprocess.exec + type: test + params: + working_dir: src + env: + PROJECT_DIRECTORY: ${PROJECT_DIRECTORY} + MONGODB_URI: ${MONGODB_URI} + DRIVERS_TOOLS: ${DRIVERS_TOOLS} + MONGODB_CLIENT_OPTIONS: ${MONGODB_CLIENT_OPTIONS} + NEW_BENCH: 'true' + binary: bash + args: + - ${PROJECT_DIRECTORY}/.evergreen/run-benchmarks.sh run x509 auth tests: - command: subprocess.exec type: test @@ -3305,6 +3319,24 @@ tasks: - command: perf.send params: file: src/results.json + - name: run-spec-benchmark-tests-node-server-new + tags: + - run-spec-benchmark-tests + - performance + exec_timeout_secs: 3600 + commands: + - command: expansions.update + type: setup + params: + updates: + - {key: NODE_LTS_VERSION, value: v22.11.0} + - {key: VERSION, value: v6.0-perf} + - {key: TOPOLOGY, value: server} + - {key: AUTH, value: noauth} + - {key: MONGODB_CLIENT_OPTIONS, value: '{}'} + - func: install dependencies + - func: bootstrap mongo-orchestration + - func: run new spec driver benchmarks - name: run-unit-tests-node-16 tags: - unit-tests @@ -5122,6 +5154,7 @@ buildvariants: - run-spec-benchmark-tests-node-server-timeoutMS-0 - run-spec-benchmark-tests-node-server-monitorCommands-true - run-spec-benchmark-tests-node-server-logging + - run-spec-benchmark-tests-node-server-new - name: rhel8-custom-dependency-tests display_name: Custom Dependency Version Test run_on: rhel80-large diff --git a/.evergreen/generate_evergreen_tasks.js b/.evergreen/generate_evergreen_tasks.js index b263dc053cb..943796c76d3 100644 --- a/.evergreen/generate_evergreen_tasks.js +++ b/.evergreen/generate_evergreen_tasks.js @@ -212,7 +212,7 @@ TASKS.push( { func: 'run socks5 tests' } ] } - ] + ] ); TASKS.push({ @@ -432,8 +432,8 @@ for (const { const expansions = { NODE_LTS_VERSION, NPM_VERSION }; const taskNames = tasks.map(({ name }) => name); - expansions.CLIENT_ENCRYPTION = String(!!clientEncryption) - expansions.TEST_CSFLE = expansions.CLIENT_ENCRYPTION + expansions.CLIENT_ENCRYPTION = String(!!clientEncryption); + expansions.TEST_CSFLE = expansions.CLIENT_ENCRYPTION; BUILD_VARIANTS.push({ name, display_name, run_on, expansions, tasks: taskNames }); } @@ -752,6 +752,28 @@ function addPerformanceTasks() { ] }); + // TODO(NODE-6729): Remove this task when the original tasks are using the new runner + const makePerfTaskNEW = (name, MONGODB_CLIENT_OPTIONS) => ({ + name, + tags: ['run-spec-benchmark-tests', 'performance'], + exec_timeout_secs: 3600, + commands: [ + updateExpansions({ + NODE_LTS_VERSION: 'v22.11.0', + VERSION: 'v6.0-perf', + TOPOLOGY: 'server', + AUTH: 'noauth', + MONGODB_CLIENT_OPTIONS: JSON.stringify(MONGODB_CLIENT_OPTIONS) + }), + ...[ + 'install dependencies', + 'bootstrap mongo-orchestration', + 'run new spec driver benchmarks' + ].map(func => ({ func })) + // No perf send! just testing + ] + }); + const tasks = [ makePerfTask('run-spec-benchmark-tests-node-server', {}), makePerfTask('run-spec-benchmark-tests-node-server-timeoutMS-120000', { timeoutMS: 120000 }), @@ -762,7 +784,8 @@ function addPerformanceTasks() { makePerfTask('run-spec-benchmark-tests-node-server-logging', { mongodbLogPath: 'stderr', mongodbLogComponentSeverities: { default: 'trace' } - }) + }), + makePerfTaskNEW('run-spec-benchmark-tests-node-server-new', {}) ]; TASKS.push(...tasks); diff --git a/.evergreen/run-benchmarks.sh b/.evergreen/run-benchmarks.sh index c81d6924716..9593c0db41c 100644 --- a/.evergreen/run-benchmarks.sh +++ b/.evergreen/run-benchmarks.sh @@ -11,6 +11,12 @@ export MONGODB_CLIENT_OPTIONS=$MONGODB_CLIENT_OPTIONS npm run build:ts +if [[ "${NEW_BENCH:-}" == "true" ]]; then + pushd test/benchmarks/driver_bench + npm start + popd + exit 0 +fi # If MONGODB_CLIENT_OPTIONS contains mongodbLogComponentSeverities redirect stderr to a file if [[ $MONGODB_CLIENT_OPTIONS == *"mongodbLogComponentSeverities"* ]]; then diff --git a/test/benchmarks/driver_bench/.gitignore b/test/benchmarks/driver_bench/.gitignore new file mode 100644 index 00000000000..ce54f56f966 --- /dev/null +++ b/test/benchmarks/driver_bench/.gitignore @@ -0,0 +1,3 @@ +results.json +results_*.json +package-lock.json diff --git a/test/benchmarks/driver_bench/package.json b/test/benchmarks/driver_bench/package.json new file mode 100644 index 00000000000..1f39320e570 --- /dev/null +++ b/test/benchmarks/driver_bench/package.json @@ -0,0 +1,9 @@ +{ + "name": "driver_bench", + "version": "0.0.0", + "private": true, + "scripts": { + "prestart": "tsc", + "start": "node lib/main.mjs" + } +} diff --git a/test/benchmarks/driver_bench/src/driver.mts b/test/benchmarks/driver_bench/src/driver.mts new file mode 100644 index 00000000000..6c399ffa162 --- /dev/null +++ b/test/benchmarks/driver_bench/src/driver.mts @@ -0,0 +1,200 @@ +import child_process from 'node:child_process'; +import fs from 'node:fs/promises'; +import module from 'node:module'; +import path from 'node:path'; +import process from 'node:process'; + +const __dirname = import.meta.dirname; +const require = module.createRequire(__dirname); + +/** + * The path to the MongoDB Node.js driver. + * This MUST be set to the directory the driver is installed in + * NOT the file "lib/index.js" that is the driver's export. + */ +export const MONGODB_DRIVER_PATH = (() => { + let driverPath = process.env.MONGODB_DRIVER_PATH; + if (!driverPath?.length) { + driverPath = path.resolve(__dirname, '../../../..'); + } + return driverPath; +})(); + +/** Grab the version from the package.json */ +export const { version: MONGODB_DRIVER_VERSION } = require( + path.join(MONGODB_DRIVER_PATH, 'package.json') +); + +/** + * Use git to optionally determine the git revision, + * but the benchmarks could be run against an npm installed version so this should be allowed to fail + */ +export const MONGODB_DRIVER_REVISION = (() => { + try { + return child_process + .execSync('git rev-parse --short HEAD', { + cwd: MONGODB_DRIVER_PATH, + encoding: 'utf8' + }) + .trim(); + } catch { + return 'unknown revision'; + } +})(); + +/** + * Find the BSON dependency inside the driver PATH given and grab the version from the package.json. + */ +export const MONGODB_BSON_PATH = path.join(MONGODB_DRIVER_PATH, 'node_modules', 'bson'); +export const { version: MONGODB_BSON_VERSION } = require( + path.join(MONGODB_BSON_PATH, 'package.json') +); + +/** + * If you need to test BSON changes, you should clone, checkout and build BSON. + * run: `npm link` with no arguments to register the link. + * Then in the driver you are testing run `npm link bson` to use your local build. + * + * This will symlink the BSON into the driver's node_modules directory. So here + * we can find the revision of the BSON we are testing against if .git exists. + */ +export const MONGODB_BSON_REVISION = await (async () => { + const bsonGitExists = await fs.access(path.join(MONGODB_BSON_PATH, '.git')).then( + () => true, + () => false + ); + if (!bsonGitExists) { + return 'installed from npm'; + } + try { + return child_process + .execSync('git rev-parse --short HEAD', { + cwd: path.join(MONGODB_BSON_PATH), + encoding: 'utf8' + }) + .trim(); + } catch { + return 'unknown revision'; + } +})(); + +export const MONGODB_CLIENT_OPTIONS = (() => { + const optionsString = process.env.MONGODB_CLIENT_OPTIONS; + let options = undefined; + if (optionsString?.length) { + options = JSON.parse(optionsString); + } + return { ...options }; +})(); + +export const MONGODB_URI = (() => { + if (process.env.MONGODB_URI?.length) return process.env.MONGODB_URI; + return 'mongodb://127.0.0.1:27017'; +})(); + +export function snakeToCamel(name: string) { + return name + .split('_') + .map((s, i) => (i !== 0 ? s[0].toUpperCase() + s.slice(1) : s)) + .join(''); +} + +import type mongodb from '../../../../mongodb.js'; +export type { mongodb }; + +const { MongoClient, GridFSBucket } = require(path.join(MONGODB_DRIVER_PATH)); + +const DB_NAME = 'perftest'; +const COLLECTION_NAME = 'corpus'; + +const SPEC_DIRECTORY = path.resolve(__dirname, '..', '..', 'driverBench', 'spec'); + +export function metrics(test_name: string, result: number, count: number) { + return { + info: { + test_name, + // Args can only be a map of string -> int32. So if its a number leave it be, + // if it is anything else test for truthiness and set to 1 or 0. + args: Object.fromEntries( + Object.entries(MONGODB_CLIENT_OPTIONS).map(([key, value]) => [ + key, + typeof value === 'number' ? value : value ? 1 : 0 + ]) + ) + }, + metrics: [ + { name: 'megabytes_per_second', value: result }, + // Reporting the count so we can verify programmatically or in UI how many iterations we reached + { name: 'count', value: count } + ] + } as const; +} + +/** + * This class exists to abstract some of the driver API so we can gloss over version differences. + * For use in setup/teardown mostly. + */ +export class DriverTester { + public client: mongodb.MongoClient; + constructor() { + this.client = new MongoClient(MONGODB_URI, MONGODB_CLIENT_OPTIONS); + } + + public get db() { + return this.client.db(DB_NAME); + } + + public get collection() { + return this.client.db(DB_NAME).collection(COLLECTION_NAME); + } + + public get bucket(): mongodb.GridFSBucket { + return new GridFSBucket(this.db); + } + + async drop() { + const utilClient = new MongoClient(MONGODB_URI, MONGODB_CLIENT_OPTIONS); + const db = utilClient.db(DB_NAME); + const collection = db.collection(COLLECTION_NAME); + await collection.drop().catch(() => null); + await db.dropDatabase().catch(() => null); + await utilClient.close(); + } + + async create() { + const utilClient = new MongoClient(MONGODB_URI, MONGODB_CLIENT_OPTIONS); + try { + await utilClient.db(DB_NAME).createCollection(COLLECTION_NAME); + } finally { + await utilClient.close(); + } + } + + async load(filePath: string, type: 'json' | 'string' | 'buffer'): Promise { + const content = await fs.readFile(path.join(SPEC_DIRECTORY, filePath)); + if (type === 'buffer') return content; + const string = content.toString('utf8'); + if (type === 'string') return string; + if (type === 'json') return JSON.parse(string); + throw new Error('unknown type: ' + type); + } + + async insertManyOf(document: Record, length: number, addId = false) { + const utilClient = new MongoClient(MONGODB_URI, MONGODB_CLIENT_OPTIONS); + const db = utilClient.db(DB_NAME); + const collection = db.collection(COLLECTION_NAME); + try { + await collection.insertMany( + Array.from({ length }, (_, _id) => ({ ...(addId ? { _id } : {}), ...document })) as any[] + ); + } finally { + await utilClient.close(); + } + } + + async close() { + await this.client.close(); + } +} + +export const driver = new DriverTester(); diff --git a/test/benchmarks/driver_bench/src/main.mts b/test/benchmarks/driver_bench/src/main.mts new file mode 100644 index 00000000000..2801ff421af --- /dev/null +++ b/test/benchmarks/driver_bench/src/main.mts @@ -0,0 +1,109 @@ +/* eslint-disable no-console */ +import child_process from 'node:child_process'; +import events from 'node:events'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import util from 'node:util'; + +import { + MONGODB_BSON_PATH, + MONGODB_BSON_REVISION, + MONGODB_BSON_VERSION, + MONGODB_CLIENT_OPTIONS, + MONGODB_DRIVER_PATH, + MONGODB_DRIVER_REVISION, + MONGODB_DRIVER_VERSION, + snakeToCamel +} from './driver.mjs'; + +const __dirname = import.meta.dirname; + +export const alphabetically = (a: unknown, b: unknown) => { + const res = `${a}`.localeCompare(`${b}`, 'en-US', { + usage: 'sort', + numeric: true, + ignorePunctuation: false + }); + return res < 0 ? -1 : res > 0 ? 1 : 0; +}; + +/** Find every mjs file in the suites folder */ +async function getBenchmarks(): Promise<{ + tests: Record>; + total: number; +}> { + let total = 0; + const tests: Record> = Object.create(null); + const suites = await fs.readdir(path.join(__dirname, 'suites')); + suites.sort(alphabetically); + + for (const suite of suites) { + const benchmarks = await fs.readdir(path.join(__dirname, 'suites', suite)); + benchmarks.sort(alphabetically); + + for (const benchmark of benchmarks) { + if (!benchmark.endsWith('.mjs')) continue; + tests[suite] ??= Object.create(null); + tests[suite][benchmark] = path.join('suites', suite, benchmark); + total += 1; + } + } + return { tests, total }; +} + +const hw = os.cpus(); +const ram = os.totalmem() / 1024 ** 3; +const platform = { name: hw[0].model, cores: hw.length, ram: `${ram}GB` }; + +const { tests, total } = await getBenchmarks(); + +const earliest = new Date(Date.now() + total * 60 * 1000); // plus one min per bench +const latest = new Date(Date.now() + total * 6 * 60 * 1000); // plus six min per bench (if we overshoot the 5 min limit) + +const systemInfo = () => + [ + `\n- cpu: ${platform.name}`, + `- cores: ${platform.cores}`, + `- arch: ${os.arch()}`, + `- os: ${process.platform} (${os.release()})`, + `- ram: ${platform.ram}`, + `- node: ${process.version}`, + `- running ${total} benchmarks`, + ` - finishes soonest: ${earliest.toLocaleTimeString('en-US', { timeZoneName: 'short' })}`, + ` latest: ${latest.toLocaleTimeString('en-US', { timeZoneName: 'short' })}`, + `- driver: ${MONGODB_DRIVER_VERSION} (${MONGODB_DRIVER_REVISION}): ${MONGODB_DRIVER_PATH}`, + ` - options ${util.inspect(MONGODB_CLIENT_OPTIONS)}`, + `- bson: ${MONGODB_BSON_VERSION} (${MONGODB_BSON_REVISION}): (${MONGODB_BSON_PATH})\n` + ].join('\n'); + +console.log(systemInfo()); + +const runnerPath = path.join(__dirname, 'runner.mjs'); + +const results = []; + +for (const [suite, benchmarks] of Object.entries(tests)) { + console.group(snakeToCamel(suite)); + + for (const [benchmark, benchFile] of Object.entries(benchmarks)) { + console.log(snakeToCamel(path.basename(benchmark, '.mjs'))); + + const runner = child_process.fork(runnerPath, [benchFile], { stdio: 'inherit' }); + + const [exitCode] = await events.once(runner, 'close'); + if (exitCode !== 0) { + throw new Error(`Benchmark exited with failure: ${exitCode}`); + } + + const result = JSON.parse( + await fs.readFile(`results_${path.basename(benchmark, '.mjs')}.json`, 'utf8') + ); + + results.push(result); + } + + console.groupEnd(); +} + +await fs.writeFile('results.json', JSON.stringify(results, undefined, 2), 'utf8'); diff --git a/test/benchmarks/driver_bench/src/runner.mts b/test/benchmarks/driver_bench/src/runner.mts new file mode 100644 index 00000000000..031be53c966 --- /dev/null +++ b/test/benchmarks/driver_bench/src/runner.mts @@ -0,0 +1,105 @@ +/* eslint-disable no-console */ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import process from 'node:process'; + +import { metrics, snakeToCamel } from './driver.mjs'; + +const [, , benchmarkFile] = process.argv; + +type BenchmarkModule = { + taskSize: number; + before?: () => Promise; + beforeEach?: () => Promise; + run: () => Promise; + afterEach?: () => Promise; + after?: () => Promise; +}; + +const benchmarkName = snakeToCamel(path.basename(benchmarkFile, '.mjs')); +const benchmark: BenchmarkModule = await import(`./${benchmarkFile}`); + +if (typeof benchmark.taskSize !== 'number') throw new Error('missing taskSize'); +if (typeof benchmark.run !== 'function') throw new Error('missing run'); + +/** CRITICAL SECTION: time task took in seconds */ +async function timeTask() { + const start = performance.now(); + await benchmark.run(); + const end = performance.now(); + return (end - start) / 1000; +} + +/** 1 min in seconds */ +const ONE_MIN = 1 * 60; +/** 5 min in seconds */ +const FIVE_MIN = 5 * 60; +/** Don't run more than 100 iterations */ +const MAX_COUNT = 100; + +await benchmark.before?.(); + +// for 1/10th the max iterations +const warmupIterations = (MAX_COUNT / 10) | 0; + +// Warm Up. +for (let i = 0; i < warmupIterations; i++) { + await benchmark.beforeEach?.(); + await timeTask(); + await benchmark.afterEach?.(); +} + +// Allocate an obscene amount of space +const data = new Float64Array(10_000_000); + +// Test. +let totalDuration = 0; +let count = 0; +do { + await benchmark.beforeEach?.(); + + data[count] = await timeTask(); + + await benchmark.afterEach?.(); + + totalDuration += data[count]; // time moves up by benchmark exec time not wall clock + count += 1; + + // must run for at least one minute + if (totalDuration < ONE_MIN) continue; + + // 100 runs OR five minutes + if (count >= 100 || totalDuration >= FIVE_MIN) break; + + // count exceeds data space, we never intend to have more than a million data points let alone 10M + if (count === data.length) break; + + // else: more than one min, less than 100 iterations, less than 5min + + // eslint-disable-next-line no-constant-condition +} while (true); + +await benchmark.after?.(); + +const durations = data.subarray(0, count).toSorted((a, b) => a - b); + +function percentileIndex(percentile: number, count: number) { + return Math.max(Math.floor((count * percentile) / 100 - 1), 0); +} + +const medianExecution = durations[percentileIndex(50, count)]; +const megabytesPerSecond = benchmark.taskSize / medianExecution; + +console.log( + ' '.repeat(3), + ...['total time:', totalDuration, 'sec,'], + ...['ran:', count, 'times,'], + ...['median time per run:', medianExecution, 'sec,'], + ...['throughput:', megabytesPerSecond, 'mb/sec'] +); + +await fs.writeFile( + `results_${path.basename(benchmarkFile, '.mjs')}.json`, + JSON.stringify(metrics(benchmarkName, megabytesPerSecond, count), undefined, 2) + '\n', + 'utf8' +); diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts new file mode 100644 index 00000000000..f404c1f057c --- /dev/null +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts @@ -0,0 +1,27 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { driver, type mongodb } from '../../driver.mjs'; + +export const taskSize = 16.22; + +let collection: mongodb.Collection; + +export async function before() { + await driver.drop(); + await driver.create(); + + const tweet = await driver.load('single_and_multi_document/tweet.json', 'json'); + await driver.insertManyOf(tweet, 10000); + + collection = driver.collection; +} + +export async function run() { + for await (const doc of collection.find({})) { + // empty + } +} + +export async function after() { + await driver.drop(); + await driver.close(); +} diff --git a/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts b/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts new file mode 100644 index 00000000000..36396ec549a --- /dev/null +++ b/test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts @@ -0,0 +1,30 @@ +import { driver, type mongodb } from '../../driver.mjs'; + +export const taskSize = 2.75; + +let collection: mongodb.Collection; +let documents: any[]; +let smallDoc: any; + +export async function before() { + smallDoc = await driver.load('single_and_multi_document/small_doc.json', 'json'); +} + +export async function beforeEach() { + await driver.drop(); + await driver.create(); + + // Make new "documents" so the _id field is not carried over from the last run + documents = Array.from({ length: 10000 }, () => ({ ...smallDoc })) as any[]; + + collection = driver.collection; +} + +export async function run() { + await collection.insertMany(documents, { ordered: true }); +} + +export async function after() { + await driver.drop(); + await driver.close(); +} diff --git a/test/benchmarks/driver_bench/tsconfig.json b/test/benchmarks/driver_bench/tsconfig.json new file mode 100644 index 00000000000..d7eb0f5c29e --- /dev/null +++ b/test/benchmarks/driver_bench/tsconfig.json @@ -0,0 +1,34 @@ +{ + "compilerOptions": { + "allowJs": true, + "checkJs": false, + "strict": true, + "alwaysStrict": true, + "target": "ESNext", + "verbatimModuleSyntax": true, + "module": "NodeNext", + "moduleResolution": "NodeNext", + "skipLibCheck": true, + // We don't make use of tslib helpers, all syntax used is supported by target engine + "importHelpers": false, + "noEmitHelpers": true, + // Never emit error filled code + "noEmitOnError": true, + "outDir": "lib", + // We want the sourcemaps in a separate file + "inlineSourceMap": false, + "sourceMap": true, + // we include sources in the release + "inlineSources": false, + // Prevents web types from being suggested by vscode. + "types": [ + "node" + ], + "forceConsistentCasingInFileNames": true, + "noImplicitOverride": true, + "noImplicitReturns": true, + }, + "include": [ + "src/**/*" + ] +}