test(NODE-6705): isolate and warmup benchmarks

Author: nbbeeken

test/benchmarks/driver_bench/.gitignore

@@ -0,0 +1,3 @@
+results.json
+results_*.json
+package-lock.json

test/benchmarks/driver_bench/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "driver_bench",
+  "version": "1.0.0",
+  "scripts": {
+    "prestart": "tsc",
+    "start": "node lib/main.js"
+  },
+  "devDependencies": {
+    "@types/node": "^22.13.0"
+  }
+}

test/benchmarks/driver_bench/src/driver.mts

@@ -0,0 +1,181 @@
+import child_process from 'node:child_process';
+import fs from 'node:fs/promises';
+import module from 'node:module';
+import path from 'node:path';
+import process from 'node:process';
+
+const __dirname = import.meta.dirname;
+const require = module.createRequire(__dirname);
+
+/**
+ * The path to the MongoDB Node.js driver.
+ * This MUST be set to the directory the driver is installed in
+ * NOT the file "lib/index.js" that is the driver's export.
+ */
+export const MONGODB_DRIVER_PATH = (() => {
+  let driverPath = process.env.MONGODB_DRIVER_PATH;
+  if (!driverPath?.length) {
+    driverPath = path.resolve(__dirname, '../../../..');
+  }
+  return driverPath;
+})();
+
+/** Grab the version from the package.json */
+export const { version: MONGODB_DRIVER_VERSION } = require(
+  path.join(MONGODB_DRIVER_PATH, 'package.json')
+);
+
+/**
+ * Use git to optionally determine the git revision,
+ * but the benchmarks could be run against an npm installed version so this should be allowed to fail
+ */
+export const MONGODB_DRIVER_REVISION = (() => {
+  try {
+    return child_process
+      .execSync('git rev-parse --short HEAD', {
+        cwd: MONGODB_DRIVER_PATH,
+        encoding: 'utf8'
+      })
+      .trim();
+  } catch {
+    return 'unknown revision';
+  }
+})();
+
+/**
+ * Find the BSON dependency inside the driver PATH given and grab the version from the package.json.
+ */
+export const MONGODB_BSON_PATH = path.join(MONGODB_DRIVER_PATH, 'node_modules', 'bson');
+export const { version: MONGODB_BSON_VERSION } = require(
+  path.join(MONGODB_BSON_PATH, 'package.json')
+);
+
+/**
+ * If you need to test BSON changes, you should clone, checkout and build BSON.
+ * run: `npm link` with no arguments to register the link.
+ * Then in the driver you are testing run `npm link bson` to use your local build.
+ *
+ * This will symlink the BSON into the driver's node_modules directory. So here
+ * we can find the revision of the BSON we are testing against if .git exists.
+ */
+export const MONGODB_BSON_REVISION = await (async () => {
+  const bsonGitExists = await fs.access(path.join(MONGODB_BSON_PATH, '.git')).then(
+    () => true,
+    () => false
+  );
+  if (!bsonGitExists) {
+    return 'installed from npm';
+  }
+  try {
+    return child_process
+      .execSync('git rev-parse --short HEAD', {
+        cwd: path.join(MONGODB_BSON_PATH),
+        encoding: 'utf8'
+      })
+      .trim();
+  } catch {
+    return 'unknown revision';
+  }
+})();
+
+export const MONGODB_CLIENT_OPTIONS = (() => {
+  const optionsString = process.env.MONGODB_CLIENT_OPTIONS;
+  let options = undefined;
+  if (optionsString?.length) {
+    options = JSON.parse(optionsString);
+  }
+  return { ...options };
+})();
+
+export const MONGODB_URI = (() => {
+  if (process.env.MONGODB_URI?.length) return process.env.MONGODB_URI;
+  return 'mongodb://127.0.0.1:27017';
+})();
+
+export function snakeToCamel(name: string) {
+  return name
+    .split('_')
+    .map((s, i) => (i !== 0 ? s[0].toUpperCase() + s.slice(1) : s))
+    .join('');
+}
+
+import type mongodb from '../../../../mongodb.js';
+export type { mongodb };
+
+const { MongoClient /* GridFSBucket */ } = require(path.join(MONGODB_DRIVER_PATH));
+
+const DB_NAME = 'perftest';
+const COLLECTION_NAME = 'corpus';
+
+const SPEC_DIRECTORY = path.resolve(__dirname, '..', '..', 'driverBench', 'spec');
+
+export function metrics(test_name: string, result: number, count: number) {
+  return {
+    info: {
+      test_name,
+      // Args can only be a map of string -> int32. So if its a number leave it be,
+      // if it is anything else test for truthiness and set to 1 or 0.
+      args: Object.fromEntries(
+        Object.entries(MONGODB_CLIENT_OPTIONS).map(([key, value]) => [
+          key,
+          typeof value === 'number' ? value : value ? 1 : 0
+        ])
+      )
+    },
+    metrics: [
+      { name: 'megabytes_per_second', value: result },
+      { name: 'count', value: count }
+    ]
+  } as const;
+}
+
+/**
+ * This class exists to abstract some of the driver API so we can gloss over version differences.
+ * For use in setup/teardown mostly.
+ */
+export class DriverTester {
+  private utilClient: mongodb.MongoClient;
+  private client: mongodb.MongoClient;
+  constructor() {
+    this.utilClient = new MongoClient(MONGODB_URI, MONGODB_CLIENT_OPTIONS);
+    this.client = new MongoClient(MONGODB_URI, MONGODB_CLIENT_OPTIONS);
+  }
+
+  get ns() {
+    return this.utilClient.db(DB_NAME).collection(COLLECTION_NAME);
+  }
+
+  async drop() {
+    await this.ns.drop().catch(() => null);
+    await this.utilClient
+      .db(DB_NAME)
+      .dropDatabase()
+      .catch(() => null);
+  }
+
+  async create() {
+    return await this.utilClient.db(DB_NAME).createCollection(COLLECTION_NAME);
+  }
+
+  async load(filePath: string, type: 'json' | 'string' | 'buffer'): Promise<any> {
+    const content = await fs.readFile(path.join(SPEC_DIRECTORY, filePath));
+    if (type === 'buffer') return content;
+    const string = content.toString('utf8');
+    if (type === 'string') return string;
+    if (type === 'json') return JSON.parse(string);
+    throw new Error('unknown type: ' + type);
+  }
+
+  async insertManyOf(document: Record<string, any>, length: number, addId = false) {
+    await this.ns.insertMany(
+      Array.from({ length }, (_, _id) => ({ ...(addId ? { _id } : {}), ...document })) as any[]
+    );
+  }
+
+  async close() {
+    await this.client.close();
+    await this.utilClient.close();
+  }
+}
+
+export const driver = new DriverTester();

test/benchmarks/driver_bench/src/main.mts

@@ -0,0 +1,89 @@
+/* eslint-disable no-console */
+import child_process from 'node:child_process';
+import events from 'node:events';
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import util from 'node:util';
+
+import {
+  MONGODB_BSON_PATH,
+  MONGODB_BSON_REVISION,
+  MONGODB_BSON_VERSION,
+  MONGODB_CLIENT_OPTIONS,
+  MONGODB_DRIVER_PATH,
+  MONGODB_DRIVER_REVISION,
+  MONGODB_DRIVER_VERSION,
+  snakeToCamel
+} from './driver.mjs';
+
+const __dirname = import.meta.dirname;
+
+/** Find every mjs file in the suites folder */
+async function getBenchmarks(): Promise<
+  Record<string, Record<string, { benchFile: string } & Record<string, any>>>
+> {
+  const tests: Record<
+    string,
+    Record<string, { benchFile: string } & Record<string, any>>
+  > = Object.create(null);
+  const suites = await fs.readdir(path.join(__dirname, 'suites'));
+  for (const suite of suites) {
+    const benchmarks = await fs.readdir(path.join(__dirname, 'suites', suite));
+    for (const benchmark of benchmarks) {
+      if (!benchmark.endsWith('.mjs')) continue;
+      tests[suite] ??= Object.create(null);
+      tests[suite][benchmark] = { benchFile: path.join('suites', suite, benchmark) };
+    }
+  }
+  return tests;
+}
+
+const hw = os.cpus();
+const ram = os.totalmem() / 1024 ** 3;
+const platform = { name: hw[0].model, cores: hw.length, ram: `${ram}GB` };
+
+const systemInfo = () =>
+  [
+    `\n- cpu: ${platform.name}`,
+    `- cores: ${platform.cores}`,
+    `- arch: ${os.arch()}`,
+    `- os: ${process.platform} (${os.release()})`,
+    `- ram: ${platform.ram}`,
+    `- node: ${process.version}`,
+    `- driver: ${MONGODB_DRIVER_VERSION} (${MONGODB_DRIVER_REVISION}): ${MONGODB_DRIVER_PATH}`,
+    `  - options ${util.inspect(MONGODB_CLIENT_OPTIONS)}`,
+    `- bson: ${MONGODB_BSON_VERSION} (${MONGODB_BSON_REVISION}): (${MONGODB_BSON_PATH})\n`
+  ].join('\n');
+
+console.log(systemInfo());
+
+const tests = await getBenchmarks();
+const runnerPath = path.join(__dirname, 'runner.mjs');
+
+const results = [];
+
+for (const [suite, benchmarks] of Object.entries(tests)) {
+  console.group(snakeToCamel(suite));
+
+  for (const [benchmark, { benchFile }] of Object.entries(benchmarks)) {
+    console.log(snakeToCamel(path.basename(benchmark, '.mjs')));
+
+    const runner = child_process.fork(runnerPath, [benchFile], { stdio: 'inherit' });
+
+    const [exitCode] = await events.once(runner, 'close');
+    if (exitCode !== 0) {
+      throw new Error(`Benchmark exited with failure: ${exitCode}`);
+    }
+
+    const result = JSON.parse(
+      await fs.readFile(`results_${path.basename(benchmark, '.mjs')}.json`, 'utf8')
+    );
+
+    results.push(result);
+  }
+
+  console.groupEnd();
+}
+
+await fs.writeFile('results.json', JSON.stringify(results, undefined, 2), 'utf8');

test/benchmarks/driver_bench/src/runner.mts

@@ -0,0 +1,111 @@
+/* eslint-disable no-console */
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import process from 'node:process';
+
+import { metrics, snakeToCamel } from './driver.mjs';
+
+const [, , benchmarkFile] = process.argv;
+
+type BenchmarkModule = {
+  taskSize: number;
+  before?: () => Promise<void>;
+  beforeEach?: () => Promise<void>;
+  run: () => Promise<void>;
+  afterEach?: () => Promise<void>;
+  after?: () => Promise<void>;
+};
+
+const benchmarkName = snakeToCamel(path.basename(benchmarkFile, '.mjs'));
+const benchmark: BenchmarkModule = await import(`./${benchmarkFile}`);
+
+if (typeof benchmark.taskSize !== 'number') throw new Error('missing taskSize');
+if (typeof benchmark.run !== 'function') throw new Error('missing run');
+
+/** CRITICAL SECTION: time task took in seconds */
+async function timeTask() {
+  const start = performance.now();
+  await benchmark.run();
+  const end = performance.now();
+  return (end - start) / 1000;
+}
+
+/** 1 min in seconds */
+const ONE_MIN = 1 * 60;
+/** 5 min in seconds */
+const FIVE_MIN = 5 * 60;
+/** Don't run more than 100 iterations */
+const MAX_COUNT = 100;
+
+await benchmark.before?.();
+
+// for 1/10th the max iterations
+const warmupIterations = (MAX_COUNT / 10) | 0;
+
+// Warm Up.
+for (let i = 0; i < warmupIterations; i++) {
+  await benchmark.beforeEach?.();
+  await timeTask();
+  await benchmark.afterEach?.();
+}
+
+// Allocate an obscene amount of space
+const data = new Float64Array(10_000_000);
+
+// Test.
+let totalDuration = 0;
+let count = 0;
+do {
+  await benchmark.beforeEach?.();
+
+  data[count] = await timeTask();
+
+  await benchmark.afterEach?.();
+
+  totalDuration += data[count]; // time moves up by benchmark exec time not wall clock
+  count += 1;
+
+  // must run for at least one minute
+  if (totalDuration < ONE_MIN) continue;
+
+  // 100 runs OR five minutes
+  if (count > 100 || totalDuration > FIVE_MIN) break;
+
+  // count exceeds data space, we never intend to have more than a million data points let alone 10M
+  if (count === data.length) break;
+
+  // else: more than one min, less than 100 iterations, less than 5min
+
+  // eslint-disable-next-line no-constant-condition
+} while (true);
+
+await benchmark.after?.();
+
+const durations = data.subarray(0, count).toSorted((a, b) => a - b);
+
+function percentileIndex(percentile: number, count: number) {
+  return Math.max(Math.floor((count * percentile) / 100 - 1), 0);
+}
+
+const medianExecution = durations[percentileIndex(50, count)];
+
+console.log(
+  '   ',
+  benchmarkName,
+  'finished in',
+  totalDuration,
+  'sec and ran',
+  count,
+  'iterations.',
+  'median exec time',
+  medianExecution,
+  'sec',
+  benchmark.taskSize / medianExecution,
+  'mb/sec'
+);
+
+await fs.writeFile(
+  `results_${path.basename(benchmarkFile, '.mjs')}.json`,
+  JSON.stringify(metrics(benchmarkName, medianExecution, count), undefined, 2) + '\n',
+  'utf8'
+);