test(NODE-6723): isolate and warmup benchmarks (#4398)

Author: nbbeeken

.evergreen/config.in.yml

@@ -142,7 +142,7 @@ functions:
         working_dir: "src"
         timeout_secs: 300
         binary: bash
-        args: 
+        args:
           - .evergreen/run-tests.sh
 
   "run serverless tests":
@@ -158,11 +158,11 @@ functions:
         timeout_secs: 300
         working_dir: src
         binary: bash
-        env: 
+        env:
           AUTH: 'auth'
           SSL: 'ssl'
         add_expansions_to_env: true
-        args: 
+        args:
           - .evergreen/run-serverless-tests.sh
 
   "start-load-balancer":
@@ -882,6 +882,23 @@ functions:
         binary: bash
         args:
           - ${PROJECT_DIRECTORY}/.evergreen/run-benchmarks.sh
+
+  # TODO(NODE-6729): Remove this task when the original tasks are using the new runner
+  "run new spec driver benchmarks":
+    - command: subprocess.exec
+      type: test
+      params:
+        working_dir: "src"
+        env:
+          PROJECT_DIRECTORY: ${PROJECT_DIRECTORY}
+          MONGODB_URI: ${MONGODB_URI}
+          DRIVERS_TOOLS: ${DRIVERS_TOOLS}
+          MONGODB_CLIENT_OPTIONS: ${MONGODB_CLIENT_OPTIONS}
+          NEW_BENCH: "true"
+        binary: bash
+        args:
+          - ${PROJECT_DIRECTORY}/.evergreen/run-benchmarks.sh
+
   "run x509 auth tests":
     - command: subprocess.exec
       type: test
@@ -1191,7 +1208,7 @@ task_groups:
           binary: bash
           args:
             - ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh
-            
+
     tasks:
       - ".serverless"
 

.evergreen/config.yml

@@ -825,6 +825,20 @@ functions:
         binary: bash
         args:
           - ${PROJECT_DIRECTORY}/.evergreen/run-benchmarks.sh
+  run new spec driver benchmarks:
+    - command: subprocess.exec
+      type: test
+      params:
+        working_dir: src
+        env:
+          PROJECT_DIRECTORY: ${PROJECT_DIRECTORY}
+          MONGODB_URI: ${MONGODB_URI}
+          DRIVERS_TOOLS: ${DRIVERS_TOOLS}
+          MONGODB_CLIENT_OPTIONS: ${MONGODB_CLIENT_OPTIONS}
+          NEW_BENCH: 'true'
+        binary: bash
+        args:
+          - ${PROJECT_DIRECTORY}/.evergreen/run-benchmarks.sh
   run x509 auth tests:
     - command: subprocess.exec
       type: test
@@ -3305,6 +3319,24 @@ tasks:
       - command: perf.send
         params:
           file: src/results.json
+  - name: run-spec-benchmark-tests-node-server-new
+    tags:
+      - run-spec-benchmark-tests
+      - performance
+    exec_timeout_secs: 3600
+    commands:
+      - command: expansions.update
+        type: setup
+        params:
+          updates:
+            - {key: NODE_LTS_VERSION, value: v22.11.0}
+            - {key: VERSION, value: v6.0-perf}
+            - {key: TOPOLOGY, value: server}
+            - {key: AUTH, value: noauth}
+            - {key: MONGODB_CLIENT_OPTIONS, value: '{}'}
+      - func: install dependencies
+      - func: bootstrap mongo-orchestration
+      - func: run new spec driver benchmarks
   - name: run-unit-tests-node-16
     tags:
       - unit-tests
@@ -5122,6 +5154,7 @@ buildvariants:
       - run-spec-benchmark-tests-node-server-timeoutMS-0
       - run-spec-benchmark-tests-node-server-monitorCommands-true
       - run-spec-benchmark-tests-node-server-logging
+      - run-spec-benchmark-tests-node-server-new
   - name: rhel8-custom-dependency-tests
     display_name: Custom Dependency Version Test
     run_on: rhel80-large

.evergreen/generate_evergreen_tasks.js

@@ -212,7 +212,7 @@ TASKS.push(
         { func: 'run socks5 tests' }
       ]
     }
-    ]
+  ]
 );
 
 TASKS.push({
@@ -432,8 +432,8 @@ for (const {
     const expansions = { NODE_LTS_VERSION, NPM_VERSION };
     const taskNames = tasks.map(({ name }) => name);
 
-    expansions.CLIENT_ENCRYPTION = String(!!clientEncryption)
-    expansions.TEST_CSFLE = expansions.CLIENT_ENCRYPTION
+    expansions.CLIENT_ENCRYPTION = String(!!clientEncryption);
+    expansions.TEST_CSFLE = expansions.CLIENT_ENCRYPTION;
 
     BUILD_VARIANTS.push({ name, display_name, run_on, expansions, tasks: taskNames });
   }
@@ -752,6 +752,28 @@ function addPerformanceTasks() {
     ]
   });
 
+  // TODO(NODE-6729): Remove this task when the original tasks are using the new runner
+  const makePerfTaskNEW = (name, MONGODB_CLIENT_OPTIONS) => ({
+    name,
+    tags: ['run-spec-benchmark-tests', 'performance'],
+    exec_timeout_secs: 3600,
+    commands: [
+      updateExpansions({
+        NODE_LTS_VERSION: 'v22.11.0',
+        VERSION: 'v6.0-perf',
+        TOPOLOGY: 'server',
+        AUTH: 'noauth',
+        MONGODB_CLIENT_OPTIONS: JSON.stringify(MONGODB_CLIENT_OPTIONS)
+      }),
+      ...[
+        'install dependencies',
+        'bootstrap mongo-orchestration',
+        'run new spec driver benchmarks'
+      ].map(func => ({ func }))
+      // No perf send! just testing
+    ]
+  });
+
   const tasks = [
     makePerfTask('run-spec-benchmark-tests-node-server', {}),
     makePerfTask('run-spec-benchmark-tests-node-server-timeoutMS-120000', { timeoutMS: 120000 }),
@@ -762,7 +784,8 @@ function addPerformanceTasks() {
     makePerfTask('run-spec-benchmark-tests-node-server-logging', {
       mongodbLogPath: 'stderr',
       mongodbLogComponentSeverities: { default: 'trace' }
-    })
+    }),
+    makePerfTaskNEW('run-spec-benchmark-tests-node-server-new', {})
   ];
 
   TASKS.push(...tasks);

.evergreen/run-benchmarks.sh

@@ -11,6 +11,12 @@ export MONGODB_CLIENT_OPTIONS=$MONGODB_CLIENT_OPTIONS
 
 npm run build:ts
 
+if [[ "${NEW_BENCH:-}" == "true" ]]; then
+  pushd test/benchmarks/driver_bench
+  npm start
+  popd
+  exit 0
+fi
 
 # If MONGODB_CLIENT_OPTIONS contains mongodbLogComponentSeverities redirect stderr to a file
 if [[ $MONGODB_CLIENT_OPTIONS == *"mongodbLogComponentSeverities"* ]]; then

test/benchmarks/driver_bench/.gitignore

@@ -0,0 +1,3 @@
+results.json
+results_*.json
+package-lock.json

test/benchmarks/driver_bench/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "driver_bench",
+  "version": "0.0.0",
+  "private": true,
+  "scripts": {
+    "prestart": "tsc",
+    "start": "node lib/main.mjs"
+  }
+}

test/benchmarks/driver_bench/src/driver.mts

@@ -0,0 +1,200 @@
+import child_process from 'node:child_process';
+import fs from 'node:fs/promises';
+import module from 'node:module';
+import path from 'node:path';
+import process from 'node:process';
+
+const __dirname = import.meta.dirname;
+const require = module.createRequire(__dirname);
+
+/**
+ * The path to the MongoDB Node.js driver.
+ * This MUST be set to the directory the driver is installed in
+ * NOT the file "lib/index.js" that is the driver's export.
+ */
+export const MONGODB_DRIVER_PATH = (() => {
+  let driverPath = process.env.MONGODB_DRIVER_PATH;
+  if (!driverPath?.length) {
+    driverPath = path.resolve(__dirname, '../../../..');
+  }
+  return driverPath;
+})();
+
+/** Grab the version from the package.json */
+export const { version: MONGODB_DRIVER_VERSION } = require(
+  path.join(MONGODB_DRIVER_PATH, 'package.json')
+);
+
+/**
+ * Use git to optionally determine the git revision,
+ * but the benchmarks could be run against an npm installed version so this should be allowed to fail
+ */
+export const MONGODB_DRIVER_REVISION = (() => {
+  try {
+    return child_process
+      .execSync('git rev-parse --short HEAD', {
+        cwd: MONGODB_DRIVER_PATH,
+        encoding: 'utf8'
+      })
+      .trim();
+  } catch {
+    return 'unknown revision';
+  }
+})();
+
+/**
+ * Find the BSON dependency inside the driver PATH given and grab the version from the package.json.
+ */
+export const MONGODB_BSON_PATH = path.join(MONGODB_DRIVER_PATH, 'node_modules', 'bson');
+export const { version: MONGODB_BSON_VERSION } = require(
+  path.join(MONGODB_BSON_PATH, 'package.json')
+);
+
+/**
+ * If you need to test BSON changes, you should clone, checkout and build BSON.
+ * run: `npm link` with no arguments to register the link.
+ * Then in the driver you are testing run `npm link bson` to use your local build.
+ *
+ * This will symlink the BSON into the driver's node_modules directory. So here
+ * we can find the revision of the BSON we are testing against if .git exists.
+ */
+export const MONGODB_BSON_REVISION = await (async () => {
+  const bsonGitExists = await fs.access(path.join(MONGODB_BSON_PATH, '.git')).then(
+    () => true,
+    () => false
+  );
+  if (!bsonGitExists) {
+    return 'installed from npm';
+  }
+  try {
+    return child_process
+      .execSync('git rev-parse --short HEAD', {
+        cwd: path.join(MONGODB_BSON_PATH),
+        encoding: 'utf8'
+      })
+      .trim();
+  } catch {
+    return 'unknown revision';
+  }
+})();
+
+export const MONGODB_CLIENT_OPTIONS = (() => {
+  const optionsString = process.env.MONGODB_CLIENT_OPTIONS;
+  let options = undefined;
+  if (optionsString?.length) {
+    options = JSON.parse(optionsString);
+  }
+  return { ...options };
+})();
+
+export const MONGODB_URI = (() => {
+  if (process.env.MONGODB_URI?.length) return process.env.MONGODB_URI;
+  return 'mongodb://127.0.0.1:27017';
+})();
+
+export function snakeToCamel(name: string) {
+  return name
+    .split('_')
+    .map((s, i) => (i !== 0 ? s[0].toUpperCase() + s.slice(1) : s))
+    .join('');
+}
+
+import type mongodb from '../../../../mongodb.js';
+export type { mongodb };
+
+const { MongoClient, GridFSBucket } = require(path.join(MONGODB_DRIVER_PATH));
+
+const DB_NAME = 'perftest';
+const COLLECTION_NAME = 'corpus';
+
+const SPEC_DIRECTORY = path.resolve(__dirname, '..', '..', 'driverBench', 'spec');
+
+export function metrics(test_name: string, result: number, count: number) {
+  return {
+    info: {
+      test_name,
+      // Args can only be a map of string -> int32. So if its a number leave it be,
+      // if it is anything else test for truthiness and set to 1 or 0.
+      args: Object.fromEntries(
+        Object.entries(MONGODB_CLIENT_OPTIONS).map(([key, value]) => [
+          key,
+          typeof value === 'number' ? value : value ? 1 : 0
+        ])
+      )
+    },
+    metrics: [
+      { name: 'megabytes_per_second', value: result },
+      // Reporting the count so we can verify programmatically or in UI how many iterations we reached
+      { name: 'count', value: count }
+    ]
+  } as const;
+}
+
+/**
+ * This class exists to abstract some of the driver API so we can gloss over version differences.
+ * For use in setup/teardown mostly.
+ */
+export class DriverTester {
+  public client: mongodb.MongoClient;
+  constructor() {
+    this.client = new MongoClient(MONGODB_URI, MONGODB_CLIENT_OPTIONS);
+  }
+
+  public get db() {
+    return this.client.db(DB_NAME);
+  }
+
+  public get collection() {
+    return this.client.db(DB_NAME).collection(COLLECTION_NAME);
+  }
+
+  public get bucket(): mongodb.GridFSBucket {
+    return new GridFSBucket(this.db);
+  }
+
+  async drop() {
+    const utilClient = new MongoClient(MONGODB_URI, MONGODB_CLIENT_OPTIONS);
+    const db = utilClient.db(DB_NAME);
+    const collection = db.collection(COLLECTION_NAME);
+    await collection.drop().catch(() => null);
+    await db.dropDatabase().catch(() => null);
+    await utilClient.close();
+  }
+
+  async create() {
+    const utilClient = new MongoClient(MONGODB_URI, MONGODB_CLIENT_OPTIONS);
+    try {
+      await utilClient.db(DB_NAME).createCollection(COLLECTION_NAME);
+    } finally {
+      await utilClient.close();
+    }
+  }
+
+  async load(filePath: string, type: 'json' | 'string' | 'buffer'): Promise<any> {
+    const content = await fs.readFile(path.join(SPEC_DIRECTORY, filePath));
+    if (type === 'buffer') return content;
+    const string = content.toString('utf8');
+    if (type === 'string') return string;
+    if (type === 'json') return JSON.parse(string);
+    throw new Error('unknown type: ' + type);
+  }
+
+  async insertManyOf(document: Record<string, any>, length: number, addId = false) {
+    const utilClient = new MongoClient(MONGODB_URI, MONGODB_CLIENT_OPTIONS);
+    const db = utilClient.db(DB_NAME);
+    const collection = db.collection(COLLECTION_NAME);
+    try {
+      await collection.insertMany(
+        Array.from({ length }, (_, _id) => ({ ...(addId ? { _id } : {}), ...document })) as any[]
+      );
+    } finally {
+      await utilClient.close();
+    }
+  }
+
+  async close() {
+    await this.client.close();
+  }
+}
+
+export const driver = new DriverTester();