ci(NODE-4589): add benchmark testing task to CI (#3385)

baileympearson · web-flow · commit 6658433b26c1 · 2022-08-31T02:38:04.000+02:00
diff --git a/.evergreen/config.in.yml b/.evergreen/config.in.yml
@@ -879,11 +879,42 @@ functions:
 
           aws s3 cp output/lcov-report s3://mciuploads/mongo-node-driver/${revision}/${version_id}//lcov-report/
 
+  "run spec driver benchmarks":
+    - command: subprocess.exec
+      type: test
+      params:
+        working_dir: "src"
+        env:
+          PROJECT_DIRECTORY: ${PROJECT_DIRECTORY}
+          MONGODB_URI: ${MONGODB_URI}
+        binary: bash
+        args:
+          - ${PROJECT_DIRECTORY}/.evergreen/run-benchmarks.sh
+
 tasks:
-    - name: "test-serverless"
-      tags: ["serverless"]
-      commands:
-        - func: "run serverless tests"
+  - name: "test-serverless"
+    tags: ["serverless"]
+    commands:
+      - func: "run serverless tests"
+
+  - name: run-spec-benchmark-tests
+    tags:
+      - run-spec-benchmark-tests
+      - performance
+    exec_timeout_secs: 3600
+    commands:
+      - func: install dependencies
+        vars:
+          NODE_LTS_NAME: gallium
+      - func: bootstrap mongo-orchestration
+        vars:
+          VERSION: rapid
+          TOPOLOGY: server
+          AUTH: noauth
+      - func: run spec driver benchmarks
+      - command: perf.send
+        params:
+          file: src/results.json
 
 task_groups:
   - name: serverless_task_group
@@ -935,3 +966,9 @@ post:
 
 ignore:
   - '*.md'
+buildvariants:
+  - name: performance-tests
+    display_name: Performance Test
+    run_on: ubuntu1804-large
+    tasks:
+      - run-spec-benchmark-tests
diff --git a/.evergreen/config.yml b/.evergreen/config.yml
@@ -834,12 +834,41 @@ functions:
           npx nyc report -t merged-coverage --reporter=html --report-dir output
 
           aws s3 cp output/lcov-report s3://mciuploads/mongo-node-driver/${revision}/${version_id}//lcov-report/
+  run spec driver benchmarks:
+    - command: subprocess.exec
+      type: test
+      params:
+        working_dir: src
+        env:
+          PROJECT_DIRECTORY: ${PROJECT_DIRECTORY}
+          MONGODB_URI: ${MONGODB_URI}
+        binary: bash
+        args:
+          - ${PROJECT_DIRECTORY}/.evergreen/run-benchmarks.sh
 tasks:
   - name: test-serverless
     tags:
       - serverless
     commands:
       - func: run serverless tests
+  - name: run-spec-benchmark-tests
+    tags:
+      - run-spec-benchmark-tests
+      - performance
+    exec_timeout_secs: 3600
+    commands:
+      - func: install dependencies
+        vars:
+          NODE_LTS_NAME: gallium
+      - func: bootstrap mongo-orchestration
+        vars:
+          VERSION: rapid
+          TOPOLOGY: server
+          AUTH: noauth
+      - func: run spec driver benchmarks
+      - command: perf.send
+        params:
+          file: src/results.json
   - name: test-latest-server
     tags:
       - latest
@@ -2205,6 +2234,11 @@ post:
 ignore:
   - '*.md'
 buildvariants:
+  - name: performance-tests
+    display_name: Performance Test
+    run_on: ubuntu1804-large
+    tasks:
+      - run-spec-benchmark-tests
   - name: ubuntu-18.04-erbium
     display_name: Ubuntu 18.04 Node Erbium
     run_on: ubuntu1804-large
diff --git a/.evergreen/run-benchmarks.sh b/.evergreen/run-benchmarks.sh
@@ -0,0 +1,8 @@
+#! /bin/bash
+
+[ -s "$PROJECT_DIRECTORY/node-artifacts/nvm/nvm.sh" ] && source "$PROJECT_DIRECTORY"/node-artifacts/nvm/nvm.sh
+
+export MONGODB_URI=$MONGODB_URI
+
+npm run build:ts
+npm run check:bench
diff --git a/test/benchmarks/driverBench/common.js b/test/benchmarks/driverBench/common.js
@@ -10,8 +10,6 @@ const COLLECTION_NAME = 'corpus';
 
 const SPEC_DIRECTORY = path.resolve(__dirname, 'spec');
 
-const MONGODB_URL = process.env.MONGODB_URL || 'mongodb://localhost:27017';
-
 function loadSpecFile(filePath, encoding) {
   const fp = [SPEC_DIRECTORY].concat(filePath);
   return fs.readFileSync(path.join.apply(path, fp), encoding);
@@ -22,7 +20,7 @@ function loadSpecString(filePath) {
 }
 
 function makeClient() {
-  this.client = new MongoClient(MONGODB_URL);
+  this.client = new MongoClient(process.env.MONGODB_URI || 'mongodb://localhost:27017');
 }
 
 function connectClient() {
diff --git a/test/benchmarks/driverBench/index.js b/test/benchmarks/driverBench/index.js
@@ -1,12 +1,11 @@
 'use strict';
 
 const MongoBench = require('../mongoBench');
+const { writeFile } = require('fs/promises');
 
 const Runner = MongoBench.Runner;
 const commonHelpers = require('./common');
 
-const { LEGACY_HELLO_COMMAND } = require('../../../src/constants');
-
 let BSON = require('bson');
 
 try {
@@ -16,6 +15,7 @@ try {
 }
 
 const { EJSON } = require('bson');
+const { inspect } = require('util');
 
 const makeClient = commonHelpers.makeClient;
 const connectClient = commonHelpers.connectClient;
@@ -112,9 +112,7 @@ function runCommand(done) {
     if (_id > 10000) {
       return done();
     }
-    return this.db.command({ [LEGACY_HELLO_COMMAND]: true }, err =>
-      err ? done(err) : loop(_id + 1)
-    );
+    return this.db.command({ hello: true }, err => (err ? done(err) : loop(_id + 1)));
   };
 
   return loop(1);
@@ -333,6 +331,8 @@ const benchmarkRunner = new Runner()
 benchmarkRunner
   .run()
   .then(microBench => {
+    // TODO - test against different BSON versions in CI
+    // const bsonType = BSON.serialize.toString().includes('native code') ? 'bson-ext' : 'js-bson';
     const bsonBench = average(Object.values(microBench.bsonBench));
     const singleBench = average([
       microBench.singleBench.findOne,
@@ -357,21 +357,31 @@ benchmarkRunner
       microBench.multiBench.gridFsUpload
       // TODO: Add parallelBench write benchmarks
     ]);
-    const driverBench = average([readBench, writeBench]);
 
-    return {
-      microBench,
+    const benchmarkResults = {
       bsonBench,
       singleBench,
       multiBench,
       parallelBench,
       readBench,
       writeBench,
-      driverBench
+      ...microBench.bsonBench,
+      ...microBench.singleBench,
+      ...microBench.multiBench
     };
+
+    return Object.entries(benchmarkResults).map(([benchmarkName, result]) => {
+      return {
+        info: {
+          test_name: benchmarkName
+        },
+        metrics: [{ name: 'megabytes_per_second', value: result }]
+      };
+    });
   })
   .then(data => {
-    data.bsonType = BSON.serialize.toString().includes('native code') ? 'bson-ext' : 'js-bson';
-    console.log(data);
+    const results = JSON.stringify(data, undefined, 2);
+    console.error(inspect(results, { depth: Infinity }));
+    return writeFile('results.json', results);
   })
   .catch(err => console.error(err));
diff --git a/test/benchmarks/mongoBench/benchmark.js b/test/benchmarks/mongoBench/benchmark.js
@@ -1,7 +1,5 @@
 'use strict';
 
-const util = require('./util');
-
 class Benchmark {
   constructor() {
     // The Task itself
@@ -165,10 +163,24 @@ class Benchmark {
   }
 
   /**
+   * Converts an array of task functions into a single async function, that awaits each
+   * task function and resolves once all are completed.
+   *
+   * The returned function will reject if any of the task functions fails.
+   *
    * @param {any} arr
+   * @return {() => Promise<void>}
    */
   _convertArrayToAsyncPipeFn(arr) {
-    return arr.length ? util.asyncChain(arr) : util.noop;
+    const array = arr.length ? arr : [];
+    return async function () {
+      // copy the array to guard against modification
+      const chain = [].concat(array);
+      const context = this;
+      for (const fn of chain) {
+        await fn.call(context);
+      }
+    };
   }
 }
 
diff --git a/test/benchmarks/mongoBench/constants.js b/test/benchmarks/mongoBench/constants.js
@@ -1,30 +1,18 @@
 'use strict';
 
-const SECOND_TO_NS = 1e9;
 const MINUTE_TO_MS = 60 * 1e3;
 const FIVE_MINUTES_TO_MS = MINUTE_TO_MS * 5;
 
-function parseIntOrDefault(val, def) {
-  if (!val) {
-    return def;
-  }
+const DEFAULT_MIN_EXECUTION_TIME =
+  Number.parseInt(process.env.DRIVER_BENCH_MIN_EX_TIME, 10) || MINUTE_TO_MS;
 
-  const parsed = Number.parseInt(val, 10);
-  return parsed ? parsed : def;
-}
+const DEFAULT_MAX_EXECUTION_TIME =
+  Number.parseInt(process.env.DRIVER_BENCH_MAX_EX_TIME, 10) || FIVE_MINUTES_TO_MS;
 
-const DEFAULT_MIN_EXECUTION_TIME = parseIntOrDefault(
-  process.env.DRIVER_BENCH_MIN_EX_TIME,
-  MINUTE_TO_MS
-);
-const DEFAULT_MAX_EXECUTION_TIME = parseIntOrDefault(
-  process.env.DRIVER_BENCH_MAX_EX_TIME,
-  FIVE_MINUTES_TO_MS
-);
-const DEFAULT_MIN_EXECUTION_COUNT = parseIntOrDefault(process.env.DRIVER_BENCH_MIN_EX_COUNT, 100);
+const DEFAULT_MIN_EXECUTION_COUNT =
+  Number.parseInt(process.env.DRIVER_BENCH_MIN_EX_COUNT, 10) || 100;
 
 module.exports = {
-  SECOND_TO_NS,
   DEFAULT_MIN_EXECUTION_COUNT,
   DEFAULT_MIN_EXECUTION_TIME,
   DEFAULT_MAX_EXECUTION_TIME
diff --git a/test/benchmarks/mongoBench/index.js b/test/benchmarks/mongoBench/index.js
@@ -1,6 +1,5 @@
 'use strict';
 
 module.exports = {
-  Runner: require('./runner'),
-  asyncChain: require('./util').asyncChain
+  Runner: require('./runner')
 };
diff --git a/test/benchmarks/mongoBench/runner.js b/test/benchmarks/mongoBench/runner.js
diff --git a/test/benchmarks/mongoBench/suite.js b/test/benchmarks/mongoBench/suite.js
diff --git a/test/benchmarks/mongoBench/util.js b/test/benchmarks/mongoBench/util.js

Original file line number	Diff line number	Diff line change
`@@ -10,8 +10,6 @@ const COLLECTION_NAME = 'corpus';`
`10`	`10`
`11`	`11`	`const SPEC_DIRECTORY = path.resolve(__dirname, 'spec');`
`12`	`12`
`13`		`-const MONGODB_URL = process.env.MONGODB_URL \|\| 'mongodb://localhost:27017';`
`14`		`-`
`15`	`13`	`function loadSpecFile(filePath, encoding) {`
`16`	`14`	`const fp = [SPEC_DIRECTORY].concat(filePath);`
`17`	`15`	`return fs.readFileSync(path.join.apply(path, fp), encoding);`
`@@ -22,7 +20,7 @@ function loadSpecString(filePath) {`
`22`	`20`	`}`
`23`	`21`
`24`	`22`	`function makeClient() {`
`25`		`- this.client = new MongoClient(MONGODB_URL);`
	`23`	`+ this.client = new MongoClient(process.env.MONGODB_URI \|\| 'mongodb://localhost:27017');`
`26`	`24`	`}`
`27`	`25`
`28`	`26`	`function connectClient() {`