From 3e6a07b446746ebf62e6ee13e7bb932848adeed4 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Tue, 5 Nov 2024 08:57:38 -0700 Subject: [PATCH 1/6] adjust config --- .evergreen/config.in.yml | 4 ++-- .evergreen/config.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.evergreen/config.in.yml b/.evergreen/config.in.yml index fa4884a05ba..deeafb7e372 100644 --- a/.evergreen/config.in.yml +++ b/.evergreen/config.in.yml @@ -1147,7 +1147,7 @@ tasks: - func: bootstrap kms servers - func: "run serverless tests" - - name: run-spec-benchmark-tests-node-18-server-6.0 + - name: run-spec-benchmark-tests-node-server tags: - run-spec-benchmark-tests - performance @@ -1157,7 +1157,7 @@ tasks: type: setup params: updates: - - { key: NODE_LTS_VERSION, value: v18.16.0 } + - { key: NODE_LTS_VERSION, value: v22.11.0 } - { key: NPM_VERSION, value: "9" } - { key: VERSION, value: v6.0-perf } - { key: TOPOLOGY, value: server } diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 01306ba9bc8..1164202eb1e 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1108,7 +1108,7 @@ tasks: - func: install dependencies - func: bootstrap kms servers - func: run serverless tests - - name: run-spec-benchmark-tests-node-18-server-6.0 + - name: run-spec-benchmark-tests-node-server tags: - run-spec-benchmark-tests - performance @@ -1118,7 +1118,7 @@ tasks: type: setup params: updates: - - {key: NODE_LTS_VERSION, value: v18.16.0} + - {key: NODE_LTS_VERSION, value: v22.11.0} - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: v6.0-perf} - {key: TOPOLOGY, value: server} From cd4b0bf52c0a88cf2e5b581cd4839c415e8eec09 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Tue, 5 Nov 2024 09:03:01 -0700 Subject: [PATCH 2/6] fix task --- .evergreen/config.in.yml | 2 +- .evergreen/config.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.in.yml b/.evergreen/config.in.yml index deeafb7e372..9cbe6150b88 100644 --- a/.evergreen/config.in.yml +++ b/.evergreen/config.in.yml @@ -1616,4 +1616,4 @@ buildvariants: display_name: Performance Test run_on: rhel90-dbx-perf-large tasks: - - run-spec-benchmark-tests-node-18-server-6.0 + - run-spec-benchmark-tests-node-server diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 1164202eb1e..03130313f71 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -4547,7 +4547,7 @@ buildvariants: display_name: Performance Test run_on: rhel90-dbx-perf-large tasks: - - run-spec-benchmark-tests-node-18-server-6.0 + - run-spec-benchmark-tests-node-server - name: rhel80-large-gallium display_name: rhel8 Node16 run_on: rhel80-large From bf11f12739ded0a37fb9f9a8f7f7fcafb1c1ace2 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Tue, 5 Nov 2024 09:10:45 -0700 Subject: [PATCH 3/6] don't pin npm --- .evergreen/config.in.yml | 1 - .evergreen/config.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.evergreen/config.in.yml b/.evergreen/config.in.yml index 9cbe6150b88..45e10f5f66f 100644 --- a/.evergreen/config.in.yml +++ b/.evergreen/config.in.yml @@ -1158,7 +1158,6 @@ tasks: params: updates: - { key: NODE_LTS_VERSION, value: v22.11.0 } - - { key: NPM_VERSION, value: "9" } - { key: VERSION, value: v6.0-perf } - { key: TOPOLOGY, value: server } - { key: AUTH, value: noauth } diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 03130313f71..b99fcdef22a 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1119,7 +1119,6 @@ tasks: params: updates: - {key: NODE_LTS_VERSION, value: v22.11.0} - - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: v6.0-perf} - {key: TOPOLOGY, value: server} - {key: AUTH, value: noauth} From ab4de3d092be025b72d09a4e5d923705cd10502c Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Tue, 5 Nov 2024 09:18:09 -0700 Subject: [PATCH 4/6] add retry threshold --- test/benchmarks/mongoBench/runner.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/benchmarks/mongoBench/runner.js b/test/benchmarks/mongoBench/runner.js index 064f51dec58..4c5a00966c4 100644 --- a/test/benchmarks/mongoBench/runner.js +++ b/test/benchmarks/mongoBench/runner.js @@ -162,12 +162,17 @@ class Runner { const minExecutionCount = this.minExecutionCount; const minExecutionTime = this.minExecutionTime; const maxExecutionTime = this.maxExecutionTime; + const maxExecutionAttempts = 10000000; let time = performance.now() - start; let count = 1; const taskTimer = benchmark._taskType === 'sync' ? timeSyncTask : timeAsyncTask; - while (time < maxExecutionTime && (time < minExecutionTime || count < minExecutionCount)) { + while ( + time < maxExecutionTime && + (time < minExecutionTime || count < minExecutionCount) && + count < maxExecutionAttempts + ) { await benchmark.beforeTask.call(ctx); const executionTime = await taskTimer(benchmark.task, ctx); rawData.push(executionTime); From bba94c6b05578dd1eb201295ef1cd43eec4eded8 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Tue, 5 Nov 2024 10:41:08 -0700 Subject: [PATCH 5/6] make max execution count configurable --- test/benchmarks/mongoBench/constants.js | 6 +++++- test/benchmarks/mongoBench/runner.js | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/test/benchmarks/mongoBench/constants.js b/test/benchmarks/mongoBench/constants.js index 305f8c562f5..4065d97719a 100644 --- a/test/benchmarks/mongoBench/constants.js +++ b/test/benchmarks/mongoBench/constants.js @@ -12,8 +12,12 @@ const DEFAULT_MAX_EXECUTION_TIME = const DEFAULT_MIN_EXECUTION_COUNT = Number.parseInt(process.env.DRIVER_BENCH_MIN_EX_COUNT, 10) || 100; +const DEFAULT_MAX_EXECUTION_COUNT = + Number.parseInt(process.env.DRIVER_BENCH_MAX_EX_COUNT, 10) || 10000000; + module.exports = { DEFAULT_MIN_EXECUTION_COUNT, DEFAULT_MIN_EXECUTION_TIME, - DEFAULT_MAX_EXECUTION_TIME + DEFAULT_MAX_EXECUTION_TIME, + DEFAULT_MAX_EXECUTION_COUNT }; diff --git a/test/benchmarks/mongoBench/runner.js b/test/benchmarks/mongoBench/runner.js index 4c5a00966c4..0d5dbdcbf2f 100644 --- a/test/benchmarks/mongoBench/runner.js +++ b/test/benchmarks/mongoBench/runner.js @@ -57,6 +57,7 @@ class Runner { this.minExecutionTime = options.minExecutionTime || CONSTANTS.DEFAULT_MIN_EXECUTION_TIME; this.maxExecutionTime = options.maxExecutionTime || CONSTANTS.DEFAULT_MAX_EXECUTION_TIME; this.minExecutionCount = options.minExecutionCount || CONSTANTS.DEFAULT_MIN_EXECUTION_COUNT; + this.minExecutionCount = options.maxExecutionCount || CONSTANTS.DEFAULT_MAX_EXECUTION_COUNT; this.reporter = options.reporter || function () { @@ -162,7 +163,7 @@ class Runner { const minExecutionCount = this.minExecutionCount; const minExecutionTime = this.minExecutionTime; const maxExecutionTime = this.maxExecutionTime; - const maxExecutionAttempts = 10000000; + const maxExecutionAttempts = this.minExecutionCount; let time = performance.now() - start; let count = 1; From 92b52eb4c4417e2ebe7b85768a19b8d602c04123 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Tue, 5 Nov 2024 11:34:00 -0700 Subject: [PATCH 6/6] comments --- test/benchmarks/mongoBench/runner.js | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test/benchmarks/mongoBench/runner.js b/test/benchmarks/mongoBench/runner.js index 0d5dbdcbf2f..745cefa74e8 100644 --- a/test/benchmarks/mongoBench/runner.js +++ b/test/benchmarks/mongoBench/runner.js @@ -39,7 +39,7 @@ function calculateMicroBench(benchmark, data) { const rawData = data.rawData; const count = data.count; - const sortedData = [].concat(rawData).sort(); + const sortedData = [].concat(rawData).sort((a, b) => a - b); const percentiles = PERCENTILES.reduce((acc, pct) => { acc[pct] = sortedData[percentileIndex(pct, count)]; @@ -57,7 +57,7 @@ class Runner { this.minExecutionTime = options.minExecutionTime || CONSTANTS.DEFAULT_MIN_EXECUTION_TIME; this.maxExecutionTime = options.maxExecutionTime || CONSTANTS.DEFAULT_MAX_EXECUTION_TIME; this.minExecutionCount = options.minExecutionCount || CONSTANTS.DEFAULT_MIN_EXECUTION_COUNT; - this.minExecutionCount = options.maxExecutionCount || CONSTANTS.DEFAULT_MAX_EXECUTION_COUNT; + this.maxExecutionCount = options.maxExecutionCount || CONSTANTS.DEFAULT_MAX_EXECUTION_COUNT; this.reporter = options.reporter || function () { @@ -127,6 +127,7 @@ class Runner { for (const [name, benchmark] of benchmarks) { this.reporter(` Executing Benchmark "${name}"`); result[name] = await this._runBenchmark(benchmark); + this.reporter(` Executed Benchmark "${name}" =`, result[name]); } return result; @@ -163,7 +164,7 @@ class Runner { const minExecutionCount = this.minExecutionCount; const minExecutionTime = this.minExecutionTime; const maxExecutionTime = this.maxExecutionTime; - const maxExecutionAttempts = this.minExecutionCount; + const maxExecutionAttempts = this.maxExecutionCount; let time = performance.now() - start; let count = 1; @@ -172,7 +173,7 @@ class Runner { while ( time < maxExecutionTime && (time < minExecutionTime || count < minExecutionCount) && - count < maxExecutionAttempts + count <= maxExecutionAttempts ) { await benchmark.beforeTask.call(ctx); const executionTime = await taskTimer(benchmark.task, ctx); @@ -187,9 +188,12 @@ class Runner { }; } - _errorHandler(e) { - console.error(e); - return NaN; + _errorHandler(error) { + this.reporter(`Error: ${error.name} - ${error.message} - ${error.stack}`); + for (let error = error.cause; error != null; error = error.cause) { + this.reporter(`Caused by: ${error.name} - ${error.message} - ${error.stack}`); + } + throw error; } }