chore: sync 3.6 changes to 3.7 (#2963)

Author: dariakp

.evergreen/config.yml

@@ -525,6 +525,18 @@ functions:
 
           UNIFIED=${UNIFIED} MONGODB_URI="${MONGODB_URI}" bash ${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh
 
+  "run custom snappy tests":
+    - command: subprocess.exec
+      params:
+        working_dir: "src"
+        timeout_secs: 60
+        env:
+          MONGODB_URI: ${MONGODB_URI}
+          PROJECT_DIRECTORY: ${PROJECT_DIRECTORY}
+        binary: bash
+        args:
+          - "${PROJECT_DIRECTORY}/.evergreen/run-snappy-version-test.sh"
+
   "upload test results":
     - command: attach.xunit_results
       params:

.evergreen/config.yml.in

@@ -6,10 +6,10 @@ const yaml = require('js-yaml');
 
 const LATEST_EFFECTIVE_VERSION = '5.0';
 const LEGACY_MONGODB_VERSIONS = new Set(['4.4', '4.2', '4.0', '3.6', '3.4', '3.2', '3.0', '2.6']);
-const MONGODB_VERSIONS = ['latest'].concat(Array.from(LEGACY_MONGODB_VERSIONS));
-const AWS_AUTH_VERSIONS = ['latest', '4.4'];
-const OCSP_VERSIONS = ['latest', '4.4'];
-const TLS_VERSIONS = ['latest', '4.2']; // also test on 4.2 because 4.4+ currently skipped on windows
+const MONGODB_VERSIONS = ['latest', '5.0'].concat(Array.from(LEGACY_MONGODB_VERSIONS));
+const AWS_AUTH_VERSIONS = ['latest', '5.0', '4.4'];
+const OCSP_VERSIONS = ['latest', '5.0', '4.4'];
+const TLS_VERSIONS = ['latest', '5.0', '4.2']; // also test on 4.2 because 4.4+ currently skipped on windows
 const NODE_VERSIONS = ['fermium', 'erbium', 'dubnium', 'carbon', 'boron', 'argon'];
 const LEGACY_TOPOLOGIES = new Set(['server', 'replica_set', 'sharded_cluster']);
 const UNIFIED_TOPOLOGIES = Array.from(LEGACY_TOPOLOGIES).map(topology => `${topology}-unified`);
@@ -154,7 +154,8 @@ TASKS.push(
     tags: ['auth', 'kerberos', 'legacy'],
     commands: [
       { func: 'install dependencies' },
-      { func: 'run kerberos tests',
+      {
+        func: 'run kerberos tests',
         vars: {
           UNIFIED: 0
         }
@@ -166,7 +167,8 @@ TASKS.push(
     tags: ['auth', 'kerberos', 'unified'],
     commands: [
       { func: 'install dependencies' },
-      { func: 'run kerberos tests',
+      {
+        func: 'run kerberos tests',
         vars: {
           UNIFIED: 1
         }
@@ -325,7 +327,7 @@ OCSP_VERSIONS.forEach(VERSION => {
 const AWS_AUTH_TASKS = [];
 
 AWS_AUTH_VERSIONS.forEach(VERSION => {
-  const name = (ex) => `aws-${VERSION}-auth-test-${ex.split(' ').join('-')}`;
+  const name = ex => `aws-${VERSION}-auth-test-${ex.split(' ').join('-')}`;
   // AWS_AUTH_TASKS.push(name);
 
   const aws_funcs = [
@@ -354,10 +356,10 @@ AWS_AUTH_VERSIONS.forEach(VERSION => {
       { func: 'setup aws env' },
       fn
     ]
-  }))
+  }));
 
   TASKS.push(...aws_tasks);
-  AWS_AUTH_TASKS.push(...aws_tasks.map(t => t.name))
+  AWS_AUTH_TASKS.push(...aws_tasks.map(t => t.name));
 });
 
 const BUILD_VARIANTS = [];
@@ -371,31 +373,33 @@ const getTaskList = (() => {
       return memo[key];
     }
     const taskList = BASE_TASKS.concat(TASKS);
-    const ret = taskList.filter(task => {
-      if (task.name.match(/^aws/)) return false;
+    const ret = taskList
+      .filter(task => {
+        if (task.name.match(/^aws/)) return false;
 
-      // skip unsupported tasks on windows
-      if (os.match(/^windows/) && task.tags.filter(tag => WINDOWS_SKIP_TAGS.has(tag)).length) {
-        return false;
-      }
+        // skip unsupported tasks on windows
+        if (os.match(/^windows/) && task.tags.filter(tag => WINDOWS_SKIP_TAGS.has(tag)).length) {
+          return false;
+        }
 
-      const tasksWithVars = task.commands.filter(task => !!task.vars);
-      if (!tasksWithVars.length) {
-        return true;
-      }
+        const tasksWithVars = task.commands.filter(task => !!task.vars);
+        if (!tasksWithVars.length) {
+          return true;
+        }
 
-      // kerberos tests don't require mongo orchestration
-      if (task.tags.filter(tag => tag === 'kerberos').length) {
-        return true;
-      }
+        // kerberos tests don't require mongo orchestration
+        if (task.tags.filter(tag => tag === 'kerberos').length) {
+          return true;
+        }
 
-      const { VERSION } = tasksWithVars[0].vars || {};
-      if (VERSION === 'latest') {
-        return semver.satisfies(semver.coerce(LATEST_EFFECTIVE_VERSION), mongoVersion);
-      }
+        const { VERSION } = tasksWithVars[0].vars || {};
+        if (VERSION === 'latest') {
+          return semver.satisfies(semver.coerce(LATEST_EFFECTIVE_VERSION), mongoVersion);
+        }
 
-      return semver.satisfies(semver.coerce(VERSION), mongoVersion);
-    }).map(x => x.name);
+        return semver.satisfies(semver.coerce(VERSION), mongoVersion);
+      })
+      .map(x => x.name);
 
     memo[key] = ret;
     return ret;
@@ -448,38 +452,48 @@ SINGLETON_TASKS.push({
   ]
 });
 
-SINGLETON_TASKS.push({
-  name: 'run-custom-csfle-tests',
-  tags: ['run-custom-csfle-tests'],
+const oneOffFuncs = [
+  { func: 'run custom csfle tests', vars: { UNIFIED: 1, NODE_LTS_NAME: 'erbium' } },
+  { func: 'run custom snappy tests', vars: { UNIFIED: 1, NODE_LTS_NAME: 'erbium' } }
+];
+
+const oneOffFuncAsTasks = oneOffFuncs.map(oneOffFunc => ({
+  name: `${oneOffFunc.func.split(' ').join('-')}`,
+  tags: ['run-custom-dependency-tests'],
   commands: [
     {
       func: 'install dependencies',
       vars: {
-        NODE_LTS_NAME: 'erbium',
-      },
+        NODE_LTS_NAME: 'erbium'
+      }
     },
     {
       func: 'bootstrap mongo-orchestration',
       vars: {
-        VERSION: '4.4',
+        VERSION: '5.0',
         TOPOLOGY: 'server'
       }
     },
-    { func: 'run custom csfle tests' }
+    oneOffFunc
   ]
-});
+}));
 
-BUILD_VARIANTS.push({
-  name: 'lint',
-  display_name: 'lint',
-  run_on: 'rhel70',
-  tasks: ['run-checks']
-}, {
-  name: 'ubuntu1804-custom-csfle-tests',
-  display_name: 'Custom FLE Version Test',
-  run_on: 'ubuntu1804-test',
-  tasks: ['run-custom-csfle-tests']
-});
+SINGLETON_TASKS.push(...oneOffFuncAsTasks);
+
+BUILD_VARIANTS.push(
+  {
+    name: 'lint',
+    display_name: 'lint',
+    run_on: 'rhel70',
+    tasks: ['run-checks']
+  },
+  {
+    name: 'ubuntu1804-custom-dependency-tests',
+    display_name: 'Custom Dependency Version Test',
+    run_on: 'ubuntu1804-large',
+    tasks: oneOffFuncAsTasks.map(({ name }) => name)
+  }
+);
 
 // special case for MONGODB-AWS authentication
 BUILD_VARIANTS.push({
@@ -493,7 +507,7 @@ BUILD_VARIANTS.push({
 });
 
 const fileData = yaml.safeLoad(fs.readFileSync(`${__dirname}/config.yml.in`, 'utf8'));
-fileData.tasks = (fileData.tasks || []).concat(BASE_TASKS).concat(TASKS).concat(SINGLETON_TASKS);
+fileData.tasks = (fileData.tasks || []).concat(BASE_TASKS, TASKS, SINGLETON_TASKS);
 fileData.buildvariants = (fileData.buildvariants || []).concat(BUILD_VARIANTS);
 
 fs.writeFileSync(`${__dirname}/config.yml`, yaml.safeDump(fileData, { lineWidth: 120 }), 'utf8');

.evergreen/generate_evergreen_tasks.js

@@ -0,0 +1,19 @@
+#! /usr/bin/env bash
+
+export PATH="/opt/mongodbtoolchain/v2/bin:$PATH"
+NODE_ARTIFACTS_PATH="${PROJECT_DIRECTORY}/node-artifacts"
+export NVM_DIR="${NODE_ARTIFACTS_PATH}/nvm"
+
+if [[ "$OS" == "Windows_NT" ]]; then
+    NVM_HOME=$(cygpath -w "$NVM_DIR")
+    export NVM_HOME
+    NVM_SYMLINK=$(cygpath -w "$NODE_ARTIFACTS_PATH/bin")
+    export NVM_SYMLINK
+    NVM_ARTIFACTS_PATH=$(cygpath -w "$NODE_ARTIFACTS_PATH/bin")
+    export NVM_ARTIFACTS_PATH
+    PATH=$(cygpath $NVM_SYMLINK):$(cygpath $NVM_HOME):$PATH
+    export PATH
+    echo "updated path on windows PATH=$PATH"
+else
+    [ -s "$NVM_DIR/nvm.sh" ] && source "$NVM_DIR/nvm.sh"
+fi

.evergreen/init-nvm.sh

@@ -0,0 +1,12 @@
+#! /usr/bin/env bash
+
+source "${PROJECT_DIRECTORY}/.evergreen/init-nvm.sh"
+export MONGODB_URI="${MONGODB_URI}"
+
+npm i --no-save snappy@6
+
+npx mocha test/unit/snappy.test.js
+
+npm i --no-save snappy@7
+
+npx mocha test/unit/snappy.test.js

.evergreen/run-snappy-version-test.sh

@@ -2,6 +2,22 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [3.6.12](https://github.com/mongodb/node-mongodb-native/compare/v3.6.11...v3.6.12) (2021-08-30)
+
+
+### Bug Fixes
+
+* **NODE-3487:** check for nullish aws mechanism property ([#2957](https://github.com/mongodb/node-mongodb-native/issues/2957)) ([5902b4c](https://github.com/mongodb/node-mongodb-native/commit/5902b4c13a977c659af94b1fbcbcfbe5e7ca4db4))
+* **NODE-3528:** add support for snappy v7 ([#2947](https://github.com/mongodb/node-mongodb-native/issues/2947)) ([54f5c2d](https://github.com/mongodb/node-mongodb-native/commit/54f5c2d682828bc751242cf4e90ea73f0342c842))
+
+### [3.6.11](https://github.com/mongodb/node-mongodb-native/compare/v3.6.10...v3.6.11) (2021-08-05)
+
+
+### Bug Fixes
+
+* **NODE-1843:** bulk operations ignoring provided sessions [PORT] ([#2898](https://github.com/mongodb/node-mongodb-native/issues/2898)) ([9244b17](https://github.com/mongodb/node-mongodb-native/commit/9244b1771e538f7b685fd6d4aa83d9da84b20093))
+* **NODE-3199:** unable to bundle driver due to uncaught require ([#2903](https://github.com/mongodb/node-mongodb-native/issues/2903)) ([60efe9d](https://github.com/mongodb/node-mongodb-native/commit/60efe9d0030477da462d326c2e2ddc5fe6c0ffff))
+
 ### [3.6.10](https://github.com/mongodb/node-mongodb-native/compare/v3.6.9...v3.6.10) (2021-07-06)
 
 

HISTORY.md

@@ -1176,8 +1176,11 @@ class BulkOperationBase {
    * @param {function} callback
    */
   bulkExecute(_writeConcern, options, callback) {
-    if (typeof options === 'function') (callback = options), (options = {});
-    options = options || {};
+    if (typeof options === 'function') {
+      callback = options;
+    }
+
+    const finalOptions = Object.assign({}, this.s.options, options);
 
     if (typeof _writeConcern === 'function') {
       callback = _writeConcern;
@@ -1203,7 +1206,7 @@ class BulkOperationBase {
       const emptyBatchError = toError('Invalid Operation, no operations specified');
       return this._handleEarlyError(emptyBatchError, callback);
     }
-    return { options, callback };
+    return { options: finalOptions, callback };
   }
 
   /**

lib/bulk/common.js

@@ -58,7 +58,10 @@ class MongoCredentials {
         this.password = process.env.AWS_SECRET_ACCESS_KEY;
       }
 
-      if (!this.mechanismProperties.AWS_SESSION_TOKEN && process.env.AWS_SESSION_TOKEN) {
+      if (
+        this.mechanismProperties.AWS_SESSION_TOKEN == null &&
+        process.env.AWS_SESSION_TOKEN != null
+      ) {
         this.mechanismProperties.AWS_SESSION_TOKEN = process.env.AWS_SESSION_TOKEN;
       }
     }

lib/core/auth/mongo_credentials.js

@@ -9,6 +9,7 @@ const url = require('url');
 
 let aws4;
 try {
+  // Ensure you always wrap an optional require in the try block NODE-3199
   aws4 = require('aws4');
 } catch (e) {
   // don't do anything;
@@ -50,12 +51,21 @@ class MongoDBAWS extends AuthProvider {
       return;
     }
 
-    const username = credentials.username;
-    const password = credentials.password;
     const db = credentials.source;
-    const token = credentials.mechanismProperties.AWS_SESSION_TOKEN;
     const bson = this.bson;
 
+    const accessKeyId = credentials.username;
+    const secretAccessKey = credentials.password;
+    const sessionToken = credentials.mechanismProperties.AWS_SESSION_TOKEN;
+
+    // If all three defined, include sessionToken, else include username and pass, else no credentials
+    const awsCredentials =
+      accessKeyId && secretAccessKey && sessionToken
+        ? { accessKeyId, secretAccessKey, sessionToken }
+        : accessKeyId && secretAccessKey
+        ? { accessKeyId, secretAccessKey }
+        : undefined;
+
     crypto.randomBytes(32, (err, nonce) => {
       if (err) {
         callback(err);
@@ -108,18 +118,14 @@ class MongoDBAWS extends AuthProvider {
             path: '/',
             body
           },
-          {
-            accessKeyId: username,
-            secretAccessKey: password,
-            token
-          }
+          awsCredentials
         );
 
         const authorization = options.headers.Authorization;
         const date = options.headers['X-Amz-Date'];
         const payload = { a: authorization, d: date };
-        if (token) {
-          payload.t = token;
+        if (sessionToken) {
+          payload.t = sessionToken;
         }
 
         const saslContinue = {
@@ -163,6 +169,7 @@ function makeTempCredentials(credentials, callback) {
   if (process.env.AWS_CONTAINER_CREDENTIALS_RELATIVE_URI) {
     request(
       `${AWS_RELATIVE_URI}${process.env.AWS_CONTAINER_CREDENTIALS_RELATIVE_URI}`,
+      undefined,
       (err, res) => {
         if (err) return callback(err);
         done(res);
@@ -214,11 +221,6 @@ function deriveRegion(host) {
 }
 
 function request(uri, options, callback) {
-  if (typeof options === 'function') {
-    callback = options;
-    options = {};
-  }
-
   options = Object.assign(
     {
       method: 'GET',

lib/core/auth/mongodb_aws.js

@@ -11,6 +11,7 @@ const Binary = BSON.Binary;
 
 let saslprep;
 try {
+  // Ensure you always wrap an optional require in the try block NODE-3199
   saslprep = require('saslprep');
 } catch (e) {
   // don't do anything;