Merge branch 'main' into NODE-6733-cpu-bench

Author: W-A-James

.evergreen/config.in.yml

@@ -307,8 +307,7 @@ AWS_LAMBDA_HANDLER_TASKS.push({
     }),
     { func: 'install dependencies' },
     { func: 'bootstrap mongo-orchestration' },
-    { func: 'add aws auth variables to file' },
-    { func: 'setup aws env' },
+    { func: 'assume secrets manager rule' },
     { func: 'run lambda handler example tests with aws auth' }
   ]
 });
@@ -365,8 +364,7 @@ for (const VERSION of AWS_AUTH_VERSIONS) {
       }),
       { func: 'install dependencies' },
       { func: 'bootstrap mongo-orchestration' },
-      { func: 'add aws auth variables to file' },
-      { func: 'setup aws env' },
+      { func: 'assume secrets manager rule' },
       { func: fn.func }
     ]
   }));
@@ -385,9 +383,7 @@ for (const VERSION of AWS_AUTH_VERSIONS) {
         }),
         { func: 'install dependencies' },
         { func: 'bootstrap mongo-orchestration' },
-        { func: 'add aws auth variables to file' },
-        { func: 'setup aws env' },
-        { func: 'remove aws-credential-providers' },
+        { func: 'assume secrets manager rule' },
         { func: fn.func }
       ]
     }));
@@ -684,6 +680,7 @@ for (const version of ['5.0', 'rapid', 'latest']) {
       { func: 'bootstrap mongo-orchestration' },
       { func: 'bootstrap kms servers' },
       { func: 'install mongodb-client-encryption' },
+      { func: 'assume secrets manager rule' },
       { func: 'run custom csfle tests' }
     ]
   });

.evergreen/config.yml

@@ -0,0 +1,31 @@
+#! /usr/bin/env bash
+
+AUTH_AWS_DIR=${DRIVERS_TOOLS}/.evergreen/auth_aws
+ECS_SRC_DIR=$AUTH_AWS_DIR/src
+
+bash $DRIVERS_TOOLS/.evergreen/auth_aws/setup-secrets.sh
+
+# pack up project directory to ssh it to the container
+mkdir -p $ECS_SRC_DIR/.evergreen
+set -ex
+
+# write test file
+echo "export MONGODB_AWS_SDK=$MONGODB_AWS_SDK" >>$PROJECT_DIRECTORY/.evergreen/run-mongodb-aws-ecs-test.sh
+echo "if [ $MONGODB_AWS_SDK = 'false' ]; then rm -rf ./node_modules/@aws-sdk/credential-providers; fi" >>$PROJECT_DIRECTORY/.evergreen/run-mongodb-aws-ecs-test.sh
+echo "npm run check:aws" >>$PROJECT_DIRECTORY/.evergreen/run-mongodb-aws-ecs-test.sh
+
+# copy test file to AWS ecs test directory
+cp $PROJECT_DIRECTORY/.evergreen/run-mongodb-aws-ecs-test.sh $ECS_SRC_DIR/.evergreen/
+
+cat $ECS_SRC_DIR/.evergreen/run-mongodb-aws-ecs-test.sh
+
+# tar the file and drivers tools and do the same
+cd ..
+tar -czf src.tgz src drivers-tools
+mv src.tgz $ECS_SRC_DIR/src.tgz
+
+export MONGODB_BINARIES="${MONGODB_BINARIES}"
+
+export PROJECT_DIRECTORY=$ECS_SRC_DIR
+
+bash $AUTH_AWS_DIR/aws_setup.sh ecs

.evergreen/generate_evergreen_tasks.js

@@ -1,24 +1,13 @@
 #!/bin/bash
 # set -o xtrace   # Write all commands first to stderr
-set -o errexit  # Exit the script with error if any of the commands fail
-
-MONGODB_URI=${MONGODB_URI:-}
-
-# ensure no secrets are printed in log files
-set +x
-
-# load node.js environment
-source $DRIVERS_TOOLS/.evergreen/init-node-and-npm-env.sh
+set -o errexit # Exit the script with error if any of the commands fail
 
 # the default connection string, may be overridden by the environment script
 export MONGODB_URI="mongodb://localhost:27017/aws"
 
-# load the script
-shopt -s expand_aliases # needed for `urlencode` alias
-[ -s "$PROJECT_DIRECTORY/prepare_mongodb_aws.sh" ] && source "$PROJECT_DIRECTORY/prepare_mongodb_aws.sh"
+source .evergreen/setup-mongodb-aws-auth-tests.sh
 
-# revert to show test output
-set -x
+# load node.js environment
+source $DRIVERS_TOOLS/.evergreen/init-node-and-npm-env.sh
 
-npm install aws4
 npm run check:lambda:aws

.evergreen/prepare-mongodb-aws-ecs-auth.sh

@@ -1,25 +1,12 @@
 #!/bin/bash
 # set -o xtrace   # Write all commands first to stderr
-set -o errexit  # Exit the script with error if any of the commands fail
+set -o errexit # Exit the script with error if any of the commands fail
 
 MONGODB_URI=${MONGODB_URI:-}
 
-# ensure no secrets are printed in log files
-set +x
+source .evergreen/setup-mongodb-aws-auth-tests.sh
 
 # load node.js environment
 source $DRIVERS_TOOLS/.evergreen/init-node-and-npm-env.sh
 
-# the default connection string, may be overridden by the environment script
-export MONGODB_URI="mongodb://localhost:27017/aws?authMechanism=MONGODB-AWS"
-
-# load the script
-shopt -s expand_aliases # needed for `urlencode` alias
-[ -s "$PROJECT_DIRECTORY/prepare_mongodb_aws.sh" ] && source "$PROJECT_DIRECTORY/prepare_mongodb_aws.sh"
-
-# revert to show test output
-set -x
-
-npm install aws4
-if [ $MONGODB_AWS_SDK = 'false' ]; then rm -rf ./node_modules/@aws-sdk/credential-providers; fi
 npm run check:aws

.evergreen/run-lambda-aws-tests.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# set -o xtrace   # Write all commands first to stderr
+set -o errexit # Exit the script with error if any of the commands fail
+
+# ensure no secrets are printed in log files
+set +x
+
+if [ -z ${MONGODB_URI+omitted} ]; then echo "MONGODB_URI is unset" && exit 1; fi
+if [ -z ${DRIVERS_TOOLS+omitted} ]; then echo "DRIVERS_TOOLS is unset" && exit 1; fi
+if [ -z ${AWS_CREDENTIAL_TYPE+omitted} ]; then echo "AWS_CREDENTIAL_TYPE is unset" && exit 1; fi
+if [ -z ${MONGODB_AWS_SDK+omitted} ]; then echo "MONGODB_AWS_SDK is unset" && exit 1; fi
+
+bash $DRIVERS_TOOLS/.evergreen/auth_aws/setup-secrets.sh
+
+BEFORE=$(pwd)
+
+cd $DRIVERS_TOOLS/.evergreen/auth_aws
+
+# Create a python virtual environment.
+. ./activate-authawsvenv.sh
+# Source the environment variables. Configure the environment and the server.
+. aws_setup.sh $AWS_CREDENTIAL_TYPE
+
+cd $BEFORE
+
+npm install --no-save aws4
+
+if [ $MONGODB_AWS_SDK = 'false' ]; then rm -rf ./node_modules/@aws-sdk/credential-providers; fi
+
+# revert to show test output
+set -x

.evergreen/run-mongodb-aws-test.sh

@@ -1,10 +1,10 @@
-import type { Document, ObjectId } from './bson';
+import type { Document } from './bson';
 import {
   type ClientBulkWriteError,
   type ClientBulkWriteResult
 } from './operations/client_bulk_write/common';
 import type { ServerType } from './sdam/common';
-import type { ServerDescription, TopologyVersion } from './sdam/server_description';
+import type { TopologyVersion } from './sdam/server_description';
 import type { TopologyDescription } from './sdam/topology_description';
 
 /** @public */
@@ -355,16 +355,8 @@ export class MongoStalePrimaryError extends MongoRuntimeError {
    *
    * @public
    **/
-  constructor(
-    serverDescription: ServerDescription,
-    maxSetVersion: number | null,
-    maxElectionId: ObjectId | null,
-    options?: { cause?: Error }
-  ) {
-    super(
-      `primary marked stale due to electionId/setVersion mismatch: server setVersion: ${serverDescription.setVersion}, server electionId: ${serverDescription.electionId}, topology setVersion: ${maxSetVersion}, topology electionId: ${maxElectionId}`,
-      options
-    );
+  constructor(message: string, options?: { cause?: Error }) {
+    super(message, options);
   }
 
   override get name(): string {

.evergreen/setup-mongodb-aws-auth-tests.sh

@@ -376,6 +376,19 @@ function updateRsFromPrimary(
   maxSetVersion: number | null = null,
   maxElectionId: ObjectId | null = null
 ): [TopologyType, string | null, number | null, ObjectId | null] {
+  const setVersionElectionIdMismatch = (
+    serverDescription: ServerDescription,
+    maxSetVersion: number | null,
+    maxElectionId: ObjectId | null
+  ) => {
+    return (
+      `primary marked stale due to electionId/setVersion mismatch:` +
+      ` server setVersion: ${serverDescription.setVersion},` +
+      ` server electionId: ${serverDescription.electionId},` +
+      ` topology setVersion: ${maxSetVersion},` +
+      ` topology electionId: ${maxElectionId}`
+    );
+  };
   setName = setName || serverDescription.setName;
   if (setName !== serverDescription.setName) {
     serverDescriptions.delete(serverDescription.address);
@@ -401,7 +414,9 @@ function updateRsFromPrimary(
       serverDescriptions.set(
         serverDescription.address,
         new ServerDescription(serverDescription.address, undefined, {
-          error: new MongoStalePrimaryError(serverDescription, maxSetVersion, maxElectionId)
+          error: new MongoStalePrimaryError(
+            setVersionElectionIdMismatch(serverDescription, maxSetVersion, maxElectionId)
+          )
         })
       );
 
@@ -419,7 +434,9 @@ function updateRsFromPrimary(
           serverDescriptions.set(
             serverDescription.address,
             new ServerDescription(serverDescription.address, undefined, {
-              error: new MongoStalePrimaryError(serverDescription, maxSetVersion, maxElectionId)
+              error: new MongoStalePrimaryError(
+                setVersionElectionIdMismatch(serverDescription, maxSetVersion, maxElectionId)
+              )
             })
           );
 
@@ -445,7 +462,9 @@ function updateRsFromPrimary(
       serverDescriptions.set(
         address,
         new ServerDescription(server.address, undefined, {
-          error: new MongoStalePrimaryError(serverDescription, maxSetVersion, maxElectionId)
+          error: new MongoStalePrimaryError(
+            'primary marked stale due to discovery of newer primary'
+          )
         })
       );
 

src/error.ts

@@ -39,6 +39,7 @@ about the types of tests and how to run them.
       - [Search Indexes](#search-indexes)
       - [Deployed Lambda Tests](#deployed-lambda-tests)
     - [Kerberos Tests](#kerberos-tests)
+    - [AWS Authentication tests](#aws-authentication-tests)
     - [TODO Special Env Sections](#todo-special-env-sections)
   - [Testing driver changes with mongosh](#testing-driver-changes-with-mongosh)
     - [Point mongosh to the driver](#point-mongosh-to-the-driver)
@@ -617,11 +618,38 @@ TODO(NODE-6698): Update deployed lambda test section.
 
 ### Kerberos Tests
 
+You must be in an office or connected to the VPN to run these tests.
+
 Run `.evergreen/run-kerberos-tests.sh`.
 
+### AWS Authentication tests
+
+> [!NOTE]
+> AWS ECS tests have a different set up process.  Don't even bother running these locally, just pray to the CI gods that things work and you never have to touch these tests.
+
+AWS tests require a cluster configured with MONGODB_AWS auth enabled.  This is easy to set up using drivers-evergreen-tools
+by specifying the `aws-auth.json` orchestration file (this is what CI does).
+
+1. Set up your cluster and export the URI of your cluster as MONGODB_URI.
+2. Choose your configuration and set the relevant environment variables.
+
+Do you want the AWS SDK to be installed while running auth?  If not, set MONGODB_AWS_SDK to false.
+
+Choose your AWS authentication credential type and export the `AWS_CREDENTIAL_TYPE` type with the chosen value:
+
+| AWS Credential Type | Explanation                                                                                     |
+| ------------------- | ----------------------------------------------------------------------------------------------- |
+| regular             | The AWS credentials are present in the URI as username:password                                 |
+| env-creds           | AWS credentials are loaded into the environment as AWS_ACCESS_KEY_ID and  AWS_SECRET_ACCESS_KEY |
+| assume-role         | The machine assumes a particular authentication role, associated with the machine               |
+| ec2                 | The driver authenticates against a local endpoint (on an AWS ec2 instance)                      |
+| web-identity        | Credentials are sourced from an AssumeRoleWithWebIdentity                                       |
+| session-creds       | Similar to env-creds, but the credentials are temporary and include a session token             |
+
+1. Run the `bash .evergreen/run-mongodb-aws-tests.sh`.
+
 ### TODO Special Env Sections
 
-- AWS Authentication
 - TLS
 - Atlas Data Lake
 - LDAP