feat: make use of the "localhost exception" for creating users

This should heavily reduce the time needed to create users
This also changes when "ephemeralForTest" is used to be able to use auth properly

fixes #670
fixes #671

Author: hasezoey

packages/mongodb-memory-server-core/src/MongoMemoryReplSet.ts

@@ -48,7 +48,7 @@ export interface ReplSetOpts {
    * enable auth ("--auth" / "--noauth")
    * @default false
    */
-  auth?: boolean | AutomaticAuth;
+  auth?: boolean | AutomaticAuth; // TODO: remove "boolean" option next major version
   /**
    * additional command line arguments passed to `mongod`
    * @default []
@@ -255,29 +255,58 @@ export class MongoMemoryReplSet extends EventEmitter implements ManagerAdvanced
 
     assertion(this._replSetOpts.count > 0, new ReplsetCountLowError(this._replSetOpts.count));
 
-    if (typeof this._replSetOpts.auth === 'object') {
+    // setting this for sanity
+    if (typeof this._replSetOpts.auth === 'boolean') {
+      this._replSetOpts.auth = { disable: !this._replSetOpts.auth };
+    }
+
+    // do not set default when "disable" is "true" to save execution and memory
+    if (!this._replSetOpts.auth.disable) {
       this._replSetOpts.auth = authDefault(this._replSetOpts.auth);
     }
   }
 
+  /**
+   * Helper function to determine if "auth" should be enabled
+   * This function expectes to be run after the auth object has been transformed to a object
+   * @returns "true" when "auth" should be enabled
+   */
+  protected enableAuth(): boolean {
+    if (isNullOrUndefined(this._replSetOpts.auth)) {
+      return false;
+    }
+
+    assertion(typeof this._replSetOpts.auth === 'object', new AuthNotObjectError());
+
+    return typeof this._replSetOpts.auth.disable === 'boolean' // if "this._replSetOpts.auth.disable" is defined, use that
+      ? !this._replSetOpts.auth.disable // invert the disable boolean, because "auth" should only be disabled if "disabled = true"
+      : true; // if "this._replSetOpts.auth.disable" is not defined, default to true because "this._replSetOpts.auth" is defined
+  }
+
   /**
    * Returns instance options suitable for a MongoMemoryServer.
    * @param baseOpts Options to merge with
+   * @param keyfileLocation The Keyfile location if "auth" is used
    */
-  protected getInstanceOpts(baseOpts: MongoMemoryInstanceOptsBase = {}): MongoMemoryInstanceOpts {
+  protected getInstanceOpts(
+    baseOpts: MongoMemoryInstanceOptsBase = {},
+    keyfileLocation?: string
+  ): MongoMemoryInstanceOpts {
+    const enableAuth: boolean = this.enableAuth();
+
     const opts: MongoMemoryInstanceOpts = {
-      // disable "auth" if replsetopts has an object-auth
-      auth:
-        typeof this._replSetOpts.auth === 'object' && !this._ranCreateAuth
-          ? false
-          : !!this._replSetOpts.auth,
+      auth: enableAuth,
       args: this._replSetOpts.args,
       dbName: this._replSetOpts.dbName,
       ip: this._replSetOpts.ip,
       replSet: this._replSetOpts.name,
       storageEngine: this._replSetOpts.storageEngine,
     };
 
+    if (!isNullOrUndefined(keyfileLocation)) {
+      opts.keyfileLocation = keyfileLocation;
+    }
+
     if (baseOpts.args) {
       opts.args = this._replSetOpts.args.concat(baseOpts.args);
     }
@@ -410,6 +439,12 @@ export class MongoMemoryReplSet extends EventEmitter implements ManagerAdvanced
       return;
     }
 
+    let keyfilePath: string | undefined = undefined;
+
+    if (this.enableAuth()) {
+      keyfilePath = resolve((await this.ensureKeyFile()).name, 'keyfile');
+    }
+
     // Any servers defined within `_instanceOpts` should be started first as
     // the user could have specified a `dbPath` in which case we would want to perform
     // the `replSetInitiate` command against that server.
@@ -420,15 +455,15 @@ export class MongoMemoryReplSet extends EventEmitter implements ManagerAdvanced
         }" from instanceOpts (count: ${this.servers.length + 1}):`,
         opts
       );
-      this.servers.push(this._initServer(this.getInstanceOpts(opts)));
+      this.servers.push(this._initServer(this.getInstanceOpts(opts, keyfilePath)));
     });
     while (this.servers.length < this._replSetOpts.count) {
       log(
         `initAllServers: starting extra server "${this.servers.length + 1}" of "${
           this._replSetOpts.count
         }" (count: ${this.servers.length + 1})`
       );
-      this.servers.push(this._initServer(this.getInstanceOpts()));
+      this.servers.push(this._initServer(this.getInstanceOpts(undefined, keyfilePath)));
     }
 
     log('initAllServers: waiting for all servers to finish starting');
@@ -638,15 +673,15 @@ export class MongoMemoryReplSet extends EventEmitter implements ManagerAdvanced
     const uris = this.servers.map((server) => server.getUri());
     const isInMemory = this.servers[0].instanceInfo?.storageEngine === 'ephemeralForTest';
 
-    let con: MongoClient = await MongoClient.connect(uris[0], {
+    const con: MongoClient = await MongoClient.connect(uris[0], {
       // somehow since mongodb-nodejs 4.0, this option is needed when the server is set to be in a replset
       directConnection: true,
     });
     log('_initReplSet: connected');
 
     // try-finally to close connection in any case
     try {
-      let adminDb = con.db('admin');
+      const adminDb = con.db('admin');
 
       const members = uris.map((uri, index) => ({
         _id: index,
@@ -682,34 +717,9 @@ export class MongoMemoryReplSet extends EventEmitter implements ManagerAdvanced
             new InstanceInfoError('_initReplSet authIsObject primary')
           );
 
+          await con.close(); // just ensuring that no timeouts happen or conflicts happen
           await primary.createAuth(primary.instanceInfo);
           this._ranCreateAuth = true;
-
-          // TODO: maybe change the static "isInMemory" to be for each server individually, based on "storageEngine", not just the first one
-          if (!isInMemory) {
-            log('_initReplSet: closing connection for restart');
-            await con.close(); // close connection in preparation for "stop"
-            await this.stop({ doCleanup: false, force: false }); // stop all servers for enabling auth
-            log('_initReplSet: starting all server again with auth');
-            await this.initAllServers(); // start all servers again with "auth" enabled
-            await this._waitForPrimary(); // wait for a primary to come around, because otherwise server selection may time out, this may take more than 30s
-
-            con = await MongoClient.connect(this.getUri(), {
-              authSource: 'admin',
-              authMechanism: 'SCRAM-SHA-256',
-              auth: {
-                username: this._replSetOpts.auth.customRootName as string, // cast because these are existing
-                password: this._replSetOpts.auth.customRootPwd as string,
-              },
-            });
-            adminDb = con.db('admin');
-            log('_initReplSet: auth restart finished');
-          } else {
-            console.warn(
-              'Not Restarting ReplSet for Auth\n' +
-                'Storage engine of current PRIMARY is ephemeralForTest, which does not write data on shutdown, and mongodb does not allow changing "auth" runtime'
-            );
-          }
         }
       } catch (e) {
         if (e instanceof MongoError && e.errmsg == 'already initialized') {

packages/mongodb-memory-server-core/src/MongoMemoryServer.ts

@@ -82,6 +82,7 @@ export interface StartupInstanceData {
   storageEngine: NonNullable<MongoMemoryInstanceOpts['storageEngine']>;
   replSet?: NonNullable<MongoMemoryInstanceOpts['replSet']>;
   tmpDir?: tmp.DirResult;
+  keyfileLocation?: NonNullable<MongoMemoryInstanceOpts['keyfileLocation']>;
 }
 
 /**
@@ -193,6 +194,7 @@ export interface CreateUser extends CreateUserMongoDB {
 }
 
 export interface MongoMemoryServerGetStartOptions {
+  /** Defines wheter should {@link MongoMemoryServer.createAuth} be run */
   createAuth: boolean;
   data: StartupInstanceData;
   mongodOptions: Partial<MongodOpts>;
@@ -361,6 +363,7 @@ export class MongoMemoryServer extends EventEmitter implements ManagerAdvanced {
       replSet: instOpts.replSet,
       dbPath: instOpts.dbPath,
       tmpDir: undefined,
+      keyfileLocation: instOpts.keyfileLocation,
     };
 
     if (isNullOrUndefined(this._instanceInfo)) {
@@ -386,12 +389,16 @@ export class MongoMemoryServer extends EventEmitter implements ManagerAdvanced {
       isNew = false;
     }
 
-    const createAuth: boolean =
+    const enableAuth: boolean =
       (typeof instOpts.auth === 'boolean' ? instOpts.auth : true) && // check if auth is even meant to be enabled
       !isNullOrUndefined(this.auth) && // check if "this.auth" is defined
-      !this.auth.disable && // check that "this.auth.disable" is falsey
+      !this.auth.disable; // check that "this.auth.disable" is falsey
+
+    const createAuth: boolean =
+      enableAuth && // re-use all the checks from "enableAuth"
+      !isNullOrUndefined(this.auth) && // needs to be re-checked because typescript complains
       (this.auth.force || isNew) && // check that either "isNew" or "this.auth.force" is "true"
-      !instOpts.replSet; // dont run "createAuth" when its an replset
+      !instOpts.replSet; // dont run "createAuth" when its an replset, it will be run by the replset controller
 
     return {
       data: data,
@@ -400,7 +407,7 @@ export class MongoMemoryServer extends EventEmitter implements ManagerAdvanced {
         instance: {
           ...data,
           args: instOpts.args,
-          auth: createAuth ? false : instOpts.auth, // disable "auth" for "createAuth"
+          auth: enableAuth,
         },
         binary: this.opts.binary,
         spawn: this.opts.spawn,
@@ -436,24 +443,28 @@ export class MongoMemoryServer extends EventEmitter implements ManagerAdvanced {
     const instance = await MongoInstance.create(mongodOptions);
     this.debug(`_startUpInstance: Instance Started, createAuth: "${createAuth}"`);
 
+    this._instanceInfo = {
+      ...data,
+      dbPath: data.dbPath as string, // because otherwise the types would be incompatible
+      instance,
+    };
+
+    // always set the "extraConnectionOptions" when "auth" is enabled, regardless of if "createAuth" gets run
+    if (!isNullOrUndefined(this.auth) && !isNullOrUndefined(mongodOptions.instance?.auth)) {
+      instance.extraConnectionOptions = {
+        authSource: 'admin',
+        authMechanism: 'SCRAM-SHA-256',
+        auth: {
+          username: this.auth.customRootName,
+          password: this.auth.customRootPwd,
+        },
+      };
+    }
+
     // "isNullOrUndefined" because otherwise typescript complains about "this.auth" possibly being not defined
     if (!isNullOrUndefined(this.auth) && createAuth) {
       this.debug(`_startUpInstance: Running "createAuth" (force: "${this.auth.force}")`);
       await this.createAuth(data);
-
-      if (data.storageEngine !== 'ephemeralForTest') {
-        this.debug('_startUpInstance: Killing No-Auth instance');
-        await instance.stop();
-
-        this.debug('_startUpInstance: Starting Auth Instance');
-        instance.instanceOpts.auth = true;
-        await instance.start();
-      } else {
-        console.warn(
-          'Not Restarting MongoInstance for Auth\n' +
-            'Storage engine is "ephemeralForTest", which does not write data on shutdown, and mongodb does not allow changing "auth" runtime'
-        );
-      }
     } else {
       // extra "if" to log when "disable" is set to "true"
       if (this.opts.auth?.disable) {
@@ -462,12 +473,6 @@ export class MongoMemoryServer extends EventEmitter implements ManagerAdvanced {
         );
       }
     }
-
-    this._instanceInfo = {
-      ...data,
-      dbPath: data.dbPath as string, // because otherwise the types would be incompatible
-      instance,
-    };
   }
 
   /**
@@ -715,7 +720,7 @@ export class MongoMemoryServer extends EventEmitter implements ManagerAdvanced {
   }
 
   /**
-   * Create Users and restart instance to enable auth
+   * Create the Root user and additional users using the [localhost exception](https://www.mongodb.com/docs/manual/core/localhost-exception/#std-label-localhost-exception)
    * This Function assumes "this.opts.auth" is already processed into "this.auth"
    * @param data Used to get "ip" and "port"
    * @internal
@@ -725,11 +730,9 @@ export class MongoMemoryServer extends EventEmitter implements ManagerAdvanced {
       !isNullOrUndefined(this.auth),
       new Error('"createAuth" got called, but "this.auth" is undefined!')
     );
+    assertionInstanceInfo(this._instanceInfo);
     this.debug('createAuth: options:', this.auth);
-    const con: MongoClient = await MongoClient.connect(
-      uriTemplate(data.ip, data.port, 'admin'),
-      {}
-    );
+    let con: MongoClient = await MongoClient.connect(uriTemplate(data.ip, data.port, 'admin'));
 
     try {
       let db = con.db('admin'); // just to ensure it is actually the "admin" database AND to have the "Db" data
@@ -761,6 +764,14 @@ export class MongoMemoryServer extends EventEmitter implements ManagerAdvanced {
           return a.database === b.database ? 0 : 1; // "0" to sort all databases that are the same after each other, and "1" to for pushing it back
         });
 
+        // reconnecting the database because the root user now exists and the "localhost exception" only allows the first user
+        await con.close();
+        con = await MongoClient.connect(
+          this.getUri('admin'),
+          this._instanceInfo.instance.extraConnectionOptions ?? {}
+        );
+        db = con.db('admin');
+
         for (const user of this.auth.extraUsers) {
           user.database = isNullOrUndefined(user.database) ? 'admin' : user.database;
 
@@ -782,6 +793,10 @@ export class MongoMemoryServer extends EventEmitter implements ManagerAdvanced {
             authenticationRestrictions: user.authenticationRestrictions ?? [],
             mechanisms: user.mechanisms ?? ['SCRAM-SHA-256'],
             digestPassword: user.digestPassword ?? true,
+            // "writeConcern" is needced, otherwise replset servers might fail with "auth failed: such user does not exist"
+            writeConcern: {
+              w: 'majority',
+            },
           } as CreateUserMongoDB);
         }
       }