feat(oidc-mock-provider,mongodb-runner): make OIDC mocks more broadly usable COMPASS-10034 (#589)

Internal TSEs have requested making oidc-mock-provider available for
internal testing with OIDC. While it cannot replicate every aspect
of real-world identity providers, it is an easily spun up local
equivalent of those, and provides flexibility that those real-world
identity providers lack in terms of configurability.

This change widens the array of CLI options provided for the
oidc-mock-provider CLI, and integrates it into mongodb-runner
so that the latter can spin up a joint OIDC-IdP-and-mongod-cluster
environment on Linux, if that is desired.

Author: addaleax

packages/mongodb-runner/README.md

@@ -2,6 +2,11 @@
 
 Helper for spinning up MongoDB servers and clusters for testing.
 
+## Requirements
+
+Node.js >= 20.19.5, npm >= 11.6.0. Running as `npx mongodb-runner ...`
+is typically the easiest way to install/run this tool.
+
 ## Example usage
 
 > Note: Version 5 of mongodb-runner is a full re-write. Many things work
@@ -11,6 +16,7 @@ Helper for spinning up MongoDB servers and clusters for testing.
 $ npx mongodb-runner start -t sharded
 $ npx mongodb-runner start -t replset -- --port 27017
 $ npx mongodb-runner start -t replset -- --setParameter allowDiskUseByDefault=true
+$ npx mongodb-runner start -t replset --version 8.2.x-enterprise --oidc='--payload={"groups":["x"],"sub":"y","aud":"aud"} --expiry=60 --skip-refresh-token'
 $ npx mongodb-runner stop --all
 $ npx mongodb-runner exec -t standalone -- sh -c 'mongosh $MONGODB_URI'
 $ npx mongodb-runner exec -t standalone -- --setParameter allowDiskUseByDefault=true -- sh -c 'mongosh $MONGODB_URI'

packages/mongodb-runner/package.json

@@ -57,9 +57,10 @@
   },
   "dependencies": {
     "@mongodb-js/mongodb-downloader": "^1.0.0",
+    "@mongodb-js/oidc-mock-provider": "^0.11.5",
+    "@mongodb-js/saslprep": "^1.3.2",
     "debug": "^4.4.0",
     "mongodb": "^6.9.0",
-    "@mongodb-js/saslprep": "^1.3.2",
     "mongodb-connection-string-url": "^3.0.0",
     "yargs": "^17.7.2"
   },

packages/mongodb-runner/src/cli.ts

@@ -1,12 +1,11 @@
 /* eslint-disable no-console */
 import yargs from 'yargs';
-import { MongoCluster } from './mongocluster';
 import os from 'os';
 import path from 'path';
-import { spawn } from 'child_process';
 import createDebug from 'debug';
-import { once } from 'events';
 import * as utilities from './index';
+import { ConnectionString } from 'mongodb-connection-string-url';
+import type { MongoClientOptions } from 'mongodb';
 
 (async function () {
   const defaultRunnerDir = path.join(os.homedir(), '.mongodb', 'runner2');
@@ -71,6 +70,10 @@ import * as utilities from './index';
       type: 'boolean',
       describe: 'for `stop`: stop all clusters',
     })
+    .option('oidc', {
+      type: 'string',
+      describe: 'Configure OIDC authentication on the server',
+    })
     .option('debug', { type: 'boolean', describe: 'Enable debug output' })
     .command('start', 'Start a MongoDB instance')
     .command('stop', 'Stop a MongoDB instance')
@@ -87,9 +90,29 @@ import * as utilities from './index';
     createDebug.enable('mongodb-runner');
   }
 
+  if (argv.oidc && process.platform !== 'linux') {
+    console.warn(
+      'OIDC authentication is currently only supported on Linux platforms.',
+    );
+  }
+  if (argv.oidc && !argv.version?.includes('enterprise')) {
+    console.warn(
+      'OIDC authentication is currently only supported on Enterprise server versions.',
+    );
+  }
+
   async function start() {
     const { cluster, id } = await utilities.start(argv, args);
-    console.log(`Server started and running at ${cluster.connectionString}`);
+    const cs = new ConnectionString(cluster.connectionString);
+    console.log(`Server started and running at ${cs.toString()}`);
+    if (cluster.oidcIssuer) {
+      cs.typedSearchParams<MongoClientOptions>().set(
+        'authMechanism',
+        'MONGODB-OIDC',
+      );
+      console.log(`OIDC provider started and running at ${cluster.oidcIssuer}`);
+      console.log(`Server connection string with OIDC auth: ${cs.toString()}`);
+    }
     console.log('Run the following command to stop the instance:');
     console.log(
       `${argv.$0} stop --id=${id}` +
@@ -118,37 +141,7 @@ import * as utilities from './index';
   }
 
   async function exec() {
-    let mongodArgs: string[];
-    let execArgs: string[];
-
-    const doubleDashIndex = args.indexOf('--');
-    if (doubleDashIndex !== -1) {
-      mongodArgs = args.slice(0, doubleDashIndex);
-      execArgs = args.slice(doubleDashIndex + 1);
-    } else {
-      mongodArgs = [];
-      execArgs = args;
-    }
-    const cluster = await MongoCluster.start({
-      ...argv,
-      args: mongodArgs,
-    });
-    try {
-      const [prog, ...progArgs] = execArgs;
-      const child = spawn(prog, progArgs, {
-        stdio: 'inherit',
-        env: {
-          ...process.env,
-          // both spellings since otherwise I'd end up misspelling these half of the time
-          MONGODB_URI: cluster.connectionString,
-          MONGODB_URL: cluster.connectionString,
-          MONGODB_HOSTPORT: cluster.hostport,
-        },
-      });
-      [process.exitCode] = await once(child, 'exit');
-    } finally {
-      await cluster.close();
-    }
+    await utilities.exec(argv, args);
   }
 
   // eslint-disable-next-line @typescript-eslint/require-await

packages/mongodb-runner/src/index.ts

@@ -2,4 +2,4 @@ export { MongoServer, MongoServerOptions } from './mongoserver';
 
 export { MongoCluster, MongoClusterOptions } from './mongocluster';
 export type { ConnectionString } from 'mongodb-connection-string-url';
-export { prune, start, stop, instances } from './runner-helpers';
+export { prune, start, stop, exec, instances } from './runner-helpers';

packages/mongodb-runner/src/mongocluster.ts

@@ -6,6 +6,7 @@ import { downloadMongoDb } from '@mongodb-js/mongodb-downloader';
 import type { MongoClientOptions } from 'mongodb';
 import { MongoClient } from 'mongodb';
 import { sleep, range, uuid, debug } from './util';
+import { OIDCMockProviderProcess } from './oidc';
 
 export interface MongoClusterOptions
   extends Pick<
@@ -19,13 +20,15 @@ export interface MongoClusterOptions
   version?: string;
   downloadDir?: string;
   downloadOptions?: DownloadOptions;
+  oidc?: string;
 }
 
 export class MongoCluster {
   private topology: MongoClusterOptions['topology'] = 'standalone';
   private replSetName?: string;
   private servers: MongoServer[] = []; // mongod/mongos
   private shards: MongoCluster[] = []; // replsets
+  private oidcMockProviderProcess?: OIDCMockProviderProcess;
 
   private constructor() {
     /* see .start() */
@@ -50,6 +53,7 @@ export class MongoCluster {
       replSetName: this.replSetName,
       servers: this.servers.map((srv) => srv.serialize()),
       shards: this.shards.map((shard) => shard.serialize()),
+      oidcMockProviderProcess: this.oidcMockProviderProcess?.serialize(),
     };
   }
 
@@ -67,6 +71,9 @@ export class MongoCluster {
     cluster.shards = await Promise.all(
       serialized.shards.map((shard: any) => MongoCluster.deserialize(shard)),
     );
+    cluster.oidcMockProviderProcess = serialized.oidcMockProviderProcess
+      ? OIDCMockProviderProcess.deserialize(serialized.oidcMockProviderProcess)
+      : undefined;
     return cluster;
   }
 
@@ -75,9 +82,17 @@ export class MongoCluster {
   }
 
   get connectionString(): string {
-    return `mongodb://${this.hostport}/${
-      this.replSetName ? `?replicaSet=${this.replSetName}` : ''
-    }`;
+    const cs = new ConnectionString(`mongodb://${this.hostport}/`);
+    if (this.replSetName)
+      cs.typedSearchParams<MongoClientOptions>().set(
+        'replicaSet',
+        this.replSetName,
+      );
+    return cs.toString();
+  }
+
+  get oidcIssuer(): string | undefined {
+    return this.oidcMockProviderProcess?.issuer;
   }
 
   get connectionStringUrl(): ConnectionString {
@@ -105,6 +120,31 @@ export class MongoCluster {
       );
     }
 
+    if (options.oidc !== undefined) {
+      cluster.oidcMockProviderProcess = await OIDCMockProviderProcess.start(
+        options.oidc || '--port=0',
+      );
+      const oidcServerConfig = [
+        {
+          issuer: cluster.oidcMockProviderProcess.issuer,
+          audience: cluster.oidcMockProviderProcess.audience,
+          authNamePrefix: 'dev',
+          clientId: 'cid',
+          authorizationClaim: 'groups',
+        },
+      ];
+      delete options.oidc;
+      options.args = [
+        ...(options.args ?? []),
+        '--setParameter',
+        `oidcIdentityProviders=${JSON.stringify(oidcServerConfig)}`,
+        '--setParameter',
+        'authenticationMechanisms=SCRAM-SHA-256,MONGODB-OIDC',
+        '--setParameter',
+        'enableTestCommands=true',
+      ];
+    }
+
     if (options.topology === 'standalone') {
       cluster.servers.push(
         await MongoServer.start({
@@ -233,7 +273,9 @@ export class MongoCluster {
 
   async close(): Promise<void> {
     await Promise.all(
-      [...this.servers, ...this.shards].map((closable) => closable.close()),
+      [...this.servers, ...this.shards, this.oidcMockProviderProcess].map(
+        (closable) => closable?.close(),
+      ),
     );
     this.servers = [];
     this.shards = [];

packages/mongodb-runner/src/oidc.ts

@@ -0,0 +1,125 @@
+import { spawn } from 'child_process';
+import { once } from 'events';
+import { parseCLIArgs, OIDCMockProvider } from '@mongodb-js/oidc-mock-provider';
+import { debug } from './util';
+
+if (process.env.RUN_OIDC_MOCK_PROVIDER !== undefined) {
+  (async function main() {
+    const uuid = crypto.randomUUID();
+    debug('starting OIDC mock provider with UUID', uuid);
+    const config = parseCLIArgs(process.env.RUN_OIDC_MOCK_PROVIDER);
+    const sampleTokenConfig = await config.getTokenPayload({
+      client_id: 'cid',
+      scope: 'scope',
+    });
+    debug('sample OIDC token config', sampleTokenConfig, uuid);
+    const audience = sampleTokenConfig.payload.aud;
+    const provider = await OIDCMockProvider.create({
+      ...config,
+      overrideRequestHandler(url, req, res) {
+        if (new URL(url).pathname === `/shutdown/${uuid}`) {
+          res.on('close', () => {
+            process.exit();
+          });
+          res.writeHead(200);
+          res.end();
+          return;
+        }
+        return config.overrideRequestHandler?.(url, req, res);
+      },
+    });
+    debug('started OIDC mock provider with UUID', {
+      issuer: provider.issuer,
+      uuid,
+      audience,
+    });
+    process.send?.({
+      issuer: provider.issuer,
+      uuid,
+      audience,
+    });
+  })().catch((error) => {
+    // eslint-disable-next-line no-console
+    console.error('Error starting OIDC mock identity provider server:', error);
+    process.exitCode = 1;
+  });
+}
+
+export class OIDCMockProviderProcess {
+  pid?: number;
+  issuer?: string;
+  uuid?: string;
+  audience?: string;
+
+  serialize(): unknown /* JSON-serializable */ {
+    return {
+      pid: this.pid,
+      issuer: this.issuer,
+      uuid: this.uuid,
+      audience: this.audience,
+    };
+  }
+
+  static deserialize(serialized: any): OIDCMockProviderProcess {
+    const process = new OIDCMockProviderProcess();
+    process.pid = serialized.pid;
+    process.issuer = serialized.issuer;
+    process.uuid = serialized.uuid;
+    process.audience = serialized.audience;
+    return process;
+  }
+
+  private constructor() {
+    /* see .start() */
+  }
+
+  static async start(args: string): Promise<OIDCMockProviderProcess> {
+    const oidcProc = new this();
+    debug('spawning OIDC child process', [process.execPath, __filename], args);
+    const proc = spawn(process.execPath, [__filename], {
+      stdio: ['inherit', 'inherit', 'inherit', 'ipc'],
+      env: {
+        ...process.env,
+        RUN_OIDC_MOCK_PROVIDER: args,
+      },
+      detached: true,
+      serialization: 'advanced',
+    });
+    await once(proc, 'spawn');
+    try {
+      oidcProc.pid = proc.pid;
+      const [msg] = await Promise.race([
+        once(proc, 'message'),
+        once(proc, 'exit').then(() => {
+          throw new Error(
+            `OIDC mock provider process exited before sending message (${String(proc.exitCode)}, ${String(proc.signalCode)})`,
+          );
+        }),
+      ]);
+      debug('received message from OIDC child process', msg);
+      oidcProc.issuer = msg.issuer;
+      oidcProc.uuid = msg.uuid;
+      oidcProc.audience = msg.audience;
+    } catch (err) {
+      proc.kill();
+      throw err;
+    }
+    proc.unref();
+    proc.channel?.unref();
+    debug('OIDC setup complete, uuid =', oidcProc.uuid);
+    return oidcProc;
+  }
+
+  async close(): Promise<void> {
+    try {
+      if (this.pid) process.kill(this.pid, 0);
+    } catch (e) {
+      if (typeof e === 'object' && e && 'code' in e && e.code === 'ESRCH')
+        return; // process already exited
+    }
+
+    if (!this.issuer || !this.uuid) return;
+    await fetch(new URL(this.issuer, `/shutdown/${this.uuid}`));
+    this.uuid = undefined;
+  }
+}