chore(cli-repl): add snapshot package list tests MONGOSH-1736 (#1885)

Add the ability to introspect the compiled mongosh executables for
what is part of the snapshot and what is not, and then use that
information in e2e tests to verify that it is accurate.

Author: addaleax

packages/cli-repl/package.json

@@ -31,8 +31,8 @@
     "check": "npm run lint && npm run depcheck",
     "depcheck": "depcheck",
     "prepublish": "npm run compile",
-    "webpack-build": "npm run compile && webpack --mode production",
-    "webpack-build-dev": "npm run compile && webpack --mode development",
+    "webpack-build": "npm run compile && webpack --mode production && cat dist/add-module-mapping.js >> dist/mongosh.js",
+    "webpack-build-dev": "npm run compile && webpack --mode development && cat dist/add-module-mapping.js >> dist/mongosh.js",
     "start-snapshot": "rm -f snapshot.blob && node --snapshot-blob snapshot.blob --build-snapshot dist/mongosh.js && node --snapshot-blob snapshot.blob dist/mongosh.js",
     "prettier": "prettier",
     "reformat": "npm run prettier -- --write . && npm run eslint --fix"

packages/cli-repl/src/run.ts

@@ -30,6 +30,7 @@ import crypto from 'crypto';
 import net from 'net';
 import v8 from 'v8';
 import { TimingCategories } from '@mongosh/types';
+import './webpack-self-inspection';
 
 // TS does not yet have type definitions for v8.startupSnapshot
 if ((v8 as any)?.startupSnapshot?.isBuildingSnapshot?.()) {

packages/cli-repl/src/webpack-self-inspection.ts

@@ -0,0 +1,47 @@
+import v8 from 'v8';
+
+// Allow us to inspect the set of loaded modules at a few interesting points in time, e.g.
+// startup, snapshot, and after entering VM/REPL execution mode
+declare const __webpack_module_cache__: Record<string, unknown> | undefined;
+declare const __webpack_modules__: Record<string, unknown> | undefined;
+declare const __webpack_reverse_module_lookup__:
+  | (() => Record<string | number, string>)
+  | undefined;
+
+// Return all ids of modules loaded at the current time
+function enumerateLoadedModules(): (string | number)[] | null {
+  if (typeof __webpack_module_cache__ !== 'undefined') {
+    return Object.keys(__webpack_module_cache__);
+  }
+  return null;
+}
+// Return all ids of modules that can be loaded/are known to webpack
+function enumerateAllModules(): (string | number)[] | null {
+  if (typeof __webpack_modules__ !== 'undefined') {
+    return Object.keys(__webpack_modules__);
+  }
+  return null;
+}
+// Perform a reverse lookup to determine the "natural" name for a given
+// module id (i.e. original filename, if available).
+// Calling this the first time is potentially expensive.
+function lookupNaturalModuleName(id: string | number): string | null {
+  let lookupTable = null;
+  if (typeof __webpack_reverse_module_lookup__ !== 'undefined')
+    lookupTable = __webpack_reverse_module_lookup__();
+  return lookupTable?.[id] ?? null;
+}
+Object.defineProperty(process, '__mongosh_webpack_stats', {
+  value: {
+    enumerateLoadedModules,
+    enumerateAllModules,
+    lookupNaturalModuleName,
+  },
+});
+if ((v8 as any)?.startupSnapshot?.isBuildingSnapshot?.()) {
+  (v8 as any).startupSnapshot.addSerializeCallback(() => {
+    const atSnapshotTime = enumerateLoadedModules();
+    (process as any).__mongosh_webpack_stats.enumerateSnapshotModules = () =>
+      atSnapshotTime;
+  });
+}

packages/cli-repl/webpack.config.js

@@ -5,6 +5,9 @@ const crypto = require('crypto');
 const { merge } = require('webpack-merge');
 const path = require('path');
 const { WebpackDependenciesPlugin } = require('@mongodb-js/sbom-tools');
+const {
+  WebpackEnableReverseModuleLookupPlugin,
+} = require('../../scripts/webpack-enable-reverse-module-lookup-plugin.js');
 
 const baseWebpackConfig = require('../../config/webpack.base.config');
 
@@ -29,6 +32,11 @@ const webpackDependenciesPlugin = new WebpackDependenciesPlugin({
   includeExternalProductionDependencies: true,
 });
 
+const enableReverseModuleLookupPlugin =
+  new WebpackEnableReverseModuleLookupPlugin({
+    outputFilename: path.resolve(__dirname, 'dist', 'add-module-mapping.js'),
+  });
+
 /** @type import('webpack').Configuration */
 const config = {
   output: {
@@ -41,7 +49,7 @@ const config = {
       type: 'var',
     },
   },
-  plugins: [webpackDependenciesPlugin],
+  plugins: [webpackDependenciesPlugin, enableReverseModuleLookupPlugin],
   entry: './lib/run.js',
   resolve: {
     alias: {
@@ -83,7 +91,13 @@ module.exports = merge(baseWebpackConfig, config);
 // startup that should depend on runtime state.
 function makeLazyForwardModule(pkg) {
   const S = JSON.stringify;
-  const tmpdir = path.resolve(__dirname, '..', 'tmp', 'lazy-webpack-modules');
+  const tmpdir = path.resolve(
+    __dirname,
+    '..',
+    '..',
+    'tmp',
+    'lazy-webpack-modules'
+  );
   fs.mkdirSync(tmpdir, { recursive: true });
   const filename = path.join(
     tmpdir,

packages/e2e-tests/test/e2e-snapshot.spec.ts

@@ -0,0 +1,147 @@
+import {
+  skipIfApiStrict,
+  startSharedTestServer,
+} from '../../../testing/integration-testing-hooks';
+import { TestShell } from './test-shell';
+import { expect } from 'chai';
+
+const setDifference = <T>(a: T[], b: T[]) => a.filter((e) => !b.includes(e));
+const expectIsSubset = <T>(a: T[], b: T[]) =>
+  expect(setDifference(a, b)).to.have.lengthOf(0);
+const commonPrefix = (a: string, b: string): string =>
+  a.startsWith(b)
+    ? b
+    : b.startsWith(a)
+    ? a
+    : b && commonPrefix(a, b.slice(0, -1));
+
+describe('e2e startup banners', function () {
+  skipIfApiStrict();
+  afterEach(TestShell.cleanup);
+
+  const testServer = startSharedTestServer();
+
+  context('modules included in snapshots', function () {
+    it('includes the right modules at the right point in time', async function () {
+      if (!process.env.MONGOSH_TEST_EXECUTABLE_PATH) return this.skip();
+
+      const connectionString = await testServer.connectionString();
+      const helperScript = `
+      const S = process.__mongosh_webpack_stats;
+      const L = (list) => list.map(S.lookupNaturalModuleName).filter(name => name && !name.endsWith('.json'));
+      `;
+      const commonArgs = ['--quiet', '--json=relaxed', '--eval', helperScript];
+      const argLists = [
+        [...commonArgs, '--nodb', '--eval', 'L(S.enumerateAllModules())'],
+        [...commonArgs, '--nodb', '--eval', 'L(S.enumerateSnapshotModules())'],
+        [...commonArgs, '--nodb', '--eval', 'L(S.enumerateLoadedModules())'],
+        [
+          ...commonArgs,
+          connectionString,
+          '--eval',
+          'L(S.enumerateLoadedModules())',
+        ],
+        [
+          ...commonArgs,
+          connectionString,
+          '--jsContext=repl',
+          '--eval',
+          'L(S.enumerateLoadedModules())',
+        ],
+      ];
+      const [
+        all,
+        atSnapshotTime,
+        atNodbEvalTime,
+        atDbEvalTime,
+        atReplEvalTime,
+      ] = (
+        await Promise.all(
+          argLists.map((args) =>
+            TestShell.runAndGetOutputWithoutErrors({ args })
+          )
+        )
+      ).map((output) =>
+        (JSON.parse(output) as string[])
+          .sort()
+          .map((pkg) => pkg.replace(/\\/g, '/'))
+      );
+
+      // Ensure that: atSnapshotTime ⊆ atNodbEvalTime ⊆ atDbEvalTime ⊆ atReplEvalTime ⊆ all
+      expectIsSubset(atSnapshotTime, atNodbEvalTime);
+      expectIsSubset(atNodbEvalTime, atDbEvalTime);
+      expectIsSubset(atDbEvalTime, atReplEvalTime);
+      expectIsSubset(atReplEvalTime, all);
+
+      const prefix = all.reduce(commonPrefix);
+      const stripPrefix = (s: string) =>
+        s.startsWith(prefix) ? s.replace(prefix, '') : s;
+
+      const categorized = [
+        ...atSnapshotTime.map(stripPrefix).map((m) => [m, 'snapshot'] as const),
+        ...setDifference(atNodbEvalTime, atSnapshotTime)
+          .map(stripPrefix)
+          .map((m) => [m, 'nodb-eval'] as const),
+        ...setDifference(atDbEvalTime, atNodbEvalTime)
+          .map(stripPrefix)
+          .map((m) => [m, 'db-eval'] as const),
+        ...setDifference(atReplEvalTime, atDbEvalTime)
+          .map(stripPrefix)
+          .map((m) => [m, 'repl-eval'] as const),
+        ...setDifference(all, atReplEvalTime)
+          .map(stripPrefix)
+          .map((m) => [m, 'not-loaded'] as const),
+      ];
+
+      // This is very helpful for inspecting snapshotted contents manually:
+      // console.table(categorized.map(([m, c]) => [m.replace(prefix, ''), c]));
+      const verifyAllInCategoryMatch = (
+        category: (typeof categorized)[number][1],
+        re: RegExp
+      ) => {
+        for (const [module, cat] of categorized) {
+          if (cat === category) {
+            expect(module).to.match(
+              re,
+              `Found unexpected '${module}' in category '${cat}'`
+            );
+          }
+        }
+      };
+      const verifyAllThatMatchAreInCategory = (
+        category: (typeof categorized)[number][1],
+        re: RegExp
+      ) => {
+        for (const [module, cat] of categorized) {
+          if (re.test(module)) {
+            expect(cat).to.equal(
+              category,
+              `Expected '${module}' to be in category '${category}', actual category is '${cat}'`
+            );
+          }
+        }
+      };
+
+      // The core test: Verify that in the categories beyond 'not loaded at all'
+      // and 'part of the snapshot', only a very specific set of modules is present,
+      // and that some modules are only in specific categories.
+      verifyAllInCategoryMatch('repl-eval', /^node_modules\/pretty-repl\//);
+      verifyAllInCategoryMatch(
+        'db-eval',
+        /^node_modules\/(kerberos|os-dns-native|resolve-mongodb-srv)\//
+      );
+      verifyAllInCategoryMatch(
+        'nodb-eval',
+        /^node_modules\/(kerberos|mongodb-client-encryption)\//
+      );
+      verifyAllThatMatchAreInCategory(
+        'not-loaded',
+        /^node_modules\/(express|openid-client|qs|send|jose|execa|body-parser|@babel\/highlight|@babel\/code-frame)\//
+      );
+      verifyAllThatMatchAreInCategory(
+        'snapshot',
+        /^node_modules\/(@babel\/types|@babel\/traverse|@mongodb-js\/devtools-connect|mongodb)\/|^packages\//
+      );
+    });
+  });
+});

packages/e2e-tests/test/test-shell.ts

@@ -30,22 +30,22 @@ function matches(str: string, pattern: string | RegExp): boolean {
     : pattern.test(str);
 }
 
+export interface TestShellOptions {
+  args: string[];
+  env?: Record<string, string>;
+  removeSigintListeners?: boolean;
+  cwd?: string;
+  forceTerminal?: boolean;
+  consumeStdio?: boolean;
+}
+
 /**
  * Test shell helper class.
  */
 export class TestShell {
   private static _openShells: TestShell[] = [];
 
-  static start(
-    options: {
-      args: string[];
-      env?: Record<string, string>;
-      removeSigintListeners?: boolean;
-      cwd?: string;
-      forceTerminal?: boolean;
-      consumeStdio?: boolean;
-    } = { args: [] }
-  ): TestShell {
+  static start(options: TestShellOptions = { args: [] }): TestShell {
     let shellProcess: ChildProcessWithoutNullStreams;
 
     let env = options.env || process.env;
@@ -95,6 +95,15 @@ export class TestShell {
     return shell;
   }
 
+  static async runAndGetOutputWithoutErrors(
+    options: TestShellOptions
+  ): Promise<string> {
+    const shell = this.start(options);
+    await shell.waitForExit();
+    shell.assertNoErrors();
+    return shell.output;
+  }
+
   static async killall(): Promise<void> {
     const exitPromises: Promise<unknown>[] = [];
     while (TestShell._openShells.length) {

scripts/webpack-enable-reverse-module-lookup-plugin.js

@@ -0,0 +1,30 @@
+'use strict';
+const path = require('path');
+const fs = require('fs');
+const zlib = require('zlib');
+const { promisify } = require('util');
+
+class WebpackEnableReverseModuleLookupPlugin {
+  outputFilename;
+  constructor({ outputFilename }) { this.outputFilename = outputFilename; }
+
+  apply(compiler) {
+    compiler.hooks.emit.tapPromise('EnableReverseModuleLookupPlugin', async(compilation) => {
+      const map = Object.create(null);
+      for (const module of compilation.modules) {
+        const id = compilation.chunkGraph.getModuleId(module);
+        if (id && module.resource) {
+          map[id] = module.resource;
+        }
+      }
+      const data = (await promisify(zlib.brotliCompress)(JSON.stringify(map))).toString('base64');
+      await fs.promises.mkdir(path.dirname(this.outputFilename), { recursive: true });
+      await fs.promises.writeFile(this.outputFilename, `function __webpack_reverse_module_lookup__() {
+        return __webpack_reverse_module_lookup__.data ??= JSON.parse(
+          require("zlib").brotliDecompressSync(Buffer.from(${JSON.stringify(data)}, 'base64')));
+      }`);
+    })
+  }
+}
+
+module.exports = { WebpackEnableReverseModuleLookupPlugin };