build(node-runtime-worker-thread, compass-shell): Do not inline worker-runtime modules (#588)

* fix(node-runtime-worker-thread): Do not polyfill node built-ins; Externalize optional mongodb deps

* fix(node-runtime-worker-thread): Do not inline worker and child process proxy during the build

* build(compass-shell): Make node-runtime-worker-thread a peer dependency

This allows us to avoid inlining source of node-runtime-worker-thread when
used in compass, but has a small downside of manually installing runtime in
compass main repo

* chore(node-runtime-worker-thread): Remove unused loader

* chore(compas-shell): Clean-up unused leftovers in deps

* build(compass-shell): Move node-runtime back to direct dependencies and explicitly exclude it for all builds

Author: gribnoysup

packages/compass-shell/config/webpack.base.config.js

@@ -128,5 +128,11 @@ module.exports = {
         exclude: /(node_modules)/
       }
     ]
+  },
+  externals: {
+    // Runtime implementation depends on worker file existing near the library
+    // main import and for that reason it needs to stay external to the
+    // compass-shell
+    '@mongosh/node-runtime-worker-thread': 'commonjs2 @mongosh/node-runtime-worker-thread',
   }
 };

packages/compass-shell/config/webpack.dev.config.js

@@ -7,6 +7,7 @@ const { spawn } = require('child_process');
 const baseWebpackConfig = require('./webpack.base.config');
 const project = require('./project');
 
+/** @type import('webpack').Configuration */
 const config = {
   mode: 'development',
   target: 'electron-renderer',
@@ -76,6 +77,21 @@ const config = {
         .on('close', () => process.exit(0))
         .on('error', spawnError => console.error(spawnError)); // eslint-disable-line no-console
     }
+  },
+  resolve: {
+    // Without this alias, in dev mode symlinked browser-repl breaks the code
+    // by having two reacts loadeded on the page
+    alias: {
+      'react': require.resolve('react'),
+      'react-dom': require.resolve('@hot-loader/react-dom'),
+    }
+  },
+  externals: {
+    // "Optional" mongodb dependencies that should stay out of the build in dev
+    // mode
+    'mongodb-client-encryption': 'commonjs2 mongodb-client-encryption',
+    kerberos: 'commonjs2 kerberos',
+    snappy: 'commonjs2 snappy'
   }
 };
 

packages/node-runtime-worker-thread/loaders/inline-entry-loader.js

@@ -13,16 +13,20 @@
  *
  * @see {@link https://github.com/nodejs/node/pull/36344}
  */
-
 import { once } from 'events';
 import { SHARE_ENV, Worker } from 'worker_threads';
+import fs from 'fs';
+import path from 'path';
 import { exposeAll, createCaller } from './rpc';
 
-/**
- * The source has to be inlined to allow the runtime to be bundled in a single
- * file when used by compass shell
- */
-import workerRuntimeSrc from 'inline-entry-loader!./worker-runtime';
+// eslint-disable-next-line no-sync
+const workerRuntimeSrc = fs.readFileSync(
+  path.resolve(
+    process.env.NODE_RUNTIME_WORKER_THREAD_PARENT_DIRNAME || __dirname,
+    'worker-runtime.js'
+  ),
+  'utf-8'
+);
 
 const workerProcess = new Worker(workerRuntimeSrc, {
   eval: true,

packages/node-runtime-worker-thread/src/child-process-proxy.ts

@@ -4,11 +4,12 @@
 import { ChildProcess, SpawnOptionsWithoutStdio } from 'child_process';
 import { MongoClientOptions } from '@mongosh/service-provider-core';
 import { Runtime, RuntimeEvaluationListener, RuntimeEvaluationResult } from '@mongosh/browser-runtime-core';
+import { promises as fs } from 'fs';
+import path from 'path';
 import spawnChildFromSource, { kill } from './spawn-child-from-source';
 import { Caller, createCaller } from './rpc';
 import { ChildProcessEvaluationListener } from './child-process-evaluation-listener';
 import type { WorkerRuntime as WorkerThreadWorkerRuntime } from './worker-runtime';
-import childProcessProxySrc from 'inline-entry-loader!./child-process-proxy';
 import { deserializeEvaluationResult } from './serializer';
 
 type ChildProcessRuntime = Caller<WorkerThreadWorkerRuntime>;
@@ -43,11 +44,22 @@ class WorkerRuntime implements Runtime {
   private async initWorker() {
     const { uri, driverOptions, cliOptions, spawnOptions } = this.initOptions;
 
-    this.childProcess = await spawnChildFromSource(
-      childProcessProxySrc,
-      spawnOptions
+    const childProcessProxySrc = await fs.readFile(
+      path.resolve(__dirname, 'child-process-proxy.js'),
+      'utf-8'
     );
 
+    this.childProcess = await spawnChildFromSource(childProcessProxySrc, {
+      ...spawnOptions,
+      env: {
+        // Proxy child process and worker_threads worker are inlined and as such
+        // they are not aware of the dirname (which child process will need to
+        // read worker source)
+        NODE_RUNTIME_WORKER_THREAD_PARENT_DIRNAME: __dirname,
+        ...spawnOptions.env
+      }
+    });
+
     this.childProcessRuntime = createCaller(
       ['init', 'evaluate', 'getCompletions', 'setEvaluationListener'],
       this.childProcess

packages/node-runtime-worker-thread/src/index.ts

@@ -5,8 +5,6 @@ const TerserPlugin = require('terser-webpack-plugin');
 const config = {
   target: 'node',
 
-  devtool: false,
-
   output: {
     path: path.resolve(__dirname, 'dist'),
     filename: '[name].js',
@@ -27,10 +25,6 @@ const config = {
     extensions: ['.ts', '.js']
   },
 
-  resolveLoader: {
-    modules: ['node_modules', path.resolve(__dirname, 'loaders')]
-  },
-
   optimization: {
     minimizer: [
       new TerserPlugin({
@@ -40,6 +34,14 @@ const config = {
         }
       })
     ]
+  },
+
+  node: false,
+
+  externals: {
+    'mongodb-client-encryption': 'commonjs2 mongodb-client-encryption',
+    kerberos: 'commonjs2 kerberos',
+    snappy: 'commonjs2 snappy'
   }
 };