feat(babel): add parallel processing via worker threads

Add a `parallel` option that processes files concurrently using Node.js worker threads. This reduces build times for large projects where Babel transformation is a bottleneck. This is similar to the existing parallel behavior of `@rollup/plugin-terser`.

This required some fairly significant refactoring, because we can only pass serializable objects between the main thread and the worker threads. It also required changes to the plugin's own build config, so that we can generate a dedicated worker entrypoint.

Validations are added to ensure that unserializable config (e.g. inline babel plugins) cannot be used alongside the new parallel mode. For people using dedicated babel config files, this isn't a problem, because they are loaded directly by babel in the worker thread itself.

The worker threads do have a setup cost, so this only makes sense for large projects. In Discourse, enabling this parallel mode cuts our overall vite (rolldown) build time by about 45% (from ~11s to ~6s) on my machine.

Author: davidtaylorhq

packages/babel/README.md

@@ -135,6 +135,15 @@ Default: `false`
 
 Before transpiling your input files this plugin also transpile a short piece of code **for each** input file. This is used to validate some misconfiguration errors, but for sufficiently big projects it can slow your build times so if you are confident about your configuration then you might disable those checks with this option.
 
+### `parallel`
+
+Type: `Boolean | number`
+Default: `false`
+
+Enable parallel processing of files in worker threads. This has a setup cost, so is best suited for larger projects. Pass an integer to set the number of workers. Set `true` for the default number of workers (4).
+
+This option cannot be used alongside custom overrides or non-serializable Babel options.
+
 ### External dependencies
 
 Ideally, you should only be transforming your source code, rather than running all of your external dependencies through Babel (to ignore external dependencies from being handled by this plugin you might use `exclude: 'node_modules/**'` option). If you have a dependency that exposes untranspiled ES6 source code that doesn't run in your target environment, then you may need to break this rule, but it often causes problems with unusual `.babelrc` files or mismatched versions of Babel.

packages/babel/rollup.config.mjs

@@ -1,14 +1,37 @@
 import { readFileSync } from 'fs';
 
-import { createConfig } from '../../shared/rollup.config.mjs';
+import { createConfig, emitModulePackageFile } from '../../shared/rollup.config.mjs';
 
 import { babel } from './src/index.js';
 
 const pkg = JSON.parse(readFileSync(new URL('./package.json', import.meta.url), 'utf8'));
 
 export default {
   ...createConfig({ pkg }),
-  input: './src/index.js',
+  input: {
+    index: './src/index.js',
+    worker: './src/worker.js'
+  },
+  output: [
+    {
+      format: 'cjs',
+      dir: 'dist/cjs',
+      exports: 'named',
+      footer(chunkInfo) {
+        if (chunkInfo.name === 'index') {
+          return 'module.exports = Object.assign(exports.default, exports);';
+        }
+        return null;
+      },
+      sourcemap: true
+    },
+    {
+      format: 'es',
+      dir: 'dist/es',
+      plugins: [emitModulePackageFile()],
+      sourcemap: true
+    }
+  ],
   plugins: [
     babel({
       presets: [['@babel/preset-env', { targets: { node: 14 } }]],

packages/babel/src/index.js

@@ -2,10 +2,9 @@ import * as babel from '@babel/core';
 import { createFilter } from '@rollup/pluginutils';
 
 import { BUNDLED, HELPERS } from './constants.js';
-import bundledHelpersPlugin from './bundledHelpersPlugin.js';
-import preflightCheck from './preflightCheck.js';
 import transformCode from './transformCode.js';
-import { addBabelPlugin, escapeRegExpCharacters, warnOnce } from './utils.js';
+import { escapeRegExpCharacters, warnOnce } from './utils.js';
+import WorkerPool from './workerPool.js';
 
 const unpackOptions = ({
   extensions = babel.DEFAULT_EXTENSIONS,
@@ -100,6 +99,24 @@ const returnObject = () => {
   return {};
 };
 
+function isSerializable(value) {
+  if (value === null) {
+    return true;
+  } else if (Array.isArray(value)) {
+    return value.every(isSerializable);
+  }
+  switch (typeof value) {
+    case 'string':
+    case 'number':
+    case 'boolean':
+      return true;
+    case 'object':
+      return Object.keys(value).every((key) => isSerializable(value[key]));
+    default:
+      return false;
+  }
+}
+
 function createBabelInputPluginFactory(customCallback = returnObject) {
   const overrides = customCallback(babel);
 
@@ -109,13 +126,16 @@ function createBabelInputPluginFactory(customCallback = returnObject) {
       overrides
     );
 
+    let workerPool;
+
     const {
       exclude,
       extensions,
       babelHelpers,
       include,
       filter: customFilter,
       skipPreflightCheck,
+      parallel,
       ...babelOptions
     } = unpackInputPluginOptions(pluginOptionsWithOverrides);
 
@@ -129,6 +149,23 @@ function createBabelInputPluginFactory(customCallback = returnObject) {
       typeof customFilter === 'function' ? customFilter : createFilter(include, exclude);
     const filter = (id, code) => extensionRegExp.test(id) && userDefinedFilter(id, code);
 
+    if (parallel) {
+      const parallelAllowed =
+        isSerializable(babelOptions) && !overrides?.config && !overrides?.result;
+
+      if (!parallelAllowed) {
+        throw new Error(
+          'Cannot use "parallel" mode alongside custom overrides or non-serializable Babel options.'
+        );
+      }
+
+      const parallelWorkerCount = typeof parallel === 'number' ? parallel : 4;
+      workerPool = new WorkerPool(
+        new URL('./worker.js', import.meta.url).pathname,
+        parallelWorkerCount
+      );
+    }
+
     const helpersFilter = { id: new RegExp(`^${escapeRegExpCharacters(HELPERS)}$`) };
 
     return {
@@ -162,22 +199,39 @@ function createBabelInputPluginFactory(customCallback = returnObject) {
           if (!(await filter(filename, code))) return null;
           if (filename === HELPERS) return null;
 
-          return transformCode(
-            code,
-            { ...babelOptions, filename },
-            overrides,
+          if (parallel) {
+            return workerPool.runTask({
+              inputCode: code,
+              babelOptions: { ...babelOptions, filename },
+              runPreflightCheck: !skipPreflightCheck,
+              babelHelpers
+            });
+          }
+
+          return transformCode({
+            inputCode: code,
+            babelOptions: { ...babelOptions, filename },
+            overrides: {
+              config: overrides.config?.bind(this),
+              result: overrides.result?.bind(this)
+            },
             customOptions,
-            this,
-            async (transformOptions) => {
-              if (!skipPreflightCheck) {
-                await preflightCheck(this, babelHelpers, transformOptions);
-              }
-
-              return babelHelpers === BUNDLED
-                ? addBabelPlugin(transformOptions, bundledHelpersPlugin)
-                : transformOptions;
-            }
-          );
+            error: this.error.bind(this),
+            runPreflightCheck: !skipPreflightCheck,
+            babelHelpers
+          });
+        }
+      },
+
+      async closeBundle() {
+        if (parallel && !this.meta.watchMode) {
+          await workerPool.terminate();
+        }
+      },
+
+      async closeWatcher() {
+        if (parallel) {
+          await workerPool.terminate();
         }
       }
     };
@@ -257,7 +311,16 @@ function createBabelOutputPluginFactory(customCallback = returnObject) {
           }
         }
 
-        return transformCode(code, babelOptions, overrides, customOptions, this);
+        return transformCode({
+          inputCode: code,
+          babelOptions,
+          overrides: {
+            config: overrides.config?.bind(this),
+            result: overrides.result?.bind(this)
+          },
+          customOptions,
+          error: this.error.bind(this)
+        });
       }
     };
   };

packages/babel/src/preflightCheck.js

@@ -37,27 +37,27 @@ const mismatchError = (actual, expected, filename) =>
 // Revert to /\/helpers\/(esm\/)?inherits/ when Babel 8 gets released, this was fixed in https://github.com/babel/babel/issues/14185
 const inheritsHelperRe = /[\\/]+helpers[\\/]+(esm[\\/]+)?inherits/;
 
-export default async function preflightCheck(ctx, babelHelpers, transformOptions) {
+export default async function preflightCheck(error, babelHelpers, transformOptions) {
   const finalOptions = addBabelPlugin(transformOptions, helpersTestTransform);
   const check = (await babel.transformAsync(PREFLIGHT_INPUT, finalOptions)).code;
 
   // Babel sometimes splits ExportDefaultDeclaration into 2 statements, so we also check for ExportNamedDeclaration
   if (!/export (d|{)/.test(check)) {
-    ctx.error(MODULE_ERROR);
+    error(MODULE_ERROR);
   }
 
   if (inheritsHelperRe.test(check)) {
     if (babelHelpers === RUNTIME) {
       return;
     }
-    ctx.error(mismatchError(RUNTIME, babelHelpers, transformOptions.filename));
+    error(mismatchError(RUNTIME, babelHelpers, transformOptions.filename));
   }
 
   if (check.includes('babelHelpers.inherits')) {
     if (babelHelpers === EXTERNAL) {
       return;
     }
-    ctx.error(mismatchError(EXTERNAL, babelHelpers, transformOptions.filename));
+    error(mismatchError(EXTERNAL, babelHelpers, transformOptions.filename));
   }
 
   // test unminifiable string content
@@ -66,12 +66,12 @@ export default async function preflightCheck(ctx, babelHelpers, transformOptions
       return;
     }
     if (babelHelpers === RUNTIME && !transformOptions.plugins.length) {
-      ctx.error(
+      error(
         `You must use the \`@babel/plugin-transform-runtime\` plugin when \`babelHelpers\` is "${RUNTIME}".\n`
       );
     }
-    ctx.error(mismatchError(INLINE, babelHelpers, transformOptions.filename));
+    error(mismatchError(INLINE, babelHelpers, transformOptions.filename));
   }
 
-  ctx.error(UNEXPECTED_ERROR);
+  error(UNEXPECTED_ERROR);
 }

packages/babel/src/transformCode.js

@@ -1,13 +1,19 @@
 import * as babel from '@babel/core';
 
-export default async function transformCode(
+import bundledHelpersPlugin from './bundledHelpersPlugin.js';
+import preflightCheck from './preflightCheck.js';
+import { BUNDLED } from './constants.js';
+import { addBabelPlugin } from './utils.js';
+
+export default async function transformCode({
   inputCode,
   babelOptions,
   overrides,
   customOptions,
-  ctx,
-  finalizeOptions
-) {
+  error,
+  runPreflightCheck,
+  babelHelpers
+}) {
   // loadPartialConfigAsync has become available in @babel/core@7.8.0
   const config = await (babel.loadPartialConfigAsync || babel.loadPartialConfig)(babelOptions);
 
@@ -16,18 +22,23 @@ export default async function transformCode(
     return null;
   }
 
-  let transformOptions = !overrides.config
+  let transformOptions = !overrides?.config
     ? config.options
-    : await overrides.config.call(ctx, config, {
+    : await overrides.config(config, {
         code: inputCode,
         customOptions
       });
 
-  if (finalizeOptions) {
-    transformOptions = await finalizeOptions(transformOptions);
+  if (runPreflightCheck) {
+    await preflightCheck(error, babelHelpers, transformOptions);
   }
 
-  if (!overrides.result) {
+  transformOptions =
+    babelHelpers === BUNDLED
+      ? addBabelPlugin(transformOptions, bundledHelpersPlugin)
+      : transformOptions;
+
+  if (!overrides?.result) {
     const { code, map } = await babel.transformAsync(inputCode, transformOptions);
     return {
       code,
@@ -36,7 +47,7 @@ export default async function transformCode(
   }
 
   const result = await babel.transformAsync(inputCode, transformOptions);
-  const { code, map } = await overrides.result.call(ctx, result, {
+  const { code, map } = await overrides.result(result, {
     code: inputCode,
     customOptions,
     config,

packages/babel/src/worker.js

@@ -0,0 +1,25 @@
+import { parentPort } from 'worker_threads';
+
+import transformCode from './transformCode.js';
+
+parentPort.on('message', async (opts) => {
+  try {
+    const result = await transformCode({
+      ...opts,
+      error: (msg) => {
+        throw new Error(msg);
+      }
+    });
+    parentPort.postMessage({
+      result
+    });
+  } catch (error) {
+    parentPort.postMessage({
+      error: {
+        message: error.message,
+        stack: error.stack,
+        name: error.name
+      }
+    });
+  }
+});