perf(linter/plugins): lazy-load tokens parsing code (#16011)

Tokens APIs (#15861 and further PRs) utilize TS-ESLint for parsing. TS-ESLint uses TypeScript internally. So this pulls in a ton of code (8 MB unminified).

Lazy-load this code only when tokens APIs are used for the first time, so that users don't pay for it if their plugins don't use these APIs (or plugins only use tokens APIs for fixes, and their code has no errors that require fixes).

Do this by compiling `@typescript-eslint/typescript-estree` into a separate bundle. It's bundled as CommonJS, so it can be synchonously `require`-ed without needing `require(esm)` support.

Additionally, minify this bundle, which cuts the increase in package size due to TS-ESLint from 8 MB to 3.7 MB. That's still a lot, but not *completely* unacceptable, given that the `oxlint` binary is around 9.3 MB. As soon as we can, we'll re-implement token-generation in Oxc parser, and then we'll be able to remove TS-ESLint again.

Author: overlookmotel

apps/oxlint/src-js/plugins/tokens.ts

@@ -2,7 +2,7 @@
  * `SourceCode` methods related to tokens.
  */
 
-import { parse } from '@typescript-eslint/typescript-estree';
+import { createRequire } from 'node:module';
 import { sourceText, initSourceText } from './source_code.js';
 import { debugAssertIsNonNull } from '../utils/asserts.js';
 
@@ -139,16 +139,30 @@ let tokens: Token[] | null = null;
 let comments: CommentToken[] | null = null;
 let tokensWithComments: Token[] | null = null;
 
+// TS-ESLint `parse` method.
+// Lazy-loaded only when needed, as it's a lot of code.
+// Bundle contains both `@typescript-eslint/typescript-estree` and `typescript`.
+let tsEslintParse: typeof import('@typescript-eslint/typescript-estree').parse | null = null;
+
 /**
  * Initialize TS-ESLint tokens for current file.
  */
 function initTokens() {
   debugAssertIsNonNull(sourceText);
-  ({ tokens, comments } = parse(sourceText, {
+
+  // Lazy-load TS-ESLint.
+  // `./ts_eslint.cjs` is path to the bundle in `dist` directory, as well as relative path in `src-js`,
+  // so is valid both in bundled `dist` output, and in unit tests.
+  if (tsEslintParse === null) {
+    const require = createRequire(import.meta.url);
+    tsEslintParse = (require('./ts_eslint.cjs') as typeof import('@typescript-eslint/typescript-estree')).parse;
+  }
+
+  ({ tokens, comments } = tsEslintParse(sourceText, {
     sourceType: 'module',
     tokens: true,
     comment: true,
-    // TODO: Enable JSX only when needed
+    // TODO: Set this option dependent on source type
     jsx: true,
   }));
 }

apps/oxlint/src-js/plugins/ts_eslint.cjs

@@ -0,0 +1,3 @@
+'use strict';
+
+module.exports = require('@typescript-eslint/typescript-estree');

apps/oxlint/tsdown.config.ts

@@ -7,39 +7,55 @@ import type { Plugin } from 'rolldown';
 // This is the build used in tests.
 const DEBUG = process.env.DEBUG === 'true' || process.env.DEBUG === '1';
 
-export default defineConfig({
-  entry: ['src-js/cli.ts', 'src-js/index.ts'],
-  format: 'esm',
+const commonConfig = defineConfig({
   platform: 'node',
   target: 'node20',
   outDir: 'dist',
   clean: true,
   unbundle: false,
   hash: false,
-  external: [
-    // External native bindings
-    './oxlint.*.node',
-    '@oxlint/*',
-  ],
   fixedExtension: false,
-  // Handle `__filename`. Needed to bundle `typescript` for token methods.
-  shims: true,
-  // At present only compress syntax.
-  // Don't mangle identifiers or remove whitespace, so `dist` code remains somewhat readable.
-  minify: {
-    compress: { keepNames: { function: true, class: true } },
-    mangle: false,
-    codegen: { removeWhitespace: false },
+});
+
+export default defineConfig([
+  // Main build
+  {
+    ...commonConfig,
+    entry: ['src-js/cli.ts', 'src-js/index.ts'],
+    format: 'esm',
+    external: [
+      // External native bindings
+      './oxlint.*.node',
+      '@oxlint/*',
+    ],
+    // At present only compress syntax.
+    // Don't mangle identifiers or remove whitespace, so `dist` code remains somewhat readable.
+    minify: {
+      compress: { keepNames: { function: true, class: true } },
+      mangle: false,
+      codegen: { removeWhitespace: false },
+    },
+    dts: { resolve: true },
+    attw: true,
+    define: { DEBUG: DEBUG ? 'true' : 'false' },
+    plugins: DEBUG ? [] : [createReplaceAssertsPlugin()],
+    inputOptions: {
+      // For `replaceAssertsPlugin`
+      experimental: { nativeMagicString: true },
+    },
   },
-  dts: { resolve: true },
-  attw: true,
-  define: { DEBUG: DEBUG ? 'true' : 'false' },
-  plugins: DEBUG ? [] : [createReplaceAssertsPlugin()],
-  inputOptions: {
-    // For `replaceAssertsPlugin`
-    experimental: { nativeMagicString: true },
+  // TS-ESLint parser.
+  // Bundled separately and lazy-loaded, as it's a lot of code.
+  // Bundle contains both `@typescript-eslint/typescript-estree` and `typescript`.
+  {
+    ...commonConfig,
+    entry: 'src-js/plugins/ts_eslint.cjs',
+    format: 'commonjs',
+    // Minify as this bundle is just dependencies. We don't need to be able to debug it.
+    // Minification halves the size of the bundle.
+    minify: true,
   },
-});
+]);
 
 /**
  * Create a plugin to remove imports of `assert*` functions from `src-js/utils/asserts.ts`,