refactor(lazy-compilation): use POST request to transfer ids of active modules (#12678)

* refactor: use post transfer module ids

* test: skip lazy active method

* test: add lazy post request help

* test: add large module id container

* test: add case introduction

* refactor: remove about controller

* test: we are using post method now

* refactor: rename

* fix body parser (vibe-kanban 09797829)

1. 如果其他中间件一个解析好了body，并挂载在req.body 上的花就直直接使用 req.body
2. packages/rspack/src/builtin-plugin/lazy-compilation/middleware.ts body 解析的时候不能简单的使用 string 的拼接，需要考虑多字节符号的截断的问题。

* refactor read module ids from body (vibe-kanban a7874ac1)

packages/rspack/src/builtin-plugin/lazy-compilation/middleware.ts 当req.body 非空时，直接假body中时一组 module ids，如果没有 body 自行拼接完，之后也 parse json string，返回 module ids 数组

* refactor: downgrading to lower web api

* test: ✅ add lazy compilation active cors cases

* chore: update test case doc

* fix: we all need cors header no matter is simple request or not

* refactor:delete cors header setting

* refactor: set cors header should set by user

* Update packages/rspack/hot/lazy-compilation-web.js

Co-authored-by: neverland <chenjiahan.jait@bytedance.com>

* test: fix case name

* chore: api-extract update

* fix: memory leak of requst listeners

* refactor: remove event source handle logic

* fix: jsdom XMLHTTPRequets need strict cors header

* refactor: node lazy compilation client use post too

---------

Co-authored-by: neverland <chenjiahan.jait@bytedance.com>

Author: stormslowly

AGENTS.md

@@ -36,6 +36,8 @@ Rspack is a high-performance JavaScript bundler written in Rust that offers stro
 - Update snapshots: `npm run test -- -u`
 - Filter tests: `npm run test -- -t configCases/asset`
 
+Depends on what you have modified, you need to rebuild by `pnpm run build:js` or `pnpm run build:binding:dev` or `pnpm run build:cli:dev` first, then run testing commands to verify the modification.
+
 ## Debugging
 
 - **VS Code**: `.vscode/launch.json` with `Debug Rspack` and `Attach` options

packages/rspack-test-tools/src/plugin/lazy-compilation-test-plugin.ts

@@ -38,6 +38,9 @@ export class LazyCompilationTestPlugin {
         resolve(null);
       });
       server.on('request', (req, res) => {
+        // Set CORS headers for jsdom's XMLHttpRequest
+        res.setHeader('Access-Control-Allow-Origin', '*');
+
         middleware(req, res, () => {});
       });
       server.on('connection', (socket) => {

packages/rspack-test-tools/src/runner/web/index.ts

@@ -175,7 +175,12 @@ export class WebRunner extends NodeRunner {
   protected createBaseModuleScope() {
     const moduleScope = super.createBaseModuleScope();
     moduleScope.EventSource = EventSource;
-    moduleScope.fetch = async (url: string) => {
+    moduleScope.fetch = async (url: string, options: any) => {
+      // For Lazy Compilation Proxy the POST request to the real dev server.
+      if (options?.method === 'POST') {
+        return fetch(url, options as any);
+      }
+
       try {
         const filePath = this.urlToPath(url);
         this.log(`fetch: ${url} -> ${filePath}`);

packages/rspack/hot/lazy-compilation-node.js

@@ -1,4 +1,79 @@
 var urlBase = decodeURIComponent(__resourceQuery.slice(1));
+var compiling = new Set();
+var errorHandlers = new Set();
+
+/** @type {import("http").ClientRequest | undefined} */
+var pendingRequest;
+/** @type {boolean} */
+var hasPendingUpdate = false;
+
+function sendRequest() {
+  if (compiling.size === 0) {
+    hasPendingUpdate = false;
+    return;
+  }
+
+  var modules = Array.from(compiling);
+  var data = modules.join('\n');
+
+  var httpModule = urlBase.startsWith('https')
+    ? require('https')
+    : require('http');
+
+  var request = httpModule.request(
+    urlBase,
+    {
+      method: 'POST',
+      agent: false,
+      headers: {
+        'Content-Type': 'text/plain',
+      },
+    },
+    function (res) {
+      pendingRequest = undefined;
+      if (res.statusCode < 200 || res.statusCode >= 300) {
+        var error = new Error(
+          'Problem communicating active modules to the server: HTTP ' +
+            res.statusCode,
+        );
+        errorHandlers.forEach(function (onError) {
+          onError(error);
+        });
+      }
+      // Consume response data to free up memory
+      res.resume();
+      if (hasPendingUpdate) {
+        hasPendingUpdate = false;
+        sendRequest();
+      }
+    },
+  );
+
+  pendingRequest = request;
+
+  request.on('error', function (err) {
+    pendingRequest = undefined;
+    var error = new Error(
+      'Problem communicating active modules to the server: ' + err.message,
+    );
+    errorHandlers.forEach(function (onError) {
+      onError(error);
+    });
+  });
+
+  request.write(data);
+  request.end();
+}
+
+function sendActiveRequest() {
+  hasPendingUpdate = true;
+
+  // If no request is pending, start one
+  if (!pendingRequest) {
+    hasPendingUpdate = false;
+    sendRequest();
+  }
+}
 
 /**
  * @param {{ data: string, onError: (err: Error) => void, active: boolean, module: module }} options options
@@ -9,42 +84,23 @@ exports.activate = function (options) {
   var onError = options.onError;
   var active = options.active;
   var module = options.module;
-  /** @type {import("http").IncomingMessage} */
-  var response;
-  var request = (
-    urlBase.startsWith('https') ? require('https') : require('http')
-  ).request(
-    urlBase + encodeURIComponent(data),
-    {
-      agent: false,
-      headers: { accept: 'text/event-stream' },
-    },
-    function (res) {
-      response = res;
-      response.on('error', errorHandler);
-      if (!active && !module.hot) {
-        console.log(
-          'Hot Module Replacement is not enabled. Waiting for process restart...',
-        );
-      }
-    },
-  );
 
-  /**
-   * @param {Error} err error
-   */
-  function errorHandler(err) {
-    err.message =
-      'Problem communicating active modules to the server' +
-      (err.message ? ': ' + err.message : '') +
-      '\nRequest: ' +
-      urlBase +
-      data;
-    onError(err);
+  errorHandlers.add(onError);
+
+  if (!compiling.has(data)) {
+    compiling.add(data);
+    sendActiveRequest();
   }
-  request.on('error', errorHandler);
-  request.end();
+
+  if (!active && !module.hot) {
+    console.log(
+      'Hot Module Replacement is not enabled. Waiting for process restart...',
+    );
+  }
+
   return function () {
-    response.destroy();
+    errorHandlers.delete(onError);
+    compiling.delete(data);
+    sendActiveRequest();
   };
 };

packages/rspack/hot/lazy-compilation-web.js

@@ -1,47 +1,73 @@
-if (typeof EventSource !== 'function') {
+if (typeof XMLHttpRequest === 'undefined') {
   throw new Error(
-    "Environment doesn't support lazy compilation (requires EventSource)",
+    "Environment doesn't support lazy compilation (requires XMLHttpRequest)",
   );
 }
 
 var urlBase = decodeURIComponent(__resourceQuery.slice(1));
-/** @type {EventSource | undefined} */
-var activeEventSource;
 var compiling = new Set();
 var errorHandlers = new Set();
 
-var updateEventSource = function updateEventSource() {
-  if (activeEventSource) activeEventSource.close();
-  if (compiling.size) {
-    activeEventSource = new EventSource(
-      urlBase +
-        Array.from(compiling, function (module) {
-          return encodeURIComponent(module);
-        }).join('@'),
-    );
-    /**
-     * @this {EventSource}
-     * @param {Event & { message?: string, filename?: string, lineno?: number, colno?: number, error?: Error }} event event
-     */
-    activeEventSource.onerror = function (event) {
-      errorHandlers.forEach(function (onError) {
-        onError(
-          new Error(
-            'Problem communicating active modules to the server' +
-              (event.message ? `: ${event.message} ` : '') +
-              (event.filename ? `: ${event.filename} ` : '') +
-              (event.lineno ? `: ${event.lineno} ` : '') +
-              (event.colno ? `: ${event.colno} ` : '') +
-              (event.error ? `: ${event.error}` : ''),
-          ),
-        );
-      });
-    };
-  } else {
-    activeEventSource = undefined;
+/** @type {XMLHttpRequest | undefined} */
+var pendingXhr;
+/** @type {boolean} */
+var hasPendingUpdate = false;
+
+var sendRequest = function sendRequest() {
+  if (compiling.size === 0) {
+    hasPendingUpdate = false;
+    return;
   }
+
+  var modules = Array.from(compiling);
+  var data = modules.join('\n');
+
+  var xhr = new XMLHttpRequest();
+  pendingXhr = xhr;
+  xhr.open('POST', urlBase, true);
+  // text/plain Content-Type is simple request header
+  xhr.setRequestHeader('Content-Type', 'text/plain');
+
+  xhr.onreadystatechange = function () {
+    if (xhr.readyState === 4) {
+      pendingXhr = undefined;
+      if (xhr.status < 200 || xhr.status >= 300) {
+        var error = new Error(
+          'Problem communicating active modules to the server: HTTP ' +
+            xhr.status,
+        );
+        errorHandlers.forEach(function (onError) {
+          onError(error);
+        });
+      }
+      if (hasPendingUpdate) {
+        hasPendingUpdate = false;
+        sendRequest();
+      }
+    }
+  };
+
+  xhr.onerror = function () {
+    pendingXhr = undefined;
+    var error = new Error('Problem communicating active modules to the server');
+    errorHandlers.forEach(function (onError) {
+      onError(error);
+    });
+  };
+
+  xhr.send(data);
 };
 
+function sendActiveRequest() {
+  hasPendingUpdate = true;
+
+  // If no request is pending, start one
+  if (!pendingXhr) {
+    hasPendingUpdate = false;
+    sendRequest();
+  }
+}
+
 /**
  * @param {{ data: string, onError: (err: Error) => void, active: boolean, module: module }} options options
  * @returns {() => void} function to destroy response
@@ -55,7 +81,7 @@ exports.activate = function (options) {
 
   if (!compiling.has(data)) {
     compiling.add(data);
-    updateEventSource();
+    sendActiveRequest();
   }
 
   if (!active && !module.hot) {
@@ -67,6 +93,6 @@ exports.activate = function (options) {
   return function () {
     errorHandlers.delete(onError);
     compiling.delete(data);
-    updateEventSource();
+    sendActiveRequest();
   };
 };