implement webpack chunks file updating using ast manipulation

.prettierignore

@@ -2,3 +2,5 @@
 .wrangler
 pnpm-lock.yaml
 .vscode/setting.json
+test-fixtures
+test-snapshots

TODO.md

@@ -15,27 +15,13 @@ DONE:
 
 - `npx create-next-app@latest <app-name> --use-npm` (use npm to avoid symlinks)
 
-- update next.config.mjs as follows
-
-  ```typescript
-  /** @type {import('next').NextConfig} */
-  const nextConfig = {
-    output: "standalone",
-    experimental: {
-      serverMinification: false,
-    },
-  };
-
-  export default nextConfig;
-  ```
-
 - add the following devDependency to the package.json:
 
   ```json
   "wrangler": "^3.78.6"
   ```
 
-- add a wrangler.toml int the generated app
+- add a wrangler.toml into the generated app
 
   ```toml
   #:schema node_modules/wrangler/config-schema.json

builder/README.md

@@ -2,20 +2,6 @@
 
 ## Build your app
 
-- update the `next.config.mjs` as follows
-
-  ```typescript
-  /** @type {import('next').NextConfig} */
-  const nextConfig = {
-    output: "standalone",
-    experimental: {
-      serverMinification: false,
-    },
-  };
-
-  export default nextConfig;
-  ```
-
 - add the following `devDependency` to the `package.json`:
 
   ```json

builder/package.json

@@ -4,7 +4,9 @@
   "version": "0.0.1",
   "scripts": {
     "build": "tsup",
-    "build:watch": "tsup --watch src"
+    "build:watch": "tsup --watch src",
+    "test": "vitest --run",
+    "test:watch": "vitest"
   },
   "bin": "dist/index.mjs",
   "files": [
@@ -27,12 +29,14 @@
   },
   "homepage": "https://github.com/flarelabs-net/poc-next",
   "devDependencies": {
-    "@cloudflare/workers-types": "^4.20240909.0",
-    "@types/node": "^22.2.0",
-    "esbuild": "^0.23.0",
-    "glob": "^11.0.0",
-    "next": "14.2.5",
-    "tsup": "^8.2.4",
-    "typescript": "^5.5.4"
+    "@types/node": "catalog:",
+    "esbuild": "catalog:",
+    "glob": "catalog:",
+    "tsup": "catalog:",
+    "typescript": "catalog:",
+    "vitest": "catalog:"
+  },
+  "dependencies": {
+    "ts-morph": "catalog:"
   }
 }

builder/src/build/build-worker.ts

@@ -1,6 +1,6 @@
 import { NextjsAppPaths } from "../nextjs-paths";
 import { build, Plugin } from "esbuild";
-import { readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { readFileSync } from "node:fs";
 import { cp, readFile, writeFile } from "node:fs/promises";
 
 import { patchRequire } from "./patches/investigated/patch-require";
@@ -11,6 +11,7 @@ import { patchFindDir } from "./patches/to-investigate/patch-find-dir";
 import { inlineNextRequire } from "./patches/to-investigate/inline-next-require";
 import { inlineEvalManifest } from "./patches/to-investigate/inline-eval-manifest";
 import { patchWranglerDeps } from "./patches/to-investigate/wrangler-deps";
+import { updateWebpackChunksFile } from "./patches/investigated/update-webpack-chunks-file";
 
 /**
  * Using the Next.js build output in the `.next` directory builds a workerd compatible output
@@ -155,49 +156,6 @@ async function updateWorkerBundledCode(
   await writeFile(workerOutputFile, patchedCode);
 }
 
-/**
- * Fixes the webpack-runtime.js file by removing its webpack dynamic requires.
- *
- * This hack is especially bad for two reasons:
- *  - it requires setting `experimental.serverMinification` to `false` in the app's config file
- *  - indicates that files inside the output directory still get a hold of files from the outside: `${nextjsAppPaths.standaloneAppServerDir}/webpack-runtime.js`
- *    so this shows that not everything that's needed to deploy the application is in the output directory...
- */
-async function updateWebpackChunksFile(nextjsAppPaths: NextjsAppPaths) {
-  console.log("# updateWebpackChunksFile");
-  const webpackRuntimeFile = `${nextjsAppPaths.standaloneAppServerDir}/webpack-runtime.js`;
-
-  console.log({ webpackRuntimeFile });
-
-  const fileContent = readFileSync(webpackRuntimeFile, "utf-8");
-
-  const chunks = readdirSync(`${nextjsAppPaths.standaloneAppServerDir}/chunks`)
-    .filter((chunk) => /^\d+\.js$/.test(chunk))
-    .map((chunk) => {
-      console.log(` - chunk ${chunk}`);
-      return chunk.replace(/\.js$/, "");
-    });
-
-  const updatedFileContent = fileContent.replace(
-    "__webpack_require__.f.require = (chunkId, promises) => {",
-    `__webpack_require__.f.require = (chunkId, promises) => {
-	if (installedChunks[chunkId]) return;
-	${chunks
-    .map(
-      (chunk) => `
-	  if (chunkId === ${chunk}) {
-		installChunk(require("./chunks/${chunk}.js"));
-		return;
-	  }
-	`
-    )
-    .join("\n")}
-  `
-  );
-
-  writeFileSync(webpackRuntimeFile, updatedFileContent);
-}
-
 function createFixRequiresESBuildPlugin(templateDir: string): Plugin {
   return {
     name: "replaceRelative",

builder/src/build/patches/investigated/update-webpack-chunks-file/get-chunk-installation-identifiers.test.ts

@@ -0,0 +1,30 @@
+import { readFile } from "node:fs/promises";
+
+import { expect, test, describe } from "vitest";
+
+import { getChunkInstallationIdentifiers } from "./get-chunk-installation-identifiers";
+import { tsParseFile } from "../../../utils";
+
+describe("getChunkInstallationIdentifiers", () => {
+  test("gets chunk identifiers from unminified code", async () => {
+    const fileContent = await readFile(
+      `${import.meta.dirname}/test-fixtures/unminified-webpacks-file.js`,
+      "utf8"
+    );
+    const tsSourceFile = tsParseFile(fileContent);
+    const { installChunk, installedChunks } = await getChunkInstallationIdentifiers(tsSourceFile);
+    expect(installChunk).toEqual("installChunk");
+    expect(installedChunks).toEqual("installedChunks");
+  });
+
+  test("gets chunk identifiers from minified code", async () => {
+    const fileContent = await readFile(
+      `${import.meta.dirname}/test-fixtures/minified-webpacks-file.js`,
+      "utf8"
+    );
+    const tsSourceFile = tsParseFile(fileContent);
+    const { installChunk, installedChunks } = await getChunkInstallationIdentifiers(tsSourceFile);
+    expect(installChunk).toEqual("r");
+    expect(installedChunks).toEqual("e");
+  });
+});

builder/src/build/patches/investigated/update-webpack-chunks-file/get-chunk-installation-identifiers.ts

@@ -0,0 +1,102 @@
+import * as ts from "ts-morph";
+
+/**
+ * Gets the names of the variables that in the unminified webpack runtime file are called `installedChunks` and `installChunk`.
+ *
+ * Variables example: https://github.com/webpack/webpack/blob/dae16ad11e/examples/module-worker/README.md?plain=1#L256-L282
+ *
+ * @param sourceFile the webpack runtime file parsed with ts-morph
+ * @returns an object containing the two variable names
+ */
+export async function getChunkInstallationIdentifiers(sourceFile: ts.SourceFile): Promise<{
+  installedChunks: string;
+  installChunk: string;
+}> {
+  const installChunkDeclaration = getInstallChunkDeclaration(sourceFile);
+  const installedChunksDeclaration = getInstalledChunksDeclaration(sourceFile, installChunkDeclaration);
+
+  return {
+    installChunk: installChunkDeclaration.getName(),
+    installedChunks: installedChunksDeclaration.getName(),
+  };
+}
+
+/**
+ * Gets the declaration for what in the unminified webpack runtime file is called `installChunk`(which is a function that registers the various chunks.
+ *
+ * `installChunk` example: https://github.com/webpack/webpack/blob/dae16ad11e/examples/module-worker/README.md?plain=1#L263-L282
+ *
+ * @param sourceFile the webpack runtime file parsed with ts-morph
+ * @returns the `installChunk` declaration
+ */
+function getInstallChunkDeclaration(sourceFile: ts.SourceFile): ts.VariableDeclaration {
+  const installChunkDeclaration = sourceFile
+    .getDescendantsOfKind(ts.SyntaxKind.VariableDeclaration)
+    .find((declaration) => {
+      const arrowFunction = declaration.getInitializerIfKind(ts.SyntaxKind.ArrowFunction);
+      // we're looking for an arrow function
+      if (!arrowFunction) return false;
+
+      const functionParameters = arrowFunction.getParameters();
+      // the arrow function we're looking for has a single parameter (the chunkId)
+      if (functionParameters.length !== 1) return false;
+
+      const arrowFunctionBodyBlock = arrowFunction.getFirstChildByKind(ts.SyntaxKind.Block);
+
+      // the arrow function we're looking for has a block body
+      if (!arrowFunctionBodyBlock) return false;
+
+      const statementKinds = arrowFunctionBodyBlock.getStatements().map((statement) => statement.getKind());
+
+      // the function we're looking for has 2 for loops (a standard one and a for-in one)
+      const forInStatements = statementKinds.filter((s) => s === ts.SyntaxKind.ForInStatement);
+      const forStatements = statementKinds.filter((s) => s === ts.SyntaxKind.ForStatement);
+      if (forInStatements.length !== 1 || forStatements.length !== 1) return false;
+
+      // the function we're looking for accesses its parameter three times, and it
+      // accesses its `modules`, `ids` and `runtime` properties (in this order)
+      const parameterName = functionParameters[0].getText();
+      const functionParameterAccessedProperties = arrowFunctionBodyBlock
+        .getDescendantsOfKind(ts.SyntaxKind.PropertyAccessExpression)
+        .filter(
+          (propertyAccessExpression) => propertyAccessExpression.getExpression().getText() === parameterName
+        )
+        .map((propertyAccessExpression) => propertyAccessExpression.getName());
+      if (functionParameterAccessedProperties.join(", ") !== "modules, ids, runtime") return false;
+
+      return true;
+    });
+
+  if (!installChunkDeclaration) {
+    throw new Error("ERROR: unable to find the installChunk function declaration");
+  }
+
+  return installChunkDeclaration;
+}
+
+/**
+ * Gets the declaration for what in the unminified webpack runtime file is called `installedChunks` which is an object that holds the various registered chunks.
+ *
+ * `installedChunks` example: https://github.com/webpack/webpack/blob/dae16ad11e/examples/module-worker/README.md?plain=1#L256-L261
+ *
+ * @param sourceFile the webpack runtime file parsed with ts-morph
+ * @param installChunkDeclaration the declaration for the `installChunk` variable
+ * @returns the `installedChunks` declaration
+ */
+function getInstalledChunksDeclaration(
+  sourceFile: ts.SourceFile,
+  installChunkDeclaration: ts.VariableDeclaration
+): ts.VariableDeclaration {
+  const allVariableDeclarations = sourceFile.getDescendantsOfKind(ts.SyntaxKind.VariableDeclaration);
+  const installChunkDeclarationIdx = allVariableDeclarations.findIndex(
+    (declaration) => declaration === installChunkDeclaration
+  );
+
+  // the installedChunks declaration comes right before the installChunk one
+  const installedChunksDeclaration = allVariableDeclarations[installChunkDeclarationIdx - 1];
+
+  if (!installedChunksDeclaration?.getInitializer()?.isKind(ts.SyntaxKind.ObjectLiteralExpression)) {
+    throw new Error("ERROR: unable to find the installedChunks declaration");
+  }
+  return installedChunksDeclaration;
+}

builder/src/build/patches/investigated/update-webpack-chunks-file/get-file-content-with-updated-webpack-f-require-code.test.ts

@@ -0,0 +1,44 @@
+import { readFile } from "node:fs/promises";
+
+import { expect, test, describe } from "vitest";
+
+import { getFileContentWithUpdatedWebpackFRequireCode } from "./get-file-content-with-updated-webpack-f-require-code";
+import { tsParseFile } from "../../../utils";
+
+describe("getFileContentWithUpdatedWebpackFRequireCode", () => {
+  test("returns the updated content of the f.require function from unminified webpack runtime code", async () => {
+    const fileContent = await readFile(
+      `${import.meta.dirname}/test-fixtures/unminified-webpacks-file.js`,
+      "utf8"
+    );
+    const tsSourceFile = tsParseFile(fileContent);
+    const updatedFCode = await getFileContentWithUpdatedWebpackFRequireCode(
+      tsSourceFile,
+      { installChunk: "installChunk", installedChunks: "installedChunks" },
+      ["658"]
+    );
+    expect(unstyleCode(updatedFCode)).toContain(`if (installedChunks[chunkId]) return;`);
+    expect(unstyleCode(updatedFCode)).toContain(
+      `if (chunkId === 658) return installChunk(require("./chunks/658.js"));`
+    );
+  });
+
+  test("returns the updated content of the f.require function from minified webpack runtime code", async () => {
+    const fileContent = await readFile(
+      `${import.meta.dirname}/test-fixtures/minified-webpacks-file.js`,
+      "utf8"
+    );
+    const tsSourceFile = tsParseFile(fileContent);
+    const updatedFCode = await getFileContentWithUpdatedWebpackFRequireCode(
+      tsSourceFile,
+      { installChunk: "r", installedChunks: "e" },
+      ["658"]
+    );
+    expect(unstyleCode(updatedFCode)).toContain("if (e[o]) return;");
+    expect(unstyleCode(updatedFCode)).toContain(`if (o === 658) return r(require("./chunks/658.js"));`);
+  });
+});
+
+function unstyleCode(text: string): string {
+  return text.replace(/\n\s+/g, "\n").replace(/\n/g, " ");
+}