implement webpack chunks file updating using ast manipulation

this allows us not to have to require the use of the `serverMinification: false` option
(as we're no longer relying on known variable names)

Author: dario-piotrowicz

.prettierignore

@@ -2,3 +2,5 @@
 .wrangler
 pnpm-lock.yaml
 .vscode/setting.json
+test-fixtures
+test-snapshots

builder/package.json

@@ -4,7 +4,9 @@
   "version": "0.0.1",
   "scripts": {
     "build": "tsup",
-    "build:watch": "tsup --watch src"
+    "build:watch": "tsup --watch src",
+    "test": "vitest --run",
+    "test:watch": "vitest"
   },
   "bin": "dist/index.mjs",
   "files": [
@@ -33,6 +35,10 @@
     "glob": "^11.0.0",
     "next": "14.2.5",
     "tsup": "^8.2.4",
-    "typescript": "^5.5.4"
+    "typescript": "^5.5.4",
+    "vitest": "^2.1.1"
+  },
+  "dependencies": {
+    "ts-morph": "^23.0.0"
   }
 }

builder/src/build/build-worker.ts

@@ -1,6 +1,6 @@
 import { NextjsAppPaths } from "../nextjs-paths";
 import { build, Plugin } from "esbuild";
-import { readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { readFileSync } from "node:fs";
 import { cp, readFile, writeFile } from "node:fs/promises";
 
 import { patchRequire } from "./patches/investigated/patch-require";
@@ -11,6 +11,7 @@ import { patchFindDir } from "./patches/to-investigate/patch-find-dir";
 import { inlineNextRequire } from "./patches/to-investigate/inline-next-require";
 import { inlineEvalManifest } from "./patches/to-investigate/inline-eval-manifest";
 import { patchWranglerDeps } from "./patches/to-investigate/wrangler-deps";
+import { updateWebpackChunksFile } from "./patches/investigated/update-webpack-chunks-file";
 
 /**
  * Using the Next.js build output in the `.next` directory builds a workerd compatible output
@@ -155,49 +156,6 @@ async function updateWorkerBundledCode(
   await writeFile(workerOutputFile, patchedCode);
 }
 
-/**
- * Fixes the webpack-runtime.js file by removing its webpack dynamic requires.
- *
- * This hack is especially bad for two reasons:
- *  - it requires setting `experimental.serverMinification` to `false` in the app's config file
- *  - indicates that files inside the output directory still get a hold of files from the outside: `${nextjsAppPaths.standaloneAppServerDir}/webpack-runtime.js`
- *    so this shows that not everything that's needed to deploy the application is in the output directory...
- */
-async function updateWebpackChunksFile(nextjsAppPaths: NextjsAppPaths) {
-  console.log("# updateWebpackChunksFile");
-  const webpackRuntimeFile = `${nextjsAppPaths.standaloneAppServerDir}/webpack-runtime.js`;
-
-  console.log({ webpackRuntimeFile });
-
-  const fileContent = readFileSync(webpackRuntimeFile, "utf-8");
-
-  const chunks = readdirSync(`${nextjsAppPaths.standaloneAppServerDir}/chunks`)
-    .filter((chunk) => /^\d+\.js$/.test(chunk))
-    .map((chunk) => {
-      console.log(` - chunk ${chunk}`);
-      return chunk.replace(/\.js$/, "");
-    });
-
-  const updatedFileContent = fileContent.replace(
-    "__webpack_require__.f.require = (chunkId, promises) => {",
-    `__webpack_require__.f.require = (chunkId, promises) => {
-	if (installedChunks[chunkId]) return;
-	${chunks
-    .map(
-      (chunk) => `
-	  if (chunkId === ${chunk}) {
-		installChunk(require("./chunks/${chunk}.js"));
-		return;
-	  }
-	`
-    )
-    .join("\n")}
-  `
-  );
-
-  writeFileSync(webpackRuntimeFile, updatedFileContent);
-}
-
 function createFixRequiresESBuildPlugin(templateDir: string): Plugin {
   return {
     name: "replaceRelative",

builder/src/build/patches/investigated/update-webpack-chunks-file/get-chunk-installation-identifiers.test.ts

@@ -0,0 +1,30 @@
+import { readFile } from "node:fs/promises";
+
+import { expect, test, describe } from "vitest";
+
+import { getChunkInstallationIdentifiers } from "./get-chunk-installation-identifiers";
+import { getWebpackChunksFileTsSource } from "./get-webpack-chunks-file-ts-source";
+
+describe("getChunkInstallationIdentifiers", () => {
+  test("gets chunk identifiers from unminified code", async () => {
+    const fileContent = await readFile(
+      `${import.meta.dirname}/test-fixtures/unminified-webpacks-file.js`,
+      "utf8"
+    );
+    const tsSourceFile = getWebpackChunksFileTsSource(fileContent);
+    const { installChunk, installedChunks } = await getChunkInstallationIdentifiers(tsSourceFile);
+    expect(installChunk).toEqual("installChunk");
+    expect(installedChunks).toEqual("installedChunks");
+  });
+
+  test("gets chunk identifiers from minified code", async () => {
+    const fileContent = await readFile(
+      `${import.meta.dirname}/test-fixtures/minified-webpacks-file.js`,
+      "utf8"
+    );
+    const tsSourceFile = getWebpackChunksFileTsSource(fileContent);
+    const { installChunk, installedChunks } = await getChunkInstallationIdentifiers(tsSourceFile);
+    expect(installChunk).toEqual("r");
+    expect(installedChunks).toEqual("e");
+  });
+});

builder/src/build/patches/investigated/update-webpack-chunks-file/get-chunk-installation-identifiers.ts

@@ -0,0 +1,77 @@
+import * as ts from "ts-morph";
+
+export async function getChunkInstallationIdentifiers(sourceFile: ts.SourceFile): Promise<{
+  installedChunks: string;
+  installChunk: string;
+}> {
+  const installChunkDeclaration = getInstallChunkDeclaration(sourceFile);
+  const installedChunksDeclaration = getInstalledChunksDeclaration(sourceFile, installChunkDeclaration);
+
+  return {
+    installChunk: installChunkDeclaration.getName(),
+    installedChunks: installedChunksDeclaration.getName(),
+  };
+}
+
+function getInstallChunkDeclaration(sourceFile: ts.SourceFile): ts.VariableDeclaration {
+  const installChunkDeclaration = sourceFile
+    .getDescendantsOfKind(ts.SyntaxKind.VariableDeclaration)
+    .find((declaration) => {
+      const arrowFunction = declaration.getInitializerIfKind(ts.SyntaxKind.ArrowFunction);
+      // we're looking for an arrow function
+      if (!arrowFunction) return false;
+
+      const functionParameters = arrowFunction.getParameters();
+      // the arrow function we're looking for has a single parameter (the chunkId)
+      if (functionParameters.length !== 1) return false;
+
+      const arrowFunctionBodyBlock = arrowFunction.getFirstChildByKind(ts.SyntaxKind.Block);
+
+      // the arrow function we're looking for has a block body
+      if (!arrowFunctionBodyBlock) return false;
+
+      const statementKinds = arrowFunctionBodyBlock.getStatements().map((statement) => statement.getKind());
+
+      // the function we're looking for has 2 for loops (a standard one and a for-in one)
+      const forInStatements = statementKinds.filter((s) => s === ts.SyntaxKind.ForInStatement);
+      const forStatements = statementKinds.filter((s) => s === ts.SyntaxKind.ForStatement);
+      if (forInStatements.length !== 1 || forStatements.length !== 1) return false;
+
+      // the function we're looking for accesses its parameter three times, and it
+      // accesses its `modules`, `ids` and `runtime` properties (in this order)
+      const parameterName = functionParameters[0].getText();
+      const functionParameterAccessedProperties = arrowFunctionBodyBlock
+        .getDescendantsOfKind(ts.SyntaxKind.PropertyAccessExpression)
+        .filter(
+          (propertyAccessExpression) => propertyAccessExpression.getExpression().getText() === parameterName
+        )
+        .map((propertyAccessExpression) => propertyAccessExpression.getName());
+      if (functionParameterAccessedProperties.join(", ") !== "modules, ids, runtime") return false;
+
+      return true;
+    });
+
+  if (!installChunkDeclaration) {
+    throw new Error("ERROR: unable to find the installChunk function declaration");
+  }
+
+  return installChunkDeclaration;
+}
+
+function getInstalledChunksDeclaration(
+  sourceFile: ts.SourceFile,
+  installChunkDeclaration: ts.VariableDeclaration
+): ts.VariableDeclaration {
+  const allVariableDeclarations = sourceFile.getDescendantsOfKind(ts.SyntaxKind.VariableDeclaration);
+  const installChunkDeclarationIdx = allVariableDeclarations.findIndex(
+    (declaration) => declaration === installChunkDeclaration
+  );
+
+  // the installedChunks declaration is comes right before the installChunk one
+  const installedChunksDeclaration = allVariableDeclarations[installChunkDeclarationIdx - 1];
+
+  if (!installedChunksDeclaration?.getInitializer()?.isKind(ts.SyntaxKind.ObjectLiteralExpression)) {
+    throw new Error("ERROR: unable to find the installedChunks declaration");
+  }
+  return installedChunksDeclaration;
+}

builder/src/build/patches/investigated/update-webpack-chunks-file/get-file-content-with-updated-webpack-f-require-code.test.ts

@@ -0,0 +1,44 @@
+import { readFile } from "node:fs/promises";
+
+import { expect, test, describe } from "vitest";
+
+import { getFileContentWithUpdatedWebpackFRequireCode } from "./get-file-content-with-updated-webpack-f-require-code";
+import { getWebpackChunksFileTsSource } from "./get-webpack-chunks-file-ts-source";
+
+describe("getFileContentWithUpdatedWebpackFRequireCode", () => {
+  test("returns the updated content of the f.require function from unminified webpack runtime code", async () => {
+    const fileContent = await readFile(
+      `${import.meta.dirname}/test-fixtures/unminified-webpacks-file.js`,
+      "utf8"
+    );
+    const tsSourceFile = getWebpackChunksFileTsSource(fileContent);
+    const updatedFCode = await getFileContentWithUpdatedWebpackFRequireCode(
+      tsSourceFile,
+      { installChunk: "installChunk", installedChunks: "installedChunks" },
+      ["658"]
+    );
+    expect(unstyleCode(updatedFCode)).toContain(`if (installedChunks[chunkId]) return;`);
+    expect(unstyleCode(updatedFCode)).toContain(
+      `if (chunkId === 658) return installChunk(require("./chunks/658.js"));`
+    );
+  });
+
+  test("returns the updated content of the f.require function from minified webpack runtime code", async () => {
+    const fileContent = await readFile(
+      `${import.meta.dirname}/test-fixtures/minified-webpacks-file.js`,
+      "utf8"
+    );
+    const tsSourceFile = getWebpackChunksFileTsSource(fileContent);
+    const updatedFCode = await getFileContentWithUpdatedWebpackFRequireCode(
+      tsSourceFile,
+      { installChunk: "r", installedChunks: "e" },
+      ["658"]
+    );
+    expect(unstyleCode(updatedFCode)).toContain("if (e[o]) return;");
+    expect(unstyleCode(updatedFCode)).toContain(`if (o === 658) return r(require("./chunks/658.js"));`);
+  });
+});
+
+function unstyleCode(text: string): string {
+  return text.replace(/\n\s+/g, "\n").replace(/\n/g, " ");
+}

builder/src/build/patches/investigated/update-webpack-chunks-file/get-file-content-with-updated-webpack-f-require-code.ts

@@ -0,0 +1,73 @@
+import * as ts from "ts-morph";
+
+export async function getFileContentWithUpdatedWebpackFRequireCode(
+  sourceFile: ts.SourceFile,
+  { installChunk, installedChunks }: { installChunk: string; installedChunks: string },
+  chunks: string[]
+): Promise<string> {
+  const webpackFRequireFunction = sourceFile
+    .getDescendantsOfKind(ts.SyntaxKind.BinaryExpression)
+    .map((binaryExpression) => {
+      const binaryExpressionLeft = binaryExpression.getLeft();
+      if (!binaryExpressionLeft.getText().endsWith(".f.require")) return;
+
+      const binaryExpressionOperator = binaryExpression.getOperatorToken();
+      if (binaryExpressionOperator.getText() !== "=") return;
+
+      const binaryExpressionRight = binaryExpression.getRight();
+      const binaryExpressionRightText = binaryExpressionRight.getText();
+
+      const functionUsesChunkInstallationVariables =
+        binaryExpressionRightText.includes(installChunk) &&
+        binaryExpressionRightText.includes(installedChunks);
+      if (!functionUsesChunkInstallationVariables) return;
+
+      if (!binaryExpressionRight.isKind(ts.SyntaxKind.ArrowFunction)) return;
+
+      const arrowFunctionBody = binaryExpressionRight.getBody();
+      if (!arrowFunctionBody.isKind(ts.SyntaxKind.Block)) return;
+
+      const arrowFunction = binaryExpressionRight;
+      const functionParameters = arrowFunction.getParameters();
+      if (functionParameters.length !== 2) return;
+
+      const callsInstallChunk = arrowFunctionBody
+        .getDescendantsOfKind(ts.SyntaxKind.CallExpression)
+        .some((callExpression) => callExpression.getExpression().getText() === installChunk);
+      if (!callsInstallChunk) return;
+
+      const functionFirstParameterName = functionParameters[0]?.getName();
+      const accessesInstalledChunksUsingItsFirstParameter = arrowFunctionBody
+        .getDescendantsOfKind(ts.SyntaxKind.ElementAccessExpression)
+        .some((elementAccess) => {
+          return (
+            elementAccess.getExpression().getText() === installedChunks &&
+            elementAccess.getArgumentExpression()?.getText() === functionFirstParameterName
+          );
+        });
+      if (!accessesInstalledChunksUsingItsFirstParameter) return;
+
+      return arrowFunction;
+    })
+    .find(Boolean);
+
+  if (!webpackFRequireFunction) {
+    throw new Error("ERROR: unable to find the webpack f require function declaration");
+  }
+
+  const functionParameterNames = webpackFRequireFunction
+    .getParameters()
+    .map((parameter) => parameter.getName());
+  const chunkId = functionParameterNames[0];
+
+  const functionBody = webpackFRequireFunction.getBody() as ts.Block;
+
+  functionBody.insertStatements(0, [
+    `if (${installedChunks}[${chunkId}]) return;`,
+    ...chunks.map(
+      (chunk) => `\nif(${chunkId} === ${chunk}) return ${installChunk}(require("./chunks/${chunk}.js"));`
+    ),
+  ]);
+
+  return sourceFile.print();
+}

builder/src/build/patches/investigated/update-webpack-chunks-file/get-updated-webpack-chunks-file-content.test.ts

@@ -0,0 +1,25 @@
+import { readFile } from "node:fs/promises";
+
+import { expect, test, describe } from "vitest";
+
+import { getUpdatedWebpackChunksFileContent } from "./get-updated-webpack-chunks-file-content";
+
+describe("getUpdatedWebpackChunksFileContent", () => {
+  test("returns the updated content of a webpack runtime chunks unminified file", async () => {
+    const fileContent = await readFile(
+      `${import.meta.dirname}/test-fixtures/unminified-webpacks-file.js`,
+      "utf8"
+    );
+    const updatedContent = await getUpdatedWebpackChunksFileContent(fileContent, ["658"]);
+    expect(updatedContent).toMatchFileSnapshot("./test-snapshots/unminified-webpacks-file.js");
+  });
+
+  test("returns the updated content of a webpack runtime chunks minified file", async () => {
+    const fileContent = await readFile(
+      `${import.meta.dirname}/test-fixtures/minified-webpacks-file.js`,
+      "utf8"
+    );
+    const updatedContent = await getUpdatedWebpackChunksFileContent(fileContent, ["658"]);
+    expect(updatedContent).toMatchFileSnapshot("./test-snapshots/minified-webpacks-file.js");
+  });
+});

builder/src/build/patches/investigated/update-webpack-chunks-file/get-updated-webpack-chunks-file-content.ts

@@ -0,0 +1,22 @@
+import * as ts from "ts-morph";
+
+import { getChunkInstallationIdentifiers } from "./get-chunk-installation-identifiers";
+import { getFileContentWithUpdatedWebpackFRequireCode } from "./get-file-content-with-updated-webpack-f-require-code";
+import { getWebpackChunksFileTsSource } from "./get-webpack-chunks-file-ts-source";
+
+export async function getUpdatedWebpackChunksFileContent(
+  fileContent: string,
+  chunks: string[]
+): Promise<string> {
+  const tsSourceFile = getWebpackChunksFileTsSource(fileContent);
+
+  const chunkInstallationIdentifiers = await getChunkInstallationIdentifiers(tsSourceFile);
+
+  const updatedFileContent = getFileContentWithUpdatedWebpackFRequireCode(
+    tsSourceFile,
+    chunkInstallationIdentifiers,
+    chunks
+  );
+
+  return updatedFileContent;
+}

builder/src/build/patches/investigated/update-webpack-chunks-file/get-webpack-chunks-file-ts-source.ts

@@ -0,0 +1,17 @@
+import * as ts from "ts-morph";
+
+export function getWebpackChunksFileTsSource(fileContent: string): ts.SourceFile {
+  const project = new ts.Project({
+    compilerOptions: {
+      target: ts.ScriptTarget.ES2023,
+      lib: ["ES2023"],
+      module: ts.ModuleKind.CommonJS,
+      moduleResolution: ts.ModuleResolutionKind.NodeNext,
+      allowJs: true,
+    },
+  });
+
+  const sourceFile = project.createSourceFile("webpack-runtime.js", fileContent);
+
+  return sourceFile;
+}