Copy templates to standalone app

Author: vicb

TODO.txt

@@ -0,0 +1,26 @@
+TODO:
+
+- [] copy the template folders
+- [] figure out the assets
+- [] dependency graph
+
+DONE:
+
+
+-----
+
+wrangler alias
+
+-> % cp -r .next/standalone/node_modules/.pnpm/[email protected]_@[email protected][email protected][email protected][email protected]/node_modules/next .next/standalone/next/api/node_modules/
+-> % cp -r .next/standalone/node_modules/.pnpm/[email protected]/node_modules/react .next/standalone/next/api/node_modules/
+
+find -L . -type l -ls
+
+symlinks -rv
+
+----
+
+Need to add "node-url": "npm:url@^0.11.4"
+
+---
+

builder/src/build/build-worker/index.ts

@@ -3,20 +3,15 @@ import { build, Plugin } from "esbuild";
 import { readdirSync, readFileSync, writeFileSync } from "node:fs";
 import { cp, readFile, writeFile } from "node:fs/promises";
 
-import { resolve } from "node:path";
-
 import { patchRequire } from "./patches/investigated/patchRequire";
 import { patchUrl } from "./patches/investigated/patchUrl";
+import { copyTemplates } from "./patches/investigated/copyTemplates";
 
 import { patchReadFile } from "./patches/to-investigate/patchReadFile";
 import { patchFindDir } from "./patches/to-investigate/patchFindDir";
 import { inlineNextRequire } from "./patches/to-investigate/inlineNextRequire";
 import { inlineEvalManifest } from "./patches/to-investigate/inlineEvalManifest";
 
-import * as url from "url";
-
-const __dirname = url.fileURLToPath(new URL(".", import.meta.url));
-
 /**
  * Using the Next.js build output in the `.next` directory builds a workerd compatible output
  *
@@ -25,11 +20,12 @@ const __dirname = url.fileURLToPath(new URL(".", import.meta.url));
  */
 export async function buildWorker(
 	outputDir: string,
-	nextjsAppPaths: NextjsAppPaths
+	nextjsAppPaths: NextjsAppPaths,
+	templateSrcDir: string
 ): Promise<void> {
-	const repoRoot = resolve(`${__dirname}/../..`);
+	const templateDir = copyTemplates(templateSrcDir, nextjsAppPaths);
 
-	const workerEntrypoint = `${__dirname}/templates/worker.ts`;
+	const workerEntrypoint = `${templateDir}/worker.ts`;
 	const workerOutputFile = `${outputDir}/index.mjs`;
 	const nextConfigStr =
 		readFileSync(nextjsAppPaths.standaloneAppDir + "/server.js", "utf8")?.match(
@@ -44,27 +40,27 @@ export async function buildWorker(
 		format: "esm",
 		target: "esnext",
 		minify: false,
-		plugins: [fixRequiresESBuildPlugin],
+		plugins: [createFixRequiresESBuildPlugin(templateDir)],
 		alias: {
 			// Note: we apply an empty shim to next/dist/compiled/ws because it generates two `eval`s:
 			//   eval("require")("bufferutil");
 			//   eval("require")("utf-8-validate");
-			"next/dist/compiled/ws": `${__dirname}/templates/shims/empty.ts`,
+			"next/dist/compiled/ws": `${templateDir}/shims/empty.ts`,
 			// Note: we apply an empty shim to next/dist/compiled/edge-runtime since (amongst others) it generated the following `eval`:
 			//   eval(getModuleCode)(module, module.exports, throwingRequire, params.context, ...Object.values(params.scopedContext));
 			//   which comes from https://github.com/vercel/edge-runtime/blob/6e96b55f/packages/primitives/src/primitives/load.js#L57-L63
 			// QUESTION: Why did I encountered this but mhart didn't?
-			"next/dist/compiled/edge-runtime": `${__dirname}/templates/shims/empty.ts`,
+			"next/dist/compiled/edge-runtime": `${templateDir}/shims/empty.ts`,
 			// Note: we need to stub out `@opentelemetry/api` as that is problematic and doesn't get properly bundled...
-			critters: `${__dirname}/templates/shims/empty.ts`,
+			critters: `${templateDir}/shims/empty.ts`,
 			// Note: we need to stub out `@opentelemetry/api` as it is problematic
 			// IMPORTANT: we shim @opentelemetry/api to the throwing shim so that it will throw right away, this is so that we throw inside the
 			//            try block here: https://github.com/vercel/next.js/blob/9e8266a7/packages/next/src/server/lib/trace/tracer.ts#L27-L31
 			//            causing the code to require the 'next/dist/compiled/@opentelemetry/api' module instead (which properly works)
-			"@opentelemetry/api": `${__dirname}/templates/shims/throw.ts`,
+			"@opentelemetry/api": `${templateDir}/shims/throw.ts`,
 			// `@next/env` is a library Next.js uses for loading dotenv files, for obvious reasons we need to stub it here
 			// source: https://github.com/vercel/next.js/tree/0ac10d79720/packages/next-env
-			"@next/env": `${__dirname}/templates/shims/env.ts`,
+			"@next/env": `${templateDir}/shims/env.ts`,
 		},
 		define: {
 			// config file used by Next.js, see: https://github.com/vercel/next.js/blob/68a7128/packages/next/src/build/utils.ts#L2137-L2139
@@ -147,13 +143,19 @@ async function updateWorkerBundledCode(
  *    so this shows that not everything that's needed to deploy the application is in the output directory...
  */
 async function updateWebpackChunksFile(nextjsAppPaths: NextjsAppPaths) {
+	console.log("# updateWebpackChunksFile");
 	const webpackRuntimeFile = `${nextjsAppPaths.standaloneAppServerDir}/webpack-runtime.js`;
 
+	console.log({ webpackRuntimeFile });
+
 	const fileContent = readFileSync(webpackRuntimeFile, "utf-8");
 
 	const chunks = readdirSync(`${nextjsAppPaths.standaloneAppServerDir}/chunks`)
 		.filter((chunk) => /^\d+\.js$/.test(chunk))
-		.map((chunk) => chunk.replace(/\.js$/, ""));
+		.map((chunk) => {
+			console.log(` - chunk ${chunk}`);
+			return chunk.replace(/\.js$/, "");
+		});
 
 	const updatedFileContent = fileContent.replace(
 		"__webpack_require__.f.require = (chunkId, promises) => {",
@@ -175,12 +177,14 @@ async function updateWebpackChunksFile(nextjsAppPaths: NextjsAppPaths) {
 	writeFileSync(webpackRuntimeFile, updatedFileContent);
 }
 
-const fixRequiresESBuildPlugin: Plugin = {
-	name: "replaceRelative",
-	setup(build) {
-		// Note: we (empty) shim require-hook modules as they generate problematic code that uses requires
-		build.onResolve({ filter: /^\.\/require-hook$/ }, (args) => ({
-			path: `${__dirname}/templates/shims/empty.ts`,
-		}));
-	},
+function createFixRequiresESBuildPlugin(templateDir: string): Plugin {
+	return {
+		name: "replaceRelative",
+		setup(build) {
+			// Note: we (empty) shim require-hook modules as they generate problematic code that uses requires
+			build.onResolve({ filter: /^\.\/require-hook$/ }, (args) => ({
+				path: `${templateDir}/shims/empty.ts`,
+			}));
+		},
+	};
 };

builder/src/build/build-worker/patches/investigated/copyTemplates.ts

@@ -0,0 +1,20 @@
+import path from "node:path";
+import { NextjsAppPaths } from "builder/src/nextjsPaths";
+import { cpSync, mkdirSync } from "node:fs";
+
+/**
+ * Copy templates in the standalone folder.
+ *
+ * We need to have the template files locally to referenced package paths
+ * to resolve to files in the the node_module of the standalone app.=
+ */
+export function copyTemplates(srcDir: string, nextjsAppPaths: NextjsAppPaths) {
+	console.log("# copyTemplates");
+	const destDir = path.join(
+		nextjsAppPaths.standaloneAppDir,
+		"node_modules/cf/templates"
+	);
+
+	cpSync(srcDir, destDir, { recursive: true });
+	return destDir;
+}

builder/src/build/build-worker/patches/investigated/patchRequire.ts

@@ -5,6 +5,7 @@
  * James on Aug 29: `module.createRequire()` is planned.
  */
 export function patchRequire(code: string): string {
+	console.log("# patchRequire");
 	return code
 		.replace(/__require\d?\(/g, "require(")
 		.replace(/__require\d?\./g, "require.");

builder/src/build/build-worker/patches/investigated/patchUrl.ts

@@ -5,6 +5,7 @@
  * Hopefully this should not be necessary after this unenv PR lands: https://github.com/unjs/unenv/pull/292
  */
 export function patchUrl(code: string): string {
+  console.log("# patchUrl");
 	return code.replace(
 		/ ([a-zA-Z0-9_]+) = require\("url"\);/g,
 		` $1 = require("url");

builder/src/build/build-worker/patches/to-investigate/inlineEvalManifest.ts

@@ -12,6 +12,7 @@ export function inlineEvalManifest(
 	code: string,
 	nextjsAppPaths: NextjsAppPaths
 ): string {
+	console.log("# inlineEvalManifest");
 	const manifestJss = globSync(
 		`${nextjsAppPaths.standaloneAppDotNextDir}/**/*_client-reference-manifest.js`
 	).map((file) => file.replace(`${nextjsAppPaths.standaloneAppDir}/`, ""));

builder/src/build/build-worker/patches/to-investigate/inlineNextRequire.ts

@@ -9,6 +9,7 @@ export function inlineNextRequire(
 	code: string,
 	nextjsAppPaths: NextjsAppPaths
 ) {
+	console.log("# inlineNextRequire");
 	const pagesManifestFile = `${nextjsAppPaths.standaloneAppServerDir}/pages-manifest.json`;
 	const appPathsManifestFile = `${nextjsAppPaths.standaloneAppServerDir}/app-paths-manifest.json`;
 

builder/src/build/build-worker/patches/to-investigate/patchFindDir.ts

@@ -11,6 +11,7 @@ export function patchFindDir(
 	code: string,
 	nextjsAppPaths: NextjsAppPaths
 ): string {
+	console.log("# patchFindDir");
 	return code.replace(
 		"function findDir(dir, name) {",
 		`function findDir(dir, name) {

builder/src/build/build-worker/patches/to-investigate/patchReadFile.ts

@@ -6,6 +6,7 @@ export function patchReadFile(
 	code: string,
 	nextjsAppPaths: NextjsAppPaths
 ): string {
+	console.log("# patchReadFile");
 	// The next-server code gets the buildId from the filesystem, resulting in a `[unenv] fs.readFileSync is not implemented yet!` error
 	// so we add an early return to the `getBuildId` function so that the `readyFileSync` is never encountered
 	// (source: https://github.com/vercel/next.js/blob/15aeb92efb34c09a36/packages/next/src/server/next-server.ts#L438-L451)

builder/src/build/index.ts

@@ -2,6 +2,9 @@ import { rm } from "node:fs/promises";
 import { buildNextjsApp } from "./build-next-app";
 import { buildWorker } from "./build-worker";
 import { getNextjsAppPaths } from "../nextjsPaths";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { cpSync, rmSync } from "node:fs";
 
 /**
  * Builds the application in a format that can be passed to workerd
@@ -17,14 +20,32 @@ export async function build(
 	opts: BuildOptions
 ): Promise<void> {
 	if (!opts.skipBuild) {
+		// Build the next app and save a copy in .next.save
 		buildNextjsApp(inputNextAppDir);
+		cpSync(`${inputNextAppDir}/.next`, `${inputNextAppDir}/save.next`, {
+			recursive: true,
+		});
+	} else {
+		// Skip the next build and restore the copy from .next.save
+		rmSync(`${inputNextAppDir}/.next`, { recursive: true, force: true });
+		cpSync(`${inputNextAppDir}/save.next`, `${inputNextAppDir}/.next`, {
+			recursive: true,
+		});
 	}
 
 	const outputDir = `${opts.outputDir ?? inputNextAppDir}/.worker-next`;
-	cleanDirectory(outputDir);
+	await cleanDirectory(outputDir);
 
 	const nextjsAppPaths = getNextjsAppPaths(inputNextAppDir);
-	await buildWorker(outputDir, nextjsAppPaths);
+
+	const templateDir = path.join(
+		path.dirname(fileURLToPath(import.meta.url)),
+		"templates"
+	);
+
+	console.log({ outputDir, nextjsAppPaths, templateDir });
+
+	await buildWorker(outputDir, nextjsAppPaths, templateDir);
 }
 
 type BuildOptions = {