refactor: use the traced file copy from OpenNext

Co-authored-by: Dario Piotrowicz <[email protected]>

Author: vicb

packages/cloudflare/package.json

@@ -73,7 +73,7 @@
   "dependencies": {
     "@ast-grep/napi": "^0.34.1",
     "@dotenvx/dotenvx": "catalog:",
-    "@opennextjs/aws": "https://pkg.pr.new/@opennextjs/aws@704",
+    "@opennextjs/aws": "https://pkg.pr.new/@opennextjs/aws@712",
     "enquirer": "^2.4.1",
     "glob": "catalog:",
     "ts-morph": "catalog:",

packages/cloudflare/src/cli/build/build.ts

@@ -1,6 +1,5 @@
-import { cpSync } from "node:fs";
 import { createRequire } from "node:module";
-import { dirname, join } from "node:path";
+import { dirname } from "node:path";
 
 import { buildNextjsApp, setStandaloneBuildMode } from "@opennextjs/aws/build/buildNextApp.js";
 import { compileCache } from "@opennextjs/aws/build/compileCache.js";
@@ -12,7 +11,7 @@ import { printHeader, showWarningOnWindows } from "@opennextjs/aws/build/utils.j
 import logger from "@opennextjs/aws/logger.js";
 
 import type { ProjectOptions } from "../config.js";
-import { containsDotNextDir, getConfig } from "../config.js";
+import { containsDotNextDir } from "../config.js";
 import { bundleServer } from "./bundle-server.js";
 import { compileEnvFiles } from "./open-next/compile-env-files.js";
 import { copyCacheAssets } from "./open-next/copyCacheAssets.js";
@@ -95,14 +94,7 @@ export async function build(projectOpts: ProjectOptions): Promise<void> {
 
   await createServerBundle(options);
 
-  // TODO: drop this copy.
-  // Copy the .next directory to the output directory so it can be mutated.
-  cpSync(join(projectOpts.sourceDir, ".next"), join(projectOpts.outputDir, ".next"), { recursive: true });
-
-  const projConfig = getConfig(projectOpts);
-
-  // TODO: rely on options only.
-  await bundleServer(projConfig, options);
+  await bundleServer(options);
 
   if (!projectOpts.skipWranglerConfigCheck) {
     await createWranglerConfigIfNotExistent(projectOpts);

packages/cloudflare/src/cli/build/bundle-server.ts

@@ -4,11 +4,10 @@ import path from "node:path";
 import { fileURLToPath } from "node:url";
 
 import { Lang, parse } from "@ast-grep/napi";
-import type { BuildOptions } from "@opennextjs/aws/build/helper.js";
+import { type BuildOptions, getPackagePath } from "@opennextjs/aws/build/helper.js";
 import { getCrossPlatformPathRegex } from "@opennextjs/aws/utils/regex.js";
 import { build, Plugin } from "esbuild";
 
-import { Config } from "../config.js";
 import { patchOptionalDependencies } from "./patches/ast/optional-deps.js";
 import * as patches from "./patches/index.js";
 import { normalizePath, patchCodeWithValidations } from "./utils/index.js";
@@ -19,22 +18,25 @@ const packageDistDir = path.join(path.dirname(fileURLToPath(import.meta.url)), "
 /**
  * Bundle the Open Next server.
  */
-export async function bundleServer(config: Config, openNextOptions: BuildOptions): Promise<void> {
-  patches.copyPackageCliFiles(packageDistDir, config, openNextOptions);
-
-  const nextConfigStr =
-    fs
-      .readFileSync(path.join(config.paths.output.standaloneApp, "server.js"), "utf8")
-      ?.match(/const nextConfig = ({.+?})\n/)?.[1] ?? {};
+export async function bundleServer(buildOpts: BuildOptions): Promise<void> {
+  patches.copyPackageCliFiles(packageDistDir, buildOpts);
+
+  const { appPath, outputDir, monorepoRoot } = buildOpts;
+  const serverFiles = path.join(
+    outputDir,
+    "server-functions/default",
+    getPackagePath(buildOpts),
+    ".next/required-server-files.json"
+  );
+  const nextConfig = JSON.parse(fs.readFileSync(serverFiles, "utf-8")).config;
 
   console.log(`\x1b[35m⚙️ Bundling the OpenNext server...\n\x1b[0m`);
 
-  patches.patchWranglerDeps(config);
-  patches.updateWebpackChunksFile(config);
+  patches.patchWranglerDeps(buildOpts);
+  patches.updateWebpackChunksFile(buildOpts);
 
-  const { appBuildOutputPath, appPath, outputDir, monorepoRoot } = openNextOptions;
   const outputPath = path.join(outputDir, "server-functions", "default");
-  const packagePath = path.relative(monorepoRoot, appBuildOutputPath);
+  const packagePath = getPackagePath(buildOpts);
   const openNextServer = path.join(outputPath, packagePath, `index.mjs`);
   const openNextServerBundle = path.join(outputPath, packagePath, `handler.mjs`);
 
@@ -45,25 +47,28 @@ export async function bundleServer(config: Config, openNextOptions: BuildOptions
     format: "esm",
     target: "esnext",
     minify: false,
-    plugins: [createFixRequiresESBuildPlugin(config)],
+    plugins: [createFixRequiresESBuildPlugin(buildOpts)],
     external: ["./middleware/handler.mjs", "caniuse-lite"],
     alias: {
       // Note: we apply an empty shim to next/dist/compiled/ws because it generates two `eval`s:
       //   eval("require")("bufferutil");
       //   eval("require")("utf-8-validate");
-      "next/dist/compiled/ws": path.join(config.paths.internal.templates, "shims", "empty.js"),
+      "next/dist/compiled/ws": path.join(buildOpts.outputDir, "cloudflare-templates/shims/empty.js"),
       // Note: we apply an empty shim to next/dist/compiled/edge-runtime since (amongst others) it generated the following `eval`:
       //   eval(getModuleCode)(module, module.exports, throwingRequire, params.context, ...Object.values(params.scopedContext));
       //   which comes from https://github.com/vercel/edge-runtime/blob/6e96b55f/packages/primitives/src/primitives/load.js#L57-L63
       // QUESTION: Why did I encountered this but mhart didn't?
-      "next/dist/compiled/edge-runtime": path.join(config.paths.internal.templates, "shims", "empty.js"),
+      "next/dist/compiled/edge-runtime": path.join(
+        buildOpts.outputDir,
+        "cloudflare-templates/shims/empty.js"
+      ),
       // `@next/env` is a library Next.js uses for loading dotenv files, for obvious reasons we need to stub it here
       // source: https://github.com/vercel/next.js/tree/0ac10d79720/packages/next-env
-      "@next/env": path.join(config.paths.internal.templates, "shims", "env.js"),
+      "@next/env": path.join(buildOpts.outputDir, "cloudflare-templates/shims/env.js"),
     },
     define: {
       // config file used by Next.js, see: https://github.com/vercel/next.js/blob/68a7128/packages/next/src/build/utils.ts#L2137-L2139
-      "process.env.__NEXT_PRIVATE_STANDALONE_CONFIG": JSON.stringify(nextConfigStr),
+      "process.env.__NEXT_PRIVATE_STANDALONE_CONFIG": `${JSON.stringify(nextConfig)}`,
       // Next.js tried to access __dirname so we need to define it
       __dirname: '""',
       // Note: we need the __non_webpack_require__ variable declared as it is used by next-server:
@@ -117,7 +122,7 @@ globalThis.__BUILD_TIMESTAMP_MS__ = ${Date.now()};
     },
   });
 
-  await updateWorkerBundledCode(openNextServerBundle, config, openNextOptions);
+  await updateWorkerBundledCode(openNextServerBundle, buildOpts);
 
   const isMonorepo = monorepoRoot !== appPath;
   if (isMonorepo) {
@@ -127,35 +132,26 @@ globalThis.__BUILD_TIMESTAMP_MS__ = ${Date.now()};
     );
   }
 
-  console.log(`\x1b[35mWorker saved in \`${getOutputWorkerPath(openNextOptions)}\` 🚀\n\x1b[0m`);
+  console.log(`\x1b[35mWorker saved in \`${getOutputWorkerPath(buildOpts)}\` 🚀\n\x1b[0m`);
 }
 
 /**
- * This function applies string replacements on the bundled worker code necessary to get it to run in workerd
- *
- * Needless to say all the logic in this function is something we should avoid as much as possible!
- *
- * @param workerOutputFile
- * @param config
+ * This function applies patches required for the code to run on workers.
  */
-async function updateWorkerBundledCode(
-  workerOutputFile: string,
-  config: Config,
-  openNextOptions: BuildOptions
-): Promise<void> {
+async function updateWorkerBundledCode(workerOutputFile: string, buildOpts: BuildOptions): Promise<void> {
   const code = await readFile(workerOutputFile, "utf8");
 
   const patchedCode = await patchCodeWithValidations(code, [
     ["require", patches.patchRequire],
-    ["`buildId` function", (code) => patches.patchBuildId(code, config)],
-    ["`loadManifest` function", (code) => patches.patchLoadManifest(code, config)],
-    ["next's require", (code) => patches.inlineNextRequire(code, config)],
-    ["`findDir` function", (code) => patches.patchFindDir(code, config)],
-    ["`evalManifest` function", (code) => patches.inlineEvalManifest(code, config)],
-    ["cacheHandler", (code) => patches.patchCache(code, openNextOptions)],
+    ["`buildId` function", (code) => patches.patchBuildId(code, buildOpts)],
+    ["`loadManifest` function", (code) => patches.patchLoadManifest(code, buildOpts)],
+    ["next's require", (code) => patches.inlineNextRequire(code, buildOpts)],
+    ["`findDir` function", (code) => patches.patchFindDir(code, buildOpts)],
+    ["`evalManifest` function", (code) => patches.inlineEvalManifest(code, buildOpts)],
+    ["cacheHandler", (code) => patches.patchCache(code, buildOpts)],
     [
       "'require(this.middlewareManifestPath)'",
-      (code) => patches.inlineMiddlewareManifestRequire(code, config),
+      (code) => patches.inlineMiddlewareManifestRequire(code, buildOpts),
     ],
     ["exception bubbling", patches.patchExceptionBubbling],
     ["`loadInstrumentationModule` function", patches.patchLoadInstrumentationModule],
@@ -185,15 +181,15 @@ async function updateWorkerBundledCode(
   await writeFile(workerOutputFile, bundle.commitEdits(edits));
 }
 
-function createFixRequiresESBuildPlugin(config: Config): Plugin {
+function createFixRequiresESBuildPlugin(options: BuildOptions): Plugin {
   return {
     name: "replaceRelative",
     setup(build) {
       // Note: we (empty) shim require-hook modules as they generate problematic code that uses requires
       build.onResolve(
         { filter: getCrossPlatformPathRegex(String.raw`^\./require-hook$`, { escape: false }) },
         () => ({
-          path: path.join(config.paths.internal.templates, "shims", "empty.js"),
+          path: path.join(options.outputDir, "cloudflare-templates/shims/empty.js"),
         })
       );
     },
@@ -203,9 +199,9 @@ function createFixRequiresESBuildPlugin(config: Config): Plugin {
 /**
  * Gets the path of the worker.js file generated by the build process
  *
- * @param openNextOptions the open-next build options
+ * @param buildOpts the open-next build options
  * @returns the path of the worker.js file that the build process generates
  */
-export function getOutputWorkerPath(openNextOptions: BuildOptions): string {
-  return path.join(openNextOptions.outputDir, "worker.js");
+export function getOutputWorkerPath(buildOpts: BuildOptions): string {
+  return path.join(buildOpts.outputDir, "worker.js");
 }

packages/cloudflare/src/cli/build/open-next/compile-env-files.ts

@@ -8,11 +8,11 @@ import { extractProjectEnvVars } from "../utils/index.js";
 /**
  * Compiles the values extracted from the project's env files to the output directory for use in the worker.
  */
-export function compileEnvFiles(options: BuildOptions) {
+export function compileEnvFiles(buildOpts: BuildOptions) {
   ["production", "development", "test"].forEach((mode) =>
     fs.appendFileSync(
-      path.join(options.outputDir, `.env.mjs`),
-      `export const ${mode} = ${JSON.stringify(extractProjectEnvVars(mode, options))};\n`
+      path.join(buildOpts.outputDir, `.env.mjs`),
+      `export const ${mode} = ${JSON.stringify(extractProjectEnvVars(mode, buildOpts))};\n`
     )
   );
 }

packages/cloudflare/src/cli/build/patches/investigated/copy-package-cli-files.ts

@@ -3,21 +3,21 @@ import path from "node:path";
 
 import type { BuildOptions } from "@opennextjs/aws/build/helper.js";
 
-import { Config } from "../../../config.js";
 import { getOutputWorkerPath } from "../../bundle-server.js";
 
 /**
- * Copies the template files present in the cloudflare adapter package into the standalone node_modules folder
+ * Copies
+ * - the template files present in the cloudflare adapter package to `.open-next/cloudflare-templates`
+ * - `worker.js` to `.open-next/`
  */
-export function copyPackageCliFiles(packageDistDir: string, config: Config, openNextOptions: BuildOptions) {
+export function copyPackageCliFiles(packageDistDir: string, buildOpts: BuildOptions) {
   console.log("# copyPackageTemplateFiles");
-  const sourceDir = path.join(packageDistDir, "cli");
-  const destinationDir = path.join(config.paths.internal.package, "cli");
+  const sourceDir = path.join(packageDistDir, "cli/templates");
 
+  const destinationDir = path.join(buildOpts.outputDir, "cloudflare-templates");
+
+  fs.mkdirSync(destinationDir, { recursive: true });
   fs.cpSync(sourceDir, destinationDir, { recursive: true });
 
-  fs.copyFileSync(
-    path.join(packageDistDir, "cli", "templates", "worker.js"),
-    getOutputWorkerPath(openNextOptions)
-  );
+  fs.copyFileSync(path.join(packageDistDir, "cli/templates/worker.js"), getOutputWorkerPath(buildOpts));
 }

packages/cloudflare/src/cli/build/patches/investigated/patch-cache.ts

@@ -1,6 +1,6 @@
 import path from "node:path";
 
-import type { BuildOptions } from "@opennextjs/aws/build/helper.js";
+import { type BuildOptions, getPackagePath } from "@opennextjs/aws/build/helper.js";
 
 import { normalizePath } from "../../utils/index.js";
 
@@ -15,17 +15,17 @@ import { normalizePath } from "../../utils/index.js";
  * build-time. Therefore, we have to manually override the default way that the cache handler is
  * instantiated with a dynamic require that uses a string literal for the path.
  */
-export async function patchCache(code: string, openNextOptions: BuildOptions): Promise<string> {
-  const { appBuildOutputPath, outputDir, monorepoRoot } = openNextOptions;
+export async function patchCache(code: string, buildOpts: BuildOptions): Promise<string> {
+  const { outputDir } = buildOpts;
 
   // TODO: switch to cache.mjs
-  const outputPath = path.join(outputDir, "server-functions", "default");
-  const packagePath = path.relative(monorepoRoot, appBuildOutputPath);
-  const cacheFile = path.join(outputPath, packagePath, "cache.cjs");
+  const outputPath = path.join(outputDir, "server-functions/default");
+  const cacheFile = path.join(outputPath, getPackagePath(buildOpts), "cache.cjs");
 
   return code.replace(
     "const { cacheHandler } = this.nextConfig;",
-    `const cacheHandler = null;
+    `
+const cacheHandler = null;
 CacheHandler = require('${normalizePath(cacheFile)}').default;
 `
   );

packages/cloudflare/src/cli/build/patches/investigated/patch-require.ts

@@ -1,8 +1,5 @@
 /**
- * ESBuild does not support CJS format
- * See https://github.com/evanw/esbuild/issues/1921 and linked issues
- * Some of the solutions are based on `module.createRequire()` not implemented in workerd.
- * James on Aug 29: `module.createRequire()` is planned.
+ * Replaces webpack `__require` with actual `require`
  */
 export function patchRequire(code: string): string {
   return code.replace(/__require\d?\(/g, "require(").replace(/__require\d?\./g, "require.");

packages/cloudflare/src/cli/build/patches/investigated/update-webpack-chunks-file/index.ts

@@ -1,22 +1,30 @@
 import { readdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 
-import { Config } from "../../../../config.js";
+import { type BuildOptions, getPackagePath } from "@opennextjs/aws/build/helper.js";
+
 import { getUpdatedWebpackChunksFileContent } from "./get-updated-webpack-chunks-file-content.js";
 
 /**
  * Fixes the webpack-runtime.js file by removing its webpack dynamic requires.
- *
- * This hack is particularly bad as it indicates that files inside the output directory still get a hold of files from the outside: `${nextjsAppPaths.standaloneAppServerDir}/webpack-runtime.js`
- *    so this shows that not everything that's needed to deploy the application is in the output directory...
  */
-export async function updateWebpackChunksFile(config: Config) {
+export async function updateWebpackChunksFile(buildOpts: BuildOptions) {
   console.log("# updateWebpackChunksFile");
-  const webpackRuntimeFile = join(config.paths.output.standaloneAppServer, "webpack-runtime.js");
+
+  const { outputDir } = buildOpts;
+
+  const dotNextServerDir = join(
+    outputDir,
+    "server-functions/default",
+    getPackagePath(buildOpts),
+    ".next/server"
+  );
+
+  const webpackRuntimeFile = join(dotNextServerDir, "webpack-runtime.js");
 
   const fileContent = readFileSync(webpackRuntimeFile, "utf-8");
 
-  const chunks = readdirSync(join(config.paths.output.standaloneAppServer, "chunks"))
+  const chunks = readdirSync(join(dotNextServerDir, "chunks"))
     .filter((chunk) => /^\d+\.js$/.test(chunk))
     .map((chunk) => {
       console.log(` - chunk ${chunk}`);