From c9475a7c1fba159bc662c6f940e76005a398af21 Mon Sep 17 00:00:00 2001 From: Victor Berchet Date: Tue, 24 Sep 2024 11:28:50 +0200 Subject: [PATCH] Add a basic KV cache handler --- packages/cloudflare/package.json | 2 + packages/cloudflare/src/build/build-worker.ts | 106 +++++++-------- packages/cloudflare/src/build/build.ts | 17 ++- .../patches/investigated/copy-package.ts | 11 ++ .../patches/investigated/copy-templates.ts | 17 --- .../build/patches/investigated/patch-cache.ts | 25 ++++ .../update-webpack-chunks-file/index.ts | 9 +- .../to-investigate/inline-eval-manifest.ts | 11 +- .../to-investigate/inline-next-require.ts | 13 +- .../patches/to-investigate/patch-find-dir.ts | 13 +- .../patches/to-investigate/patch-read-file.ts | 13 +- .../patches/to-investigate/wrangler-deps.ts | 10 +- packages/cloudflare/src/cache-handler.ts | 63 +++++++++ packages/cloudflare/src/config.ts | 121 ++++++++++++++++++ packages/cloudflare/src/nextjs-paths.ts | 101 --------------- packages/cloudflare/src/templates/worker.ts | 1 - packages/cloudflare/tsconfig.json | 2 +- packages/cloudflare/tsup.config.ts | 2 +- pnpm-lock.yaml | 24 ++-- pnpm-workspace.yaml | 1 + 20 files changed, 344 insertions(+), 218 deletions(-) create mode 100644 packages/cloudflare/src/build/patches/investigated/copy-package.ts delete mode 100644 packages/cloudflare/src/build/patches/investigated/copy-templates.ts create mode 100644 packages/cloudflare/src/build/patches/investigated/patch-cache.ts create mode 100644 packages/cloudflare/src/cache-handler.ts create mode 100644 packages/cloudflare/src/config.ts delete mode 100644 packages/cloudflare/src/nextjs-paths.ts diff --git a/packages/cloudflare/package.json b/packages/cloudflare/package.json index 3cb1dcf6..a7c805f2 100644 --- a/packages/cloudflare/package.json +++ b/packages/cloudflare/package.json @@ -29,9 +29,11 @@ }, "homepage": "https://github.com/opennextjs/opennextjs-cloudflare", "devDependencies": { + "@cloudflare/workers-types": "catalog:", "@types/node": "catalog:", "esbuild": "catalog:", "glob": "catalog:", + "next": "catalog:", "tsup": "catalog:", "typescript": "catalog:", "vitest": "catalog:" diff --git a/packages/cloudflare/src/build/build-worker.ts b/packages/cloudflare/src/build/build-worker.ts index 2594cfe8..3051feb8 100644 --- a/packages/cloudflare/src/build/build-worker.ts +++ b/packages/cloudflare/src/build/build-worker.ts @@ -1,12 +1,12 @@ -import { NextjsAppPaths } from "../nextjs-paths"; +import { Config } from "../config"; import { build, Plugin } from "esbuild"; -import { existsSync, readFileSync } from "node:fs"; +import { existsSync, readFileSync, cpSync } from "node:fs"; import { cp, readFile, writeFile } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { patchRequire } from "./patches/investigated/patch-require"; -import { copyTemplates } from "./patches/investigated/copy-templates"; +import { copyPackage } from "./patches/investigated/copy-package"; import { patchReadFile } from "./patches/to-investigate/patch-read-file"; import { patchFindDir } from "./patches/to-investigate/patch-find-dir"; @@ -14,34 +14,53 @@ import { inlineNextRequire } from "./patches/to-investigate/inline-next-require" import { inlineEvalManifest } from "./patches/to-investigate/inline-eval-manifest"; import { patchWranglerDeps } from "./patches/to-investigate/wrangler-deps"; import { updateWebpackChunksFile } from "./patches/investigated/update-webpack-chunks-file"; +import { patchCache } from "./patches/investigated/patch-cache"; /** The directory containing the Cloudflare template files. */ -const templateSrcDir = path.join(path.dirname(fileURLToPath(import.meta.url)), "templates"); +const packageDir = path.dirname(fileURLToPath(import.meta.url)); /** * Using the Next.js build output in the `.next` directory builds a workerd compatible output * * @param outputDir the directory where to save the output - * @param nextjsAppPaths + * @param config */ -export async function buildWorker( - appDir: string, - outputDir: string, - nextjsAppPaths: NextjsAppPaths -): Promise { - const templateDir = copyTemplates(templateSrcDir, nextjsAppPaths); - - const workerEntrypoint = `${templateDir}/worker.ts`; - const workerOutputFile = `${outputDir}/index.mjs`; +export async function buildWorker(config: Config): Promise { + console.log(`\x1b[35m⚙️ Copying files...\n\x1b[0m`); + + // Copy over client-side generated files + await cp( + path.join(config.paths.dotNext, "static"), + path.join(config.paths.builderOutput, "assets", "_next", "static"), + { + recursive: true, + } + ); + + // Copy over any static files (e.g. images) from the source project + const publicDir = path.join(config.paths.nextApp, "public"); + if (existsSync(publicDir)) { + await cp(publicDir, path.join(config.paths.builderOutput, "assets"), { + recursive: true, + }); + } + + copyPackage(packageDir, config); + + const templateDir = path.join(config.paths.internalPackage, "templates"); + + const workerEntrypoint = path.join(templateDir, "worker.ts"); + const workerOutputFile = path.join(config.paths.builderOutput, "index.mjs"); + const nextConfigStr = - readFileSync(nextjsAppPaths.standaloneAppDir + "/server.js", "utf8")?.match( + readFileSync(path.join(config.paths.standaloneApp, "/server.js"), "utf8")?.match( /const nextConfig = ({.+?})\n/ )?.[1] ?? {}; console.log(`\x1b[35m⚙️ Bundling the worker file...\n\x1b[0m`); - patchWranglerDeps(nextjsAppPaths); - updateWebpackChunksFile(nextjsAppPaths); + patchWranglerDeps(config); + updateWebpackChunksFile(config); await build({ entryPoints: [workerEntrypoint], @@ -55,15 +74,15 @@ export async function buildWorker( // Note: we apply an empty shim to next/dist/compiled/ws because it generates two `eval`s: // eval("require")("bufferutil"); // eval("require")("utf-8-validate"); - "next/dist/compiled/ws": `${templateDir}/shims/empty.ts`, + "next/dist/compiled/ws": path.join(templateDir, "shims", "empty.ts"), // Note: we apply an empty shim to next/dist/compiled/edge-runtime since (amongst others) it generated the following `eval`: // eval(getModuleCode)(module, module.exports, throwingRequire, params.context, ...Object.values(params.scopedContext)); // which comes from https://github.com/vercel/edge-runtime/blob/6e96b55f/packages/primitives/src/primitives/load.js#L57-L63 // QUESTION: Why did I encountered this but mhart didn't? - "next/dist/compiled/edge-runtime": `${templateDir}/shims/empty.ts`, + "next/dist/compiled/edge-runtime": path.join(templateDir, "shims", "empty.ts"), // `@next/env` is a library Next.js uses for loading dotenv files, for obvious reasons we need to stub it here // source: https://github.com/vercel/next.js/tree/0ac10d79720/packages/next-env - "@next/env": `${templateDir}/shims/env.ts`, + "@next/env": path.join(templateDir, "shims", "env.ts"), }, define: { // config file used by Next.js, see: https://github.com/vercel/next.js/blob/68a7128/packages/next/src/build/utils.ts#L2137-L2139 @@ -98,22 +117,21 @@ export async function buildWorker( // Do not crash on cache not supported // https://github.com/cloudflare/workerd/pull/2434 // compatibility flag "cache_option_enabled" -> does not support "force-cache" -let isPatchedAlready = globalThis.fetch.__nextPatched; const curFetch = globalThis.fetch; globalThis.fetch = (input, init) => { - console.log("globalThis.fetch", input); - if (init) delete init.cache; + if (init) { + delete init.cache; + } return curFetch(input, init); }; import { Readable } from 'node:stream'; -globalThis.fetch.__nextPatched = isPatchedAlready; fetch = globalThis.fetch; const CustomRequest = class extends globalThis.Request { constructor(input, init) { - console.log("CustomRequest", input); if (init) { delete init.cache; if (init.body?.__node_stream__ === true) { + // https://github.com/cloudflare/workerd/issues/2746 init.body = Readable.toWeb(init.body); } } @@ -122,25 +140,11 @@ const CustomRequest = class extends globalThis.Request { }; globalThis.Request = CustomRequest; Request = globalThis.Request; - `, +`, }, }); - await updateWorkerBundledCode(workerOutputFile, nextjsAppPaths); - - console.log(`\x1b[35m⚙️ Copying asset files...\n\x1b[0m`); - - // Copy over client-side generated files - await cp(`${nextjsAppPaths.dotNextDir}/static`, `${outputDir}/assets/_next/static`, { - recursive: true, - }); - - // Copy over any static files (e.g. images) from the source project - if (existsSync(`${appDir}/public`)) { - await cp(`${appDir}/public`, `${outputDir}/assets`, { - recursive: true, - }); - } + await updateWorkerBundledCode(workerOutputFile, config); console.log(`\x1b[35mWorker saved in \`${workerOutputFile}\` 🚀\n\x1b[0m`); } @@ -151,21 +155,19 @@ Request = globalThis.Request; * Needless to say all the logic in this function is something we should avoid as much as possible! * * @param workerOutputFile - * @param nextjsAppPaths + * @param config */ -async function updateWorkerBundledCode( - workerOutputFile: string, - nextjsAppPaths: NextjsAppPaths -): Promise { +async function updateWorkerBundledCode(workerOutputFile: string, config: Config): Promise { const originalCode = await readFile(workerOutputFile, "utf8"); let patchedCode = originalCode; patchedCode = patchRequire(patchedCode); - patchedCode = patchReadFile(patchedCode, nextjsAppPaths); - patchedCode = inlineNextRequire(patchedCode, nextjsAppPaths); - patchedCode = patchFindDir(patchedCode, nextjsAppPaths); - patchedCode = inlineEvalManifest(patchedCode, nextjsAppPaths); + patchedCode = patchReadFile(patchedCode, config); + patchedCode = inlineNextRequire(patchedCode, config); + patchedCode = patchFindDir(patchedCode, config); + patchedCode = inlineEvalManifest(patchedCode, config); + patchedCode = patchCache(patchedCode, config); await writeFile(workerOutputFile, patchedCode); } @@ -176,10 +178,10 @@ function createFixRequiresESBuildPlugin(templateDir: string): Plugin { setup(build) { // Note: we (empty) shim require-hook modules as they generate problematic code that uses requires build.onResolve({ filter: /^\.\/require-hook$/ }, (args) => ({ - path: `${templateDir}/shims/empty.ts`, + path: path.join(templateDir, "shims", "empty.ts"), })); build.onResolve({ filter: /\.\/lib\/node-fs-methods$/ }, (args) => ({ - path: `${templateDir}/shims/node-fs.ts`, + path: path.join(templateDir, "shims", "empty.ts"), })); }, }; diff --git a/packages/cloudflare/src/build/build.ts b/packages/cloudflare/src/build/build.ts index 5f70a9a9..34f1c9f3 100644 --- a/packages/cloudflare/src/build/build.ts +++ b/packages/cloudflare/src/build/build.ts @@ -1,9 +1,9 @@ import { rm } from "node:fs/promises"; import { buildNextjsApp } from "./build-next-app"; import { buildWorker } from "./build-worker"; -import { getNextjsAppPaths } from "../nextjs-paths"; +import { containsDotNextDir, getConfig } from "../config"; import { cpSync } from "node:fs"; -import { resolve } from "node:path"; +import path from "node:path"; /** * Builds the application in a format that can be passed to workerd @@ -20,15 +20,20 @@ export async function build(appDir: string, opts: BuildOptions): Promise { buildNextjsApp(appDir); } + if (!containsDotNextDir(appDir)) { + throw new Error(`.next folder not found in ${appDir}`); + } + // Create a clean output directory - const outputDir = resolve(opts.outputDir ?? appDir, ".worker-next"); + const outputDir = path.resolve(opts.outputDir ?? appDir, ".worker-next"); await cleanDirectory(outputDir); // Copy the .next directory to the output directory so it can be mutated. - cpSync(resolve(`${appDir}/.next`), resolve(`${outputDir}/.next`), { recursive: true }); - const nextjsAppPaths = getNextjsAppPaths(outputDir); + cpSync(path.join(appDir, ".next"), path.join(outputDir, ".next"), { recursive: true }); + + const config = getConfig(appDir, outputDir); - await buildWorker(appDir, outputDir, nextjsAppPaths); + await buildWorker(config); } type BuildOptions = { diff --git a/packages/cloudflare/src/build/patches/investigated/copy-package.ts b/packages/cloudflare/src/build/patches/investigated/copy-package.ts new file mode 100644 index 00000000..5c499521 --- /dev/null +++ b/packages/cloudflare/src/build/patches/investigated/copy-package.ts @@ -0,0 +1,11 @@ +import path from "node:path"; +import { Config } from "../../../config"; +import { cpSync } from "node:fs"; + +/** + * Copy the builder package in the standalone node_modules folder. + */ +export function copyPackage(srcDir: string, config: Config) { + console.log("# copyPackage"); + cpSync(srcDir, config.paths.internalPackage, { recursive: true }); +} diff --git a/packages/cloudflare/src/build/patches/investigated/copy-templates.ts b/packages/cloudflare/src/build/patches/investigated/copy-templates.ts deleted file mode 100644 index fd144db0..00000000 --- a/packages/cloudflare/src/build/patches/investigated/copy-templates.ts +++ /dev/null @@ -1,17 +0,0 @@ -import path from "node:path"; -import { NextjsAppPaths } from "../../../nextjs-paths"; -import { cpSync } from "node:fs"; - -/** - * Copy templates in the standalone folder. - * - * We need to have the template files locally to referenced package paths - * to resolve to files in the the node_module of the standalone app.= - */ -export function copyTemplates(srcDir: string, nextjsAppPaths: NextjsAppPaths) { - console.log("# copyTemplates"); - const destDir = path.join(nextjsAppPaths.standaloneAppDir, "node_modules/cf/templates"); - - cpSync(srcDir, destDir, { recursive: true }); - return destDir; -} diff --git a/packages/cloudflare/src/build/patches/investigated/patch-cache.ts b/packages/cloudflare/src/build/patches/investigated/patch-cache.ts new file mode 100644 index 00000000..9224d7b5 --- /dev/null +++ b/packages/cloudflare/src/build/patches/investigated/patch-cache.ts @@ -0,0 +1,25 @@ +import path from "node:path"; +import { Config } from "../../../config"; + +/** + * Install the cloudflare KV cache handler + */ +export function patchCache(code: string, config: Config): string { + console.log("# patchCached"); + + const cacheHandler = path.join(config.paths.internalPackage, "cache-handler.mjs"); + + const patchedCode = code.replace( + "const { cacheHandler } = this.nextConfig;", + `const cacheHandler = null; +CacheHandler = (await import('${cacheHandler}')).default; +CacheHandler.maybeKVNamespace = process.env["${config.cache.kvBindingName}"]; +` + ); + + if (patchedCode === code) { + throw new Error("Cache patch not applied"); + } + + return patchedCode; +} diff --git a/packages/cloudflare/src/build/patches/investigated/update-webpack-chunks-file/index.ts b/packages/cloudflare/src/build/patches/investigated/update-webpack-chunks-file/index.ts index b4e34a5d..3e2e1fc9 100644 --- a/packages/cloudflare/src/build/patches/investigated/update-webpack-chunks-file/index.ts +++ b/packages/cloudflare/src/build/patches/investigated/update-webpack-chunks-file/index.ts @@ -1,6 +1,7 @@ import { readdirSync, readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; -import { NextjsAppPaths } from "../../../../nextjs-paths"; +import { Config } from "../../../../config"; import { getUpdatedWebpackChunksFileContent } from "./get-updated-webpack-chunks-file-content"; /** @@ -9,13 +10,13 @@ import { getUpdatedWebpackChunksFileContent } from "./get-updated-webpack-chunks * This hack is particularly bad as it indicates that files inside the output directory still get a hold of files from the outside: `${nextjsAppPaths.standaloneAppServerDir}/webpack-runtime.js` * so this shows that not everything that's needed to deploy the application is in the output directory... */ -export async function updateWebpackChunksFile(nextjsAppPaths: NextjsAppPaths) { +export async function updateWebpackChunksFile(config: Config) { console.log("# updateWebpackChunksFile"); - const webpackRuntimeFile = `${nextjsAppPaths.standaloneAppServerDir}/webpack-runtime.js`; + const webpackRuntimeFile = path.join(config.paths.standaloneAppServer, "webpack-runtime.js"); const fileContent = readFileSync(webpackRuntimeFile, "utf-8"); - const chunks = readdirSync(`${nextjsAppPaths.standaloneAppServerDir}/chunks`) + const chunks = readdirSync(path.join(config.paths.standaloneAppServer, "chunks")) .filter((chunk) => /^\d+\.js$/.test(chunk)) .map((chunk) => { console.log(` - chunk ${chunk}`); diff --git a/packages/cloudflare/src/build/patches/to-investigate/inline-eval-manifest.ts b/packages/cloudflare/src/build/patches/to-investigate/inline-eval-manifest.ts index 1b0a9e55..fd6b24e9 100644 --- a/packages/cloudflare/src/build/patches/to-investigate/inline-eval-manifest.ts +++ b/packages/cloudflare/src/build/patches/to-investigate/inline-eval-manifest.ts @@ -1,5 +1,6 @@ import { globSync } from "glob"; -import { NextjsAppPaths } from "../../../nextjs-paths"; +import path from "node:path"; +import { Config } from "../../../config"; /** * `evalManifest` relies on readFileSync so we need to patch the function so that it instead returns the content of the manifest files @@ -8,11 +9,11 @@ import { NextjsAppPaths } from "../../../nextjs-paths"; * Note: we could/should probably just patch readFileSync here or something, but here the issue is that after the readFileSync call * there is a vm `runInNewContext` call which we also don't support (source: https://github.com/vercel/next.js/blob/b1e32c5d1f/packages/next/src/server/load-manifest.ts#L88) */ -export function inlineEvalManifest(code: string, nextjsAppPaths: NextjsAppPaths): string { +export function inlineEvalManifest(code: string, config: Config): string { console.log("# inlineEvalManifest"); const manifestJss = globSync( - `${nextjsAppPaths.standaloneAppDotNextDir}/**/*_client-reference-manifest.js` - ).map((file) => file.replace(`${nextjsAppPaths.standaloneAppDir}/`, "")); + path.join(config.paths.standaloneAppDotNext, "**", "*_client-reference-manifest.js") + ).map((file) => file.replace(`${config.paths.standaloneApp}/`, "")); return code.replace( /function evalManifest\((.+?), .+?\) {/, `$& @@ -20,7 +21,7 @@ export function inlineEvalManifest(code: string, nextjsAppPaths: NextjsAppPaths) .map( (manifestJs) => ` if ($1.endsWith("${manifestJs}")) { - require("${nextjsAppPaths.standaloneAppDir}/${manifestJs}"); + require("${path.join(config.paths.standaloneApp, manifestJs)}"); return { __RSC_MANIFEST: { "${manifestJs diff --git a/packages/cloudflare/src/build/patches/to-investigate/inline-next-require.ts b/packages/cloudflare/src/build/patches/to-investigate/inline-next-require.ts index a0332dc1..025cfe5f 100644 --- a/packages/cloudflare/src/build/patches/to-investigate/inline-next-require.ts +++ b/packages/cloudflare/src/build/patches/to-investigate/inline-next-require.ts @@ -1,14 +1,15 @@ import { readFileSync, existsSync } from "node:fs"; -import { NextjsAppPaths } from "../../../nextjs-paths"; +import { Config } from "../../../config"; +import path from "node:path"; /** * The following avoid various Next.js specific files `require`d at runtime since we can just read * and inline their content during build time */ -export function inlineNextRequire(code: string, nextjsAppPaths: NextjsAppPaths) { +export function inlineNextRequire(code: string, config: Config) { console.log("# inlineNextRequire"); - const pagesManifestFile = `${nextjsAppPaths.standaloneAppServerDir}/pages-manifest.json`; - const appPathsManifestFile = `${nextjsAppPaths.standaloneAppServerDir}/app-paths-manifest.json`; + const pagesManifestFile = path.join(config.paths.standaloneAppServer, "pages-manifest.json"); + const appPathsManifestFile = path.join(config.paths.standaloneAppServer, "app-paths-manifest.json"); const pagesManifestFiles = existsSync(pagesManifestFile) ? Object.values(JSON.parse(readFileSync(pagesManifestFile, "utf-8"))).map( @@ -32,7 +33,7 @@ export function inlineNextRequire(code: string, nextjsAppPaths: NextjsAppPaths) .map( (htmlPage) => ` if (pagePath.endsWith("${htmlPage}")) { - return ${JSON.stringify(readFileSync(`${nextjsAppPaths.standaloneAppDir}/${htmlPage}`, "utf-8"))}; + return ${JSON.stringify(readFileSync(path.join(config.paths.standaloneApp, htmlPage), "utf-8"))}; } ` ) @@ -41,7 +42,7 @@ export function inlineNextRequire(code: string, nextjsAppPaths: NextjsAppPaths) .map( (module) => ` if (pagePath.endsWith("${module}")) { - return require("${nextjsAppPaths.standaloneAppDir}/${module}"); + return require("${path.join(config.paths.standaloneApp, module)}"); } ` ) diff --git a/packages/cloudflare/src/build/patches/to-investigate/patch-find-dir.ts b/packages/cloudflare/src/build/patches/to-investigate/patch-find-dir.ts index 03ba5b5f..ee390892 100644 --- a/packages/cloudflare/src/build/patches/to-investigate/patch-find-dir.ts +++ b/packages/cloudflare/src/build/patches/to-investigate/patch-find-dir.ts @@ -1,4 +1,5 @@ -import { NextjsAppPaths } from "../../../nextjs-paths"; +import path from "node:path"; +import { Config } from "../../../config"; import { existsSync } from "node:fs"; /** @@ -7,14 +8,18 @@ import { existsSync } from "node:fs"; * (usage source: https://github.com/vercel/next.js/blob/ba995993/packages/next/src/server/next-server.ts#L450-L451) * Note: `findDir` uses `fs.existsSync` under the hood, so patching that should be enough to make this work */ -export function patchFindDir(code: string, nextjsAppPaths: NextjsAppPaths): string { +export function patchFindDir(code: string, config: Config): string { console.log("# patchFindDir"); return code.replace( "function findDir(dir, name) {", `function findDir(dir, name) { if (dir.endsWith(".next/server")) { - if (name === "app") return ${existsSync(`${nextjsAppPaths.standaloneAppServerDir}/app`)}; - if (name === "pages") return ${existsSync(`${nextjsAppPaths.standaloneAppServerDir}/pages`)}; + if (name === "app") { + return ${existsSync(`${path.join(config.paths.standaloneAppServer, "app")}`)}; + } + if (name === "pages") { + return ${existsSync(`${path.join(config.paths.standaloneAppServer, "pages")}`)}; + } } throw new Error("Unknown findDir call: " + dir + " " + name); ` diff --git a/packages/cloudflare/src/build/patches/to-investigate/patch-read-file.ts b/packages/cloudflare/src/build/patches/to-investigate/patch-read-file.ts index 19a7b4ff..e2e94eb2 100644 --- a/packages/cloudflare/src/build/patches/to-investigate/patch-read-file.ts +++ b/packages/cloudflare/src/build/patches/to-investigate/patch-read-file.ts @@ -1,8 +1,9 @@ import { readFileSync } from "node:fs"; import { globSync } from "glob"; -import { NextjsAppPaths } from "../../../nextjs-paths"; +import { Config } from "../../../config"; +import path from "node:path"; -export function patchReadFile(code: string, nextjsAppPaths: NextjsAppPaths): string { +export function patchReadFile(code: string, config: Config): string { console.log("# patchReadFile"); // The next-server code gets the buildId from the filesystem, resulting in a `[unenv] fs.readFileSync is not implemented yet!` error // so we add an early return to the `getBuildId` function so that the `readyFileSync` is never encountered @@ -11,15 +12,15 @@ export function patchReadFile(code: string, nextjsAppPaths: NextjsAppPaths): str code = code.replace( "getBuildId() {", `getBuildId() { - return ${JSON.stringify(readFileSync(`${nextjsAppPaths.standaloneAppDotNextDir}/BUILD_ID`, "utf-8"))}; + return ${JSON.stringify(readFileSync(path.join(config.paths.standaloneAppDotNext, "BUILD_ID"), "utf-8"))}; ` ); // Same as above, the next-server code loads the manifests with `readyFileSync` and we want to avoid that // (source: https://github.com/vercel/next.js/blob/15aeb92e/packages/next/src/server/load-manifest.ts#L34-L56) // Note: we could/should probably just patch readFileSync here or something! - const manifestJsons = globSync(`${nextjsAppPaths.standaloneAppDotNextDir}/**/*-manifest.json`).map((file) => - file.replace(nextjsAppPaths.standaloneAppDir + "/", "") + const manifestJsons = globSync(path.join(config.paths.standaloneAppDotNext, "**", "*-manifest.json")).map( + (file) => file.replace(config.paths.standaloneApp + "/", "") ); code = code.replace( /function loadManifest\((.+?), .+?\) {/, @@ -28,7 +29,7 @@ export function patchReadFile(code: string, nextjsAppPaths: NextjsAppPaths): str .map( (manifestJson) => ` if ($1.endsWith("${manifestJson}")) { - return ${readFileSync(`${nextjsAppPaths.standaloneAppDir}/${manifestJson}`, "utf-8")}; + return ${readFileSync(path.join(config.paths.standaloneApp, manifestJson), "utf-8")}; } ` ) diff --git a/packages/cloudflare/src/build/patches/to-investigate/wrangler-deps.ts b/packages/cloudflare/src/build/patches/to-investigate/wrangler-deps.ts index b482bcbb..cf37a29f 100644 --- a/packages/cloudflare/src/build/patches/to-investigate/wrangler-deps.ts +++ b/packages/cloudflare/src/build/patches/to-investigate/wrangler-deps.ts @@ -1,12 +1,10 @@ import path from "node:path"; import fs, { writeFileSync } from "node:fs"; -import { NextjsAppPaths } from "../../../nextjs-paths"; +import { Config } from "../../../config"; -export function patchWranglerDeps(paths: NextjsAppPaths) { +export function patchWranglerDeps(config: Config) { console.log("# patchWranglerDeps"); - console.log({ base: paths.standaloneAppDotNextDir }); - // Patch .next/standalone/node_modules/next/dist/compiled/next-server/pages.runtime.prod.js // // Remove the need for an alias in wrangler.toml: @@ -15,7 +13,7 @@ export function patchWranglerDeps(paths: NextjsAppPaths) { // # critters is `require`d from `pages.runtime.prod.js` when running wrangler dev, so we need to stub it out // "critters" = "./.next/standalone/node_modules/cf/templates/shims/empty.ts" const pagesRuntimeFile = path.join( - paths.standaloneAppDir, + config.paths.standaloneApp, "node_modules", "next", "dist", @@ -41,7 +39,7 @@ export function patchWranglerDeps(paths: NextjsAppPaths) { // # causing the code to require the 'next/dist/compiled/@opentelemetry/api' module instead (which properly works) // #"@opentelemetry/api" = "./.next/standalone/node_modules/cf/templates/shims/throw.ts" const tracerFile = path.join( - paths.standaloneAppDir, + config.paths.standaloneApp, "node_modules", "next", "dist", diff --git a/packages/cloudflare/src/cache-handler.ts b/packages/cloudflare/src/cache-handler.ts new file mode 100644 index 00000000..b5df556e --- /dev/null +++ b/packages/cloudflare/src/cache-handler.ts @@ -0,0 +1,63 @@ +import { type CacheHandler, type CacheHandlerContext } from "next/dist/server/lib/incremental-cache"; +import type { IncrementalCacheEntry, IncrementalCacheValue } from "next/dist/server/response-cache"; +import { KVNamespace } from "@cloudflare/workers-types"; + +export default class CfWorkersKvCacheHandler implements CacheHandler { + static maybeKVNamespace: KVNamespace | undefined = undefined; + + constructor(protected ctx: CacheHandlerContext) {} + + async get(key: string): Promise { + if (CfWorkersKvCacheHandler.maybeKVNamespace === undefined) { + return null; + } + + console.log(`[Cf] Getting cache[${key}]`); + + try { + return (await CfWorkersKvCacheHandler.maybeKVNamespace.get(key, "json")) ?? null; + } catch (e) { + console.error(`Failed to get value for key = ${key}: ${e}`); + return null; + } + } + + async set( + key: string, + entry: IncrementalCacheValue | null, + ctx: { + revalidate?: number | false; + fetchCache?: boolean; + fetchUrl?: string; + fetchIdx?: number; + tags?: string[]; + } + ) { + if (CfWorkersKvCacheHandler.maybeKVNamespace === undefined) { + return; + } + + console.log(`[Cf] Setting cache[${key}]`); + + try { + const data = { + lastModified: Date.now(), + value: entry, + }; + await CfWorkersKvCacheHandler.maybeKVNamespace.put(key, JSON.stringify(data)); + } catch (e) { + console.error(`Failed to set value for key = ${key}: ${e}`); + } + } + + async revalidateTag(tags: string | string[]) { + if (CfWorkersKvCacheHandler.maybeKVNamespace === undefined) { + return; + } + + tags = [tags].flat(); + console.log(`[Cf] revalidateTag ${JSON.stringify(tags)}}`); + } + + resetRequestCache(): void {} +} diff --git a/packages/cloudflare/src/config.ts b/packages/cloudflare/src/config.ts new file mode 100644 index 00000000..2bdbf084 --- /dev/null +++ b/packages/cloudflare/src/config.ts @@ -0,0 +1,121 @@ +import { readdirSync, statSync } from "node:fs"; +import path, { relative } from "node:path"; + +const PACKAGE_NAME = "@opennextjs/cloudflare"; + +// Make this user configurable +const UserConfig = { + cache: { + bindingName: "NEXT_CACHE_WORKERS_KV", + }, +}; + +export type Config = { + paths: { + // Path to the next application + nextApp: string; + // Path to the output folder + builderOutput: string; + // Path to the app's `.next` directory (where `next build` saves the build output) + dotNext: string; + // Path to the application standalone directory (where `next build` saves the standalone app) + standaloneApp: string; + // Path to the `.next` directory specific to the standalone application + standaloneAppDotNext: string; + // Path to the `server` directory specific to the standalone application + standaloneAppServer: string; + // Package in the standalone node_modules + internalPackage: string; + }; + + cache: { + kvBindingName: string; + }; + + // Internal name for the copy of the package + internalPackageName: string; +}; + +/** + * Computes the configuration. + * + * @param appDir Next app root folder + * @param outputDir Output of the cloudflare builder + * + * @returns the configuration, see `Config` + */ +export function getConfig(appDir: string, outputDir: string): Config { + const dotNext = path.join(outputDir, ".next"); + const appPath = getNextjsApplicationPath(dotNext).replace(/\/$/, ""); + const standaloneApp = path.join(dotNext, "standalone", appPath); + const standaloneAppDotNext = path.join(standaloneApp, ".next"); + const standaloneAppServer = path.join(standaloneAppDotNext, "server"); + + const nodeModules = path.join(standaloneApp, "node_modules"); + const internalPackage = path.join(nodeModules, ...PACKAGE_NAME.split("/")); + + return { + paths: { + nextApp: appDir, + builderOutput: outputDir, + dotNext, + standaloneApp, + standaloneAppDotNext, + standaloneAppServer, + internalPackage, + }, + + cache: { + kvBindingName: UserConfig.cache.bindingName, + }, + + internalPackageName: PACKAGE_NAME, + }; +} + +export function containsDotNextDir(folder: string): boolean { + try { + return statSync(path.join(folder, ".next")).isDirectory(); + } catch (e) { + return false; + } +} + +/** + * It basically tries to find the path that the application is under inside the `.next/standalone` directory, using the `.next/server` directory + * presence as the condition that needs to be met. + * + * For example: + * When I build the api application the `.next/server` directory is located in: + * `/standalone/next-apps/api/.next/server` + * and the function here given the `dotNextDir` returns `next-apps/api` + */ +function getNextjsApplicationPath(dotNextDir: string): string { + const serverPath = findServerParentPath(dotNextDir); + + if (!serverPath) { + throw new Error(`Unexpected Error: no \`.next/server\` folder could be found in \`${serverPath}\``); + } + + return relative(path.join(dotNextDir, "standalone"), serverPath); +} + +function findServerParentPath(parentPath: string): string | undefined { + try { + if (statSync(path.join(parentPath, ".next", "server")).isDirectory()) { + return parentPath; + } + } catch {} + + const folders = readdirSync(parentPath); + + for (const folder of folders) { + const subFolder = path.join(parentPath, folder); + if (statSync(path.join(parentPath, folder)).isDirectory()) { + const dirServerPath = findServerParentPath(subFolder); + if (dirServerPath) { + return dirServerPath; + } + } + } +} diff --git a/packages/cloudflare/src/nextjs-paths.ts b/packages/cloudflare/src/nextjs-paths.ts deleted file mode 100644 index 050164bd..00000000 --- a/packages/cloudflare/src/nextjs-paths.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { readdirSync, statSync } from "node:fs"; -import path, { relative } from "node:path"; - -/** - * This type includes all the paths necessary to deal with a Next.js application build - * - * NOTE: WIP, we still need to discern which paths are relevant here! - */ -export type NextjsAppPaths = { - /** - * The path to the application's `.next` directory (where `next build` saves the build output) - */ - dotNextDir: string; - - /** - * The path to the application standalone directory (where `next build` saves the standalone app when standalone mode is used) - */ - standaloneAppDir: string; - - /** - * the path to the `.next` directory specific to the standalone application - */ - standaloneAppDotNextDir: string; - - /** - * the path to the `server` directory specific to the standalone application - */ - standaloneAppServerDir: string; -}; - -/** - * Collects all the paths necessary for dealing with the Next.js applications output - * - * @param baseDir The path to the directory that contains the .next directory - * @returns the various paths. - */ -export function getNextjsAppPaths(baseDir: string): NextjsAppPaths { - const dotNextDir = getDotNextDirPath(baseDir); - - const appPath = getNextjsApplicationPath(dotNextDir).replace(/\/$/, ""); - - const standaloneAppDir = path.join(dotNextDir, "standalone", appPath); - - return { - dotNextDir, - standaloneAppDir, - standaloneAppDotNextDir: path.join(standaloneAppDir, ".next"), - standaloneAppServerDir: path.join(standaloneAppDir, ".next", "server"), - }; -} - -function getDotNextDirPath(nextAppDir: string): string { - const dotNextDirPath = `${nextAppDir}/.next`; - - try { - const dirStats = statSync(dotNextDirPath); - if (!dirStats.isDirectory()) throw new Error(); - } catch { - throw new Error(`Error: \`.next\` directory not found!`); - } - - return dotNextDirPath; -} - -/** - * It basically tries to find the path that the application is under inside the `.next/standalone` directory, using the `.next/server` directory - * presence as the condition that needs to be met. - * - * For example: - * When I build the api application the `.next/server` directory is located in: - * `/standalone/next-apps/api/.next/server` - * and the function here given the `dotNextDir` returns `next-apps/api` - */ -function getNextjsApplicationPath(dotNextDir: string): string { - const serverPath = findServerParentPath(dotNextDir); - - if (!serverPath) { - throw new Error(`Unexpected Error: no \`.next/server\` folder could be found in \`${serverPath}\``); - } - - return relative(`${dotNextDir}/standalone`, serverPath); - - function findServerParentPath(path: string): string | undefined { - try { - if (statSync(`${path}/.next/server`).isDirectory()) { - return path; - } - } catch {} - - const files = readdirSync(path); - - for (const file of files) { - if (statSync(`${path}/${file}`).isDirectory()) { - const dirServerPath = findServerParentPath(`${path}/${file}`); - if (dirServerPath) { - return dirServerPath; - } - } - } - } -} diff --git a/packages/cloudflare/src/templates/worker.ts b/packages/cloudflare/src/templates/worker.ts index 94975ac8..6b362cf3 100644 --- a/packages/cloudflare/src/templates/worker.ts +++ b/packages/cloudflare/src/templates/worker.ts @@ -102,7 +102,6 @@ function getWrappedStreams(request: Request, ctx: any) { }, set: function (val) { if (this.finished || this.headersSent) { - console.error("headers already sent"); return; } statusCode = val; diff --git a/packages/cloudflare/tsconfig.json b/packages/cloudflare/tsconfig.json index 5d889dd8..c2a686ab 100644 --- a/packages/cloudflare/tsconfig.json +++ b/packages/cloudflare/tsconfig.json @@ -7,6 +7,6 @@ "forceConsistentCasingInFileNames": true, "strict": true, "skipLibCheck": true, - "types": [] + "types": ["@cloudflare/workers-types"] } } diff --git a/packages/cloudflare/tsup.config.ts b/packages/cloudflare/tsup.config.ts index a1ed4703..cd15be8d 100644 --- a/packages/cloudflare/tsup.config.ts +++ b/packages/cloudflare/tsup.config.ts @@ -2,7 +2,7 @@ import { cp } from "fs/promises"; import { defineConfig } from "tsup"; export default defineConfig({ - entry: ["src/index.ts"], + entry: ["src/index.ts", "src/cache-handler.ts"], outDir: "dist", dts: false, format: ["esm"], diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 09578f1e..335379d9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6,6 +6,9 @@ settings: catalogs: default: + '@cloudflare/workers-types': + specifier: ^4.20240919.0 + version: 4.20240919.0 '@playwright/test': specifier: 1.47.0 version: 1.47.0 @@ -89,7 +92,7 @@ importers: version: 22.2.0 wrangler: specifier: 'catalog:' - version: 3.78.6(@cloudflare/workers-types@4.20240909.0) + version: 3.78.6(@cloudflare/workers-types@4.20240919.0) examples/create-next-app: dependencies: @@ -135,7 +138,7 @@ importers: version: 5.5.4 wrangler: specifier: 'catalog:' - version: 3.78.6(@cloudflare/workers-types@4.20240909.0) + version: 3.78.6(@cloudflare/workers-types@4.20240919.0) packages/cloudflare: dependencies: @@ -143,6 +146,9 @@ importers: specifier: 'catalog:' version: 23.0.0 devDependencies: + '@cloudflare/workers-types': + specifier: 'catalog:' + version: 4.20240919.0 '@types/node': specifier: 'catalog:' version: 22.2.0 @@ -152,6 +158,9 @@ importers: glob: specifier: 'catalog:' version: 11.0.0 + next: + specifier: 'catalog:' + version: 14.2.11(@playwright/test@1.47.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) tsup: specifier: 'catalog:' version: 8.2.4(jiti@1.21.6)(postcss@8.4.47)(tsx@4.17.0)(typescript@5.5.4)(yaml@2.5.1) @@ -206,8 +215,8 @@ packages: resolution: {integrity: sha512-Yk5Im7zsyKbzd7qi+DrL7ZJR9+bdZwq9BqZWS4muDIWA8MCUeSLsUC+C9u+jdwfPSi5It2AcQG4f0iwZr6jkkQ==} engines: {node: '>=16.7.0'} - '@cloudflare/workers-types@4.20240909.0': - resolution: {integrity: sha512-4knwtX6efxIsIxawdmPyynU9+S8A78wntU8eUIEldStWP4gNgxGkeWcfCMXulTx8oxr3DU4aevHyld9HGV8VKQ==} + '@cloudflare/workers-types@4.20240919.0': + resolution: {integrity: sha512-DZwTpZVAV+fKTLxo6ntC2zMNRL/UJwvtMKUt/U7ZyJdR+t0qcBUZGx8jLi9gOFWYxkzO3s7slajwkR2hQRPXYQ==} '@cspotcode/source-map-support@0.8.1': resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} @@ -2793,8 +2802,7 @@ snapshots: mime: 3.0.0 zod: 3.23.8 - '@cloudflare/workers-types@4.20240909.0': - optional: true + '@cloudflare/workers-types@4.20240919.0': {} '@cspotcode/source-map-support@0.8.1': dependencies: @@ -5408,7 +5416,7 @@ snapshots: '@cloudflare/workerd-linux-arm64': 1.20240909.0 '@cloudflare/workerd-windows-64': 1.20240909.0 - wrangler@3.78.6(@cloudflare/workers-types@4.20240909.0): + wrangler@3.78.6(@cloudflare/workers-types@4.20240919.0): dependencies: '@cloudflare/kv-asset-handler': 0.3.4 '@cloudflare/workers-shared': 0.5.3 @@ -5429,7 +5437,7 @@ snapshots: workerd: 1.20240909.0 xxhash-wasm: 1.0.2 optionalDependencies: - '@cloudflare/workers-types': 4.20240909.0 + '@cloudflare/workers-types': 4.20240919.0 fsevents: 2.3.3 transitivePeerDependencies: - bufferutil diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 38c8e766..8d1fb39c 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -3,6 +3,7 @@ packages: - "examples/*" catalog: + "@cloudflare/workers-types": "^4.20240919.0" "@playwright/test": 1.47.0 "@types/node": ^22.2.0 "@types/react": ^18