Skip to content

Commit 6cc9bf4

Browse files
vicbvicb
committed
Separate patches
1 parent 855699a commit 6cc9bf4

File tree

7 files changed

+238
-180
lines changed

7 files changed

+238
-180
lines changed

‎nextjs-worker-builder/src/build/build-worker/index.ts‎

Lines changed: 24 additions & 180 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,17 @@
11
import { NextjsAppPaths } from "../../nextjsPaths";
22
import { build, Plugin } from "esbuild";
3-
import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
3+
import { readdirSync, readFileSync, writeFileSync } from "node:fs";
44
import { cp, readFile, writeFile } from "node:fs/promises";
55

6-
import { globSync } from "glob";
76
import { resolve } from "node:path";
87

9-
const fixRequiresESBuildPlugin: Plugin = {
10-
name: "replaceRelative",
11-
setup(build) {
12-
// Note: we (empty) shim require-hook modules as they generate problematic code that uses requires
13-
build.onResolve({ filter: /^\.\/require-hook$/ }, (args) => ({
14-
path: `${__dirname}/templates/shims/empty.ts`,
15-
}));
16-
},
17-
};
8+
import { patchRequire } from "./patches/investigated/patchRequire";
9+
import { patchUrl } from "./patches/investigated/patchUrl";
10+
11+
import { patchReadFile } from "./patches/to-investigate/patchReadFile";
12+
import { patchFindDir } from "./patches/to-investigate/patchFindDir";
13+
import { inlineNextRequire } from "./patches/to-investigate/inlineNextRequire";
14+
import { inlineEvalManifest } from "./patches/to-investigate/inlineEvalManifest";
1815

1916
/**
2017
* Using the Next.js build output in the `.next` directory builds a workerd compatible output
@@ -129,175 +126,12 @@ async function updateWorkerBundledCode(
129126

130127
let patchedCode = originalCode;
131128

132-
// ESBuild does not support CJS format
133-
// See https://github.com/evanw/esbuild/issues/1921 and linked issues
134-
// Some of the solutions are based on `module.createRequire()` not implemented in workerd.
135-
patchedCode = patchedCode
136-
.replace(/__require\d?\(/g, "require(")
137-
.replace(/__require\d?\./g, "require.");
138-
139-
// The next-server code gets the buildId from the filesystem, resulting in a `[unenv] fs.readFileSync is not implemented yet!` error
140-
// so we add an early return to the `getBuildId` function so that the `readyFileSync` is never encountered
141-
// (source: https://github.com/vercel/next.js/blob/15aeb92efb34c09a36/packages/next/src/server/next-server.ts#L438-L451)
142-
// Note: we could/should probably just patch readFileSync here or something!
143-
patchedCode = patchedCode.replace(
144-
"getBuildId() {",
145-
`getBuildId() {
146-
return ${JSON.stringify(
147-
readFileSync(
148-
`${nextjsAppPaths.standaloneAppDotNextDir}/BUILD_ID`,
149-
"utf-8"
150-
)
151-
)};
152-
`
153-
);
154-
155-
// Same as above, the next-server code loads the manifests with `readyFileSync` and we want to avoid that
156-
// (source: https://github.com/vercel/next.js/blob/15aeb92e/packages/next/src/server/load-manifest.ts#L34-L56)
157-
// Note: we could/should probably just patch readFileSync here or something!
158-
const manifestJsons = globSync(
159-
`${nextjsAppPaths.standaloneAppDotNextDir}/**/*-manifest.json`
160-
).map((file) => file.replace(nextjsAppPaths.standaloneAppDir + "/", ""));
161-
patchedCode = patchedCode.replace(
162-
/function loadManifest\((.+?), .+?\) {/,
163-
`$&
164-
${manifestJsons
165-
.map(
166-
(manifestJson) => `
167-
if ($1.endsWith("${manifestJson}")) {
168-
return ${readFileSync(
169-
`${nextjsAppPaths.standaloneAppDir}/${manifestJson}`,
170-
"utf-8"
171-
)};
172-
}
173-
`
174-
)
175-
.join("\n")}
176-
throw new Error("Unknown loadManifest: " + $1);
177-
`
178-
);
179-
180-
// This solves the fact that the workerd URL parsing is not compatible with the node.js one
181-
// VERY IMPORTANT: this required the following dependency to be part of the application!!!! (this is very bad!!!)
182-
// "node-url": "npm:url@^0.11.4"
183-
// Hopefully this should not be necessary after this unenv PR lands: https://github.com/unjs/unenv/pull/292
184-
patchedCode = patchedCode.replace(
185-
/ ([a-zA-Z0-9_]+) = require\("url"\);/g,
186-
` $1 = require("url");
187-
const nodeUrl = require("node-url");
188-
$1.parse = nodeUrl.parse.bind(nodeUrl);
189-
$1.format = nodeUrl.format.bind(nodeUrl);
190-
$1.pathToFileURL = (path) => {
191-
console.log("url.pathToFileURL", path);
192-
return new URL("file://" + path);
193-
}
194-
`
195-
);
196-
197-
// The following avoid various Next.js specific files `require`d at runtime since we can just read
198-
// and inline their content during build time
199-
const pagesManifestFile = `${nextjsAppPaths.standaloneAppServerDir}/pages-manifest.json`;
200-
const appPathsManifestFile = `${nextjsAppPaths.standaloneAppServerDir}/app-paths-manifest.json`;
201-
202-
const pagesManifestFiles = existsSync(pagesManifestFile)
203-
? Object.values(JSON.parse(readFileSync(pagesManifestFile, "utf-8"))).map(
204-
(file) => ".next/server/" + file
205-
)
206-
: [];
207-
const appPathsManifestFiles = existsSync(appPathsManifestFile)
208-
? Object.values(
209-
JSON.parse(readFileSync(appPathsManifestFile, "utf-8"))
210-
).map((file) => ".next/server/" + file)
211-
: [];
212-
const allManifestFiles = pagesManifestFiles.concat(appPathsManifestFiles);
213-
214-
const htmlPages = allManifestFiles.filter((file) => file.endsWith(".html"));
215-
const pageModules = allManifestFiles.filter((file) => file.endsWith(".js"));
216-
217-
patchedCode = patchedCode.replace(
218-
/const pagePath = getPagePath\(.+?\);/,
219-
`$&
220-
${htmlPages
221-
.map(
222-
(htmlPage) => `
223-
if (pagePath.endsWith("${htmlPage}")) {
224-
return ${JSON.stringify(
225-
readFileSync(
226-
`${nextjsAppPaths.standaloneAppDir}/${htmlPage}`,
227-
"utf-8"
228-
)
229-
)};
230-
}
231-
`
232-
)
233-
.join("\n")}
234-
${pageModules
235-
.map(
236-
(module) => `
237-
if (pagePath.endsWith("${module}")) {
238-
return require("${nextjsAppPaths.standaloneAppDir}/${module}");
239-
}
240-
`
241-
)
242-
.join("\n")}
243-
throw new Error("Unknown pagePath: " + pagePath);
244-
`
245-
);
246-
247-
// Here we patch `findDir` so that the next server can detect whether the `app` or `pages` directory exists
248-
// (source: https://github.com/vercel/next.js/blob/ba995993/packages/next/src/lib/find-pages-dir.ts#L4-L13)
249-
// (usage source: https://github.com/vercel/next.js/blob/ba995993/packages/next/src/server/next-server.ts#L450-L451)
250-
// Note: `findDir` uses `fs.existsSync` under the hood, so patching that should be enough to make this work
251-
patchedCode = patchedCode.replace(
252-
"function findDir(dir, name) {",
253-
`function findDir(dir, name) {
254-
if (dir.endsWith(".next/server")) {
255-
if (name === "app") return ${existsSync(
256-
`${nextjsAppPaths.standaloneAppServerDir}/app`
257-
)};
258-
if (name === "pages") return ${existsSync(
259-
`${nextjsAppPaths.standaloneAppServerDir}/pages`
260-
)};
261-
}
262-
throw new Error("Unknown findDir call: " + dir + " " + name);
263-
`
264-
);
265-
266-
// `evalManifest` relies on readFileSync so we need to patch the function so that it instead returns the content of the manifest files
267-
// which are known at build time
268-
// (source: https://github.com/vercel/next.js/blob/b1e32c5d1f/packages/next/src/server/load-manifest.ts#L72)
269-
// Note: we could/should probably just patch readFileSync here or something, but here the issue is that after the readFileSync call
270-
// there is a vm `runInNewContext` call which we also don't support (source: https://github.com/vercel/next.js/blob/b1e32c5d1f/packages/next/src/server/load-manifest.ts#L88)
271-
const manifestJss = globSync(
272-
`${nextjsAppPaths.standaloneAppDotNextDir}/**/*_client-reference-manifest.js`
273-
).map((file) => file.replace(`${nextjsAppPaths.standaloneAppDir}/`, ""));
274-
patchedCode = patchedCode.replace(
275-
/function evalManifest\((.+?), .+?\) {/,
276-
`$&
277-
${manifestJss
278-
.map(
279-
(manifestJs) => `
280-
if ($1.endsWith("${manifestJs}")) {
281-
require("${nextjsAppPaths.standaloneAppDir}/${manifestJs}");
282-
return {
283-
__RSC_MANIFEST: {
284-
"${manifestJs
285-
.replace(".next/server/app", "")
286-
.replace(
287-
"_client-reference-manifest.js",
288-
""
289-
)}": globalThis.__RSC_MANIFEST["${manifestJs
290-
.replace(".next/server/app", "")
291-
.replace("_client-reference-manifest.js", "")}"],
292-
},
293-
};
294-
}
295-
`
296-
)
297-
.join("\n")}
298-
throw new Error("Unknown evalManifest: " + $1);
299-
`
300-
);
129+
patchedCode = patchRequire(patchedCode);
130+
patchedCode = patchReadFile(patchedCode, nextjsAppPaths);
131+
patchedCode = patchUrl(patchedCode);
132+
patchedCode = inlineNextRequire(patchedCode, nextjsAppPaths);
133+
patchedCode = patchFindDir(patchedCode, nextjsAppPaths);
134+
patchedCode = inlineEvalManifest(patchedCode, nextjsAppPaths);
301135

302136
await writeFile(workerOutputFile, patchedCode);
303137
}
@@ -338,3 +172,13 @@ async function updateWebpackChunksFile(nextjsAppPaths: NextjsAppPaths) {
338172

339173
writeFileSync(webpackRuntimeFile, updatedFileContent);
340174
}
175+
176+
const fixRequiresESBuildPlugin: Plugin = {
177+
name: "replaceRelative",
178+
setup(build) {
179+
// Note: we (empty) shim require-hook modules as they generate problematic code that uses requires
180+
build.onResolve({ filter: /^\.\/require-hook$/ }, (args) => ({
181+
path: `${__dirname}/templates/shims/empty.ts`,
182+
}));
183+
},
184+
};

‎nextjs-worker-builder/src/build/build-worker/patches/investigated/patchRequire.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
/**
2+
* ESBuild does not support CJS format
3+
* See https://github.com/evanw/esbuild/issues/1921 and linked issues
4+
* Some of the solutions are based on `module.createRequire()` not implemented in workerd.
5+
* James on Aug 29: `module.createRequire()` is planned.
6+
*/
7+
export function patchRequire(code: string): string {
8+
return code
9+
.replace(/__require\d?\(/g, "require(")
10+
.replace(/__require\d?\./g, "require.");
11+
}

‎nextjs-worker-builder/src/build/build-worker/patches/investigated/patchUrl.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/**
2+
* This solves the fact that the workerd URL parsing is not compatible with the node.js one
3+
* VERY IMPORTANT: this required the following dependency to be part of the application!!!! (this is very bad!!!)
4+
* "node-url": "npm:url@^0.11.4"
5+
* Hopefully this should not be necessary after this unenv PR lands: https://github.com/unjs/unenv/pull/292
6+
*/
7+
export function patchUrl(code: string): string {
8+
return code.replace(
9+
/ ([a-zA-Z0-9_]+) = require\("url"\);/g,
10+
` $1 = require("url");
11+
const nodeUrl = require("node-url");
12+
$1.parse = nodeUrl.parse.bind(nodeUrl);
13+
$1.format = nodeUrl.format.bind(nodeUrl);
14+
$1.pathToFileURL = (path) => {
15+
console.log("url.pathToFileURL", path);
16+
return new URL("file://" + path);
17+
}
18+
`
19+
);
20+
}

‎nextjs-worker-builder/src/build/build-worker/patches/to-investigate/inlineEvalManifest.ts‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
import { globSync } from "glob";
2+
import { NextjsAppPaths } from "../../../../nextjsPaths";
3+
4+
/**
5+
* `evalManifest` relies on readFileSync so we need to patch the function so that it instead returns the content of the manifest files
6+
* which are known at build time
7+
* (source: https://github.com/vercel/next.js/blob/b1e32c5d1f/packages/next/src/server/load-manifest.ts#L72)
8+
* Note: we could/should probably just patch readFileSync here or something, but here the issue is that after the readFileSync call
9+
* there is a vm `runInNewContext` call which we also don't support (source: https://github.com/vercel/next.js/blob/b1e32c5d1f/packages/next/src/server/load-manifest.ts#L88)
10+
*/
11+
export function inlineEvalManifest(
12+
code: string,
13+
nextjsAppPaths: NextjsAppPaths
14+
): string {
15+
const manifestJss = globSync(
16+
`${nextjsAppPaths.standaloneAppDotNextDir}/**/*_client-reference-manifest.js`
17+
).map((file) => file.replace(`${nextjsAppPaths.standaloneAppDir}/`, ""));
18+
return code.replace(
19+
/function evalManifest\((.+?), .+?\) {/,
20+
`$&
21+
${manifestJss
22+
.map(
23+
(manifestJs) => `
24+
if ($1.endsWith("${manifestJs}")) {
25+
require("${nextjsAppPaths.standaloneAppDir}/${manifestJs}");
26+
return {
27+
__RSC_MANIFEST: {
28+
"${manifestJs
29+
.replace(".next/server/app", "")
30+
.replace(
31+
"_client-reference-manifest.js",
32+
""
33+
)}": globalThis.__RSC_MANIFEST["${manifestJs
34+
.replace(".next/server/app", "")
35+
.replace("_client-reference-manifest.js", "")}"],
36+
},
37+
};
38+
}
39+
`
40+
)
41+
.join("\n")}
42+
throw new Error("Unknown evalManifest: " + $1);
43+
`
44+
);
45+
}

‎nextjs-worker-builder/src/build/build-worker/patches/to-investigate/inlineNextRequire.ts‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
import { readFileSync, existsSync } from "node:fs";
2+
import { NextjsAppPaths } from "../../../../nextjsPaths";
3+
4+
/**
5+
* The following avoid various Next.js specific files `require`d at runtime since we can just read
6+
* and inline their content during build time
7+
*/
8+
export function inlineNextRequire(
9+
code: string,
10+
nextjsAppPaths: NextjsAppPaths
11+
) {
12+
const pagesManifestFile = `${nextjsAppPaths.standaloneAppServerDir}/pages-manifest.json`;
13+
const appPathsManifestFile = `${nextjsAppPaths.standaloneAppServerDir}/app-paths-manifest.json`;
14+
15+
const pagesManifestFiles = existsSync(pagesManifestFile)
16+
? Object.values(JSON.parse(readFileSync(pagesManifestFile, "utf-8"))).map(
17+
(file) => ".next/server/" + file
18+
)
19+
: [];
20+
const appPathsManifestFiles = existsSync(appPathsManifestFile)
21+
? Object.values(
22+
JSON.parse(readFileSync(appPathsManifestFile, "utf-8"))
23+
).map((file) => ".next/server/" + file)
24+
: [];
25+
const allManifestFiles = pagesManifestFiles.concat(appPathsManifestFiles);
26+
27+
const htmlPages = allManifestFiles.filter((file) => file.endsWith(".html"));
28+
const pageModules = allManifestFiles.filter((file) => file.endsWith(".js"));
29+
30+
return code.replace(
31+
/const pagePath = getPagePath\(.+?\);/,
32+
`$&
33+
${htmlPages
34+
.map(
35+
(htmlPage) => `
36+
if (pagePath.endsWith("${htmlPage}")) {
37+
return ${JSON.stringify(
38+
readFileSync(
39+
`${nextjsAppPaths.standaloneAppDir}/${htmlPage}`,
40+
"utf-8"
41+
)
42+
)};
43+
}
44+
`
45+
)
46+
.join("\n")}
47+
${pageModules
48+
.map(
49+
(module) => `
50+
if (pagePath.endsWith("${module}")) {
51+
return require("${nextjsAppPaths.standaloneAppDir}/${module}");
52+
}
53+
`
54+
)
55+
.join("\n")}
56+
throw new Error("Unknown pagePath: " + pagePath);
57+
`
58+
);
59+
}

‎nextjs-worker-builder/src/build/build-worker/patches/to-investigate/patchFindDir.ts‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
import { NextjsAppPaths } from "../../../../nextjsPaths";
2+
import { existsSync } from "node:fs";
3+
4+
/**
5+
* Here we patch `findDir` so that the next server can detect whether the `app` or `pages` directory exists
6+
* (source: https://github.com/vercel/next.js/blob/ba995993/packages/next/src/lib/find-pages-dir.ts#L4-L13)
7+
* (usage source: https://github.com/vercel/next.js/blob/ba995993/packages/next/src/server/next-server.ts#L450-L451)
8+
* Note: `findDir` uses `fs.existsSync` under the hood, so patching that should be enough to make this work
9+
*/
10+
export function patchFindDir(
11+
code: string,
12+
nextjsAppPaths: NextjsAppPaths
13+
): string {
14+
return code.replace(
15+
"function findDir(dir, name) {",
16+
`function findDir(dir, name) {
17+
if (dir.endsWith(".next/server")) {
18+
if (name === "app") return ${existsSync(
19+
`${nextjsAppPaths.standaloneAppServerDir}/app`
20+
)};
21+
if (name === "pages") return ${existsSync(
22+
`${nextjsAppPaths.standaloneAppServerDir}/pages`
23+
)};
24+
}
25+
throw new Error("Unknown findDir call: " + dir + " " + name);
26+
`
27+
);
28+
}

0 commit comments

Comments
 (0)