diff --git a/packages/cloudflare/src/api/config.ts b/packages/cloudflare/src/api/config.ts index 1e165b942..cd0eb5924 100644 --- a/packages/cloudflare/src/api/config.ts +++ b/packages/cloudflare/src/api/config.ts @@ -45,6 +45,8 @@ export function defineCloudflareConfig(config: CloudflareOverrides = {}): OpenNe queue: resolveQueue(queue), }, }, + // node:crypto is used to compute cache keys + edgeExternals: ["node:crypto"], }; } diff --git a/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts b/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts index d3140e5bd..7a2e40f1f 100644 --- a/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts +++ b/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts @@ -1,3 +1,5 @@ +import { createHash } from "node:crypto"; + import { error } from "@opennextjs/aws/adapters/logger.js"; import type { CacheValue, IncrementalCache, WithLastModified } from "@opennextjs/aws/types/overrides.js"; import { IgnorableError } from "@opennextjs/aws/utils/error.js"; @@ -9,6 +11,17 @@ export const NAME = "cf-kv-incremental-cache"; export const BINDING_NAME = "NEXT_INC_CACHE_KV"; +export type KeyOptions = { + isFetch?: boolean; + buildId?: string; +}; + +export function computeCacheKey(key: string, options: KeyOptions) { + const { isFetch = false, buildId = FALLBACK_BUILD_ID } = options; + const hash = createHash("sha256").update(key).digest("hex"); + return `${buildId}/${hash}.${isFetch ? "fetch" : "cache"}`.replace(/\/+/g, "/"); +} + /** * Open Next cache based on Cloudflare KV. * @@ -93,8 +106,10 @@ class KVIncrementalCache implements IncrementalCache { } protected getKVKey(key: string, isFetch?: boolean): string { - const buildId = process.env.NEXT_BUILD_ID ?? FALLBACK_BUILD_ID; - return `${buildId}/${key}.${isFetch ? "fetch" : "cache"}`.replace(/\/+/g, "/"); + return computeCacheKey(key, { + buildId: process.env.NEXT_BUILD_ID, + isFetch, + }); } } diff --git a/packages/cloudflare/src/api/overrides/incremental-cache/r2-incremental-cache.ts b/packages/cloudflare/src/api/overrides/incremental-cache/r2-incremental-cache.ts index 2fdd32b97..75a4c269b 100644 --- a/packages/cloudflare/src/api/overrides/incremental-cache/r2-incremental-cache.ts +++ b/packages/cloudflare/src/api/overrides/incremental-cache/r2-incremental-cache.ts @@ -1,3 +1,5 @@ +import { createHash } from "node:crypto"; + import { error } from "@opennextjs/aws/adapters/logger.js"; import type { CacheValue, IncrementalCache, WithLastModified } from "@opennextjs/aws/types/overrides.js"; import { IgnorableError } from "@opennextjs/aws/utils/error.js"; @@ -12,6 +14,18 @@ export const BINDING_NAME = "NEXT_INC_CACHE_R2_BUCKET"; export const PREFIX_ENV_NAME = "NEXT_INC_CACHE_R2_PREFIX"; export const DEFAULT_PREFIX = "incremental-cache"; +export type KeyOptions = { + isFetch?: boolean; + directory?: string; + buildId?: string; +}; + +export function computeCacheKey(key: string, options: KeyOptions) { + const { isFetch = false, directory = DEFAULT_PREFIX, buildId = FALLBACK_BUILD_ID } = options; + const hash = createHash("sha256").update(key).digest("hex"); + return `${directory}/${buildId}/${hash}.${isFetch ? "fetch" : "cache"}`.replace(/\/+/g, "/"); +} + /** * An instance of the Incremental Cache that uses an R2 bucket (`NEXT_INC_CACHE_R2_BUCKET`) as it's * underlying data store. @@ -76,12 +90,11 @@ class R2IncrementalCache implements IncrementalCache { } protected getR2Key(key: string, isFetch?: boolean): string { - const directory = getCloudflareContext().env[PREFIX_ENV_NAME] ?? DEFAULT_PREFIX; - - return `${directory}/${process.env.NEXT_BUILD_ID ?? FALLBACK_BUILD_ID}/${key}.${isFetch ? "fetch" : "cache"}`.replace( - /\/+/g, - "/" - ); + return computeCacheKey(key, { + directory: getCloudflareContext().env[PREFIX_ENV_NAME], + buildId: process.env.NEXT_BUILD_ID, + isFetch, + }); } } diff --git a/packages/cloudflare/src/cli/build/utils/ensure-cf-config.ts b/packages/cloudflare/src/cli/build/utils/ensure-cf-config.ts index 30bdae56f..59a64dd65 100644 --- a/packages/cloudflare/src/cli/build/utils/ensure-cf-config.ts +++ b/packages/cloudflare/src/cli/build/utils/ensure-cf-config.ts @@ -22,6 +22,7 @@ export function ensureCloudflareConfig(config: OpenNextConfig) { config.default?.override?.queue === "direct" || typeof config.default?.override?.queue === "function", mwIsMiddlewareIntegrated: config.middleware === undefined, + hasCryptoExternal: config.edgeExternals?.includes("node:crypto"), }; if (config.default?.override?.queue === "direct") { @@ -42,6 +43,7 @@ export function ensureCloudflareConfig(config: OpenNextConfig) { queue: "dummy" | "direct" | function, }, }, + edgeExternals: ["node:crypto"], }\n\n`.replace(/^ {8}/gm, "") ); } diff --git a/packages/cloudflare/src/cli/commands/populate-cache.spec.ts b/packages/cloudflare/src/cli/commands/populate-cache.spec.ts new file mode 100644 index 000000000..528d13b1e --- /dev/null +++ b/packages/cloudflare/src/cli/commands/populate-cache.spec.ts @@ -0,0 +1,70 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import path from "node:path"; + +import type { BuildOptions } from "@opennextjs/aws/build/helper"; +import mockFs from "mock-fs"; +import { afterAll, beforeAll, describe, expect, test } from "vitest"; + +import { getCacheAssets } from "./populate-cache"; + +describe("getCacheAssets", () => { + beforeAll(() => { + mockFs(); + + const fetchBaseDir = "/base/path/cache/__fetch/buildID"; + const cacheDir = "/base/path/cache/buildID/path/to"; + + mkdirSync(fetchBaseDir, { recursive: true }); + mkdirSync(cacheDir, { recursive: true }); + + for (let i = 0; i < 3; i++) { + writeFileSync(path.join(fetchBaseDir, `${i}`), "", { encoding: "utf-8" }); + writeFileSync(path.join(cacheDir, `${i}.cache`), "", { encoding: "utf-8" }); + } + }); + + afterAll(() => mockFs.restore()); + + test("list cache assets", () => { + expect(getCacheAssets({ outputDir: "/base/path" } as BuildOptions)).toMatchInlineSnapshot(` + [ + { + "buildId": "buildID", + "fullPath": "/base/path/cache/buildID/path/to/2.cache", + "isFetch": false, + "key": "/path/to/2", + }, + { + "buildId": "buildID", + "fullPath": "/base/path/cache/buildID/path/to/1.cache", + "isFetch": false, + "key": "/path/to/1", + }, + { + "buildId": "buildID", + "fullPath": "/base/path/cache/buildID/path/to/0.cache", + "isFetch": false, + "key": "/path/to/0", + }, + { + "buildId": "buildID", + "fullPath": "/base/path/cache/__fetch/buildID/2", + "isFetch": true, + "key": "/2", + }, + { + "buildId": "buildID", + "fullPath": "/base/path/cache/__fetch/buildID/1", + "isFetch": true, + "key": "/1", + }, + { + "buildId": "buildID", + "fullPath": "/base/path/cache/__fetch/buildID/0", + "isFetch": true, + "key": "/0", + }, + ] + `); + }); +}); diff --git a/packages/cloudflare/src/cli/commands/populate-cache.ts b/packages/cloudflare/src/cli/commands/populate-cache.ts index 2d6e3e2db..04cd2684b 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.ts @@ -16,11 +16,12 @@ import { unstable_readConfig } from "wrangler"; import { BINDING_NAME as KV_CACHE_BINDING_NAME, + computeCacheKey as computeKVCacheKey, NAME as KV_CACHE_NAME, } from "../../api/overrides/incremental-cache/kv-incremental-cache.js"; import { BINDING_NAME as R2_CACHE_BINDING_NAME, - DEFAULT_PREFIX as R2_CACHE_DEFAULT_PREFIX, + computeCacheKey as computeR2CacheKey, NAME as R2_CACHE_NAME, PREFIX_ENV_NAME as R2_CACHE_PREFIX_ENV_NAME, } from "../../api/overrides/incremental-cache/r2-incremental-cache.js"; @@ -45,22 +46,50 @@ async function resolveCacheName( return typeof value === "function" ? (await value()).name : value; } -function getCacheAssetPaths(opts: BuildOptions) { - return globSync(path.join(opts.outputDir, "cache/**/*"), { +export type CacheAsset = { isFetch: boolean; fullPath: string; key: string; buildId: string }; + +export function getCacheAssets(opts: BuildOptions): CacheAsset[] { + const allFiles = globSync(path.join(opts.outputDir, "cache/**/*"), { withFileTypes: true, windowsPathsNoEscape: true, - }) - .filter((f) => f.isFile()) - .map((f) => { - const relativePath = path.relative(path.join(opts.outputDir, "cache"), f.fullpathPosix()); - - return { - fsPath: f.fullpathPosix(), - destPath: relativePath.startsWith("__fetch") - ? `${relativePath.replace("__fetch/", "")}.fetch` - : relativePath, - }; - }); + }).filter((f) => f.isFile()); + + const assets: CacheAsset[] = []; + + for (const file of allFiles) { + const fullPath = file.fullpathPosix(); + const relativePath = path.relative(path.join(opts.outputDir, "cache"), fullPath); + + if (relativePath.startsWith("__fetch")) { + const [__fetch, buildId, ...keyParts] = relativePath.split("/"); + + if (__fetch !== "__fetch" || buildId === undefined || keyParts.length === 0) { + throw new Error(`Invalid path for a Cache Asset file: ${relativePath}`); + } + + assets.push({ + isFetch: true, + fullPath, + key: `/${keyParts.join("/")}`, + buildId, + }); + } else { + const [buildId, ...keyParts] = relativePath.slice(0, -".cache".length).split("/"); + + if (!relativePath.endsWith(".cache") || buildId === undefined || keyParts.length === 0) { + throw new Error(`Invalid path for a Cache Asset file: ${relativePath}`); + } + + assets.push({ + isFetch: false, + fullPath, + key: `/${keyParts.join("/")}`, + buildId, + }); + } + } + + return assets; } function populateR2IncrementalCache( @@ -81,17 +110,18 @@ function populateR2IncrementalCache( throw new Error(`R2 binding ${JSON.stringify(R2_CACHE_BINDING_NAME)} should have a 'bucket_name'`); } - const assets = getCacheAssetPaths(options); - for (const { fsPath, destPath } of tqdm(assets)) { - const fullDestPath = path.join( - bucket, - process.env[R2_CACHE_PREFIX_ENV_NAME] ?? R2_CACHE_DEFAULT_PREFIX, - destPath - ); + const assets = getCacheAssets(options); + + for (const { fullPath, key, buildId, isFetch } of tqdm(assets)) { + const cacheKey = computeR2CacheKey(key, { + directory: process.env[R2_CACHE_PREFIX_ENV_NAME], + buildId, + isFetch, + }); runWrangler( options, - ["r2 object put", JSON.stringify(fullDestPath), `--file ${JSON.stringify(fsPath)}`], + ["r2 object put", JSON.stringify(path.join(bucket, cacheKey)), `--file ${JSON.stringify(fullPath)}`], // NOTE: R2 does not support the environment flag and results in the following error: // Incorrect type for the 'cacheExpiry' field on 'HttpMetadata': the provided value is not of type 'date'. { target: populateCacheOptions.target, excludeRemoteFlag: true, logging: "error" } @@ -113,15 +143,21 @@ function populateKVIncrementalCache( throw new Error(`No KV binding ${JSON.stringify(KV_CACHE_BINDING_NAME)} found!`); } - const assets = getCacheAssetPaths(options); - for (const { fsPath, destPath } of tqdm(assets)) { + const assets = getCacheAssets(options); + + for (const { fullPath, key, buildId, isFetch } of tqdm(assets)) { + const cacheKey = computeKVCacheKey(key, { + buildId, + isFetch, + }); + runWrangler( options, [ "kv key put", - JSON.stringify(destPath), + JSON.stringify(cacheKey), `--binding ${JSON.stringify(KV_CACHE_BINDING_NAME)}`, - `--path ${JSON.stringify(fsPath)}`, + `--path ${JSON.stringify(fullPath)}`, ], { ...populateCacheOptions, logging: "error" } );