Skip to content

Commit 930ab83

Browse files
vicbvicb
authored
Hash cache keys to limit their length (#553)
1 parent 31b31dd commit 930ab83

File tree

6 files changed

+173
-35
lines changed

6 files changed

+173
-35
lines changed

‎packages/cloudflare/src/api/config.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ export function defineCloudflareConfig(config: CloudflareOverrides = {}): OpenNe
4545
queue: resolveQueue(queue),
4646
},
4747
},
48+
// node:crypto is used to compute cache keys
49+
edgeExternals: ["node:crypto"],
4850
};
4951
}
5052

‎packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import { createHash } from "node:crypto";
2+
13
import { error } from "@opennextjs/aws/adapters/logger.js";
24
import type { CacheValue, IncrementalCache, WithLastModified } from "@opennextjs/aws/types/overrides.js";
35
import { IgnorableError } from "@opennextjs/aws/utils/error.js";
@@ -9,6 +11,17 @@ export const NAME = "cf-kv-incremental-cache";
911

1012
export const BINDING_NAME = "NEXT_INC_CACHE_KV";
1113

14+
export type KeyOptions = {
15+
isFetch?: boolean;
16+
buildId?: string;
17+
};
18+
19+
export function computeCacheKey(key: string, options: KeyOptions) {
20+
const { isFetch = false, buildId = FALLBACK_BUILD_ID } = options;
21+
const hash = createHash("sha256").update(key).digest("hex");
22+
return `${buildId}/${hash}.${isFetch ? "fetch" : "cache"}`.replace(/\/+/g, "/");
23+
}
24+
1225
/**
1326
* Open Next cache based on Cloudflare KV.
1427
*
@@ -93,8 +106,10 @@ class KVIncrementalCache implements IncrementalCache {
93106
}
94107

95108
protected getKVKey(key: string, isFetch?: boolean): string {
96-
const buildId = process.env.NEXT_BUILD_ID ?? FALLBACK_BUILD_ID;
97-
return `${buildId}/${key}.${isFetch ? "fetch" : "cache"}`.replace(/\/+/g, "/");
109+
return computeCacheKey(key, {
110+
buildId: process.env.NEXT_BUILD_ID,
111+
isFetch,
112+
});
98113
}
99114
}
100115

‎packages/cloudflare/src/api/overrides/incremental-cache/r2-incremental-cache.ts‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import { createHash } from "node:crypto";
2+
13
import { error } from "@opennextjs/aws/adapters/logger.js";
24
import type { CacheValue, IncrementalCache, WithLastModified } from "@opennextjs/aws/types/overrides.js";
35
import { IgnorableError } from "@opennextjs/aws/utils/error.js";
@@ -12,6 +14,18 @@ export const BINDING_NAME = "NEXT_INC_CACHE_R2_BUCKET";
1214
export const PREFIX_ENV_NAME = "NEXT_INC_CACHE_R2_PREFIX";
1315
export const DEFAULT_PREFIX = "incremental-cache";
1416

17+
export type KeyOptions = {
18+
isFetch?: boolean;
19+
directory?: string;
20+
buildId?: string;
21+
};
22+
23+
export function computeCacheKey(key: string, options: KeyOptions) {
24+
const { isFetch = false, directory = DEFAULT_PREFIX, buildId = FALLBACK_BUILD_ID } = options;
25+
const hash = createHash("sha256").update(key).digest("hex");
26+
return `${directory}/${buildId}/${hash}.${isFetch ? "fetch" : "cache"}`.replace(/\/+/g, "/");
27+
}
28+
1529
/**
1630
* An instance of the Incremental Cache that uses an R2 bucket (`NEXT_INC_CACHE_R2_BUCKET`) as it's
1731
* underlying data store.
@@ -76,12 +90,11 @@ class R2IncrementalCache implements IncrementalCache {
7690
}
7791

7892
protected getR2Key(key: string, isFetch?: boolean): string {
79-
const directory = getCloudflareContext().env[PREFIX_ENV_NAME] ?? DEFAULT_PREFIX;
80-
81-
return `${directory}/${process.env.NEXT_BUILD_ID ?? FALLBACK_BUILD_ID}/${key}.${isFetch ? "fetch" : "cache"}`.replace(
82-
/\/+/g,
83-
"/"
84-
);
93+
return computeCacheKey(key, {
94+
directory: getCloudflareContext().env[PREFIX_ENV_NAME],
95+
buildId: process.env.NEXT_BUILD_ID,
96+
isFetch,
97+
});
8598
}
8699
}
87100

‎packages/cloudflare/src/cli/build/utils/ensure-cf-config.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ export function ensureCloudflareConfig(config: OpenNextConfig) {
2222
config.default?.override?.queue === "direct" ||
2323
typeof config.default?.override?.queue === "function",
2424
mwIsMiddlewareIntegrated: config.middleware === undefined,
25+
hasCryptoExternal: config.edgeExternals?.includes("node:crypto"),
2526
};
2627

2728
if (config.default?.override?.queue === "direct") {
@@ -42,6 +43,7 @@ export function ensureCloudflareConfig(config: OpenNextConfig) {
4243
queue: "dummy" | "direct" | function,
4344
},
4445
},
46+
edgeExternals: ["node:crypto"],
4547
}\n\n`.replace(/^ {8}/gm, "")
4648
);
4749
}

‎packages/cloudflare/src/cli/commands/populate-cache.spec.ts‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
import { mkdirSync, writeFileSync } from "node:fs";
2+
import path from "node:path";
3+
4+
import type { BuildOptions } from "@opennextjs/aws/build/helper";
5+
import mockFs from "mock-fs";
6+
import { afterAll, beforeAll, describe, expect, test } from "vitest";
7+
8+
import { getCacheAssets } from "./populate-cache";
9+
10+
describe("getCacheAssets", () => {
11+
beforeAll(() => {
12+
mockFs();
13+
14+
const fetchBaseDir = "/base/path/cache/__fetch/buildID";
15+
const cacheDir = "/base/path/cache/buildID/path/to";
16+
17+
mkdirSync(fetchBaseDir, { recursive: true });
18+
mkdirSync(cacheDir, { recursive: true });
19+
20+
for (let i = 0; i < 3; i++) {
21+
writeFileSync(path.join(fetchBaseDir, `${i}`), "", { encoding: "utf-8" });
22+
writeFileSync(path.join(cacheDir, `${i}.cache`), "", { encoding: "utf-8" });
23+
}
24+
});
25+
26+
afterAll(() => mockFs.restore());
27+
28+
test("list cache assets", () => {
29+
expect(getCacheAssets({ outputDir: "/base/path" } as BuildOptions)).toMatchInlineSnapshot(`
30+
[
31+
{
32+
"buildId": "buildID",
33+
"fullPath": "/base/path/cache/buildID/path/to/2.cache",
34+
"isFetch": false,
35+
"key": "/path/to/2",
36+
},
37+
{
38+
"buildId": "buildID",
39+
"fullPath": "/base/path/cache/buildID/path/to/1.cache",
40+
"isFetch": false,
41+
"key": "/path/to/1",
42+
},
43+
{
44+
"buildId": "buildID",
45+
"fullPath": "/base/path/cache/buildID/path/to/0.cache",
46+
"isFetch": false,
47+
"key": "/path/to/0",
48+
},
49+
{
50+
"buildId": "buildID",
51+
"fullPath": "/base/path/cache/__fetch/buildID/2",
52+
"isFetch": true,
53+
"key": "/2",
54+
},
55+
{
56+
"buildId": "buildID",
57+
"fullPath": "/base/path/cache/__fetch/buildID/1",
58+
"isFetch": true,
59+
"key": "/1",
60+
},
61+
{
62+
"buildId": "buildID",
63+
"fullPath": "/base/path/cache/__fetch/buildID/0",
64+
"isFetch": true,
65+
"key": "/0",
66+
},
67+
]
68+
`);
69+
});
70+
});

‎packages/cloudflare/src/cli/commands/populate-cache.ts‎

Lines changed: 63 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,12 @@ import { unstable_readConfig } from "wrangler";
1616

1717
import {
1818
BINDING_NAME as KV_CACHE_BINDING_NAME,
19+
computeCacheKey as computeKVCacheKey,
1920
NAME as KV_CACHE_NAME,
2021
} from "../../api/overrides/incremental-cache/kv-incremental-cache.js";
2122
import {
2223
BINDING_NAME as R2_CACHE_BINDING_NAME,
23-
DEFAULT_PREFIX as R2_CACHE_DEFAULT_PREFIX,
24+
computeCacheKey as computeR2CacheKey,
2425
NAME as R2_CACHE_NAME,
2526
PREFIX_ENV_NAME as R2_CACHE_PREFIX_ENV_NAME,
2627
} from "../../api/overrides/incremental-cache/r2-incremental-cache.js";
@@ -45,22 +46,50 @@ async function resolveCacheName(
4546
return typeof value === "function" ? (await value()).name : value;
4647
}
4748

48-
function getCacheAssetPaths(opts: BuildOptions) {
49-
return globSync(path.join(opts.outputDir, "cache/**/*"), {
49+
export type CacheAsset = { isFetch: boolean; fullPath: string; key: string; buildId: string };
50+
51+
export function getCacheAssets(opts: BuildOptions): CacheAsset[] {
52+
const allFiles = globSync(path.join(opts.outputDir, "cache/**/*"), {
5053
withFileTypes: true,
5154
windowsPathsNoEscape: true,
52-
})
53-
.filter((f) => f.isFile())
54-
.map((f) => {
55-
const relativePath = path.relative(path.join(opts.outputDir, "cache"), f.fullpathPosix());
56-
57-
return {
58-
fsPath: f.fullpathPosix(),
59-
destPath: relativePath.startsWith("__fetch")
60-
? `${relativePath.replace("__fetch/", "")}.fetch`
61-
: relativePath,
62-
};
63-
});
55+
}).filter((f) => f.isFile());
56+
57+
const assets: CacheAsset[] = [];
58+
59+
for (const file of allFiles) {
60+
const fullPath = file.fullpathPosix();
61+
const relativePath = path.relative(path.join(opts.outputDir, "cache"), fullPath);
62+
63+
if (relativePath.startsWith("__fetch")) {
64+
const [__fetch, buildId, ...keyParts] = relativePath.split("/");
65+
66+
if (__fetch !== "__fetch" || buildId === undefined || keyParts.length === 0) {
67+
throw new Error(`Invalid path for a Cache Asset file: ${relativePath}`);
68+
}
69+
70+
assets.push({
71+
isFetch: true,
72+
fullPath,
73+
key: `/${keyParts.join("/")}`,
74+
buildId,
75+
});
76+
} else {
77+
const [buildId, ...keyParts] = relativePath.slice(0, -".cache".length).split("/");
78+
79+
if (!relativePath.endsWith(".cache") || buildId === undefined || keyParts.length === 0) {
80+
throw new Error(`Invalid path for a Cache Asset file: ${relativePath}`);
81+
}
82+
83+
assets.push({
84+
isFetch: false,
85+
fullPath,
86+
key: `/${keyParts.join("/")}`,
87+
buildId,
88+
});
89+
}
90+
}
91+
92+
return assets;
6493
}
6594

6695
function populateR2IncrementalCache(
@@ -81,17 +110,18 @@ function populateR2IncrementalCache(
81110
throw new Error(`R2 binding ${JSON.stringify(R2_CACHE_BINDING_NAME)} should have a 'bucket_name'`);
82111
}
83112

84-
const assets = getCacheAssetPaths(options);
85-
for (const { fsPath, destPath } of tqdm(assets)) {
86-
const fullDestPath = path.join(
87-
bucket,
88-
process.env[R2_CACHE_PREFIX_ENV_NAME] ?? R2_CACHE_DEFAULT_PREFIX,
89-
destPath
90-
);
113+
const assets = getCacheAssets(options);
114+
115+
for (const { fullPath, key, buildId, isFetch } of tqdm(assets)) {
116+
const cacheKey = computeR2CacheKey(key, {
117+
directory: process.env[R2_CACHE_PREFIX_ENV_NAME],
118+
buildId,
119+
isFetch,
120+
});
91121

92122
runWrangler(
93123
options,
94-
["r2 object put", JSON.stringify(fullDestPath), `--file ${JSON.stringify(fsPath)}`],
124+
["r2 object put", JSON.stringify(path.join(bucket, cacheKey)), `--file ${JSON.stringify(fullPath)}`],
95125
// NOTE: R2 does not support the environment flag and results in the following error:
96126
// Incorrect type for the 'cacheExpiry' field on 'HttpMetadata': the provided value is not of type 'date'.
97127
{ target: populateCacheOptions.target, excludeRemoteFlag: true, logging: "error" }
@@ -113,15 +143,21 @@ function populateKVIncrementalCache(
113143
throw new Error(`No KV binding ${JSON.stringify(KV_CACHE_BINDING_NAME)} found!`);
114144
}
115145

116-
const assets = getCacheAssetPaths(options);
117-
for (const { fsPath, destPath } of tqdm(assets)) {
146+
const assets = getCacheAssets(options);
147+
148+
for (const { fullPath, key, buildId, isFetch } of tqdm(assets)) {
149+
const cacheKey = computeKVCacheKey(key, {
150+
buildId,
151+
isFetch,
152+
});
153+
118154
runWrangler(
119155
options,
120156
[
121157
"kv key put",
122-
JSON.stringify(destPath),
158+
JSON.stringify(cacheKey),
123159
`--binding ${JSON.stringify(KV_CACHE_BINDING_NAME)}`,
124-
`--path ${JSON.stringify(fsPath)}`,
160+
`--path ${JSON.stringify(fullPath)}`,
125161
],
126162
{ ...populateCacheOptions, logging: "error" }
127163
);

0 commit comments

Comments
 (0)