Skip to content

Commit df21f84

Browse files
committed
feat: r2 adapter for the incremental cache
1 parent ced3fcf commit df21f84

File tree

6 files changed

+243
-8
lines changed

6 files changed

+243
-8
lines changed

.changeset/weak-houses-divide.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@opennextjs/cloudflare": patch
3+
---
4+
5+
feat: r2 adapter for the incremental cache
Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
import { defineCloudflareConfig } from "@opennextjs/cloudflare";
22
import d1TagCache from "@opennextjs/cloudflare/d1-tag-cache";
3-
import kvIncrementalCache from "@opennextjs/cloudflare/kv-cache";
3+
// import kvIncrementalCache from "@opennextjs/cloudflare/kv-cache";
4+
import r2IncrementalCache from "@opennextjs/cloudflare/r2-incremental-cache";
45
import memoryQueue from "@opennextjs/cloudflare/memory-queue";
56

67
export default defineCloudflareConfig({
7-
incrementalCache: kvIncrementalCache,
8+
incrementalCache: r2IncrementalCache,
89
tagCache: d1TagCache,
910
queue: memoryQueue,
1011
});

examples/e2e/app-router/wrangler.json

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,5 +20,12 @@
2020
"database_id": "NEXT_CACHE_D1",
2121
"database_name": "NEXT_CACHE_D1"
2222
}
23+
],
24+
"r2_buckets": [
25+
{
26+
"binding": "NEXT_CACHE_R2_BUCKET",
27+
"bucket_name": "NEXT_CACHE_R2_BUCKET",
28+
"preview_bucket_name": "NEXT_CACHE_R2_BUCKET"
29+
}
2330
]
2431
}

packages/cloudflare/src/api/cloudflare-context.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ declare global {
66
NEXT_CACHE_D1?: D1Database;
77
NEXT_CACHE_D1_TAGS_TABLE?: string;
88
NEXT_CACHE_D1_REVALIDATIONS_TABLE?: string;
9+
NEXT_CACHE_R2_BUCKET?: R2Bucket;
10+
NEXT_CACHE_R2_DIRECTORY?: string;
911
ASSETS?: Fetcher;
1012
}
1113
}
Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
import { debug, error } from "@opennextjs/aws/adapters/logger.js";
2+
import type { CacheValue, IncrementalCache, WithLastModified } from "@opennextjs/aws/types/overrides.js";
3+
import { IgnorableError } from "@opennextjs/aws/utils/error.js";
4+
5+
import { getCloudflareContext } from "./cloudflare-context.js";
6+
7+
type Entry<IsFetch extends boolean> = {
8+
value: CacheValue<IsFetch>;
9+
lastModified: number;
10+
};
11+
12+
const ONE_YEAR_IN_SECONDS = 31536000;
13+
14+
/**
15+
* An instance of the Incremental Cache that uses an R2 bucket (`NEXT_CACHE_R2_BUCKET`) as it's
16+
* underlying data store.
17+
*
18+
* The directory that the cache entries are stored in can be confused with the `NEXT_CACHE_R2_DIRECTORY`
19+
* environment variable, and defaults to `incremental-cache`.
20+
*
21+
* The cache uses an instance of the Cache API (`incremental-cache`) to store a local version of the
22+
* R2 cache entry to enable fast retrieval, with the cache being updated from R2 in the background.
23+
*/
24+
class R2IncrementalCache implements IncrementalCache {
25+
readonly name = "r2-incremental-cache";
26+
27+
protected localCache: Cache | undefined;
28+
29+
async get<IsFetch extends boolean = false>(
30+
key: string,
31+
isFetch?: IsFetch
32+
): Promise<WithLastModified<CacheValue<IsFetch>> | null> {
33+
const r2 = getCloudflareContext().env.NEXT_CACHE_R2_BUCKET;
34+
if (!r2) throw new IgnorableError("No R2 bucket");
35+
36+
debug(`Get ${key}`);
37+
38+
try {
39+
const r2Response = r2.get(this.getR2Key(key));
40+
41+
const localCacheKey = this.getLocalCacheKey(key, isFetch);
42+
43+
// Check for a cached entry as this will be faster than R2.
44+
const cachedResponse = await this.getFromLocalCache(localCacheKey);
45+
if (cachedResponse) {
46+
debug(` -> Cached response`);
47+
// Update the local cache after the R2 fetch has completed.
48+
getCloudflareContext().ctx.waitUntil(
49+
Promise.resolve(r2Response).then(async (res) => {
50+
if (res) {
51+
const entry: Entry<IsFetch> = await res.json();
52+
await this.putToLocalCache(localCacheKey, JSON.stringify(entry), entry.value.revalidate);
53+
}
54+
})
55+
);
56+
57+
return cachedResponse.json();
58+
}
59+
60+
const r2Object = await r2Response;
61+
if (!r2Object) return null;
62+
const entry: Entry<IsFetch> = await r2Object.json();
63+
64+
// Update the locale cache after retrieving from R2.
65+
getCloudflareContext().ctx.waitUntil(
66+
this.putToLocalCache(localCacheKey, JSON.stringify(entry), entry.value.revalidate)
67+
);
68+
69+
return entry;
70+
} catch (e) {
71+
error(`Failed to get from cache`, e);
72+
return null;
73+
}
74+
}
75+
76+
async set<IsFetch extends boolean = false>(
77+
key: string,
78+
value: CacheValue<IsFetch>,
79+
isFetch?: IsFetch
80+
): Promise<void> {
81+
const r2 = getCloudflareContext().env.NEXT_CACHE_R2_BUCKET;
82+
if (!r2) throw new IgnorableError("No R2 bucket");
83+
84+
debug(`Set ${key}`);
85+
86+
try {
87+
const entry: Entry<IsFetch> = {
88+
value,
89+
// Note: `Date.now()` returns the time of the last IO rather than the actual time.
90+
// See https://developers.cloudflare.com/workers/reference/security-model/
91+
lastModified: Date.now(),
92+
};
93+
94+
await Promise.all([
95+
r2.put(this.getR2Key(key, isFetch), JSON.stringify(entry)),
96+
// Update the locale cache for faster retrieval.
97+
this.putToLocalCache(
98+
this.getLocalCacheKey(key, isFetch),
99+
JSON.stringify(entry),
100+
entry.value.revalidate
101+
),
102+
]);
103+
} catch (e) {
104+
error(`Failed to set to cache`, e);
105+
}
106+
}
107+
108+
async delete(key: string): Promise<void> {
109+
const r2 = getCloudflareContext().env.NEXT_CACHE_R2_BUCKET;
110+
if (!r2) throw new IgnorableError("No R2 bucket");
111+
112+
debug(`Delete ${key}`);
113+
114+
try {
115+
await Promise.all([
116+
r2.delete(this.getR2Key(key)),
117+
this.deleteFromLocalCache(this.getLocalCacheKey(key)),
118+
]);
119+
} catch (e) {
120+
error(`Failed to delete from cache`, e);
121+
}
122+
}
123+
124+
protected getBaseCacheKey(key: string, isFetch?: boolean): string {
125+
return `${process.env.NEXT_BUILD_ID ?? "no-build-id"}/${key}.${isFetch ? "fetch" : "cache"}`;
126+
}
127+
128+
protected getR2Key(key: string, isFetch?: boolean): string {
129+
const directory = getCloudflareContext().env.NEXT_CACHE_R2_DIRECTORY ?? "incremental-cache";
130+
return `${directory}/${this.getBaseCacheKey(key, isFetch)}`;
131+
}
132+
133+
protected getLocalCacheKey(key: string, isFetch?: boolean) {
134+
return new Request(new URL(this.getBaseCacheKey(key, isFetch), "http://cache.local"));
135+
}
136+
137+
protected async getLocalCacheInstance(): Promise<Cache> {
138+
if (this.localCache) return this.localCache;
139+
140+
this.localCache = await caches.open("incremental-cache");
141+
return this.localCache;
142+
}
143+
144+
protected async getFromLocalCache(key: Request) {
145+
const cache = await this.getLocalCacheInstance();
146+
return cache.match(key);
147+
}
148+
149+
protected async putToLocalCache(
150+
key: Request,
151+
entry: string,
152+
revalidate: number | false | undefined
153+
): Promise<void> {
154+
const cache = await this.getLocalCacheInstance();
155+
await cache.put(
156+
key,
157+
new Response(entry, {
158+
headers: new Headers({
159+
"cache-control": `max-age=${revalidate || ONE_YEAR_IN_SECONDS}`,
160+
}),
161+
})
162+
);
163+
}
164+
165+
protected async deleteFromLocalCache(key: Request) {
166+
const cache = await this.getLocalCacheInstance();
167+
await cache.delete(key);
168+
}
169+
}
170+
171+
export default new R2IncrementalCache();

packages/cloudflare/src/cli/build/utils/populate-cache.ts

Lines changed: 55 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import type {
1111
OpenNextConfig,
1212
} from "@opennextjs/aws/types/open-next.js";
1313
import type { IncrementalCache, TagCache } from "@opennextjs/aws/types/overrides.js";
14+
import { globSync } from "glob";
1415

1516
export type CacheBindingMode = "local" | "remote";
1617

@@ -24,10 +25,20 @@ async function resolveCacheName(
2425
return typeof value === "function" ? (await value()).name : value;
2526
}
2627

27-
function runWrangler(opts: BuildOptions, mode: CacheBindingMode, args: string[]) {
28+
function runWrangler(
29+
opts: BuildOptions,
30+
wranglerOpts: { mode: CacheBindingMode; excludeRemoteFlag?: boolean },
31+
args: string[]
32+
) {
2833
const result = spawnSync(
2934
opts.packager,
30-
["exec", "wrangler", ...args, mode === "remote" && "--remote"].filter((v): v is string => !!v),
35+
[
36+
"exec",
37+
"wrangler",
38+
...args,
39+
wranglerOpts.mode === "remote" && !wranglerOpts.excludeRemoteFlag && "--remote",
40+
wranglerOpts.mode === "local" && "--local",
41+
].filter((v): v is string => !!v),
3142
{
3243
shell: true,
3344
stdio: ["ignore", "ignore", "inherit"],
@@ -37,11 +48,24 @@ function runWrangler(opts: BuildOptions, mode: CacheBindingMode, args: string[])
3748
if (result.status !== 0) {
3849
logger.error("Failed to populate cache");
3950
process.exit(1);
40-
} else {
41-
logger.info("Successfully populated cache");
4251
}
4352
}
4453

54+
function getCacheAssetPaths(opts: BuildOptions) {
55+
return globSync(path.join(opts.outputDir, "cache/**/*"), { withFileTypes: true })
56+
.filter((f) => f.isFile())
57+
.map((f) => {
58+
const relativePath = path.relative(path.join(opts.outputDir, "cache"), f.fullpathPosix());
59+
60+
return {
61+
fsPath: f.fullpathPosix(),
62+
destPath: relativePath.startsWith("__fetch")
63+
? `${relativePath.replace("__fetch/", "")}.fetch`
64+
: relativePath,
65+
};
66+
});
67+
}
68+
4569
export async function populateCache(opts: BuildOptions, config: OpenNextConfig, mode: CacheBindingMode) {
4670
const { incrementalCache, tagCache } = config.default.override ?? {};
4771

@@ -51,7 +75,31 @@ export async function populateCache(opts: BuildOptions, config: OpenNextConfig,
5175
}
5276

5377
if (!config.dangerous?.disableIncrementalCache && incrementalCache) {
54-
logger.info("Incremental cache does not need populating");
78+
const name = await resolveCacheName(incrementalCache);
79+
switch (name) {
80+
case "r2-incremental-cache": {
81+
logger.info("\nPopulating R2 incremental cache...");
82+
83+
const assets = getCacheAssetPaths(opts);
84+
assets.forEach(({ fsPath, destPath }) => {
85+
const fullDestPath = path.join(
86+
"NEXT_CACHE_R2_BUCKET",
87+
process.env.NEXT_CACHE_R2_DIRECTORY ?? "incremental-cache",
88+
destPath
89+
);
90+
91+
runWrangler(opts, { mode, excludeRemoteFlag: true }, [
92+
"r2 object put",
93+
JSON.stringify(fullDestPath),
94+
`--file ${JSON.stringify(fsPath)}`,
95+
]);
96+
});
97+
logger.info(`Successfully populated cache with ${assets.length} assets`);
98+
break;
99+
}
100+
default:
101+
logger.info("INcremental cache does not need populating");
102+
}
55103
}
56104

57105
if (!config.dangerous?.disableTagCache && !config.dangerous?.disableIncrementalCache && tagCache) {
@@ -60,11 +108,12 @@ export async function populateCache(opts: BuildOptions, config: OpenNextConfig,
60108
case "d1-tag-cache": {
61109
logger.info("\nPopulating D1 tag cache...");
62110

63-
runWrangler(opts, mode, [
111+
runWrangler(opts, { mode }, [
64112
"d1 execute",
65113
"NEXT_CACHE_D1",
66114
`--file ${JSON.stringify(path.join(opts.outputDir, "cloudflare/cache-assets-manifest.sql"))}`,
67115
]);
116+
logger.info("Successfully populated cache");
68117
break;
69118
}
70119
default:

0 commit comments

Comments
 (0)