From 2deac38d4b3be16f4f196f35ae7394e48ec4eb05 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 7 Apr 2025 20:15:51 +0100 Subject: [PATCH 01/18] delete original kv inc cache --- .../incremental-cache/kv-incremental-cache.ts | 163 ------------------ 1 file changed, 163 deletions(-) delete mode 100644 packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts diff --git a/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts b/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts deleted file mode 100644 index 1335eeda..00000000 --- a/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts +++ /dev/null @@ -1,163 +0,0 @@ -import type { CacheValue, IncrementalCache, WithLastModified } from "@opennextjs/aws/types/overrides"; -import { IgnorableError, RecoverableError } from "@opennextjs/aws/utils/error.js"; - -import { getCloudflareContext } from "../../cloudflare-context.js"; - -export const CACHE_ASSET_DIR = "cdn-cgi/_next_cache"; - -export const STATUS_DELETED = 1; - -export const NAME = "cf-kv-incremental-cache"; - -/** - * Open Next cache based on cloudflare KV and Assets. - * - * Note: The class is instantiated outside of the request context. - * The cloudflare context and process.env are not initialized yet - * when the constructor is called. - */ -class KVIncrementalCache implements IncrementalCache { - readonly name = NAME; - - async get( - key: string, - isFetch?: IsFetch - ): Promise> | null> { - const cfEnv = getCloudflareContext().env; - const kv = cfEnv.NEXT_INC_CACHE_KV; - const assets = cfEnv.ASSETS; - - if (!(kv || assets)) { - throw new IgnorableError(`No KVNamespace nor Fetcher`); - } - - this.debug(`Get ${key}`); - - try { - let entry: { - value?: CacheValue; - lastModified?: number; - status?: number; - } | null = null; - - if (kv) { - this.debug(`- From KV`); - const kvKey = this.getKVKey(key, isFetch); - entry = await kv.get(kvKey, "json"); - if (entry?.status === STATUS_DELETED) { - return null; - } - } - - if (!entry && assets) { - this.debug(`- From Assets`); - const url = this.getAssetUrl(key, isFetch); - const response = await assets.fetch(url); - if (response.ok) { - // TODO: consider populating KV with the asset value if faster. - // This could be optional as KV writes are $$. - // See https://github.com/opennextjs/opennextjs-cloudflare/pull/194#discussion_r1893166026 - entry = { - value: await response.json(), - // __BUILD_TIMESTAMP_MS__ is injected by ESBuild. - lastModified: (globalThis as { __BUILD_TIMESTAMP_MS__?: number }).__BUILD_TIMESTAMP_MS__, - }; - } - if (!kv) { - // The cache can not be updated when there is no KV - // As we don't want to keep serving stale data for ever, - // we pretend the entry is not in cache - if ( - entry?.value && - "kind" in entry.value && - entry.value.kind === "FETCH" && - entry.value.data?.headers?.expires - ) { - const expiresTime = new Date(entry.value.data.headers.expires).getTime(); - if (!isNaN(expiresTime) && expiresTime <= Date.now()) { - this.debug(`found expired entry (expire time: ${entry.value.data.headers.expires})`); - return null; - } - } - } - } - - this.debug(entry ? `-> hit` : `-> miss`); - return { value: entry?.value, lastModified: entry?.lastModified }; - } catch { - throw new RecoverableError(`Failed to get cache [${key}]`); - } - } - - async set( - key: string, - value: CacheValue, - isFetch?: IsFetch - ): Promise { - const kv = getCloudflareContext().env.NEXT_INC_CACHE_KV; - - if (!kv) { - throw new IgnorableError(`No KVNamespace`); - } - - this.debug(`Set ${key}`); - - try { - const kvKey = this.getKVKey(key, isFetch); - // Note: We can not set a TTL as we might fallback to assets, - // still removing old data (old BUILD_ID) could help avoiding - // the cache growing too big. - await kv.put( - kvKey, - JSON.stringify({ - value, - // Note: `Date.now()` returns the time of the last IO rather than the actual time. - // See https://developers.cloudflare.com/workers/reference/security-model/ - lastModified: Date.now(), - }) - ); - } catch { - throw new RecoverableError(`Failed to set cache [${key}]`); - } - } - - async delete(key: string): Promise { - const kv = getCloudflareContext().env.NEXT_INC_CACHE_KV; - - if (!kv) { - throw new IgnorableError(`No KVNamespace`); - } - - this.debug(`Delete ${key}`); - - try { - const kvKey = this.getKVKey(key, /* isFetch= */ false); - // Do not delete the key as we would then fallback to the assets. - await kv.put(kvKey, JSON.stringify({ status: STATUS_DELETED })); - } catch { - throw new RecoverableError(`Failed to delete cache [${key}]`); - } - } - - protected getKVKey(key: string, isFetch?: boolean): string { - return `${this.getBuildId()}/${key}.${isFetch ? "fetch" : "cache"}`.replace(/\/+/g, "/"); - } - - protected getAssetUrl(key: string, isFetch?: boolean): string { - return isFetch - ? `http://assets.local/${CACHE_ASSET_DIR}/__fetch/${this.getBuildId()}/${key}` - : `http://assets.local/${CACHE_ASSET_DIR}/${this.getBuildId()}/${key}.cache`; - } - - protected debug(...args: unknown[]) { - if (process.env.NEXT_PRIVATE_DEBUG_CACHE) { - console.log(`[Cache ${this.name}] `, ...args); - } - } - - protected getBuildId() { - return process.env.NEXT_BUILD_ID ?? "no-build-id"; - } -} - -export default new KVIncrementalCache(); From d77da467d57f70354f48c6c6018adff0edbe174a Mon Sep 17 00:00:00 2001 From: James Date: Mon, 7 Apr 2025 20:16:04 +0100 Subject: [PATCH 02/18] add simplified new kv inc cache --- .../incremental-cache/kv-incremental-cache.ts | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts diff --git a/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts b/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts new file mode 100644 index 00000000..ec52fca3 --- /dev/null +++ b/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts @@ -0,0 +1,89 @@ +import { debug, error } from "@opennextjs/aws/adapters/logger.js"; +import type { CacheValue, IncrementalCache, WithLastModified } from "@opennextjs/aws/types/overrides.js"; +import { IgnorableError } from "@opennextjs/aws/utils/error.js"; + +import { getCloudflareContext } from "../../cloudflare-context.js"; +import { IncrementalCacheEntry } from "./internal.js"; + +export const NAME = "cf-kv-incremental-cache"; + +/** + * Open Next cache based on Cloudflare KV. + * + * Note: The class is instantiated outside of the request context. + * The cloudflare context and process.env are not initialized yet + * when the constructor is called. + */ +class KVIncrementalCache implements IncrementalCache { + readonly name = NAME; + + async get( + key: string, + isFetch?: IsFetch + ): Promise> | null> { + const kv = getCloudflareContext().env.NEXT_INC_CACHE_KV; + if (!kv) throw new IgnorableError("No KV Namespace"); + + debug(`Get ${key}`); + + try { + const entry = await kv.get>(this.getKVKey(key, isFetch), "json"); + + return entry; + } catch (e) { + error("Failed to get from cache", e); + return null; + } + } + + async set( + key: string, + value: CacheValue, + isFetch?: IsFetch + ): Promise { + const kv = getCloudflareContext().env.NEXT_INC_CACHE_KV; + if (!kv) throw new IgnorableError("No KV Namespace"); + + debug(`Set ${key}`); + + try { + await kv.put( + this.getKVKey(key, isFetch), + JSON.stringify({ + value, + // Note: `Date.now()` returns the time of the last IO rather than the actual time. + // See https://developers.cloudflare.com/workers/reference/security-model/ + lastModified: Date.now(), + }), + { + // When available, we only cache for the max revalidate time + ...(value.revalidate && { + expirationTtl: value.revalidate, + }), + } + ); + } catch (e) { + error("Failed to set to cache", e); + } + } + + async delete(key: string): Promise { + const kv = getCloudflareContext().env.NEXT_INC_CACHE_KV; + if (!kv) throw new IgnorableError("No KV Namespace"); + + debug(`Delete ${key}`); + + try { + await kv.delete(this.getKVKey(key, /* isFetch= */ false)); + } catch (e) { + error("Failed to delete from cache", e); + } + } + + protected getKVKey(key: string, isFetch?: boolean): string { + const buildId = process.env.NEXT_BUILD_ID ?? "no-build-id"; + return `${buildId}/${key}.${isFetch ? "fetch" : "cache"}`.replace(/\/+/g, "/"); + } +} + +export default new KVIncrementalCache(); From 2abb399c09a7bb8e52e055b2d0d1513f1b069dd3 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 7 Apr 2025 21:00:17 +0100 Subject: [PATCH 03/18] add kv population --- packages/cloudflare/src/cli/build/build.ts | 2 -- .../cli/build/open-next/copyCacheAssets.ts | 14 ------------- .../src/cli/commands/populate-cache.ts | 20 +++++++++++++++++++ 3 files changed, 20 insertions(+), 16 deletions(-) delete mode 100644 packages/cloudflare/src/cli/build/open-next/copyCacheAssets.ts diff --git a/packages/cloudflare/src/cli/build/build.ts b/packages/cloudflare/src/cli/build/build.ts index 312e1848..b0e857e2 100644 --- a/packages/cloudflare/src/cli/build/build.ts +++ b/packages/cloudflare/src/cli/build/build.ts @@ -13,7 +13,6 @@ import { bundleServer } from "./bundle-server.js"; import { compileCacheAssetsManifestSqlFile } from "./open-next/compile-cache-assets-manifest.js"; import { compileEnvFiles } from "./open-next/compile-env-files.js"; import { compileDurableObjects } from "./open-next/compileDurableObjects.js"; -import { copyCacheAssets } from "./open-next/copyCacheAssets.js"; import { createServerBundle } from "./open-next/createServerBundle.js"; import { createWranglerConfigIfNotExistent } from "./utils/index.js"; import { getVersion } from "./utils/version.js"; @@ -71,7 +70,6 @@ export async function build( if (config.dangerous?.disableIncrementalCache !== true) { const { useTagCache, metaFiles } = createCacheAssets(options); - copyCacheAssets(options); if (useTagCache) { compileCacheAssetsManifestSqlFile(options, metaFiles); diff --git a/packages/cloudflare/src/cli/build/open-next/copyCacheAssets.ts b/packages/cloudflare/src/cli/build/open-next/copyCacheAssets.ts deleted file mode 100644 index 9ea967a7..00000000 --- a/packages/cloudflare/src/cli/build/open-next/copyCacheAssets.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { cpSync, mkdirSync } from "node:fs"; -import { join } from "node:path"; - -import * as buildHelper from "@opennextjs/aws/build/helper.js"; - -import { CACHE_ASSET_DIR } from "../../../api/overrides/incremental-cache/kv-incremental-cache.js"; - -export function copyCacheAssets(options: buildHelper.BuildOptions) { - const { outputDir } = options; - const srcPath = join(outputDir, "cache"); - const dstPath = join(outputDir, "assets", CACHE_ASSET_DIR); - mkdirSync(dstPath, { recursive: true }); - cpSync(srcPath, dstPath, { recursive: true }); -} diff --git a/packages/cloudflare/src/cli/commands/populate-cache.ts b/packages/cloudflare/src/cli/commands/populate-cache.ts index 893fa0a3..347a535f 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.ts @@ -14,6 +14,7 @@ import { globSync } from "glob"; import { tqdm } from "ts-tqdm"; import { unstable_readConfig } from "wrangler"; +import { NAME as KV_CACHE_NAME } from "../../api/overrides/incremental-cache/kv-incremental-cache.js"; import { NAME as R2_CACHE_NAME } from "../../api/overrides/incremental-cache/r2-incremental-cache.js"; import { NAME as D1_TAG_NAME } from "../../api/overrides/tag-cache/d1-next-tag-cache.js"; import type { WranglerTarget } from "../utils/run-wrangler.js"; @@ -100,6 +101,25 @@ export async function populateCache( logger.info(`Successfully populated cache with ${assets.length} assets`); break; } + case KV_CACHE_NAME: { + logger.info("\nPopulating KV incremental cache..."); + + const assets = getCacheAssetPaths(options); + for (const { fsPath, destPath } of tqdm(assets)) { + runWrangler( + options, + [ + "kv key put", + JSON.stringify(destPath), + "--binding NEXT_INC_CACHE_KV", + `--path ${JSON.stringify(fsPath)}`, + ], + { ...populateCacheOptions, logging: "error" } + ); + } + logger.info(`Successfully populated cache with ${assets.length} assets`); + break; + } default: logger.info("Incremental cache does not need populating"); } From faae80f4bef54c61abb3835eb1a5efa4385fcc22 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 7 Apr 2025 21:01:17 +0100 Subject: [PATCH 04/18] changeset --- .changeset/metal-lemons-sparkle.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/metal-lemons-sparkle.md diff --git a/.changeset/metal-lemons-sparkle.md b/.changeset/metal-lemons-sparkle.md new file mode 100644 index 00000000..18edba92 --- /dev/null +++ b/.changeset/metal-lemons-sparkle.md @@ -0,0 +1,5 @@ +--- +"@opennextjs/cloudflare": patch +--- + +feat: auto-populate kv incremental cache From 4c20b8ec0d07ac29e77579538794550f1e4c6317 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 7 Apr 2025 21:15:18 +0100 Subject: [PATCH 05/18] fix e2e --- examples/ssg-app/wrangler.jsonc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/ssg-app/wrangler.jsonc b/examples/ssg-app/wrangler.jsonc index bb008505..0b9c532a 100644 --- a/examples/ssg-app/wrangler.jsonc +++ b/examples/ssg-app/wrangler.jsonc @@ -8,6 +8,12 @@ "directory": ".open-next/assets", "binding": "ASSETS" }, + "kv_namespaces": [ + { + "binding": "NEXT_INC_CACHE_KV", + "id": "" + } + ], "vars": { "APP_VERSION": "1.2.345" } From d8d49d39da31830ce0070ae1277a18ddf1565f28 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 7 Apr 2025 21:42:30 +0100 Subject: [PATCH 06/18] fix serving asset populated from build --- .../incremental-cache/kv-incremental-cache.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts b/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts index ec52fca3..86321301 100644 --- a/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts +++ b/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts @@ -27,9 +27,21 @@ class KVIncrementalCache implements IncrementalCache { debug(`Get ${key}`); try { - const entry = await kv.get>(this.getKVKey(key, isFetch), "json"); + const entry = await kv.get | CacheValue>( + this.getKVKey(key, isFetch), + "json" + ); + + if (!entry || "lastModified" in entry) { + return entry; + } - return entry; + // if there is no lastModified property, the file was stored during build-time cache population. + return { + value: entry, + // __BUILD_TIMESTAMP_MS__ is injected by ESBuild. + lastModified: (globalThis as { __BUILD_TIMESTAMP_MS__?: number }).__BUILD_TIMESTAMP_MS__, + }; } catch (e) { error("Failed to get from cache", e); return null; From b8d264edda950048377401848ca2a7e2881e8a09 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 7 Apr 2025 21:45:34 +0100 Subject: [PATCH 07/18] remove ttl for now --- .../incremental-cache/kv-incremental-cache.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts b/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts index 86321301..d8cbcab5 100644 --- a/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts +++ b/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts @@ -66,13 +66,9 @@ class KVIncrementalCache implements IncrementalCache { // Note: `Date.now()` returns the time of the last IO rather than the actual time. // See https://developers.cloudflare.com/workers/reference/security-model/ lastModified: Date.now(), - }), - { - // When available, we only cache for the max revalidate time - ...(value.revalidate && { - expirationTtl: value.revalidate, - }), - } + }) + // TODO: Figure out how to best leverage KV's TTL. + // NOTE: Ideally, the cache should operate in an SWR-like manner. ); } catch (e) { error("Failed to set to cache", e); From 1b621631ebe2f81a4ee59b0c257b8bc39d392d19 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 8 Apr 2025 07:44:32 +0100 Subject: [PATCH 08/18] shared cache debug function --- .../incremental-cache/kv-incremental-cache.ts | 10 +++++----- .../incremental-cache/r2-incremental-cache.ts | 9 +++++---- .../api/overrides/incremental-cache/regional-cache.ts | 6 +++--- .../api/overrides/{incremental-cache => }/internal.ts | 6 ++++++ .../cloudflare/src/api/overrides/queue/memory-queue.ts | 5 +++-- .../src/api/overrides/tag-cache/d1-next-tag-cache.ts | 5 +++-- .../api/overrides/tag-cache/do-sharded-tag-cache.ts | 7 ++++--- 7 files changed, 29 insertions(+), 19 deletions(-) rename packages/cloudflare/src/api/overrides/{incremental-cache => }/internal.ts (53%) diff --git a/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts b/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts index d8cbcab5..448ae69f 100644 --- a/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts +++ b/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts @@ -1,9 +1,9 @@ -import { debug, error } from "@opennextjs/aws/adapters/logger.js"; +import { error } from "@opennextjs/aws/adapters/logger.js"; import type { CacheValue, IncrementalCache, WithLastModified } from "@opennextjs/aws/types/overrides.js"; import { IgnorableError } from "@opennextjs/aws/utils/error.js"; import { getCloudflareContext } from "../../cloudflare-context.js"; -import { IncrementalCacheEntry } from "./internal.js"; +import { debugCache, IncrementalCacheEntry } from "../internal.js"; export const NAME = "cf-kv-incremental-cache"; @@ -24,7 +24,7 @@ class KVIncrementalCache implements IncrementalCache { const kv = getCloudflareContext().env.NEXT_INC_CACHE_KV; if (!kv) throw new IgnorableError("No KV Namespace"); - debug(`Get ${key}`); + debugCache(`Get ${key}`); try { const entry = await kv.get | CacheValue>( @@ -56,7 +56,7 @@ class KVIncrementalCache implements IncrementalCache { const kv = getCloudflareContext().env.NEXT_INC_CACHE_KV; if (!kv) throw new IgnorableError("No KV Namespace"); - debug(`Set ${key}`); + debugCache(`Set ${key}`); try { await kv.put( @@ -79,7 +79,7 @@ class KVIncrementalCache implements IncrementalCache { const kv = getCloudflareContext().env.NEXT_INC_CACHE_KV; if (!kv) throw new IgnorableError("No KV Namespace"); - debug(`Delete ${key}`); + debugCache(`Delete ${key}`); try { await kv.delete(this.getKVKey(key, /* isFetch= */ false)); diff --git a/packages/cloudflare/src/api/overrides/incremental-cache/r2-incremental-cache.ts b/packages/cloudflare/src/api/overrides/incremental-cache/r2-incremental-cache.ts index 725321a4..aad8b576 100644 --- a/packages/cloudflare/src/api/overrides/incremental-cache/r2-incremental-cache.ts +++ b/packages/cloudflare/src/api/overrides/incremental-cache/r2-incremental-cache.ts @@ -1,8 +1,9 @@ -import { debug, error } from "@opennextjs/aws/adapters/logger.js"; +import { error } from "@opennextjs/aws/adapters/logger.js"; import type { CacheValue, IncrementalCache, WithLastModified } from "@opennextjs/aws/types/overrides.js"; import { IgnorableError } from "@opennextjs/aws/utils/error.js"; import { getCloudflareContext } from "../../cloudflare-context.js"; +import { debugCache } from "../internal.js"; export const NAME = "cf-r2-incremental-cache"; @@ -23,7 +24,7 @@ class R2IncrementalCache implements IncrementalCache { const r2 = getCloudflareContext().env.NEXT_INC_CACHE_R2_BUCKET; if (!r2) throw new IgnorableError("No R2 bucket"); - debug(`Get ${key}`); + debugCache(`Get ${key}`); try { const r2Object = await r2.get(this.getR2Key(key, isFetch)); @@ -47,7 +48,7 @@ class R2IncrementalCache implements IncrementalCache { const r2 = getCloudflareContext().env.NEXT_INC_CACHE_R2_BUCKET; if (!r2) throw new IgnorableError("No R2 bucket"); - debug(`Set ${key}`); + debugCache(`Set ${key}`); try { await r2.put(this.getR2Key(key, isFetch), JSON.stringify(value)); @@ -60,7 +61,7 @@ class R2IncrementalCache implements IncrementalCache { const r2 = getCloudflareContext().env.NEXT_INC_CACHE_R2_BUCKET; if (!r2) throw new IgnorableError("No R2 bucket"); - debug(`Delete ${key}`); + debugCache(`Delete ${key}`); try { await r2.delete(this.getR2Key(key)); diff --git a/packages/cloudflare/src/api/overrides/incremental-cache/regional-cache.ts b/packages/cloudflare/src/api/overrides/incremental-cache/regional-cache.ts index 5c611c96..6f205366 100644 --- a/packages/cloudflare/src/api/overrides/incremental-cache/regional-cache.ts +++ b/packages/cloudflare/src/api/overrides/incremental-cache/regional-cache.ts @@ -1,8 +1,8 @@ -import { debug, error } from "@opennextjs/aws/adapters/logger.js"; +import { error } from "@opennextjs/aws/adapters/logger.js"; import { CacheValue, IncrementalCache, WithLastModified } from "@opennextjs/aws/types/overrides.js"; import { getCloudflareContext } from "../../cloudflare-context.js"; -import { IncrementalCacheEntry } from "./internal.js"; +import { debugCache, IncrementalCacheEntry } from "../internal.js"; import { NAME as KV_CACHE_NAME } from "./kv-incremental-cache.js"; const ONE_MINUTE_IN_SECONDS = 60; @@ -57,7 +57,7 @@ class RegionalCache implements IncrementalCache { // Check for a cached entry as this will be faster than the store response. const cachedResponse = await cache.match(localCacheKey); if (cachedResponse) { - debug("Get - cached response"); + debugCache("Get - cached response"); // Re-fetch from the store and update the regional cache in the background if (this.opts.shouldLazilyUpdateOnCacheHit) { diff --git a/packages/cloudflare/src/api/overrides/incremental-cache/internal.ts b/packages/cloudflare/src/api/overrides/internal.ts similarity index 53% rename from packages/cloudflare/src/api/overrides/incremental-cache/internal.ts rename to packages/cloudflare/src/api/overrides/internal.ts index 2407fef8..e343739b 100644 --- a/packages/cloudflare/src/api/overrides/incremental-cache/internal.ts +++ b/packages/cloudflare/src/api/overrides/internal.ts @@ -4,3 +4,9 @@ export type IncrementalCacheEntry = { value: CacheValue; lastModified: number; }; + +export const debugCache = (name: string, ...args: unknown[]) => { + if (process.env.NEXT_PRIVATE_DEBUG_CACHE) { + console.log(`[${name}] `, ...args); + } +}; diff --git a/packages/cloudflare/src/api/overrides/queue/memory-queue.ts b/packages/cloudflare/src/api/overrides/queue/memory-queue.ts index 60de7599..d8e4a7ce 100644 --- a/packages/cloudflare/src/api/overrides/queue/memory-queue.ts +++ b/packages/cloudflare/src/api/overrides/queue/memory-queue.ts @@ -1,8 +1,9 @@ -import { debug, error } from "@opennextjs/aws/adapters/logger.js"; +import { error } from "@opennextjs/aws/adapters/logger.js"; import type { Queue, QueueMessage } from "@opennextjs/aws/types/overrides.js"; import { IgnorableError } from "@opennextjs/aws/utils/error.js"; import { getCloudflareContext } from "../../cloudflare-context"; +import { debugCache } from "../internal"; export const DEFAULT_REVALIDATION_TIMEOUT_MS = 10_000; @@ -48,7 +49,7 @@ export class MemoryQueue implements Queue { if (response.status !== 200 || response.headers.get("x-nextjs-cache") !== "REVALIDATED") { error(`Revalidation failed for ${url} with status ${response.status}`); } - debug(`Revalidation successful for ${url}`); + debugCache(`Revalidation successful for ${url}`); } catch (e) { error(e); } finally { diff --git a/packages/cloudflare/src/api/overrides/tag-cache/d1-next-tag-cache.ts b/packages/cloudflare/src/api/overrides/tag-cache/d1-next-tag-cache.ts index 1503fcdd..a024301c 100644 --- a/packages/cloudflare/src/api/overrides/tag-cache/d1-next-tag-cache.ts +++ b/packages/cloudflare/src/api/overrides/tag-cache/d1-next-tag-cache.ts @@ -1,9 +1,10 @@ -import { debug, error } from "@opennextjs/aws/adapters/logger.js"; +import { error } from "@opennextjs/aws/adapters/logger.js"; import type { OpenNextConfig } from "@opennextjs/aws/types/open-next.js"; import type { NextModeTagCache } from "@opennextjs/aws/types/overrides.js"; import { RecoverableError } from "@opennextjs/aws/utils/error.js"; import { getCloudflareContext } from "../../cloudflare-context.js"; +import { debugCache } from "../internal.js"; export const NAME = "d1-next-mode-tag-cache"; @@ -48,7 +49,7 @@ export class D1NextModeTagCache implements NextModeTagCache { const cfEnv = getCloudflareContext().env; const db = cfEnv.NEXT_TAG_CACHE_D1; - if (!db) debug("No D1 database found"); + if (!db) debugCache("No D1 database found"); const isDisabled = !!(globalThis as unknown as { openNextConfig: OpenNextConfig }).openNextConfig .dangerous?.disableTagCache; diff --git a/packages/cloudflare/src/api/overrides/tag-cache/do-sharded-tag-cache.ts b/packages/cloudflare/src/api/overrides/tag-cache/do-sharded-tag-cache.ts index 4f0edf5a..d0cd2e9c 100644 --- a/packages/cloudflare/src/api/overrides/tag-cache/do-sharded-tag-cache.ts +++ b/packages/cloudflare/src/api/overrides/tag-cache/do-sharded-tag-cache.ts @@ -1,10 +1,11 @@ -import { debug, error } from "@opennextjs/aws/adapters/logger.js"; +import { error } from "@opennextjs/aws/adapters/logger.js"; import { generateShardId } from "@opennextjs/aws/core/routing/queue.js"; import type { OpenNextConfig } from "@opennextjs/aws/types/open-next"; import type { NextModeTagCache } from "@opennextjs/aws/types/overrides.js"; import { IgnorableError } from "@opennextjs/aws/utils/error.js"; import { getCloudflareContext } from "../../cloudflare-context"; +import { debugCache } from "../internal"; export const DEFAULT_WRITE_RETRIES = 3; export const DEFAULT_NUM_SHARDS = 4; @@ -196,7 +197,7 @@ class ShardedDOTagCache implements NextModeTagCache { const cfEnv = getCloudflareContext().env; const db = cfEnv.NEXT_TAG_CACHE_DO_SHARDED; - if (!db) debug("No Durable object found"); + if (!db) debugCache("No Durable object found"); const isDisabled = !!(globalThis as unknown as { openNextConfig: OpenNextConfig }).openNextConfig .dangerous?.disableTagCache; @@ -334,7 +335,7 @@ class ShardedDOTagCache implements NextModeTagCache { const key = await this.getCacheKey(doId, tags); await cache.delete(key); } catch (e) { - debug("Error while deleting from regional cache", e); + debugCache("Error while deleting from regional cache", e); } } } From 1b85d3bb9ab4cf7f2456278888a16a070a1f6605 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 8 Apr 2025 07:57:09 +0100 Subject: [PATCH 09/18] share names between files --- .changeset/metal-lemons-sparkle.md | 2 +- .../incremental-cache/kv-incremental-cache.ts | 12 ++++--- .../incremental-cache/r2-incremental-cache.ts | 17 ++++++---- .../incremental-cache/regional-cache.ts | 4 +-- .../cloudflare/src/api/overrides/internal.ts | 2 ++ .../overrides/tag-cache/d1-next-tag-cache.ts | 9 +++--- .../src/cli/commands/populate-cache.ts | 31 ++++++++++++------- 7 files changed, 48 insertions(+), 29 deletions(-) diff --git a/.changeset/metal-lemons-sparkle.md b/.changeset/metal-lemons-sparkle.md index 18edba92..bc5aaddf 100644 --- a/.changeset/metal-lemons-sparkle.md +++ b/.changeset/metal-lemons-sparkle.md @@ -2,4 +2,4 @@ "@opennextjs/cloudflare": patch --- -feat: auto-populate kv incremental cache +feat: populate kv incremental cache diff --git a/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts b/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts index 448ae69f..75f9bb04 100644 --- a/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts +++ b/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts @@ -3,10 +3,12 @@ import type { CacheValue, IncrementalCache, WithLastModified } from "@opennextjs import { IgnorableError } from "@opennextjs/aws/utils/error.js"; import { getCloudflareContext } from "../../cloudflare-context.js"; -import { debugCache, IncrementalCacheEntry } from "../internal.js"; +import { debugCache, FALLBACK_BUILD_ID, IncrementalCacheEntry } from "../internal.js"; export const NAME = "cf-kv-incremental-cache"; +export const BINDING_NAME = "NEXT_INC_CACHE_KV"; + /** * Open Next cache based on Cloudflare KV. * @@ -21,7 +23,7 @@ class KVIncrementalCache implements IncrementalCache { key: string, isFetch?: IsFetch ): Promise> | null> { - const kv = getCloudflareContext().env.NEXT_INC_CACHE_KV; + const kv = getCloudflareContext().env[BINDING_NAME]; if (!kv) throw new IgnorableError("No KV Namespace"); debugCache(`Get ${key}`); @@ -53,7 +55,7 @@ class KVIncrementalCache implements IncrementalCache { value: CacheValue, isFetch?: IsFetch ): Promise { - const kv = getCloudflareContext().env.NEXT_INC_CACHE_KV; + const kv = getCloudflareContext().env[BINDING_NAME]; if (!kv) throw new IgnorableError("No KV Namespace"); debugCache(`Set ${key}`); @@ -76,7 +78,7 @@ class KVIncrementalCache implements IncrementalCache { } async delete(key: string): Promise { - const kv = getCloudflareContext().env.NEXT_INC_CACHE_KV; + const kv = getCloudflareContext().env[BINDING_NAME]; if (!kv) throw new IgnorableError("No KV Namespace"); debugCache(`Delete ${key}`); @@ -89,7 +91,7 @@ class KVIncrementalCache implements IncrementalCache { } protected getKVKey(key: string, isFetch?: boolean): string { - const buildId = process.env.NEXT_BUILD_ID ?? "no-build-id"; + const buildId = process.env.NEXT_BUILD_ID ?? FALLBACK_BUILD_ID; return `${buildId}/${key}.${isFetch ? "fetch" : "cache"}`.replace(/\/+/g, "/"); } } diff --git a/packages/cloudflare/src/api/overrides/incremental-cache/r2-incremental-cache.ts b/packages/cloudflare/src/api/overrides/incremental-cache/r2-incremental-cache.ts index aad8b576..2fdd32b9 100644 --- a/packages/cloudflare/src/api/overrides/incremental-cache/r2-incremental-cache.ts +++ b/packages/cloudflare/src/api/overrides/incremental-cache/r2-incremental-cache.ts @@ -3,10 +3,15 @@ import type { CacheValue, IncrementalCache, WithLastModified } from "@opennextjs import { IgnorableError } from "@opennextjs/aws/utils/error.js"; import { getCloudflareContext } from "../../cloudflare-context.js"; -import { debugCache } from "../internal.js"; +import { debugCache, FALLBACK_BUILD_ID } from "../internal.js"; export const NAME = "cf-r2-incremental-cache"; +export const BINDING_NAME = "NEXT_INC_CACHE_R2_BUCKET"; + +export const PREFIX_ENV_NAME = "NEXT_INC_CACHE_R2_PREFIX"; +export const DEFAULT_PREFIX = "incremental-cache"; + /** * An instance of the Incremental Cache that uses an R2 bucket (`NEXT_INC_CACHE_R2_BUCKET`) as it's * underlying data store. @@ -21,7 +26,7 @@ class R2IncrementalCache implements IncrementalCache { key: string, isFetch?: IsFetch ): Promise> | null> { - const r2 = getCloudflareContext().env.NEXT_INC_CACHE_R2_BUCKET; + const r2 = getCloudflareContext().env[BINDING_NAME]; if (!r2) throw new IgnorableError("No R2 bucket"); debugCache(`Get ${key}`); @@ -45,7 +50,7 @@ class R2IncrementalCache implements IncrementalCache { value: CacheValue, isFetch?: IsFetch ): Promise { - const r2 = getCloudflareContext().env.NEXT_INC_CACHE_R2_BUCKET; + const r2 = getCloudflareContext().env[BINDING_NAME]; if (!r2) throw new IgnorableError("No R2 bucket"); debugCache(`Set ${key}`); @@ -58,7 +63,7 @@ class R2IncrementalCache implements IncrementalCache { } async delete(key: string): Promise { - const r2 = getCloudflareContext().env.NEXT_INC_CACHE_R2_BUCKET; + const r2 = getCloudflareContext().env[BINDING_NAME]; if (!r2) throw new IgnorableError("No R2 bucket"); debugCache(`Delete ${key}`); @@ -71,9 +76,9 @@ class R2IncrementalCache implements IncrementalCache { } protected getR2Key(key: string, isFetch?: boolean): string { - const directory = getCloudflareContext().env.NEXT_INC_CACHE_R2_PREFIX ?? "incremental-cache"; + const directory = getCloudflareContext().env[PREFIX_ENV_NAME] ?? DEFAULT_PREFIX; - return `${directory}/${process.env.NEXT_BUILD_ID ?? "no-build-id"}/${key}.${isFetch ? "fetch" : "cache"}`.replace( + return `${directory}/${process.env.NEXT_BUILD_ID ?? FALLBACK_BUILD_ID}/${key}.${isFetch ? "fetch" : "cache"}`.replace( /\/+/g, "/" ); diff --git a/packages/cloudflare/src/api/overrides/incremental-cache/regional-cache.ts b/packages/cloudflare/src/api/overrides/incremental-cache/regional-cache.ts index 6f205366..8c801413 100644 --- a/packages/cloudflare/src/api/overrides/incremental-cache/regional-cache.ts +++ b/packages/cloudflare/src/api/overrides/incremental-cache/regional-cache.ts @@ -2,7 +2,7 @@ import { error } from "@opennextjs/aws/adapters/logger.js"; import { CacheValue, IncrementalCache, WithLastModified } from "@opennextjs/aws/types/overrides.js"; import { getCloudflareContext } from "../../cloudflare-context.js"; -import { debugCache, IncrementalCacheEntry } from "../internal.js"; +import { debugCache, FALLBACK_BUILD_ID, IncrementalCacheEntry } from "../internal.js"; import { NAME as KV_CACHE_NAME } from "./kv-incremental-cache.js"; const ONE_MINUTE_IN_SECONDS = 60; @@ -129,7 +129,7 @@ class RegionalCache implements IncrementalCache { protected getCacheKey(key: string, isFetch?: boolean) { return new Request( new URL( - `${process.env.NEXT_BUILD_ID ?? "no-build-id"}/${key}.${isFetch ? "fetch" : "cache"}`, + `${process.env.NEXT_BUILD_ID ?? FALLBACK_BUILD_ID}/${key}.${isFetch ? "fetch" : "cache"}`, "http://cache.local" ) ); diff --git a/packages/cloudflare/src/api/overrides/internal.ts b/packages/cloudflare/src/api/overrides/internal.ts index e343739b..7dd8c523 100644 --- a/packages/cloudflare/src/api/overrides/internal.ts +++ b/packages/cloudflare/src/api/overrides/internal.ts @@ -10,3 +10,5 @@ export const debugCache = (name: string, ...args: unknown[]) => { console.log(`[${name}] `, ...args); } }; + +export const FALLBACK_BUILD_ID = "no-build-id"; diff --git a/packages/cloudflare/src/api/overrides/tag-cache/d1-next-tag-cache.ts b/packages/cloudflare/src/api/overrides/tag-cache/d1-next-tag-cache.ts index a024301c..2590a0fd 100644 --- a/packages/cloudflare/src/api/overrides/tag-cache/d1-next-tag-cache.ts +++ b/packages/cloudflare/src/api/overrides/tag-cache/d1-next-tag-cache.ts @@ -4,10 +4,12 @@ import type { NextModeTagCache } from "@opennextjs/aws/types/overrides.js"; import { RecoverableError } from "@opennextjs/aws/utils/error.js"; import { getCloudflareContext } from "../../cloudflare-context.js"; -import { debugCache } from "../internal.js"; +import { debugCache, FALLBACK_BUILD_ID } from "../internal.js"; export const NAME = "d1-next-mode-tag-cache"; +export const BINDING_NAME = "NEXT_TAG_CACHE_D1"; + export class D1NextModeTagCache implements NextModeTagCache { readonly mode = "nextMode" as const; readonly name = NAME; @@ -46,8 +48,7 @@ export class D1NextModeTagCache implements NextModeTagCache { } private getConfig() { - const cfEnv = getCloudflareContext().env; - const db = cfEnv.NEXT_TAG_CACHE_D1; + const db = getCloudflareContext().env[BINDING_NAME]; if (!db) debugCache("No D1 database found"); @@ -71,7 +72,7 @@ export class D1NextModeTagCache implements NextModeTagCache { } protected getBuildId() { - return process.env.NEXT_BUILD_ID ?? "no-build-id"; + return process.env.NEXT_BUILD_ID ?? FALLBACK_BUILD_ID; } } diff --git a/packages/cloudflare/src/cli/commands/populate-cache.ts b/packages/cloudflare/src/cli/commands/populate-cache.ts index 347a535f..bb88018d 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.ts @@ -14,9 +14,20 @@ import { globSync } from "glob"; import { tqdm } from "ts-tqdm"; import { unstable_readConfig } from "wrangler"; -import { NAME as KV_CACHE_NAME } from "../../api/overrides/incremental-cache/kv-incremental-cache.js"; -import { NAME as R2_CACHE_NAME } from "../../api/overrides/incremental-cache/r2-incremental-cache.js"; -import { NAME as D1_TAG_NAME } from "../../api/overrides/tag-cache/d1-next-tag-cache.js"; +import { + BINDING_NAME as KV_CACHE_BINDING_NAME, + NAME as KV_CACHE_NAME, +} from "../../api/overrides/incremental-cache/kv-incremental-cache.js"; +import { + BINDING_NAME as R2_CACHE_BINDING_NAME, + DEFAULT_PREFIX as R2_CACHE_DEFAULT_PREFIX, + NAME as R2_CACHE_NAME, + PREFIX_ENV_NAME as R2_CACHE_PREFIX_ENV_NAME, +} from "../../api/overrides/incremental-cache/r2-incremental-cache.js"; +import { + NAME as D1_TAG_NAME, + BINDING_NAME as D1_TAG_BINDING_NAME, +} from "../../api/overrides/tag-cache/d1-next-tag-cache.js"; import type { WranglerTarget } from "../utils/run-wrangler.js"; import { runWrangler } from "../utils/run-wrangler.js"; @@ -66,18 +77,16 @@ export async function populateCache( case R2_CACHE_NAME: { const config = unstable_readConfig({ env: populateCacheOptions.environment }); - const binding = (config.r2_buckets ?? []).find( - ({ binding }) => binding === "NEXT_INC_CACHE_R2_BUCKET" - ); + const binding = (config.r2_buckets ?? []).find(({ binding }) => binding === R2_CACHE_BINDING_NAME); if (!binding) { - throw new Error("No R2 binding 'NEXT_INC_CACHE_R2_BUCKET' found!"); + throw new Error(`No R2 binding '${R2_CACHE_BINDING_NAME}' found!`); } const bucket = binding.bucket_name; if (!bucket) { - throw new Error("R2 binding 'NEXT_INC_CACHE_R2_BUCKET' should have a 'bucket_name'"); + throw new Error(`R2 binding '${R2_CACHE_BINDING_NAME}' should have a 'bucket_name'`); } logger.info("\nPopulating R2 incremental cache..."); @@ -86,7 +95,7 @@ export async function populateCache( for (const { fsPath, destPath } of tqdm(assets)) { const fullDestPath = path.join( bucket, - process.env.NEXT_INC_CACHE_R2_PREFIX ?? "incremental-cache", + process.env[R2_CACHE_PREFIX_ENV_NAME] ?? R2_CACHE_DEFAULT_PREFIX, destPath ); @@ -111,7 +120,7 @@ export async function populateCache( [ "kv key put", JSON.stringify(destPath), - "--binding NEXT_INC_CACHE_KV", + `--binding ${JSON.stringify(KV_CACHE_BINDING_NAME)}`, `--path ${JSON.stringify(fsPath)}`, ], { ...populateCacheOptions, logging: "error" } @@ -135,7 +144,7 @@ export async function populateCache( options, [ "d1 execute", - "NEXT_TAG_CACHE_D1", + JSON.stringify(D1_TAG_BINDING_NAME), `--command "CREATE TABLE IF NOT EXISTS revalidations (tag TEXT NOT NULL, revalidatedAt INTEGER NOT NULL, UNIQUE(tag) ON CONFLICT REPLACE);"`, ], { ...populateCacheOptions, logging: "error" } From c0cb5fffcad2a1330276cd68242dc25dc014952c Mon Sep 17 00:00:00 2001 From: James Date: Tue, 8 Apr 2025 07:58:17 +0100 Subject: [PATCH 10/18] tweak error log --- packages/cloudflare/src/cli/commands/populate-cache.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cloudflare/src/cli/commands/populate-cache.ts b/packages/cloudflare/src/cli/commands/populate-cache.ts index bb88018d..13135b8b 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.ts @@ -80,13 +80,13 @@ export async function populateCache( const binding = (config.r2_buckets ?? []).find(({ binding }) => binding === R2_CACHE_BINDING_NAME); if (!binding) { - throw new Error(`No R2 binding '${R2_CACHE_BINDING_NAME}' found!`); + throw new Error(`No R2 binding ${JSON.stringify(R2_CACHE_BINDING_NAME)} found!`); } const bucket = binding.bucket_name; if (!bucket) { - throw new Error(`R2 binding '${R2_CACHE_BINDING_NAME}' should have a 'bucket_name'`); + throw new Error(`R2 binding ${JSON.stringify(R2_CACHE_BINDING_NAME)} should have a 'bucket_name'`); } logger.info("\nPopulating R2 incremental cache..."); From d07d6a083db994f98ba9895da0e95f3c555c4ff6 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 8 Apr 2025 08:00:07 +0100 Subject: [PATCH 11/18] sort imports --- packages/cloudflare/src/cli/commands/populate-cache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cloudflare/src/cli/commands/populate-cache.ts b/packages/cloudflare/src/cli/commands/populate-cache.ts index 13135b8b..398fbe00 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.ts @@ -25,8 +25,8 @@ import { PREFIX_ENV_NAME as R2_CACHE_PREFIX_ENV_NAME, } from "../../api/overrides/incremental-cache/r2-incremental-cache.js"; import { - NAME as D1_TAG_NAME, BINDING_NAME as D1_TAG_BINDING_NAME, + NAME as D1_TAG_NAME, } from "../../api/overrides/tag-cache/d1-next-tag-cache.js"; import type { WranglerTarget } from "../utils/run-wrangler.js"; import { runWrangler } from "../utils/run-wrangler.js"; From 85a82534e5d501ed7ce207506cfd444eb05ec4ec Mon Sep 17 00:00:00 2001 From: James Date: Tue, 8 Apr 2025 08:06:23 +0100 Subject: [PATCH 12/18] move cache population to individual functions --- .../src/cli/commands/populate-cache.ts | 157 ++++++++++-------- 1 file changed, 86 insertions(+), 71 deletions(-) diff --git a/packages/cloudflare/src/cli/commands/populate-cache.ts b/packages/cloudflare/src/cli/commands/populate-cache.ts index 398fbe00..e35140be 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.ts @@ -59,6 +59,86 @@ function getCacheAssetPaths(opts: BuildOptions) { }); } +function populateR2IncrementalCache( + options: BuildOptions, + populateCacheOptions: { target: WranglerTarget; environment?: string } +) { + const config = unstable_readConfig({ env: populateCacheOptions.environment }); + + const binding = (config.r2_buckets ?? []).find(({ binding }) => binding === R2_CACHE_BINDING_NAME); + + if (!binding) { + throw new Error(`No R2 binding ${JSON.stringify(R2_CACHE_BINDING_NAME)} found!`); + } + + const bucket = binding.bucket_name; + + if (!bucket) { + throw new Error(`R2 binding ${JSON.stringify(R2_CACHE_BINDING_NAME)} should have a 'bucket_name'`); + } + + logger.info("\nPopulating R2 incremental cache..."); + + const assets = getCacheAssetPaths(options); + for (const { fsPath, destPath } of tqdm(assets)) { + const fullDestPath = path.join( + bucket, + process.env[R2_CACHE_PREFIX_ENV_NAME] ?? R2_CACHE_DEFAULT_PREFIX, + destPath + ); + + runWrangler( + options, + ["r2 object put", JSON.stringify(fullDestPath), `--file ${JSON.stringify(fsPath)}`], + // NOTE: R2 does not support the environment flag and results in the following error: + // Incorrect type for the 'cacheExpiry' field on 'HttpMetadata': the provided value is not of type 'date'. + { target: populateCacheOptions.target, excludeRemoteFlag: true, logging: "error" } + ); + } + logger.info(`Successfully populated cache with ${assets.length} assets`); +} + +function populateKVIncrementalCache( + options: BuildOptions, + populateCacheOptions: { target: WranglerTarget; environment?: string } +) { + logger.info("\nPopulating KV incremental cache..."); + + const assets = getCacheAssetPaths(options); + for (const { fsPath, destPath } of tqdm(assets)) { + runWrangler( + options, + [ + "kv key put", + JSON.stringify(destPath), + `--binding ${JSON.stringify(KV_CACHE_BINDING_NAME)}`, + `--path ${JSON.stringify(fsPath)}`, + ], + { ...populateCacheOptions, logging: "error" } + ); + } + logger.info(`Successfully populated cache with ${assets.length} assets`); +} + +function populateD1TagCache( + options: BuildOptions, + populateCacheOptions: { target: WranglerTarget; environment?: string } +) { + logger.info("\nCreating D1 table if necessary..."); + + runWrangler( + options, + [ + "d1 execute", + JSON.stringify(D1_TAG_BINDING_NAME), + `--command "CREATE TABLE IF NOT EXISTS revalidations (tag TEXT NOT NULL, revalidatedAt INTEGER NOT NULL, UNIQUE(tag) ON CONFLICT REPLACE);"`, + ], + { ...populateCacheOptions, logging: "error" } + ); + + logger.info("\nSuccessfully created D1 table"); +} + export async function populateCache( options: BuildOptions, config: OpenNextConfig, @@ -74,61 +154,10 @@ export async function populateCache( if (!config.dangerous?.disableIncrementalCache && incrementalCache) { const name = await resolveCacheName(incrementalCache); switch (name) { - case R2_CACHE_NAME: { - const config = unstable_readConfig({ env: populateCacheOptions.environment }); - - const binding = (config.r2_buckets ?? []).find(({ binding }) => binding === R2_CACHE_BINDING_NAME); - - if (!binding) { - throw new Error(`No R2 binding ${JSON.stringify(R2_CACHE_BINDING_NAME)} found!`); - } - - const bucket = binding.bucket_name; - - if (!bucket) { - throw new Error(`R2 binding ${JSON.stringify(R2_CACHE_BINDING_NAME)} should have a 'bucket_name'`); - } - - logger.info("\nPopulating R2 incremental cache..."); - - const assets = getCacheAssetPaths(options); - for (const { fsPath, destPath } of tqdm(assets)) { - const fullDestPath = path.join( - bucket, - process.env[R2_CACHE_PREFIX_ENV_NAME] ?? R2_CACHE_DEFAULT_PREFIX, - destPath - ); - - runWrangler( - options, - ["r2 object put", JSON.stringify(fullDestPath), `--file ${JSON.stringify(fsPath)}`], - // NOTE: R2 does not support the environment flag and results in the following error: - // Incorrect type for the 'cacheExpiry' field on 'HttpMetadata': the provided value is not of type 'date'. - { target: populateCacheOptions.target, excludeRemoteFlag: true, logging: "error" } - ); - } - logger.info(`Successfully populated cache with ${assets.length} assets`); - break; - } - case KV_CACHE_NAME: { - logger.info("\nPopulating KV incremental cache..."); - - const assets = getCacheAssetPaths(options); - for (const { fsPath, destPath } of tqdm(assets)) { - runWrangler( - options, - [ - "kv key put", - JSON.stringify(destPath), - `--binding ${JSON.stringify(KV_CACHE_BINDING_NAME)}`, - `--path ${JSON.stringify(fsPath)}`, - ], - { ...populateCacheOptions, logging: "error" } - ); - } - logger.info(`Successfully populated cache with ${assets.length} assets`); - break; - } + case R2_CACHE_NAME: + return populateR2IncrementalCache(options, populateCacheOptions); + case KV_CACHE_NAME: + return populateKVIncrementalCache(options, populateCacheOptions); default: logger.info("Incremental cache does not need populating"); } @@ -137,22 +166,8 @@ export async function populateCache( if (!config.dangerous?.disableTagCache && !config.dangerous?.disableIncrementalCache && tagCache) { const name = await resolveCacheName(tagCache); switch (name) { - case D1_TAG_NAME: { - logger.info("\nCreating D1 table if necessary..."); - - runWrangler( - options, - [ - "d1 execute", - JSON.stringify(D1_TAG_BINDING_NAME), - `--command "CREATE TABLE IF NOT EXISTS revalidations (tag TEXT NOT NULL, revalidatedAt INTEGER NOT NULL, UNIQUE(tag) ON CONFLICT REPLACE);"`, - ], - { ...populateCacheOptions, logging: "error" } - ); - - logger.info("\nSuccessfully created D1 table"); - break; - } + case D1_TAG_NAME: + return populateD1TagCache(options, populateCacheOptions); default: logger.info("Tag cache does not need populating"); } From 6cc6d902a5c7fd56357e3b423b231a270d778f1b Mon Sep 17 00:00:00 2001 From: James Date: Tue, 8 Apr 2025 08:09:08 +0100 Subject: [PATCH 13/18] check kv binding exists --- .../cloudflare/src/cli/commands/populate-cache.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/cloudflare/src/cli/commands/populate-cache.ts b/packages/cloudflare/src/cli/commands/populate-cache.ts index e35140be..36344689 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.ts @@ -63,22 +63,20 @@ function populateR2IncrementalCache( options: BuildOptions, populateCacheOptions: { target: WranglerTarget; environment?: string } ) { - const config = unstable_readConfig({ env: populateCacheOptions.environment }); + logger.info("\nPopulating R2 incremental cache..."); - const binding = (config.r2_buckets ?? []).find(({ binding }) => binding === R2_CACHE_BINDING_NAME); + const config = unstable_readConfig({ env: populateCacheOptions.environment }); + const binding = config.r2_buckets.find(({ binding }) => binding === R2_CACHE_BINDING_NAME); if (!binding) { throw new Error(`No R2 binding ${JSON.stringify(R2_CACHE_BINDING_NAME)} found!`); } const bucket = binding.bucket_name; - if (!bucket) { throw new Error(`R2 binding ${JSON.stringify(R2_CACHE_BINDING_NAME)} should have a 'bucket_name'`); } - logger.info("\nPopulating R2 incremental cache..."); - const assets = getCacheAssetPaths(options); for (const { fsPath, destPath } of tqdm(assets)) { const fullDestPath = path.join( @@ -104,6 +102,13 @@ function populateKVIncrementalCache( ) { logger.info("\nPopulating KV incremental cache..."); + const config = unstable_readConfig({ env: populateCacheOptions.environment }); + + const binding = config.kv_namespaces.find(({ binding }) => binding === KV_CACHE_BINDING_NAME); + if (!binding) { + throw new Error(`No KV binding ${JSON.stringify(KV_CACHE_BINDING_NAME)} found!`); + } + const assets = getCacheAssetPaths(options); for (const { fsPath, destPath } of tqdm(assets)) { runWrangler( From 4c1c1395e11dead9ec66fa2401ce219e9aec6399 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 8 Apr 2025 08:09:34 +0100 Subject: [PATCH 14/18] check d1 binding exists --- packages/cloudflare/src/cli/commands/populate-cache.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/cloudflare/src/cli/commands/populate-cache.ts b/packages/cloudflare/src/cli/commands/populate-cache.ts index 36344689..3a6a1498 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.ts @@ -131,6 +131,13 @@ function populateD1TagCache( ) { logger.info("\nCreating D1 table if necessary..."); + const config = unstable_readConfig({ env: populateCacheOptions.environment }); + + const binding = config.d1_databases.find(({ binding }) => binding === D1_TAG_BINDING_NAME); + if (!binding) { + throw new Error(`No D1 binding ${JSON.stringify(D1_TAG_BINDING_NAME)} found!`); + } + runWrangler( options, [ From 12779c19768291b2d15b2c54eb32bf5d075297e0 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 8 Apr 2025 08:17:42 +0100 Subject: [PATCH 15/18] dont return from populate --- packages/cloudflare/src/cli/commands/populate-cache.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/cloudflare/src/cli/commands/populate-cache.ts b/packages/cloudflare/src/cli/commands/populate-cache.ts index 3a6a1498..6a9b6f28 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.ts @@ -167,9 +167,11 @@ export async function populateCache( const name = await resolveCacheName(incrementalCache); switch (name) { case R2_CACHE_NAME: - return populateR2IncrementalCache(options, populateCacheOptions); + populateR2IncrementalCache(options, populateCacheOptions); + break; case KV_CACHE_NAME: - return populateKVIncrementalCache(options, populateCacheOptions); + populateKVIncrementalCache(options, populateCacheOptions); + break; default: logger.info("Incremental cache does not need populating"); } @@ -179,7 +181,8 @@ export async function populateCache( const name = await resolveCacheName(tagCache); switch (name) { case D1_TAG_NAME: - return populateD1TagCache(options, populateCacheOptions); + populateD1TagCache(options, populateCacheOptions); + break; default: logger.info("Tag cache does not need populating"); } From a5b44151ce0725e84d588195bf1fc54c7215bf72 Mon Sep 17 00:00:00 2001 From: James Anderson Date: Tue, 8 Apr 2025 08:59:25 +0100 Subject: [PATCH 16/18] Update packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts --- .../api/overrides/incremental-cache/kv-incremental-cache.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts b/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts index 75f9bb04..31217836 100644 --- a/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts +++ b/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts @@ -34,7 +34,9 @@ class KVIncrementalCache implements IncrementalCache { "json" ); - if (!entry || "lastModified" in entry) { + if (!entry) return null; + + if ("lastModified" in entry) { return entry; } From 59df5998e701c3f83f584d3ec8a8cf92a80b2f5c Mon Sep 17 00:00:00 2001 From: James Anderson Date: Tue, 8 Apr 2025 09:01:27 +0100 Subject: [PATCH 17/18] Update packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts --- .../src/api/overrides/incremental-cache/kv-incremental-cache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts b/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts index 31217836..7d1ad4d8 100644 --- a/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts +++ b/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts @@ -36,7 +36,7 @@ class KVIncrementalCache implements IncrementalCache { if (!entry) return null; - if ("lastModified" in entry) { + if ("lastModified" in entry) { return entry; } From 852c73a52471e5c3ccb57eaa17d29dc37b33b629 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 8 Apr 2025 09:24:37 +0100 Subject: [PATCH 18/18] run prettier --- .../src/api/overrides/incremental-cache/kv-incremental-cache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts b/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts index 7d1ad4d8..d3140e5b 100644 --- a/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts +++ b/packages/cloudflare/src/api/overrides/incremental-cache/kv-incremental-cache.ts @@ -35,7 +35,7 @@ class KVIncrementalCache implements IncrementalCache { ); if (!entry) return null; - + if ("lastModified" in entry) { return entry; }