feat: d1 adapter for the tag cache

james-elicx · james-elicx · commit ff9c04eabc95 · 2025-02-07T11:31:57.000Z
diff --git a/.changeset/five-balloons-walk.md b/.changeset/five-balloons-walk.md
@@ -0,0 +1,5 @@
+---
+"@opennextjs/cloudflare": minor
+---
+
+feat: d1 adapter for the tag cache
diff --git a/examples/e2e/app-router/e2e/after.test.ts b/examples/e2e/app-router/e2e/after.test.ts
@@ -1,8 +1,6 @@
 import { expect, test } from "@playwright/test";
 
-// Cache is currently not supported: https://github.com/opennextjs/opennextjs-cloudflare/issues/105
-// (Note: specifically this test relied on `unstable_cache`: https://github.com/opennextjs/opennextjs-cloudflare/issues/105#issuecomment-2627074820)
-test.skip("Next after", async ({ request }) => {
+test("Next after", async ({ request }) => {
   const initialSSG = await request.get("/api/after/ssg");
   expect(initialSSG.status()).toEqual(200);
   const initialSSGJson = await initialSSG.json();
diff --git a/examples/e2e/app-router/e2e/revalidateTag.test.ts b/examples/e2e/app-router/e2e/revalidateTag.test.ts
@@ -1,7 +1,6 @@
 import { expect, test } from "@playwright/test";
 
-// Cache (and revalidateTag) is currently not supported: https://github.com/opennextjs/opennextjs-cloudflare/issues/105
-test.skip("Revalidate tag", async ({ page, request }) => {
+test("Revalidate tag", async ({ page, request }) => {
   test.setTimeout(45000);
   // We need to hit the page twice to make sure it's properly cached
   // Turbo might cache next build result, resulting in the tag being newer than the page
@@ -69,8 +68,7 @@ test.skip("Revalidate tag", async ({ page, request }) => {
   expect(nextCacheHeaderNested).toEqual("HIT");
 });
 
-// Cache (and revalidatePath) is currently not supported: https://github.com/opennextjs/opennextjs-cloudflare/issues/105
-test.skip("Revalidate path", async ({ page, request }) => {
+test("Revalidate path", async ({ page, request }) => {
   await page.goto("/revalidate-path");
 
   let elLayout = page.getByText("RequestID:");
diff --git a/examples/e2e/app-router/open-next.config.ts b/examples/e2e/app-router/open-next.config.ts
@@ -1,15 +1,15 @@
 import type { OpenNextConfig } from "@opennextjs/aws/types/open-next.js";
-import cache from "@opennextjs/cloudflare/kvCache";
+import incrementalCache from "@opennextjs/cloudflare/kvCache";
+import tagCache from "@opennextjs/cloudflare/d1-tag-cache";
 
 const config: OpenNextConfig = {
   default: {
     override: {
       wrapper: "cloudflare-node",
       converter: "edge",
-      incrementalCache: async () => cache,
+      incrementalCache: () => incrementalCache,
+      tagCache: () => tagCache,
       queue: "direct",
-      // Unused implementation
-      tagCache: "dummy",
     },
   },
 
diff --git a/examples/e2e/app-router/package.json b/examples/e2e/app-router/package.json
@@ -9,9 +9,10 @@
     "start": "next start --port 3001",
     "lint": "next lint",
     "clean": "rm -rf .turbo node_modules .next .open-next",
+    "setup:d1": "wrangler d1 execute NEXT_CACHE_D1 --command \"CREATE TABLE IF NOT EXISTS tags (tag TEXT NOT NULL, path TEXT NOT NULL, revalidatedAt INTEGER NOT NULL, UNIQUE(tag, path) ON CONFLICT REPLACE)\"",
     "build:worker": "pnpm opennextjs-cloudflare",
     "preview": "pnpm build:worker && pnpm wrangler dev",
-    "e2e": "playwright test -c e2e/playwright.config.ts"
+    "e2e": "pnpm setup:d1 && playwright test -c e2e/playwright.config.ts"
   },
   "dependencies": {
     "@opennextjs/cloudflare": "workspace:*",
diff --git a/examples/e2e/app-router/wrangler.json b/examples/e2e/app-router/wrangler.json
@@ -13,5 +13,12 @@
       "binding": "NEXT_CACHE_WORKERS_KV",
       "id": "<BINDING_ID>"
     }
+  ],
+  "d1_databases": [
+    {
+      "binding": "NEXT_CACHE_D1",
+      "database_id": "NEXT_CACHE_D1",
+      "database_name": "NEXT_CACHE_D1"
+    }
   ]
 }
diff --git a/packages/cloudflare/env.d.ts b/packages/cloudflare/env.d.ts
@@ -1,3 +1,5 @@
+import { CacheAssetsManifest } from "./src/cli/build/open-next/create-cache-assets-manifest.js";
+
 declare global {
   namespace NodeJS {
     interface ProcessEnv {
@@ -7,6 +9,7 @@ declare global {
       NEXT_PRIVATE_DEBUG_CACHE?: string;
       OPEN_NEXT_ORIGIN: string;
       NODE_ENV?: string;
+      __OPENNEXT_CACHE_TAGS_MANIFEST: CacheAssetsManifest;
     }
   }
 }
diff --git a/packages/cloudflare/src/api/cloudflare-context.ts b/packages/cloudflare/src/api/cloudflare-context.ts
@@ -3,6 +3,8 @@ import type { Context, RunningCodeOptions } from "node:vm";
 declare global {
   interface CloudflareEnv {
     NEXT_CACHE_WORKERS_KV?: KVNamespace;
+    NEXT_CACHE_D1?: D1Database;
+    NEXT_CACHE_D1_TABLE?: string;
     ASSETS?: Fetcher;
   }
 }
diff --git a/packages/cloudflare/src/api/d1-tag-cache.ts b/packages/cloudflare/src/api/d1-tag-cache.ts
@@ -0,0 +1,151 @@
+import { debug, error } from "@opennextjs/aws/adapters/logger.js";
+import type { OpenNextConfig } from "@opennextjs/aws/types/open-next.js";
+import type { TagCache } from "@opennextjs/aws/types/overrides.js";
+
+import { getCloudflareContext } from "./cloudflare-context.js";
+
+// inlined during build
+const manifest = process.env.__OPENNEXT_CACHE_TAGS_MANIFEST;
+
+class D1TagCache implements TagCache {
+  public name = "d1-tag-cache";
+
+  public async getByPath(rawPath: string): Promise<string[]> {
+    const { isDisabled, db, table } = this.getConfig();
+    if (isDisabled) return [];
+
+    const path = this.getCacheKey(rawPath);
+
+    try {
+      const { success, results } = await db
+        .prepare(`SELECT tag FROM ${table} WHERE path = ?`)
+        .bind(path)
+        .all<{ tag: string }>();
+
+      if (!success) throw new Error(`D1 select failed for ${path}`);
+
+      const tags = this.mergeTagArrays(
+        manifest.paths[path],
+        results?.map((item) => item.tag)
+      );
+
+      debug("tags for path", path, tags);
+      return tags;
+    } catch (e) {
+      error("Failed to get tags by path", e);
+      return [];
+    }
+  }
+
+  public async getByTag(rawTag: string): Promise<string[]> {
+    const { isDisabled, db, table } = this.getConfig();
+    if (isDisabled) return [];
+
+    const tag = this.getCacheKey(rawTag);
+
+    try {
+      const { success, results } = await db
+        .prepare(`SELECT path FROM ${table} WHERE tag = ?`)
+        .bind(tag)
+        .all<{ path: string }>();
+
+      if (!success) throw new Error(`D1 select failed for ${tag}`);
+
+      const paths = this.mergeTagArrays(
+        manifest.tags[tag],
+        results?.map((item) => item.path)
+      );
+
+      debug("paths for tag", tag, paths);
+      return paths;
+    } catch (e) {
+      error("Failed to get by tag", e);
+      return [];
+    }
+  }
+
+  public async getLastModified(path: string, lastModified?: number): Promise<number> {
+    const { isDisabled, db, table } = this.getConfig();
+    if (isDisabled) return lastModified ?? Date.now();
+
+    try {
+      const { success, results } = await db
+        .prepare(`SELECT tag FROM ${table} WHERE path = ? AND revalidatedAt > ?`)
+        .bind(this.getCacheKey(path), lastModified ?? 0)
+        .all<{ tag: string }>();
+
+      if (!success) throw new Error(`D1 select failed for ${path} - ${lastModified ?? 0}`);
+
+      const tags = results?.map((item) => this.removeBuildId(item.tag)) ?? [];
+      debug("revalidatedTags", tags);
+      return tags.length > 0 ? -1 : (lastModified ?? Date.now());
+    } catch (e) {
+      error("Failed to get revalidated tags", e);
+      return lastModified ?? Date.now();
+    }
+  }
+
+  public async writeTags(tags: { tag: string; path: string; revalidatedAt?: number }[]): Promise<void> {
+    const { isDisabled, db, table } = this.getConfig();
+    if (isDisabled || tags.length === 0) return;
+
+    try {
+      const results = await db.batch(
+        tags.map(({ tag, path, revalidatedAt }) =>
+          db
+            .prepare(`INSERT INTO ${table} (tag, path, revalidatedAt) VALUES(?, ?, ?)`)
+            .bind(this.getCacheKey(tag), this.getCacheKey(path), revalidatedAt ?? Date.now())
+        )
+      );
+
+      const failedResults = results.filter((res) => !res.success);
+
+      if (failedResults.length > 0) {
+        throw new Error(`${failedResults.length} tags failed to write`);
+      }
+    } catch (e) {
+      error("Failed to batch write tags", e);
+    }
+  }
+
+  private getConfig() {
+    const cfEnv = getCloudflareContext().env;
+    const db = cfEnv.NEXT_CACHE_D1;
+    const table = cfEnv.NEXT_CACHE_D1_TABLE ?? "tags";
+
+    if (!db) debug("No D1 database found");
+
+    const isDisabled = !!(globalThis as unknown as { openNextConfig: OpenNextConfig }).openNextConfig
+      .dangerous?.disableTagCache;
+
+    if (!db || isDisabled) {
+      return { isDisabled: true as const };
+    }
+
+    return { isDisabled: false as const, db, table };
+  }
+
+  protected removeBuildId(key: string) {
+    return key.replace(`${this.getBuildId()}/`, "");
+  }
+
+  protected getCacheKey(key: string) {
+    return `${this.getBuildId()}/${key}`.replaceAll("//", "/");
+  }
+
+  protected getBuildId() {
+    return process.env.NEXT_BUILD_ID ?? "no-build-id";
+  }
+
+  protected mergeTagArrays(...arrays: (string[] | undefined)[]) {
+    const set = new Set<string>();
+
+    for (const arr of arrays) {
+      arr?.forEach((v) => set.add(this.removeBuildId(v)));
+    }
+
+    return [...set.values()];
+  }
+}
+
+export default new D1TagCache();
diff --git a/packages/cloudflare/src/cli/build/bundle-server.ts b/packages/cloudflare/src/cli/build/bundle-server.ts
@@ -14,6 +14,7 @@ import { handleOptionalDependencies } from "./patches/plugins/optional-deps.js";
 import { fixRequire } from "./patches/plugins/require.js";
 import { inlineRequirePagePlugin } from "./patches/plugins/require-page.js";
 import { setWranglerExternal } from "./patches/plugins/wrangler-external.js";
+import { extractCacheAssetsManifest } from "./utils/extract-cache-assets-manifest.js";
 import { normalizePath, patchCodeWithValidations } from "./utils/index.js";
 
 /** The dist directory of the Cloudflare adapter package */
@@ -113,6 +114,7 @@ export async function bundleServer(buildOpts: BuildOptions): Promise<void> {
       "process.env.NEXT_RUNTIME": '"nodejs"',
       "process.env.NODE_ENV": '"production"',
       "process.env.NEXT_MINIMAL": "true",
+      "process.env.__OPENNEXT_CACHE_TAGS_MANIFEST": `${JSON.stringify(extractCacheAssetsManifest(buildOpts))}`,
     },
     platform: "node",
     banner: {
diff --git a/packages/cloudflare/src/cli/build/utils/ensure-cf-config.ts b/packages/cloudflare/src/cli/build/utils/ensure-cf-config.ts
@@ -13,7 +13,9 @@ export function ensureCloudflareConfig(config: OpenNextConfig) {
     dftMaybeUseCache:
       config.default?.override?.incrementalCache === "dummy" ||
       typeof config.default?.override?.incrementalCache === "function",
-    dftUseDummyTagCache: config.default?.override?.tagCache === "dummy",
+    dftMaybeUseTagCache:
+      config.default?.override?.tagCache === "dummy" ||
+      typeof config.default?.override?.incrementalCache === "function",
     dftMaybeUseQueue:
       config.default?.override?.queue === "dummy" || config.default?.override?.queue === "direct",
     disableCacheInterception: config.dangerous?.enableCacheInterception !== true,
diff --git a/packages/cloudflare/src/cli/build/utils/extract-cache-assets-manifest.ts b/packages/cloudflare/src/cli/build/utils/extract-cache-assets-manifest.ts
@@ -0,0 +1,54 @@
+import { readFileSync } from "node:fs";
+import path from "node:path";
+
+import type { BuildOptions } from "@opennextjs/aws/build/helper.js";
+import { getBuildId } from "@opennextjs/aws/build/helper.js";
+import { globSync } from "glob";
+
+type CacheInfo = {
+  type: string;
+  meta?: {
+    headers?: {
+      "x-next-cache-tags"?: string;
+    };
+  };
+};
+
+export type CacheAssetsManifest = {
+  tags: { [tag: string]: string[] };
+  paths: { [path: string]: string[] };
+};
+
+/**
+ * Extracts a map of the cache assets that were generated during the build.
+ *
+ * The mapping creates an index of each tags pointing to its paths, and each path pointing to its tags.
+ */
+export function extractCacheAssetsManifest(buildOpts: BuildOptions) {
+  const cachePath = path.join(buildOpts.outputDir, "cache");
+  const buildId = getBuildId(buildOpts);
+
+  const manifest = globSync(path.join(cachePath, buildId, "**/*.cache")).reduce<CacheAssetsManifest>(
+    (acc, p) => {
+      const { meta } = JSON.parse(readFileSync(p, "utf-8")) as CacheInfo;
+      const tags = meta?.headers?.["x-next-cache-tags"]?.split(",")?.map((tag) => `${buildId}/${tag}`) ?? [];
+
+      const routePath = path.relative(cachePath, p).replace(/\.cache$/, "");
+
+      acc.paths[routePath] = tags;
+
+      for (const tag of tags) {
+        if (!acc.tags[tag]) {
+          acc.tags[tag] = [routePath];
+        } else {
+          acc.tags[tag].push(routePath);
+        }
+      }
+
+      return acc;
+    },
+    { tags: {}, paths: {} }
+  );
+
+  return manifest;
+}
diff --git a/packages/cloudflare/tsconfig.json b/packages/cloudflare/tsconfig.json
@@ -15,5 +15,5 @@
     "target": "ES2022",
     "types": ["@cloudflare/workers-types"]
   },
-  "include": ["src/**/*.ts"]
+  "include": ["src/**/*.ts", "env.d.ts"]
 }

-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +---
 +"@opennextjs/cloudflare": minor
 +---
++
 +feat: d1 adapter for the tag cache
Original file line number	Diff line number	Diff line change
`@@ -13,5 +13,12 @@`
`13`	`13`	`"binding": "NEXT_CACHE_WORKERS_KV",`
`14`	`14`	`"id": "<BINDING_ID>"`
`15`	`15`	`}`
	`16`	`+ ],`
	`17`	`+ "d1_databases": [`
	`18`	`+ {`
	`19`	`+ "binding": "NEXT_CACHE_D1",`
	`20`	`+ "database_id": "NEXT_CACHE_D1",`
	`21`	`+ "database_name": "NEXT_CACHE_D1"`
	`22`	`+ }`
`16`	`23`	`]`
`17`	`24`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,5 @@`
	`1`	`+import { CacheAssetsManifest } from "./src/cli/build/open-next/create-cache-assets-manifest.js";`
	`2`	`+`
`1`	`3`	`declare global {`
`2`	`4`	`namespace NodeJS {`
`3`	`5`	`interface ProcessEnv {`
`@@ -7,6 +9,7 @@ declare global {`
`7`	`9`	`NEXT_PRIVATE_DEBUG_CACHE?: string;`
`8`	`10`	`OPEN_NEXT_ORIGIN: string;`
`9`	`11`	`NODE_ENV?: string;`
	`12`	`+ __OPENNEXT_CACHE_TAGS_MANIFEST: CacheAssetsManifest;`
`10`	`13`	`}`
`11`	`14`	`}`
`12`	`15`	`}`
Original file line number	Diff line number	Diff line change
`@@ -3,6 +3,8 @@ import type { Context, RunningCodeOptions } from "node:vm";`
`3`	`3`	`declare global {`
`4`	`4`	`interface CloudflareEnv {`
`5`	`5`	`NEXT_CACHE_WORKERS_KV?: KVNamespace;`
	`6`	`+ NEXT_CACHE_D1?: D1Database;`
	`7`	`+ NEXT_CACHE_D1_TABLE?: string;`
`6`	`8`	`ASSETS?: Fetcher;`
`7`	`9`	`}`
`8`	`10`	`}`