use two tables for the tag cache

Author: james-elicx

examples/e2e/app-router/package.json

@@ -9,7 +9,7 @@
     "start": "next start --port 3001",
     "lint": "next lint",
     "clean": "rm -rf .turbo node_modules .next .open-next",
-    "d1:clean": "wrangler d1 execute NEXT_CACHE_D1 --command \"DROP TABLE IF EXISTS tags\"",
+    "d1:clean": "wrangler d1 execute NEXT_CACHE_D1 --command \"DROP TABLE IF EXISTS tags; DROP TABLE IF EXISTS revalidations\"",
     "d1:setup": "wrangler d1 execute NEXT_CACHE_D1 --file .open-next/cloudflare/cache-assets-manifest.sql",
     "build:worker": "pnpm opennextjs-cloudflare",
     "preview": "pnpm build:worker && pnpm d1:clean && pnpm d1:setup && pnpm wrangler dev",

packages/cloudflare/env.d.ts

@@ -7,6 +7,8 @@ declare global {
       NEXT_PRIVATE_DEBUG_CACHE?: string;
       OPEN_NEXT_ORIGIN: string;
       NODE_ENV?: string;
+      NEXT_CACHE_D1_TAGS_TABLE?: string;
+      NEXT_CACHE_D1_REVALIDATIONS_TABLE?: string;
     }
   }
 }

packages/cloudflare/src/api/cloudflare-context.ts

@@ -4,7 +4,8 @@ declare global {
   interface CloudflareEnv {
     NEXT_CACHE_WORKERS_KV?: KVNamespace;
     NEXT_CACHE_D1?: D1Database;
-    NEXT_CACHE_D1_TABLE?: string;
+    NEXT_CACHE_D1_TAGS_TABLE?: string;
+    NEXT_CACHE_D1_REVALIDATIONS_TABLE?: string;
     ASSETS?: Fetcher;
   }
 }

packages/cloudflare/src/api/d1-tag-cache.ts

@@ -7,10 +7,17 @@ import { getCloudflareContext } from "./cloudflare-context.js";
 /**
  * An instance of the Tag Cache that uses a D1 binding (`NEXT_CACHE_D1`) as it's underlying data store.
  *
- * The table used defaults to `tags`, but can be configured with the `NEXT_CACHE_D1_TABLE`
+ * **Tag/path mappings**
+ *
+ * Information about the relation between tags and paths is stored in a `tags` table that contains
+ * two columns; `tag`, and `path`. The table name can be configured with `NEXT_CACHE_D1_TAGS_TABLE`
  * environment variable.
  *
- * There should be three columns created in the table; `tag`, `path`, and `revalidatedAt`.
+ * **Tag revalidations**
+ *
+ * Revalidation times for tags are stored in a `revalidations` table that contains two columns; `tags`,
+ * and `revalidatedAt`. The table name can be configured with `NEXT_CACHE_D1_REVALIDATIONS_TABLE`
+ * environment variable.
  */
 class D1TagCache implements TagCache {
   public readonly name = "d1-tag-cache";
@@ -23,7 +30,7 @@ class D1TagCache implements TagCache {
 
     try {
       const { success, results } = await db
-        .prepare(`SELECT tag FROM ${table} WHERE path = ?`)
+        .prepare(`SELECT tag FROM ${table.tags} WHERE path = ?`)
         .bind(path)
         .all<{ tag: string }>();
 
@@ -47,7 +54,7 @@ class D1TagCache implements TagCache {
 
     try {
       const { success, results } = await db
-        .prepare(`SELECT path FROM ${table} WHERE tag = ?`)
+        .prepare(`SELECT path FROM ${table.tags} WHERE tag = ?`)
         .bind(tag)
         .all<{ path: string }>();
 
@@ -69,7 +76,11 @@ class D1TagCache implements TagCache {
 
     try {
       const { success, results } = await db
-        .prepare(`SELECT tag FROM ${table} WHERE path = ? AND revalidatedAt > ?`)
+        .prepare(
+          `SELECT ${table.revalidations}.tag FROM ${table.revalidations}
+            INNER JOIN ${table.tags} ON ${table.revalidations}.tag = ${table.tags}.tag
+            WHERE ${table.tags}.path = ? AND ${table.revalidations}.revalidatedAt > ?;`
+        )
         .bind(this.getCacheKey(path), lastModified ?? 0)
         .all<{ tag: string }>();
 
@@ -89,11 +100,19 @@ class D1TagCache implements TagCache {
 
     try {
       const results = await db.batch(
-        tags.map(({ tag, path, revalidatedAt }) =>
-          db
-            .prepare(`INSERT INTO ${table} (tag, path, revalidatedAt) VALUES(?, ?, ?)`)
-            .bind(this.getCacheKey(tag), this.getCacheKey(path), revalidatedAt ?? Date.now())
-        )
+        tags.map(({ tag, path, revalidatedAt }) => {
+          if (revalidatedAt === 1) {
+            // new tag/path mapping from set
+            return db
+              .prepare(`INSERT INTO ${table.tags} (tag, path) VALUES (?, ?)`)
+              .bind(this.getCacheKey(tag), this.getCacheKey(path));
+          }
+
+          // tag was revalidated
+          return db
+            .prepare(`INSERT INTO ${table.revalidations} (tag, revalidatedAt) VALUES (?, ?)`)
+            .bind(this.getCacheKey(tag), revalidatedAt ?? Date.now());
+        })
       );
 
       const failedResults = results.filter((res) => !res.success);
@@ -109,7 +128,6 @@ class D1TagCache implements TagCache {
   private getConfig() {
     const cfEnv = getCloudflareContext().env;
     const db = cfEnv.NEXT_CACHE_D1;
-    const table = cfEnv.NEXT_CACHE_D1_TABLE ?? "tags";
 
     if (!db) debug("No D1 database found");
 
@@ -120,7 +138,14 @@ class D1TagCache implements TagCache {
       return { isDisabled: true as const };
     }
 
-    return { isDisabled: false as const, db, table };
+    return {
+      isDisabled: false as const,
+      db,
+      table: {
+        tags: cfEnv.NEXT_CACHE_D1_TAGS_TABLE ?? "tags",
+        revalidations: cfEnv.NEXT_CACHE_D1_REVALIDATIONS_TABLE ?? "revalidations",
+      },
+    };
   }
 
   protected removeBuildId(key: string) {

packages/cloudflare/src/cli/build/open-next/compile-cache-assets-manifest.ts

@@ -17,24 +17,25 @@ export function compileCacheAssetsManifestSqlFile(options: BuildOptions) {
   const rawManifestPath = path.join(options.outputDir, "dynamodb-provider/dynamodb-cache.json");
   const outputPath = path.join(options.outputDir, "cloudflare/cache-assets-manifest.sql");
 
-  const table = process.env.NEXT_CACHE_D1 || "tags";
+  const tagsTable = process.env.NEXT_CACHE_D1_TAGS_TABLE || "tags";
+  const revalidationsTable = process.env.NEXT_CACHE_D1_REVALIDATIONS_TABLE || "revalidations";
 
   mkdirSync(path.dirname(outputPath), { recursive: true });
   writeFileSync(
     outputPath,
-    `CREATE TABLE IF NOT EXISTS ${table} (tag TEXT NOT NULL, path TEXT NOT NULL, revalidatedAt INTEGER NOT NULL, UNIQUE(tag, path) ON CONFLICT REPLACE);\n`
+    `CREATE TABLE IF NOT EXISTS ${tagsTable} (tag TEXT NOT NULL, path TEXT NOT NULL, UNIQUE(tag, path) ON CONFLICT REPLACE);
+     CREATE TABLE IF NOT EXISTS ${revalidationsTable} (tag TEXT NOT NULL, revalidatedAt INTEGER NOT NULL, UNIQUE(tag) ON CONFLICT REPLACE);\n`
   );
 
   if (existsSync(rawManifestPath)) {
     const rawManifest: RawManifest = JSON.parse(readFileSync(rawManifestPath, "utf-8"));
 
     const values = rawManifest.map(
-      ({ tag, path, revalidatedAt }) =>
-        `(${JSON.stringify(tag.S)}, ${JSON.stringify(path.S)}, ${revalidatedAt.N})`
+      ({ tag, path }) => `(${JSON.stringify(tag.S)}, ${JSON.stringify(path.S)})`
     );
 
     if (values.length) {
-      appendFileSync(outputPath, `INSERT INTO tags (tag, path, revalidatedAt) VALUES ${values.join(", ")};`);
+      appendFileSync(outputPath, `INSERT INTO ${tagsTable} (tag, path) VALUES ${values.join(", ")};`);
     }
   }
 }