output an sql file instead

Author: james-elicx

examples/e2e/app-router/open-next.config.ts

@@ -1,7 +1,7 @@
 import type { OpenNextConfig } from "@opennextjs/aws/types/open-next.js";
-import memoryQueue from "@opennextjs/cloudflare/memory-queue";
-import incrementalCache from "@opennextjs/cloudflare/kv-cache";
 import tagCache from "@opennextjs/cloudflare/d1-tag-cache";
+import incrementalCache from "@opennextjs/cloudflare/kv-cache";
+import memoryQueue from "@opennextjs/cloudflare/memory-queue";
 
 const config: OpenNextConfig = {
   default: {

examples/e2e/app-router/package.json

@@ -9,10 +9,11 @@
     "start": "next start --port 3001",
     "lint": "next lint",
     "clean": "rm -rf .turbo node_modules .next .open-next",
-    "setup:d1": "wrangler d1 execute NEXT_CACHE_D1 --command \"CREATE TABLE IF NOT EXISTS tags (tag TEXT NOT NULL, path TEXT NOT NULL, revalidatedAt INTEGER NOT NULL, UNIQUE(tag, path) ON CONFLICT REPLACE)\"",
-    "build:worker": "pnpm opennextjs-cloudflare",
+    "d1:clean": "wrangler d1 execute NEXT_CACHE_D1 --command \"DROP TABLE IF EXISTS tags\"",
+    "d1:setup": "wrangler d1 execute NEXT_CACHE_D1 --file .open-next/cache-assets-manifest.sql",
+    "build:worker": "pnpm opennextjs-cloudflare && pnpm d1:clean && pnpm d1:setup",
     "preview": "pnpm build:worker && pnpm wrangler dev",
-    "e2e": "pnpm setup:d1 && playwright test -c e2e/playwright.config.ts"
+    "e2e": "playwright test -c e2e/playwright.config.ts"
   },
   "dependencies": {
     "@opennextjs/cloudflare": "workspace:*",

packages/cloudflare/env.d.ts

@@ -1,5 +1,3 @@
-import { CacheAssetsManifest } from "./src/cli/build/open-next/create-cache-assets-manifest.js";
-
 declare global {
   namespace NodeJS {
     interface ProcessEnv {
@@ -9,7 +7,6 @@ declare global {
       NEXT_PRIVATE_DEBUG_CACHE?: string;
       OPEN_NEXT_ORIGIN: string;
       NODE_ENV?: string;
-      __OPENNEXT_CACHE_TAGS_MANIFEST: CacheAssetsManifest;
     }
   }
 }

packages/cloudflare/src/api/d1-tag-cache.ts

@@ -4,18 +4,13 @@ import type { TagCache } from "@opennextjs/aws/types/overrides.js";
 
 import { getCloudflareContext } from "./cloudflare-context.js";
 
-// inlined during build
-const manifest = process.env.__OPENNEXT_CACHE_TAGS_MANIFEST;
-
 /**
  * An instance of the Tag Cache that uses a D1 binding (`NEXT_CACHE_D1`) as it's underlying data store.
  *
  * The table used defaults to `tags`, but can be configured with the `NEXT_CACHE_D1_TABLE`
  * environment variable.
  *
  * There should be three columns created in the table; `tag`, `path`, and `revalidatedAt`.
- *
- * A combination of the D1 entries and the build-time tags manifest is used.
  */
 class D1TagCache implements TagCache {
   public readonly name = "d1-tag-cache";
@@ -34,10 +29,7 @@ class D1TagCache implements TagCache {
 
       if (!success) throw new Error(`D1 select failed for ${path}`);
 
-      const tags = this.mergeTagArrays(
-        manifest.paths[path],
-        results?.map((item) => item.tag)
-      );
+      const tags = results?.map((item) => this.removeBuildId(item.tag));
 
       debug("tags for path", path, tags);
       return tags;
@@ -61,10 +53,7 @@ class D1TagCache implements TagCache {
 
       if (!success) throw new Error(`D1 select failed for ${tag}`);
 
-      const paths = this.mergeTagArrays(
-        manifest.tags[tag],
-        results?.map((item) => item.path)
-      );
+      const paths = results?.map((item) => this.removeBuildId(item.path));
 
       debug("paths for tag", tag, paths);
       return paths;
@@ -145,16 +134,6 @@ class D1TagCache implements TagCache {
   protected getBuildId() {
     return process.env.NEXT_BUILD_ID ?? "no-build-id";
   }
-
-  protected mergeTagArrays(...arrays: (string[] | undefined)[]) {
-    const set = new Set<string>();
-
-    for (const arr of arrays) {
-      arr?.forEach((v) => set.add(v));
-    }
-
-    return [...set.values()].map((v) => this.removeBuildId(v));
-  }
 }
 
 export default new D1TagCache();

packages/cloudflare/src/cli/build/build.ts

@@ -12,6 +12,7 @@ import logger from "@opennextjs/aws/logger.js";
 
 import type { ProjectOptions } from "../project-options.js";
 import { bundleServer } from "./bundle-server.js";
+import { compileCacheAssetsManifestSqlFile } from "./open-next/compile-cache-assets-manifest.js";
 import { compileEnvFiles } from "./open-next/compile-env-files.js";
 import { copyCacheAssets } from "./open-next/copyCacheAssets.js";
 import { createServerBundle } from "./open-next/createServerBundle.js";
@@ -88,6 +89,7 @@ export async function build(projectOpts: ProjectOptions): Promise<void> {
   if (config.dangerous?.disableIncrementalCache !== true) {
     createCacheAssets(options);
     copyCacheAssets(options);
+    compileCacheAssetsManifestSqlFile(options);
   }
 
   await createServerBundle(options);

packages/cloudflare/src/cli/build/bundle-server.ts

@@ -20,7 +20,6 @@ import { fixRequire } from "./patches/plugins/require.js";
 import { shimRequireHook } from "./patches/plugins/require-hook.js";
 import { inlineRequirePage } from "./patches/plugins/require-page.js";
 import { setWranglerExternal } from "./patches/plugins/wrangler-external.js";
-import { extractCacheAssetsManifest } from "./utils/extract-cache-assets-manifest.js";
 import { normalizePath, patchCodeWithValidations } from "./utils/index.js";
 
 /** The dist directory of the Cloudflare adapter package */
@@ -129,7 +128,6 @@ export async function bundleServer(buildOpts: BuildOptions): Promise<void> {
       "process.env.NEXT_RUNTIME": '"nodejs"',
       "process.env.NODE_ENV": '"production"',
       "process.env.NEXT_MINIMAL": "true",
-      "process.env.__OPENNEXT_CACHE_TAGS_MANIFEST": `${JSON.stringify(extractCacheAssetsManifest(buildOpts))}`,
     },
     platform: "node",
     banner: {

packages/cloudflare/src/cli/build/open-next/compile-cache-assets-manifest.ts

@@ -0,0 +1,37 @@
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import path from "node:path";
+
+import type { BuildOptions } from "@opennextjs/aws/build/helper.js";
+
+type RawManifest = {
+  tag: { S: string };
+  path: { S: string };
+  revalidatedAt: { N: string };
+}[];
+
+/**
+ * Generates SQL statements that can be used to initialise the cache assets manifest in an SQL data store.
+ */
+export function compileCacheAssetsManifestSqlFile(options: BuildOptions) {
+  // TODO: Expose the function for getting this data as an adapter-agnostic utility in AWS.
+  const rawManifestPath = path.join(options.outputDir, "dynamodb-provider", "dynamodb-cache.json");
+  const outputPath = path.join(options.outputDir, "cache-assets-manifest.sql");
+
+  const table = process.env.NEXT_CACHE_D1 || "tags";
+
+  const stmts = [
+    `CREATE TABLE IF NOT EXISTS ${table} (tag TEXT NOT NULL, path TEXT NOT NULL, revalidatedAt INTEGER NOT NULL, UNIQUE(tag, path) ON CONFLICT REPLACE);`,
+  ];
+
+  if (existsSync(rawManifestPath)) {
+    const rawManifest: RawManifest = JSON.parse(readFileSync(rawManifestPath, "utf-8"));
+
+    rawManifest.forEach(({ tag: { S: tag }, path: { S: path }, revalidatedAt: { N: revalidatedAt } }) => {
+      stmts.push(
+        `INSERT INTO ${table} (tag, path, revalidatedAt) VALUES ('${tag}', '${path}', ${revalidatedAt});`
+      );
+    });
+  }
+
+  writeFileSync(outputPath, stmts.join("\n"));
+}