feat: r2 adapter for the incremental cache (#443)

james-elicx · conico974 · web-flow · commit 54508ffc613c · 2025-03-21T11:06:22.000Z
* feat: auto-populating d1 cache data

* checks for output directory / enabled

* Update packages/cloudflare/src/cli/build/utils/populate-cache.ts

Co-authored-by: conico974 &lt;nicodorseuil@yahoo.fr&gt;

* add suggestions

* rename onlyPopulate to onlyPopulateWithoutBuilding

* feat: r2 adapter for the incremental cache

* directory -&gt; prefix

* split r2 cache and regional cache into separate things

* add timeout in revalidate path test as it runs too fast locally

* incorporate suggestions

* lazily update the regional cache in the background via option

* add comments and make the lazy boolean non-optional

* add warning comment

* add comments to env vars

* change lazy update defaults

* change fallback ttl to 30 mins

* add separate app for e2e

* comments

* undo directory change as it breaks build

---------

Co-authored-by: conico974 &lt;nicodorseuil@yahoo.fr&gt;
diff --git a/.changeset/weak-houses-divide.md b/.changeset/weak-houses-divide.md
@@ -0,0 +1,5 @@
+---
+"@opennextjs/cloudflare": patch
+---
+
+feat: r2 adapter for the incremental cache
diff --git a/examples/common/apps.ts b/examples/common/apps.ts
@@ -15,6 +15,7 @@ const apps = [
   // overrides
   "d1-tag-next",
   "memory-queue",
+  "r2-incremental-cache",
   // bugs
   "gh-119",
   "gh-219",
diff --git a/examples/e2e/app-router/open-next.config.ts b/examples/e2e/app-router/open-next.config.ts
@@ -1,8 +1,7 @@
 import { defineCloudflareConfig } from "@opennextjs/cloudflare";
-import d1TagCache from "@opennextjs/cloudflare/d1-tag-cache";
 import kvIncrementalCache from "@opennextjs/cloudflare/kv-cache";
-import doQueue from "@opennextjs/cloudflare/durable-queue";
 import shardedTagCache from "@opennextjs/cloudflare/do-sharded-tag-cache";
+import doQueue from "@opennextjs/cloudflare/durable-queue";
 
 export default defineCloudflareConfig({
   incrementalCache: kvIncrementalCache,
diff --git a/examples/overrides/r2-incremental-cache/.gitignore b/examples/overrides/r2-incremental-cache/.gitignore
@@ -0,0 +1,47 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/versions
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+.pnpm-debug.log*
+
+# env files (can opt-in for committing if needed)
+.env*
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts
+
+# playwright
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/
diff --git a/examples/overrides/r2-incremental-cache/app/favicon.ico b/examples/overrides/r2-incremental-cache/app/favicon.ico
diff --git a/examples/overrides/r2-incremental-cache/app/globals.css b/examples/overrides/r2-incremental-cache/app/globals.css
@@ -0,0 +1,14 @@
+html,
+body {
+  max-width: 100vw;
+  overflow-x: hidden;
+  height: 100vh;
+  display: flex;
+  flex-direction: column;
+}
+
+footer {
+  padding: 1rem;
+  display: flex;
+  justify-content: end;
+}
diff --git a/examples/overrides/r2-incremental-cache/app/layout.tsx b/examples/overrides/r2-incremental-cache/app/layout.tsx
@@ -0,0 +1,25 @@
+import type { Metadata } from "next";
+import "./globals.css";
+
+import { getCloudflareContext } from "@opennextjs/cloudflare";
+
+export const metadata: Metadata = {
+  title: "SSG App",
+  description: "An app in which all the routes are SSG'd",
+};
+
+export default async function RootLayout({
+  children,
+}: Readonly<{
+  children: React.ReactNode;
+}>) {
+  const cloudflareContext = await getCloudflareContext({
+    async: true,
+  });
+
+  return (
+    <html lang="en">
+      <body>{children}</body>
+    </html>
+  );
+}
diff --git a/examples/overrides/r2-incremental-cache/app/page.module.css b/examples/overrides/r2-incremental-cache/app/page.module.css
@@ -0,0 +1,17 @@
+.page {
+  display: grid;
+  grid-template-rows: 20px 1fr 20px;
+  align-items: center;
+  justify-items: center;
+  flex: 1;
+  border: 3px solid gray;
+  margin: 1rem;
+  margin-block-end: 0;
+}
+
+.main {
+  display: flex;
+  flex-direction: column;
+  gap: 32px;
+  grid-row-start: 2;
+}
diff --git a/examples/overrides/r2-incremental-cache/app/page.tsx b/examples/overrides/r2-incremental-cache/app/page.tsx
@@ -0,0 +1,16 @@
+import styles from "./page.module.css";
+
+export const revalidate = 5;
+
+export default async function Home() {
+  // We purposefully wait for 2 seconds to allow deduplication to occur
+  await new Promise((resolve) => setTimeout(resolve, 2000));
+  return (
+    <div className={styles.page}>
+      <main className={styles.main}>
+        <h1>Hello from a Statically generated page</h1>
+        <p data-testid="date-local">{Date.now()}</p>
+      </main>
+    </div>
+  );
+}
diff --git a/examples/overrides/r2-incremental-cache/e2e/base.spec.ts b/examples/overrides/r2-incremental-cache/e2e/base.spec.ts
@@ -0,0 +1,36 @@
+import { test, expect } from "@playwright/test";
+
+test.describe("r2-incremental-cache", () => {
+  test("the index page should work", async ({ page }) => {
+    await page.goto("/");
+    await expect(page.getByText("Hello from a Statically generated page")).toBeVisible();
+  });
+
+  test("the index page should revalidate", async ({ page, request }) => {
+    // We need to make sure the page is loaded and is a HIT
+    // If it is STALE, the next hit may have an updated date and thus fail the test
+    let cacheHeaders = "";
+    do {
+      const req = await request.get("/");
+      cacheHeaders = req.headers()["x-nextjs-cache"];
+      await page.waitForTimeout(500);
+    } while (cacheHeaders !== "HIT");
+
+    await page.goto("/");
+    const firstDate = await page.getByTestId("date-local").textContent();
+
+    await page.reload();
+    let newDate = await page.getByTestId("date-local").textContent();
+    expect(newDate).toBe(firstDate);
+
+    await page.waitForTimeout(5000);
+
+    do {
+      await page.reload();
+      newDate = await page.getByTestId("date-local").textContent();
+      await page.waitForTimeout(1000);
+    } while (newDate === firstDate);
+
+    expect(newDate).not.toBe(firstDate);
+  });
+});
diff --git a/examples/overrides/r2-incremental-cache/e2e/playwright.config.ts b/examples/overrides/r2-incremental-cache/e2e/playwright.config.ts
@@ -0,0 +1,8 @@
+import { configurePlaywright } from "../../../common/config-e2e";
+
+// Here we don't want to run the tests in parallel
+export default configurePlaywright("r2-incremental-cache", {
+  isCI: !!process.env.CI,
+  parallel: false,
+  multipleBrowsers: false,
+});
diff --git a/examples/overrides/r2-incremental-cache/next.config.ts b/examples/overrides/r2-incremental-cache/next.config.ts
@@ -0,0 +1,11 @@
+import type { NextConfig } from "next";
+import { initOpenNextCloudflareForDev } from "@opennextjs/cloudflare";
+
+initOpenNextCloudflareForDev();
+
+const nextConfig: NextConfig = {
+  typescript: { ignoreBuildErrors: true },
+  eslint: { ignoreDuringBuilds: true },
+};
+
+export default nextConfig;
diff --git a/examples/overrides/r2-incremental-cache/open-next.config.ts b/examples/overrides/r2-incremental-cache/open-next.config.ts
@@ -0,0 +1,14 @@
+import { defineCloudflareConfig } from "@opennextjs/cloudflare";
+import d1TagCache from "@opennextjs/cloudflare/d1-tag-cache";
+import memoryQueue from "@opennextjs/cloudflare/memory-queue";
+import r2IncrementalCache from "@opennextjs/cloudflare/r2-incremental-cache";
+import { withRegionalCache } from "@opennextjs/cloudflare/regional-cache";
+
+export default defineCloudflareConfig({
+  incrementalCache: withRegionalCache(r2IncrementalCache, {
+    mode: "long-lived",
+    shouldLazilyUpdateOnCacheHit: true,
+  }),
+  tagCache: d1TagCache,
+  queue: memoryQueue,
+});
diff --git a/examples/overrides/r2-incremental-cache/package.json b/examples/overrides/r2-incremental-cache/package.json
@@ -0,0 +1,29 @@
+{
+  "name": "r2-incremental-cache",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start",
+    "lint": "next lint",
+    "d1:clean": "wrangler d1 execute NEXT_CACHE_D1 --command \"DROP TABLE IF EXISTS tags; DROP TABLE IF EXISTS revalidations\"",
+    "build:worker": "pnpm d1:clean && pnpm opennextjs-cloudflare --populateCache=local",
+    "preview": "pnpm build:worker && pnpm wrangler dev",
+    "e2e": "playwright test -c e2e/playwright.config.ts"
+  },
+  "dependencies": {
+    "react": "catalog:e2e",
+    "react-dom": "catalog:e2e",
+    "next": "catalog:e2e"
+  },
+  "devDependencies": {
+    "@opennextjs/cloudflare": "workspace:*",
+    "@playwright/test": "catalog:",
+    "@types/node": "catalog:",
+    "@types/react": "catalog:e2e",
+    "@types/react-dom": "catalog:e2e",
+    "typescript": "catalog:",
+    "wrangler": "catalog:"
+  }
+}
diff --git a/examples/overrides/r2-incremental-cache/tsconfig.json b/examples/overrides/r2-incremental-cache/tsconfig.json
@@ -0,0 +1,27 @@
+{
+  "compilerOptions": {
+    "target": "ES2017",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "bundler",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true,
+    "plugins": [
+      {
+        "name": "next"
+      }
+    ],
+    "paths": {
+      "@/*": ["./*"]
+    }
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+  "exclude": ["node_modules"]
+}
diff --git a/examples/overrides/r2-incremental-cache/wrangler.jsonc b/examples/overrides/r2-incremental-cache/wrangler.jsonc
@@ -0,0 +1,31 @@
+{
+  "$schema": "node_modules/wrangler/config-schema.json",
+  "main": ".open-next/worker.js",
+  "name": "r2-incremental-cache",
+  "compatibility_date": "2025-02-04",
+  "compatibility_flags": ["nodejs_compat"],
+  "assets": {
+    "directory": ".open-next/assets",
+    "binding": "ASSETS"
+  },
+  "d1_databases": [
+    {
+      "binding": "NEXT_CACHE_D1",
+      "database_id": "NEXT_CACHE_D1",
+      "database_name": "NEXT_CACHE_D1"
+    }
+  ],
+  "services": [
+    {
+      "binding": "NEXT_CACHE_REVALIDATION_WORKER",
+      "service": "r2-incremental-cache"
+    }
+  ],
+  "r2_buckets": [
+    {
+      "binding": "NEXT_CACHE_R2_BUCKET",
+      "bucket_name": "NEXT_CACHE_R2_BUCKET",
+      "preview_bucket_name": "NEXT_CACHE_R2_BUCKET"
+    }
+  ]
+}
diff --git a/packages/cloudflare/src/api/cloudflare-context.ts b/packages/cloudflare/src/api/cloudflare-context.ts
@@ -15,6 +15,10 @@ declare global {
     NEXT_CACHE_D1_REVALIDATIONS_TABLE?: string;
     // Service binding for the worker itself to be able to call itself from within the worker
     NEXT_CACHE_REVALIDATION_WORKER?: Service;
+    // R2 bucket used for the incremental cache
+    NEXT_CACHE_R2_BUCKET?: R2Bucket;
+    // Prefix used for the R2 incremental cache bucket
+    NEXT_CACHE_R2_PREFIX?: string;
     // Durable Object namespace to use for the durable object queue handler
     NEXT_CACHE_REVALIDATION_DURABLE_OBJECT?: DurableObjectNamespace<DurableObjectQueueHandler>;
     // Durables object namespace to use for the sharded tag cache
diff --git a/packages/cloudflare/src/api/internal/incremental-cache.ts b/packages/cloudflare/src/api/internal/incremental-cache.ts
@@ -0,0 +1,6 @@
+import { CacheValue } from "@opennextjs/aws/types/overrides.js";
+
+export type IncrementalCacheEntry<IsFetch extends boolean> = {
+  value: CacheValue<IsFetch>;
+  lastModified: number;
+};
diff --git a/packages/cloudflare/src/api/r2-incremental-cache.ts b/packages/cloudflare/src/api/r2-incremental-cache.ts
diff --git a/packages/cloudflare/src/api/regional-cache.ts b/packages/cloudflare/src/api/regional-cache.ts
diff --git a/packages/cloudflare/src/cli/build/utils/populate-cache.ts b/packages/cloudflare/src/cli/build/utils/populate-cache.ts
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml

-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +---
 +"@opennextjs/cloudflare": patch
 +---
++
 +feat: r2 adapter for the incremental cache