feat(cloudflare): add optional R2 batch uploads via rclone for cache population

.changeset/rclone-batch-upload.md

@@ -0,0 +1,55 @@
+---
+"@opennextjs/cloudflare": minor
+---
+
+feature: optional batch upload for faster R2 cache population
+
+This update adds optional batch upload support for R2 cache population, significantly improving upload performance for large caches when enabled via environment variables.
+
+**Key Changes:**
+
+1. **Optional Batch Upload**: Configure R2 credentials via environment variables to enable faster batch uploads:
+
+   - `R2_ACCESS_KEY_ID`
+   - `R2_SECRET_ACCESS_KEY`
+   - `CF_ACCOUNT_ID`
+
+2. **Automatic Detection**: When credentials are detected, batch upload is automatically used for better performance
+
+3. **Smart Fallback**: If credentials are not configured, the CLI falls back to standard Wrangler uploads with a helpful message about enabling batch upload for better performance
+
+**All deployment commands support batch upload:**
+
+- `populateCache` - Explicit cache population
+- `deploy` - Deploy with cache population
+- `upload` - Upload version with cache population
+- `preview` - Preview with cache population
+
+**Performance Benefits (when batch upload is enabled):**
+
+- Parallel transfer capabilities (32 concurrent transfers)
+- Significantly faster for large caches
+- Reduced API calls to Cloudflare
+
+**Usage:**
+
+You can set environment variables directly:
+
+```bash
+export R2_ACCESS_KEY_ID=your_key
+export R2_SECRET_ACCESS_KEY=your_secret
+export CF_ACCOUNT_ID=your_account
+opennextjs-cloudflare deploy  # batch upload automatically used
+```
+
+Or create a `.env` file in your project root (automatically loaded):
+
+```bash
+R2_ACCESS_KEY_ID=your_key
+R2_SECRET_ACCESS_KEY=your_secret
+CF_ACCOUNT_ID=your_account
+```
+
+**Note:**
+
+You can follow documentation https://developers.cloudflare.com/r2/api/tokens/ for creating API tokens with appropriate permissions for R2 access.

benchmarking/src/cloudflare.ts

@@ -96,7 +96,7 @@ export async function deployBuiltApp(dir: string): Promise<string> {
  * Makes sure that everything is set up so that wrangler can actually deploy the applications.
  * This means that:
  *  - the user has logged in
- *  - if they have more than one account they have set a CLOUDFLARE_ACCOUNT_ID env variable
+ *  - if they have more than one account they have set a CF_ACCOUNT_ID env variable
  */
 async function ensureWranglerSetup(): Promise<void> {
 	return new Promise((resolve, reject) => {
@@ -109,10 +109,10 @@ async function ensureWranglerSetup(): Promise<void> {
 				reject(new Error("Please log in using wrangler by running `pnpm dlx wrangler login`"));
 			}
 
-			if (!(process.env as Record<string, unknown>)["CLOUDFLARE_ACCOUNT_ID"]) {
+			if (!(process.env as Record<string, unknown>)["CF_ACCOUNT_ID"]) {
 				reject(
 					new Error(
-						"Please set the CLOUDFLARE_ACCOUNT_ID environment variable to the id of the account you want to use to deploy the applications"
+						"Please set the CF_ACCOUNT_ID environment variable to the id of the account you want to use to deploy the applications"
 					)
 				);
 			}

packages/cloudflare/README.md

@@ -55,3 +55,36 @@ Deploy your application to production with the following:
   # or
   bun opennextjs-cloudflare build && bun opennextjs-cloudflare deploy
   ```
+
+### Batch Cache Population (Optional, Recommended)
+
+For improved performance with large caches, you can enable batch upload by providing R2 credentials via environment variables.
+
+You can either set environment variables directly:
+
+```bash
+export R2_ACCESS_KEY_ID=your_access_key_id
+export R2_SECRET_ACCESS_KEY=your_secret_access_key
+export CF_ACCOUNT_ID=your_account_id
+```
+
+Or create a `.env` file in your project root (automatically loaded by the CLI):
+
+```bash
+R2_ACCESS_KEY_ID=your_access_key_id
+R2_SECRET_ACCESS_KEY=your_secret_access_key
+CF_ACCOUNT_ID=your_account_id
+```
+
+**Note:**
+
+You can follow documentation https://developers.cloudflare.com/r2/api/tokens/ for creating API tokens with appropriate permissions for R2 access.
+
+**Benefits:**
+
+- Significantly faster uploads for large caches using parallel transfers
+- Reduced API calls to Cloudflare
+- Automatically enabled when credentials are provided
+
+**Fallback:**
+If these environment variables are not set, the CLI will use standard Wrangler uploads. Both methods work correctly - batch upload is simply faster for large caches.

packages/cloudflare/package.json

@@ -57,6 +57,7 @@
 		"cloudflare": "^4.4.1",
 		"enquirer": "^2.4.1",
 		"glob": "catalog:",
+		"rclone.js": "^0.6.6",
 		"ts-tqdm": "^0.8.6",
 		"yargs": "catalog:"
 	},

packages/cloudflare/src/cli/commands/populate-cache.spec.ts

@@ -1,11 +1,12 @@
+import { spawnSync } from "node:child_process";
 import { mkdirSync, writeFileSync } from "node:fs";
 import path from "node:path";
 
 import type { BuildOptions } from "@opennextjs/aws/build/helper.js";
 import mockFs from "mock-fs";
-import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { afterAll, afterEach, beforeAll, describe, expect, test, vi } from "vitest";
 
-import { getCacheAssets } from "./populate-cache.js";
+import { getCacheAssets, populateCache } from "./populate-cache.js";
 
 describe("getCacheAssets", () => {
 	beforeAll(() => {
@@ -68,3 +69,180 @@ describe("getCacheAssets", () => {
     `);
 	});
 });
+
+vi.mock("../utils/run-wrangler.js", () => ({
+	runWrangler: vi.fn(),
+}));
+
+vi.mock("./helpers.js", () => ({
+	getEnvFromPlatformProxy: vi.fn(async () => ({})),
+	quoteShellMeta: vi.fn((s) => s),
+}));
+
+vi.mock("node:child_process", () => ({
+	spawnSync: vi.fn(() => ({ status: 0, stderr: Buffer.from("") })),
+}));
+
+describe("populateCache", () => {
+	// Test fixtures
+	const createTestBuildOptions = (): BuildOptions =>
+		({
+			outputDir: "/test/output",
+		}) as BuildOptions;
+
+	const createTestOpenNextConfig = () => ({
+		default: {
+			override: {
+				incrementalCache: "cf-r2-incremental-cache",
+			},
+		},
+	});
+
+	const createTestWranglerConfig = () => ({
+		r2_buckets: [
+			{
+				binding: "NEXT_INC_CACHE_R2_BUCKET",
+				bucket_name: "test-bucket",
+			},
+		],
+	});
+
+	const createTestPopulateCacheOptions = () => ({
+		target: "local" as const,
+		shouldUsePreviewId: false,
+	});
+
+	const setupMockFileSystem = () => {
+		mockFs({
+			"/test/output": {
+				cache: {
+					buildID: {
+						path: {
+							to: {
+								"test.cache": JSON.stringify({ data: "test" }),
+							},
+						},
+					},
+				},
+			},
+		});
+	};
+
+	describe("R2 incremental cache", () => {
+		afterEach(() => {
+			mockFs.restore();
+			vi.unstubAllEnvs();
+		});
+
+		test("uses sequential upload when R2 credentials are not provided", async () => {
+			const { runWrangler } = await import("../utils/run-wrangler.js");
+
+			// Ensure no batch upload credentials are set
+			vi.stubEnv("R2_ACCESS_KEY_ID", undefined);
+			vi.stubEnv("R2_SECRET_ACCESS_KEY", undefined);
+			vi.stubEnv("CF_ACCOUNT_ID", undefined);
+
+			setupMockFileSystem();
+			vi.mocked(runWrangler).mockClear();
+
+			// Test uses partial types for simplicity - full config not needed
+			await populateCache(
+				createTestBuildOptions(),
+				createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				createTestPopulateCacheOptions()
+			);
+
+			expect(runWrangler).toHaveBeenCalled();
+			expect(spawnSync).not.toHaveBeenCalled();
+		});
+
+		test("uses batch upload with temporary config when R2 credentials are provided", async () => {
+			// Set R2 credentials to enable batch upload
+			vi.stubEnv("R2_ACCESS_KEY_ID", "test_access_key");
+			vi.stubEnv("R2_SECRET_ACCESS_KEY", "test_secret_key");
+			vi.stubEnv("CF_ACCOUNT_ID", "test_account_id");
+
+			setupMockFileSystem();
+			vi.mocked(spawnSync).mockClear();
+
+			// Test uses partial types for simplicity - full config not needed
+			await populateCache(
+				createTestBuildOptions(),
+				createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				createTestPopulateCacheOptions()
+			);
+
+			// Verify batch upload was used with correct parameters and temporary config
+			expect(spawnSync).toHaveBeenCalledWith(
+				"rclone",
+				expect.arrayContaining(["copy", expect.any(String), "r2:test-bucket", "--error-on-no-transfer"]),
+				expect.objectContaining({
+					stdio: ["inherit", "inherit", "pipe"],
+					env: expect.objectContaining({
+						RCLONE_CONFIG: expect.stringMatching(/rclone-config-\d+\.conf$/),
+					}),
+				})
+			);
+		});
+
+		test("handles rclone errors with status > 0", async () => {
+			const { runWrangler } = await import("../utils/run-wrangler.js");
+
+			// Set R2 credentials
+			vi.stubEnv("R2_ACCESS_KEY_ID", "test_access_key");
+			vi.stubEnv("R2_SECRET_ACCESS_KEY", "test_secret_key");
+			vi.stubEnv("CF_ACCOUNT_ID", "test_account_id");
+
+			setupMockFileSystem();
+
+			// Mock rclone failure without stderr output
+			vi.mocked(spawnSync).mockReturnValueOnce({
+				status: 7, // Fatal error exit code
+				stderr: "", // No stderr output
+			} as any); // eslint-disable-line @typescript-eslint/no-explicit-any
+
+			vi.mocked(runWrangler).mockClear();
+
+			await populateCache(
+				createTestBuildOptions(),
+				createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				createTestPopulateCacheOptions()
+			);
+
+			// Should fall back to sequential upload when batch upload fails
+			expect(runWrangler).toHaveBeenCalled();
+		});
+	});
+
+	test("handles rclone errors with status = 0 and stderr output", async () => {
+		const { runWrangler } = await import("../utils/run-wrangler.js");
+
+		// Set R2 credentials
+		vi.stubEnv("R2_ACCESS_KEY_ID", "test_access_key");
+		vi.stubEnv("R2_SECRET_ACCESS_KEY", "test_secret_key");
+		vi.stubEnv("CF_ACCOUNT_ID", "test_account_id");
+
+		setupMockFileSystem();
+
+		// Mock rclone error in stderr
+		vi.mocked(spawnSync).mockReturnValueOnce({
+			status: 0, // non-error exit code
+			stderr: Buffer.from("ERROR : Failed to copy: AccessDenied: Access Denied (403)"),
+		} as any); // eslint-disable-line @typescript-eslint/no-explicit-any
+
+		vi.mocked(runWrangler).mockClear();
+
+		await populateCache(
+			createTestBuildOptions(),
+			createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+			createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+			createTestPopulateCacheOptions()
+		);
+
+		// Should fall back to standard upload when batch upload fails
+		expect(runWrangler).toHaveBeenCalled();
+	});
+});