feat(cloudflare): add optional R2 batch uploads via rclone for cache population (#925)

Author: krzysztof-palka-monogo

.changeset/rclone-batch-upload.md

@@ -0,0 +1,48 @@
+---
+"@opennextjs/cloudflare": minor
+---
+
+feature: optional batch upload for faster R2 cache population
+
+This update adds optional batch upload support for R2 cache population, significantly improving upload performance for large caches when enabled via .env or environment variables.
+
+**Key Changes:**
+
+1. **Optional Batch Upload**: Configure R2 credentials via .env or environment variables to enable faster batch uploads:
+
+   - `R2_ACCESS_KEY_ID`
+   - `R2_SECRET_ACCESS_KEY`
+   - `CF_ACCOUNT_ID`
+
+2. **Automatic Detection**: When credentials are detected, batch upload is automatically used for better performance
+
+3. **Smart Fallback**: If credentials are not configured, the CLI falls back to standard Wrangler uploads with a helpful message about enabling batch upload for better performance
+
+**All deployment commands support batch upload:**
+
+- `populateCache` - Explicit cache population
+- `deploy` - Deploy with cache population
+- `upload` - Upload version with cache population
+- `preview` - Preview with cache population
+
+**Performance Benefits (when batch upload is enabled):**
+
+- Parallel transfer capabilities (32 concurrent transfers)
+- Significantly faster for large caches
+- Reduced API calls to Cloudflare
+
+**Usage:**
+
+Add the credentials in a `.env`/`.dev.vars` file in your project root:
+
+```bash
+R2_ACCESS_KEY_ID=your_key
+R2_SECRET_ACCESS_KEY=your_secret
+CF_ACCOUNT_ID=your_account
+```
+
+You can also set the environment variables for CI builds.
+
+**Note:**
+
+You can follow documentation https://developers.cloudflare.com/r2/api/tokens/ for creating API tokens with appropriate permissions for R2 access.

packages/cloudflare/README.md

@@ -55,3 +55,30 @@ Deploy your application to production with the following:
   # or
   bun opennextjs-cloudflare build && bun opennextjs-cloudflare deploy
   ```
+
+### Batch Cache Population (Optional, Recommended)
+
+For improved performance with large caches, you can enable batch upload by providing R2 credentials via .env or environment variables.
+
+Create a `.env` file in your project root (automatically loaded by the CLI):
+
+```bash
+R2_ACCESS_KEY_ID=your_access_key_id
+R2_SECRET_ACCESS_KEY=your_secret_access_key
+CF_ACCOUNT_ID=your_account_id
+```
+
+You can also set the environment variables for CI builds.
+
+**Note:**
+
+You can follow documentation https://developers.cloudflare.com/r2/api/tokens/ for creating API tokens with appropriate permissions for R2 access.
+
+**Benefits:**
+
+- Significantly faster uploads for large caches using parallel transfers
+- Reduced API calls to Cloudflare
+- Automatically enabled when credentials are provided
+
+**Fallback:**
+If these environment variables are not set, the CLI will use standard Wrangler uploads. Both methods work correctly - batch upload is simply faster for large caches.

packages/cloudflare/package.json

@@ -54,9 +54,11 @@
 	"dependencies": {
 		"@dotenvx/dotenvx": "catalog:",
 		"@opennextjs/aws": "3.8.4",
+		"@types/rclone.js": "^0.6.3",
 		"cloudflare": "^4.4.1",
 		"enquirer": "^2.4.1",
 		"glob": "catalog:",
+		"rclone.js": "^0.6.6",
 		"ts-tqdm": "^0.8.6",
 		"yargs": "catalog:"
 	},

packages/cloudflare/src/api/cloudflare-context.ts

@@ -79,8 +79,13 @@ declare global {
 		CF_PREVIEW_DOMAIN?: string;
 		// Should have the `Workers Scripts:Read` permission
 		CF_WORKERS_SCRIPTS_API_TOKEN?: string;
-		// Cloudflare account id
+
+		// Cloudflare account id - needed for skew protection and R2 batch population
 		CF_ACCOUNT_ID?: string;
+
+		// R2 API credentials for batch cache population (optional, enables faster uploads)
+		R2_ACCESS_KEY_ID?: string;
+		R2_SECRET_ACCESS_KEY?: string;
 	}
 }
 

packages/cloudflare/src/cli/commands/populate-cache.spec.ts

@@ -3,9 +3,9 @@ import path from "node:path";
 
 import type { BuildOptions } from "@opennextjs/aws/build/helper.js";
 import mockFs from "mock-fs";
-import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { afterAll, afterEach, beforeAll, describe, expect, test, vi } from "vitest";
 
-import { getCacheAssets } from "./populate-cache.js";
+import { getCacheAssets, populateCache } from "./populate-cache.js";
 
 describe("getCacheAssets", () => {
 	beforeAll(() => {
@@ -68,3 +68,217 @@ describe("getCacheAssets", () => {
     `);
 	});
 });
+
+vi.mock("../utils/run-wrangler.js", () => ({
+	runWrangler: vi.fn(),
+}));
+
+vi.mock("./helpers.js", () => ({
+	getEnvFromPlatformProxy: vi.fn(async () => ({})),
+	quoteShellMeta: vi.fn((s) => s),
+}));
+
+// Mock rclone.js promises API to simulate successful copy operations by default
+vi.mock("rclone.js", () => ({
+	default: {
+		promises: {
+			copy: vi.fn(() => Promise.resolve("")),
+		},
+	},
+}));
+
+describe("populateCache", () => {
+	// Test fixtures
+	const createTestBuildOptions = (): BuildOptions =>
+		({
+			outputDir: "/test/output",
+		}) as BuildOptions;
+
+	const createTestOpenNextConfig = () => ({
+		default: {
+			override: {
+				incrementalCache: "cf-r2-incremental-cache",
+			},
+		},
+	});
+
+	const createTestWranglerConfig = () => ({
+		r2_buckets: [
+			{
+				binding: "NEXT_INC_CACHE_R2_BUCKET",
+				bucket_name: "test-bucket",
+			},
+		],
+	});
+
+	const createTestPopulateCacheOptions = () => ({
+		target: "local" as const,
+		shouldUsePreviewId: false,
+	});
+
+	const setupMockFileSystem = () => {
+		mockFs({
+			"/test/output": {
+				cache: {
+					buildID: {
+						path: {
+							to: {
+								"test.cache": JSON.stringify({ data: "test" }),
+							},
+						},
+					},
+				},
+			},
+		});
+	};
+
+	describe("R2 incremental cache", () => {
+		afterEach(() => {
+			mockFs.restore();
+			vi.unstubAllEnvs();
+		});
+
+		test("uses sequential upload for local target (skips batch upload)", async () => {
+			const { runWrangler } = await import("../utils/run-wrangler.js");
+			const rcloneModule = (await import("rclone.js")).default;
+
+			setupMockFileSystem();
+			vi.mocked(runWrangler).mockClear();
+			vi.mocked(rcloneModule.promises.copy).mockClear();
+
+			// Test with local target - should skip batch upload even with credentials
+			await populateCache(
+				createTestBuildOptions(),
+				createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				{ target: "local" as const, shouldUsePreviewId: false },
+				{
+					R2_ACCESS_KEY_ID: "test_access_key",
+					R2_SECRET_ACCESS_KEY: "test_secret_key",
+					CF_ACCOUNT_ID: "test_account_id",
+				} as any // eslint-disable-line @typescript-eslint/no-explicit-any
+			);
+
+			// Should use sequential upload (runWrangler), not batch upload (rclone.js)
+			expect(runWrangler).toHaveBeenCalled();
+			expect(rcloneModule.promises.copy).not.toHaveBeenCalled();
+		});
+
+		test("uses sequential upload when R2 credentials are not provided", async () => {
+			const { runWrangler } = await import("../utils/run-wrangler.js");
+			const rcloneModule = (await import("rclone.js")).default;
+
+			setupMockFileSystem();
+			vi.mocked(runWrangler).mockClear();
+			vi.mocked(rcloneModule.promises.copy).mockClear();
+
+			// Test uses partial types for simplicity - full config not needed
+			// Pass empty envVars to simulate no R2 credentials
+			await populateCache(
+				createTestBuildOptions(),
+				createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				createTestPopulateCacheOptions(),
+				{} as any // eslint-disable-line @typescript-eslint/no-explicit-any
+			);
+
+			expect(runWrangler).toHaveBeenCalled();
+			expect(rcloneModule.promises.copy).not.toHaveBeenCalled();
+		});
+
+		test("uses batch upload with temporary config for remote target when R2 credentials are provided", async () => {
+			const rcloneModule = (await import("rclone.js")).default;
+
+			setupMockFileSystem();
+			vi.mocked(rcloneModule.promises.copy).mockClear();
+
+			// Test uses partial types for simplicity - full config not needed
+			// Pass envVars with R2 credentials and remote target to enable batch upload
+			await populateCache(
+				createTestBuildOptions(),
+				createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				{ target: "remote" as const, shouldUsePreviewId: false },
+				{
+					R2_ACCESS_KEY_ID: "test_access_key",
+					R2_SECRET_ACCESS_KEY: "test_secret_key",
+					CF_ACCOUNT_ID: "test_account_id",
+				} as any // eslint-disable-line @typescript-eslint/no-explicit-any
+			);
+
+			// Verify batch upload was used with correct parameters and temporary config
+			expect(rcloneModule.promises.copy).toHaveBeenCalledWith(
+				expect.any(String), // staging directory
+				"r2:test-bucket",
+				expect.objectContaining({
+					progress: true,
+					transfers: 16,
+					checkers: 8,
+					env: expect.objectContaining({
+						RCLONE_CONFIG: expect.stringMatching(/rclone-config-\d+\.conf$/),
+					}),
+				})
+			);
+		});
+
+		test("handles rclone errors with status > 0 for remote target", async () => {
+			const { runWrangler } = await import("../utils/run-wrangler.js");
+			const rcloneModule = (await import("rclone.js")).default;
+
+			setupMockFileSystem();
+
+			// Mock rclone failure - Promise rejection
+			vi.mocked(rcloneModule.promises.copy).mockRejectedValueOnce(
+				new Error("rclone copy failed with exit code 7")
+			);
+
+			vi.mocked(runWrangler).mockClear();
+
+			// Pass envVars with R2 credentials and remote target to enable batch upload (which will fail)
+			await populateCache(
+				createTestBuildOptions(),
+				createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				{ target: "remote" as const, shouldUsePreviewId: false },
+				{
+					R2_ACCESS_KEY_ID: "test_access_key",
+					R2_SECRET_ACCESS_KEY: "test_secret_key",
+					CF_ACCOUNT_ID: "test_account_id",
+				} as any // eslint-disable-line @typescript-eslint/no-explicit-any
+			);
+
+			// Should fall back to sequential upload when batch upload fails
+			expect(runWrangler).toHaveBeenCalled();
+		});
+
+		test("handles rclone errors with stderr output for remote target", async () => {
+			const { runWrangler } = await import("../utils/run-wrangler.js");
+			const rcloneModule = (await import("rclone.js")).default;
+
+			setupMockFileSystem();
+
+			// Mock rclone error - Promise rejection with stderr message
+			vi.mocked(rcloneModule.promises.copy).mockRejectedValueOnce(
+				new Error("ERROR : Failed to copy: AccessDenied: Access Denied (403)")
+			);
+
+			vi.mocked(runWrangler).mockClear();
+
+			// Pass envVars with R2 credentials and remote target to enable batch upload (which will fail)
+			await populateCache(
+				createTestBuildOptions(),
+				createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				{ target: "remote" as const, shouldUsePreviewId: false },
+				{
+					R2_ACCESS_KEY_ID: "test_access_key",
+					R2_SECRET_ACCESS_KEY: "test_secret_key",
+					CF_ACCOUNT_ID: "test_account_id",
+				} as any // eslint-disable-line @typescript-eslint/no-explicit-any
+			);
+
+			// Should fall back to standard upload when batch upload fails
+			expect(runWrangler).toHaveBeenCalled();
+		});
+	});
+});