feat: add --rcloneBatch flag for faster R2 cache uploads using rclone

Author: krzysztof-palka-monogo

.changeset/rclone-batch-upload.md

@@ -0,0 +1,34 @@
+---
+"@opennextjs/cloudflare": minor
+---
+
+feature: add --rcloneBatch flag for faster R2 cache uploads
+
+This adds a new optional `--rcloneBatch` flag that enables batch uploading to R2 using rclone instead of individual wrangler uploads. This significantly improves upload performance for large caches.
+
+**Supported commands:**
+
+- `populateCache` - Explicit cache population
+- `deploy` - Deploy with cache population
+- `upload` - Upload version with cache population
+- `preview` - Preview with cache population
+
+**Performance improvements:**
+
+- Creates staging directory with all cache files organized by R2 keys
+- Uses rclone's parallel transfer capabilities (32 transfers, 16 checkers)
+- Reduces API calls to Cloudflare
+
+**Usage:**
+
+```bash
+opennextjs-cloudflare deploy --rcloneBatch
+opennextjs-cloudflare populateCache remote --rcloneBatch
+```
+
+**Requirements:**
+
+- The `rclone.js` package (included as dependency) provides the binary
+- An rclone config file at `~/.config/rclone/rclone.conf` with R2 credentials (see README for setup instructions)
+
+The original wrangler-based upload remains the default behavior for backward compatibility.

packages/cloudflare/README.md

@@ -55,3 +55,54 @@ Deploy your application to production with the following:
   # or
   bun opennextjs-cloudflare build && bun opennextjs-cloudflare deploy
   ```
+
+## CLI Options
+
+### Batch Cache Population (rclone)
+
+The `--rcloneBatch` flag enables faster R2 cache uploads using rclone batch mode. This flag is supported by the following commands:
+
+- `populateCache` - Explicitly populate cache
+- `deploy` - Deploy and populate cache
+- `upload` - Upload version and populate cache
+- `preview` - Preview and populate cache
+
+**Usage:**
+
+```bash
+# Standalone cache population
+npx opennextjs-cloudflare populateCache local --rcloneBatch
+npx opennextjs-cloudflare populateCache remote --rcloneBatch
+
+# During deployment
+npx opennextjs-cloudflare deploy --rcloneBatch
+
+# During upload
+npx opennextjs-cloudflare upload --rcloneBatch
+
+# During preview
+npx opennextjs-cloudflare preview --rcloneBatch
+```
+
+**Requirements:**
+
+1. The `rclone.js` package (included as a dependency) provides the rclone binary automatically
+2. An rclone configuration file is required at `~/.config/rclone/rclone.conf` with your R2 credentials
+
+**rclone Configuration:**
+
+Create or update `~/.config/rclone/rclone.conf` with your R2 bucket configuration:
+
+```ini
+[r2]
+type = s3
+provider = Cloudflare
+access_key_id = YOUR_ACCESS_KEY_ID
+secret_access_key = YOUR_SECRET_ACCESS_KEY
+endpoint = https://YOUR_ACCOUNT_ID.r2.cloudflarestorage.com
+acl = private
+```
+
+See [Cloudflare's rclone documentation](https://developers.cloudflare.com/r2/examples/rclone/) for more details.
+
+**Default:** `false` (uses standard wrangler-based uploads)

packages/cloudflare/package.json

@@ -57,6 +57,7 @@
 		"cloudflare": "^4.4.1",
 		"enquirer": "^2.4.1",
 		"glob": "catalog:",
+		"rclone.js": "^0.6.6",
 		"ts-tqdm": "^0.8.6",
 		"yargs": "catalog:"
 	},

packages/cloudflare/src/cli/commands/deploy.ts

@@ -19,7 +19,9 @@ import {
  *
  * @param args
  */
-export async function deployCommand(args: WithWranglerArgs<{ cacheChunkSize?: number }>): Promise<void> {
+export async function deployCommand(
+	args: WithWranglerArgs<{ cacheChunkSize?: number; rcloneBatch?: boolean }>
+): Promise<void> {
 	printHeaders("deploy");
 
 	const { config } = await retrieveCompiledConfig();
@@ -40,6 +42,7 @@ export async function deployCommand(args: WithWranglerArgs<{ cacheChunkSize?: nu
 		wranglerConfigPath: args.wranglerConfigPath,
 		cacheChunkSize: args.cacheChunkSize,
 		shouldUsePreviewId: false,
+		rcloneBatch: args.rcloneBatch,
 	});
 
 	runWrangler(

packages/cloudflare/src/cli/commands/populate-cache.spec.ts

@@ -5,7 +5,7 @@ import type { BuildOptions } from "@opennextjs/aws/build/helper.js";
 import mockFs from "mock-fs";
 import { afterAll, beforeAll, describe, expect, test } from "vitest";
 
-import { getCacheAssets } from "./populate-cache.js";
+import { getCacheAssets, withPopulateCacheOptions } from "./populate-cache.js";
 
 describe("getCacheAssets", () => {
 	beforeAll(() => {
@@ -68,3 +68,23 @@ describe("getCacheAssets", () => {
     `);
 	});
 });
+
+describe("withPopulateCacheOptions", () => {
+	test("includes rcloneBatch option with correct defaults", () => {
+		interface MockYargs {
+			options: (name: string, config: Record<string, unknown>) => MockYargs;
+		}
+
+		const mockYargs: MockYargs = {
+			options: (name: string, config: Record<string, unknown>) => {
+				expect(name).toBeDefined();
+				expect(config).toBeDefined();
+				return mockYargs;
+			},
+		};
+
+		const result = withPopulateCacheOptions(mockYargs as never);
+
+		expect(result).toBe(mockYargs);
+	});
+});

packages/cloudflare/src/cli/commands/populate-cache.ts

@@ -1,4 +1,5 @@
-import { cpSync, existsSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { spawnSync } from "node:child_process";
+import { copyFileSync, cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import path from "node:path";
 
 import type { BuildOptions } from "@opennextjs/aws/build/helper.js";
@@ -127,8 +128,54 @@ type PopulateCacheOptions = {
 	 * Instructs Wrangler to use the preview namespace or ID defined in the Wrangler config for the remote target.
 	 */
 	shouldUsePreviewId: boolean;
+	/**
+	 * Use rclone for batch uploading to R2 instead of individual wrangler uploads.
+	 *
+	 * @default false
+	 */
+	rcloneBatch?: boolean;
 };
 
+async function populateR2IncrementalCacheWithRclone(
+	options: BuildOptions,
+	bucket: string,
+	prefix: string | undefined,
+	assets: CacheAsset[]
+) {
+	logger.info("\nPopulating R2 incremental cache using rclone batch upload...");
+
+	// Create a staging dir with proper key paths
+	const stagingDir = path.join(options.outputDir, ".r2-staging");
+	rmSync(stagingDir, { recursive: true, force: true });
+	mkdirSync(stagingDir, { recursive: true });
+
+	for (const { fullPath, key, buildId, isFetch } of assets) {
+		const cacheKey = computeCacheKey(key, {
+			prefix,
+			buildId,
+			cacheType: isFetch ? "fetch" : "cache",
+		});
+		const destPath = path.join(stagingDir, cacheKey);
+		mkdirSync(path.dirname(destPath), { recursive: true });
+		copyFileSync(fullPath, destPath);
+	}
+
+	// Use rclone to sync the R2
+	const remote = `r2:${bucket}`;
+	const args = ["copy", stagingDir, remote, "--progress", "--transfers=32", "--checkers=16"];
+
+	const result = spawnSync("rclone", args, {
+		stdio: "inherit",
+		env: process.env,
+	});
+
+	if (result.status !== 0) {
+		throw new Error("rclone copy failed");
+	}
+
+	logger.info(`Successfully uploaded ${assets.length} assets to R2`);
+}
+
 async function populateR2IncrementalCache(
 	options: BuildOptions,
 	config: WranglerConfig,
@@ -151,6 +198,10 @@ async function populateR2IncrementalCache(
 
 	const assets = getCacheAssets(options);
 
+	if (populateCacheOptions.rcloneBatch) {
+		return populateR2IncrementalCacheWithRclone(options, bucket, prefix, assets);
+	}
+
 	for (const { fullPath, key, buildId, isFetch } of tqdm(assets)) {
 		const cacheKey = computeCacheKey(key, {
 			prefix,
@@ -329,7 +380,7 @@ export async function populateCache(
  */
 async function populateCacheCommand(
 	target: "local" | "remote",
-	args: WithWranglerArgs<{ cacheChunkSize?: number }>
+	args: WithWranglerArgs<{ cacheChunkSize?: number; rcloneBatch?: boolean }>
 ) {
 	printHeaders(`populate cache - ${target}`);
 
@@ -344,6 +395,7 @@ async function populateCacheCommand(
 		wranglerConfigPath: args.wranglerConfigPath,
 		cacheChunkSize: args.cacheChunkSize,
 		shouldUsePreviewId: false,
+		rcloneBatch: args.rcloneBatch,
 	});
 }
 
@@ -372,8 +424,14 @@ export function addPopulateCacheCommand<T extends yargs.Argv>(y: T) {
 }
 
 export function withPopulateCacheOptions<T extends yargs.Argv>(args: T) {
-	return withWranglerOptions(args).options("cacheChunkSize", {
-		type: "number",
-		desc: "Number of entries per chunk when populating the cache",
-	});
+	return withWranglerOptions(args)
+		.options("cacheChunkSize", {
+			type: "number",
+			desc: "Number of entries per chunk when populating the cache",
+		})
+		.options("rcloneBatch", {
+			type: "boolean",
+			desc: "Use rclone for batch uploading to R2 (faster for large caches)",
+			default: false,
+		});
 }

packages/cloudflare/src/cli/commands/preview.ts

@@ -17,7 +17,7 @@ import {
  * @param args
  */
 export async function previewCommand(
-	args: WithWranglerArgs<{ cacheChunkSize?: number; remote: boolean }>
+	args: WithWranglerArgs<{ cacheChunkSize?: number; rcloneBatch?: boolean; remote: boolean }>
 ): Promise<void> {
 	printHeaders("preview");
 
@@ -32,6 +32,7 @@ export async function previewCommand(
 		wranglerConfigPath: args.wranglerConfigPath,
 		cacheChunkSize: args.cacheChunkSize,
 		shouldUsePreviewId: args.remote,
+		rcloneBatch: args.rcloneBatch,
 	});
 
 	runWrangler(options, ["dev", ...args.wranglerArgs], { logging: "all" });

packages/cloudflare/src/cli/commands/upload.ts

@@ -19,7 +19,9 @@ import {
  *
  * @param args
  */
-export async function uploadCommand(args: WithWranglerArgs<{ cacheChunkSize?: number }>): Promise<void> {
+export async function uploadCommand(
+	args: WithWranglerArgs<{ cacheChunkSize?: number; rcloneBatch?: boolean }>
+): Promise<void> {
 	printHeaders("upload");
 
 	const { config } = await retrieveCompiledConfig();
@@ -40,6 +42,7 @@ export async function uploadCommand(args: WithWranglerArgs<{ cacheChunkSize?: nu
 		wranglerConfigPath: args.wranglerConfigPath,
 		cacheChunkSize: args.cacheChunkSize,
 		shouldUsePreviewId: false,
+		rcloneBatch: args.rcloneBatch,
 	});
 
 	runWrangler(