refactor: not expose rclone to user and use batch upload based on env variables

Author: krzysztof-palka-monogo

.changeset/rclone-batch-upload.md

@@ -2,33 +2,41 @@
 "@opennextjs/cloudflare": minor
 ---
 
-feature: add --rcloneBatch flag for faster R2 cache uploads
+feature: optional batch upload for faster R2 cache population
 
-This adds a new optional `--rcloneBatch` flag that enables batch uploading to R2 using rclone instead of individual wrangler uploads. This significantly improves upload performance for large caches.
+This update adds optional batch upload support for R2 cache population, significantly improving upload performance for large caches when enabled via environment variables.
 
-**Supported commands:**
+**Key Changes:**
+
+1. **Optional Batch Upload**: Configure R2 credentials via environment variables to enable faster batch uploads:
+
+   - `R2_ACCESS_KEY_ID`
+   - `R2_SECRET_ACCESS_KEY`
+   - `R2_ACCOUNT_ID`
+
+2. **Automatic Detection**: When credentials are detected, batch upload is automatically used for better performance
+
+3. **Smart Fallback**: If credentials are not configured, the CLI falls back to standard Wrangler uploads with a helpful message about enabling batch upload for better performance
+
+**All deployment commands support batch upload:**
 
 - `populateCache` - Explicit cache population
 - `deploy` - Deploy with cache population
 - `upload` - Upload version with cache population
 - `preview` - Preview with cache population
 
-**Performance improvements:**
+**Performance Benefits (when batch upload is enabled):**
 
-- Creates staging directory with all cache files organized by R2 keys
-- Uses rclone's parallel transfer capabilities (32 transfers, 16 checkers)
-- Reduces API calls to Cloudflare
+- Parallel transfer capabilities (32 concurrent transfers)
+- Significantly faster for large caches
+- Reduced API calls to Cloudflare
 
 **Usage:**
 
 ```bash
-opennextjs-cloudflare deploy --rcloneBatch
-opennextjs-cloudflare populateCache remote --rcloneBatch
+# Enable batch upload by setting environment variables (recommended for large caches)
+export R2_ACCESS_KEY_ID=your_key
+export R2_SECRET_ACCESS_KEY=your_secret
+export R2_ACCOUNT_ID=your_account
+opennextjs-cloudflare deploy  # batch upload automatically used
 ```
-
-**Requirements:**
-
-- The `rclone.js` package (included as dependency) provides the binary
-- An rclone config file at `~/.config/rclone/rclone.conf` with R2 credentials (see README for setup instructions)
-
-The original wrangler-based upload remains the default behavior for backward compatibility.

packages/cloudflare/README.md

@@ -56,54 +56,21 @@ Deploy your application to production with the following:
   bun opennextjs-cloudflare build && bun opennextjs-cloudflare deploy
   ```
 
-## CLI Options
+### Batch Cache Population (Optional, Recommended)
 
-### Batch Cache Population (rclone)
-
-The `--rcloneBatch` flag enables faster R2 cache uploads using rclone batch mode.  
-This flag is supported by the following commands:
-
-- `populateCache` - Explicitly populate cache
-- `deploy` - Deploy and populate cache
-- `upload` - Upload version and populate cache
-- `preview` - Preview and populate cache
-
-**Usage:**
+For improved performance with large caches, you can enable batch upload by providing R2 credentials via environment variables:
 
 ```bash
-# Standalone cache population
-npx opennextjs-cloudflare populateCache local --rcloneBatch
-npx opennextjs-cloudflare populateCache remote --rcloneBatch
-
-# During deployment
-npx opennextjs-cloudflare deploy --rcloneBatch
-
-# During upload
-npx opennextjs-cloudflare upload --rcloneBatch
-
-# During preview
-npx opennextjs-cloudflare preview --rcloneBatch
+export R2_ACCESS_KEY_ID=your_access_key_id
+export R2_SECRET_ACCESS_KEY=your_secret_access_key
+export R2_ACCOUNT_ID=your_account_id
 ```
 
-**Requirements:**
-
-1. The `rclone.js` package (included as a dependency) provides the rclone binary automatically
-2. An rclone configuration file is required at `~/.config/rclone/rclone.conf` with your R2 credentials
-
-**rclone Configuration:**
-
-Create or update `~/.config/rclone/rclone.conf` with your R2 bucket configuration:
-
-```ini
-[r2]
-type = s3
-provider = Cloudflare
-access_key_id = YOUR_ACCESS_KEY_ID
-secret_access_key = YOUR_SECRET_ACCESS_KEY
-endpoint = https://YOUR_ACCOUNT_ID.r2.cloudflarestorage.com
-acl = private
-```
+**Benefits:**
 
-See [Cloudflare's rclone documentation](https://developers.cloudflare.com/r2/examples/rclone/) for more details.
+- Significantly faster uploads for large caches using parallel transfers
+- Reduced API calls to Cloudflare
+- Automatically enabled when credentials are provided
 
-**Default:** `false` (uses standard wrangler-based uploads)
+**Fallback:**
+If these environment variables are not set, the CLI will use standard Wrangler uploads. Both methods work correctly - batch upload is simply faster for large caches.

packages/cloudflare/src/cli/commands/deploy.ts

@@ -19,9 +19,7 @@ import {
  *
  * @param args
  */
-export async function deployCommand(
-	args: WithWranglerArgs<{ cacheChunkSize?: number; rcloneBatch?: boolean }>
-): Promise<void> {
+export async function deployCommand(args: WithWranglerArgs<{ cacheChunkSize?: number }>): Promise<void> {
 	printHeaders("deploy");
 
 	const { config } = await retrieveCompiledConfig();
@@ -42,7 +40,6 @@ export async function deployCommand(
 		wranglerConfigPath: args.wranglerConfigPath,
 		cacheChunkSize: args.cacheChunkSize,
 		shouldUsePreviewId: false,
-		rcloneBatch: args.rcloneBatch,
 	});
 
 	runWrangler(

packages/cloudflare/src/cli/commands/populate-cache.ts

@@ -1,5 +1,6 @@
 import { spawnSync } from "node:child_process";
 import { copyFileSync, cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
 import path from "node:path";
 
 import type { BuildOptions } from "@opennextjs/aws/build/helper.js";
@@ -128,52 +129,117 @@ type PopulateCacheOptions = {
 	 * Instructs Wrangler to use the preview namespace or ID defined in the Wrangler config for the remote target.
 	 */
 	shouldUsePreviewId: boolean;
+};
+
+/**
+ * Check if R2 credentials are available via environment variables for batch upload
+ */
+function hasR2EnvCredentials(): boolean {
+	return !!(process.env.R2_ACCESS_KEY_ID && process.env.R2_SECRET_ACCESS_KEY && process.env.R2_ACCOUNT_ID);
+}
+
+/**
+ * Create a temporary configuration file for batch upload from environment variables
+ * @returns Path to the temporary config file or null if env vars not available
+ */
+function createTempRcloneConfig(): string | null {
+	const accessKey = process.env.R2_ACCESS_KEY_ID;
+	const secretKey = process.env.R2_SECRET_ACCESS_KEY;
+	const accountId = process.env.R2_ACCOUNT_ID;
+
+	if (!accessKey || !secretKey || !accountId) {
+		return null;
+	}
+
+	const tempDir = tmpdir();
+	const tempConfigPath = path.join(tempDir, `rclone-config-${Date.now()}.conf`);
+
+	const configContent = `[r2]
+type = s3
+provider = Cloudflare
+access_key_id = ${accessKey}
+secret_access_key = ${secretKey}
+endpoint = https://${accountId}.r2.cloudflarestorage.com
+acl = private
+`;
+
 	/**
-	 * Use rclone for batch uploading to R2 instead of individual wrangler uploads.
+	 * 0o600 is an octal number (the 0o prefix indicates octal in JavaScript)
+	 * that represents Unix file permissions:
+	 *
+	 * - 6 (owner): read (4) + write (2) = readable and writable by the file owner
+	 * - 0 (group): no permissions for the group
+	 * - 0 (others): no permissions for anyone else
 	 *
-	 * @default false
+	 * In symbolic notation, this is: rw-------
 	 */
-	rcloneBatch?: boolean;
-};
+	writeFileSync(tempConfigPath, configContent, { mode: 0o600 });
+	return tempConfigPath;
+}
 
-async function populateR2IncrementalCacheWithRclone(
+/**
+ * Populate R2 incremental cache using batch upload for better performance
+ * Uses parallel transfers to significantly speed up cache population
+ */
+async function populateR2IncrementalCacheWithBatchUpload(
 	options: BuildOptions,
 	bucket: string,
 	prefix: string | undefined,
 	assets: CacheAsset[]
 ) {
-	logger.info("\nPopulating R2 incremental cache using rclone batch upload...");
-
-	// Create a staging dir with proper key paths
-	const stagingDir = path.join(options.outputDir, ".r2-staging");
-	rmSync(stagingDir, { recursive: true, force: true });
-	mkdirSync(stagingDir, { recursive: true });
+	logger.info("\nPopulating R2 incremental cache using batch upload...");
 
-	for (const { fullPath, key, buildId, isFetch } of assets) {
-		const cacheKey = computeCacheKey(key, {
-			prefix,
-			buildId,
-			cacheType: isFetch ? "fetch" : "cache",
-		});
-		const destPath = path.join(stagingDir, cacheKey);
-		mkdirSync(path.dirname(destPath), { recursive: true });
-		copyFileSync(fullPath, destPath);
+	// Create temporary config from env vars - required for batch upload
+	const tempConfigPath = createTempRcloneConfig();
+	if (!tempConfigPath) {
+		throw new Error(
+			"R2 credentials not found. Please set R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, and R2_ACCOUNT_ID environment variables to enable batch upload."
+		);
 	}
 
-	// Use rclone to sync the R2
-	const remote = `r2:${bucket}`;
-	const args = ["copy", stagingDir, remote, "--progress", "--transfers=32", "--checkers=16"];
+	const env = { ...process.env };
+	env.RCLONE_CONFIG = tempConfigPath;
+	logger.info("Using batch upload with R2 credentials from environment variables");
 
-	const result = spawnSync("rclone", args, {
-		stdio: "inherit",
-		env: process.env,
-	});
+	try {
+		// Create a staging dir with proper key paths
+		const stagingDir = path.join(options.outputDir, ".r2-staging");
+		rmSync(stagingDir, { recursive: true, force: true });
+		mkdirSync(stagingDir, { recursive: true });
 
-	if (result.status !== 0) {
-		throw new Error("rclone copy failed");
-	}
+		for (const { fullPath, key, buildId, isFetch } of assets) {
+			const cacheKey = computeCacheKey(key, {
+				prefix,
+				buildId,
+				cacheType: isFetch ? "fetch" : "cache",
+			});
+			const destPath = path.join(stagingDir, cacheKey);
+			mkdirSync(path.dirname(destPath), { recursive: true });
+			copyFileSync(fullPath, destPath);
+		}
 
-	logger.info(`Successfully uploaded ${assets.length} assets to R2`);
+		// Use rclone to sync the R2
+		const remote = `r2:${bucket}`;
+		const args = ["copy", stagingDir, remote, "--progress", "--transfers=32", "--checkers=16"];
+
+		const result = spawnSync("rclone", args, {
+			stdio: "inherit",
+			env,
+		});
+
+		if (result.status !== 0) {
+			throw new Error("Batch upload failed");
+		}
+
+		logger.info(`Successfully uploaded ${assets.length} assets to R2 using batch upload`);
+	} finally {
+		try {
+			// Cleanup temporary config file
+			rmSync(tempConfigPath);
+		} catch {
+			console.warn(`Failed to remove temporary config at ${tempConfigPath}`);
+		}
+	}
 }
 
 async function populateR2IncrementalCache(
@@ -198,10 +264,23 @@ async function populateR2IncrementalCache(
 
 	const assets = getCacheAssets(options);
 
-	if (populateCacheOptions.rcloneBatch) {
-		return populateR2IncrementalCacheWithRclone(options, bucket, prefix, assets);
+	// Use batch upload if R2 credentials are available (optional but recommended)
+	const canUseBatchUpload = hasR2EnvCredentials();
+
+	if (canUseBatchUpload) {
+		try {
+			return await populateR2IncrementalCacheWithBatchUpload(options, bucket, prefix, assets);
+		} catch (error) {
+			logger.warn(`Batch upload failed: ${error instanceof Error ? error.message : error}`);
+			logger.info("Falling back to standard uploads...");
+		}
+	} else {
+		logger.info(
+			"Using standard cache uploads. For faster performance with large caches, set R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, and R2_ACCOUNT_ID environment variables to enable batch upload."
+		);
 	}
 
+	// Standard upload fallback (using Wrangler)
 	for (const { fullPath, key, buildId, isFetch } of tqdm(assets)) {
 		const cacheKey = computeCacheKey(key, {
 			prefix,
@@ -380,7 +459,7 @@ export async function populateCache(
  */
 async function populateCacheCommand(
 	target: "local" | "remote",
-	args: WithWranglerArgs<{ cacheChunkSize?: number; rcloneBatch?: boolean }>
+	args: WithWranglerArgs<{ cacheChunkSize?: number }>
 ) {
 	printHeaders(`populate cache - ${target}`);
 
@@ -395,7 +474,6 @@ async function populateCacheCommand(
 		wranglerConfigPath: args.wranglerConfigPath,
 		cacheChunkSize: args.cacheChunkSize,
 		shouldUsePreviewId: false,
-		rcloneBatch: args.rcloneBatch,
 	});
 }
 
@@ -424,14 +502,8 @@ export function addPopulateCacheCommand<T extends yargs.Argv>(y: T) {
 }
 
 export function withPopulateCacheOptions<T extends yargs.Argv>(args: T) {
-	return withWranglerOptions(args)
-		.options("cacheChunkSize", {
-			type: "number",
-			desc: "Number of entries per chunk when populating the cache",
-		})
-		.options("rcloneBatch", {
-			type: "boolean",
-			desc: "Use rclone for batch uploading to R2 (faster for large caches)",
-			default: false,
-		});
+	return withWranglerOptions(args).options("cacheChunkSize", {
+		type: "number",
+		desc: "Number of entries per chunk when populating the cache",
+	});
 }

packages/cloudflare/src/cli/commands/preview.ts

@@ -17,7 +17,7 @@ import {
  * @param args
  */
 export async function previewCommand(
-	args: WithWranglerArgs<{ cacheChunkSize?: number; rcloneBatch?: boolean; remote: boolean }>
+	args: WithWranglerArgs<{ cacheChunkSize?: number; remote: boolean }>
 ): Promise<void> {
 	printHeaders("preview");
 
@@ -32,7 +32,6 @@ export async function previewCommand(
 		wranglerConfigPath: args.wranglerConfigPath,
 		cacheChunkSize: args.cacheChunkSize,
 		shouldUsePreviewId: args.remote,
-		rcloneBatch: args.rcloneBatch,
 	});
 
 	runWrangler(options, ["dev", ...args.wranglerArgs], { logging: "all" });

packages/cloudflare/src/cli/commands/upload.ts

@@ -19,9 +19,7 @@ import {
  *
  * @param args
  */
-export async function uploadCommand(
-	args: WithWranglerArgs<{ cacheChunkSize?: number; rcloneBatch?: boolean }>
-): Promise<void> {
+export async function uploadCommand(args: WithWranglerArgs<{ cacheChunkSize?: number }>): Promise<void> {
 	printHeaders("upload");
 
 	const { config } = await retrieveCompiledConfig();
@@ -42,7 +40,6 @@ export async function uploadCommand(
 		wranglerConfigPath: args.wranglerConfigPath,
 		cacheChunkSize: args.cacheChunkSize,
 		shouldUsePreviewId: false,
-		rcloneBatch: args.rcloneBatch,
 	});
 
 	runWrangler(