From d8171c515f1624cdb9bef43ffd70946afd518dfc Mon Sep 17 00:00:00 2001 From: krzysztof-palka-monogo Date: Fri, 3 Oct 2025 14:25:10 +0200 Subject: [PATCH 1/8] feat: add --rcloneBatch flag for faster R2 cache uploads using rclone --- .changeset/rclone-batch-upload.md | 34 +++++++++ packages/cloudflare/README.md | 51 ++++++++++++++ packages/cloudflare/package.json | 1 + .../cloudflare/src/cli/commands/deploy.ts | 5 +- .../src/cli/commands/populate-cache.spec.ts | 22 +++++- .../src/cli/commands/populate-cache.ts | 70 +++++++++++++++++-- .../cloudflare/src/cli/commands/preview.ts | 3 +- .../cloudflare/src/cli/commands/upload.ts | 5 +- pnpm-lock.yaml | 21 ++++++ 9 files changed, 202 insertions(+), 10 deletions(-) create mode 100644 .changeset/rclone-batch-upload.md diff --git a/.changeset/rclone-batch-upload.md b/.changeset/rclone-batch-upload.md new file mode 100644 index 000000000..356c87c2f --- /dev/null +++ b/.changeset/rclone-batch-upload.md @@ -0,0 +1,34 @@ +--- +"@opennextjs/cloudflare": minor +--- + +feature: add --rcloneBatch flag for faster R2 cache uploads + +This adds a new optional `--rcloneBatch` flag that enables batch uploading to R2 using rclone instead of individual wrangler uploads. This significantly improves upload performance for large caches. + +**Supported commands:** + +- `populateCache` - Explicit cache population +- `deploy` - Deploy with cache population +- `upload` - Upload version with cache population +- `preview` - Preview with cache population + +**Performance improvements:** + +- Creates staging directory with all cache files organized by R2 keys +- Uses rclone's parallel transfer capabilities (32 transfers, 16 checkers) +- Reduces API calls to Cloudflare + +**Usage:** + +```bash +opennextjs-cloudflare deploy --rcloneBatch +opennextjs-cloudflare populateCache remote --rcloneBatch +``` + +**Requirements:** + +- The `rclone.js` package (included as dependency) provides the binary +- An rclone config file at `~/.config/rclone/rclone.conf` with R2 credentials (see README for setup instructions) + +The original wrangler-based upload remains the default behavior for backward compatibility. diff --git a/packages/cloudflare/README.md b/packages/cloudflare/README.md index cc1f5e72a..8c7eec22f 100644 --- a/packages/cloudflare/README.md +++ b/packages/cloudflare/README.md @@ -55,3 +55,54 @@ Deploy your application to production with the following: # or bun opennextjs-cloudflare build && bun opennextjs-cloudflare deploy ``` + +## CLI Options + +### Batch Cache Population (rclone) + +The `--rcloneBatch` flag enables faster R2 cache uploads using rclone batch mode. This flag is supported by the following commands: + +- `populateCache` - Explicitly populate cache +- `deploy` - Deploy and populate cache +- `upload` - Upload version and populate cache +- `preview` - Preview and populate cache + +**Usage:** + +```bash +# Standalone cache population +npx opennextjs-cloudflare populateCache local --rcloneBatch +npx opennextjs-cloudflare populateCache remote --rcloneBatch + +# During deployment +npx opennextjs-cloudflare deploy --rcloneBatch + +# During upload +npx opennextjs-cloudflare upload --rcloneBatch + +# During preview +npx opennextjs-cloudflare preview --rcloneBatch +``` + +**Requirements:** + +1. The `rclone.js` package (included as a dependency) provides the rclone binary automatically +2. An rclone configuration file is required at `~/.config/rclone/rclone.conf` with your R2 credentials + +**rclone Configuration:** + +Create or update `~/.config/rclone/rclone.conf` with your R2 bucket configuration: + +```ini +[r2] +type = s3 +provider = Cloudflare +access_key_id = YOUR_ACCESS_KEY_ID +secret_access_key = YOUR_SECRET_ACCESS_KEY +endpoint = https://YOUR_ACCOUNT_ID.r2.cloudflarestorage.com +acl = private +``` + +See [Cloudflare's rclone documentation](https://developers.cloudflare.com/r2/examples/rclone/) for more details. + +**Default:** `false` (uses standard wrangler-based uploads) diff --git a/packages/cloudflare/package.json b/packages/cloudflare/package.json index 888110ecc..1f4884b70 100644 --- a/packages/cloudflare/package.json +++ b/packages/cloudflare/package.json @@ -57,6 +57,7 @@ "cloudflare": "^4.4.1", "enquirer": "^2.4.1", "glob": "catalog:", + "rclone.js": "^0.6.6", "ts-tqdm": "^0.8.6", "yargs": "catalog:" }, diff --git a/packages/cloudflare/src/cli/commands/deploy.ts b/packages/cloudflare/src/cli/commands/deploy.ts index 6e89d0295..0aaa0332d 100644 --- a/packages/cloudflare/src/cli/commands/deploy.ts +++ b/packages/cloudflare/src/cli/commands/deploy.ts @@ -19,7 +19,9 @@ import { * * @param args */ -export async function deployCommand(args: WithWranglerArgs<{ cacheChunkSize?: number }>): Promise { +export async function deployCommand( + args: WithWranglerArgs<{ cacheChunkSize?: number; rcloneBatch?: boolean }> +): Promise { printHeaders("deploy"); const { config } = await retrieveCompiledConfig(); @@ -40,6 +42,7 @@ export async function deployCommand(args: WithWranglerArgs<{ cacheChunkSize?: nu wranglerConfigPath: args.wranglerConfigPath, cacheChunkSize: args.cacheChunkSize, shouldUsePreviewId: false, + rcloneBatch: args.rcloneBatch, }); runWrangler( diff --git a/packages/cloudflare/src/cli/commands/populate-cache.spec.ts b/packages/cloudflare/src/cli/commands/populate-cache.spec.ts index 29fbcacd3..b8ee06b7d 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.spec.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.spec.ts @@ -5,7 +5,7 @@ import type { BuildOptions } from "@opennextjs/aws/build/helper.js"; import mockFs from "mock-fs"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; -import { getCacheAssets } from "./populate-cache.js"; +import { getCacheAssets, withPopulateCacheOptions } from "./populate-cache.js"; describe("getCacheAssets", () => { beforeAll(() => { @@ -68,3 +68,23 @@ describe("getCacheAssets", () => { `); }); }); + +describe("withPopulateCacheOptions", () => { + test("includes rcloneBatch option with correct defaults", () => { + interface MockYargs { + options: (name: string, config: Record) => MockYargs; + } + + const mockYargs: MockYargs = { + options: (name: string, config: Record) => { + expect(name).toBeDefined(); + expect(config).toBeDefined(); + return mockYargs; + }, + }; + + const result = withPopulateCacheOptions(mockYargs as never); + + expect(result).toBe(mockYargs); + }); +}); diff --git a/packages/cloudflare/src/cli/commands/populate-cache.ts b/packages/cloudflare/src/cli/commands/populate-cache.ts index 8cc68707a..d9ceef1cd 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.ts @@ -1,4 +1,5 @@ -import { cpSync, existsSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { copyFileSync, cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import path from "node:path"; import type { BuildOptions } from "@opennextjs/aws/build/helper.js"; @@ -127,8 +128,54 @@ type PopulateCacheOptions = { * Instructs Wrangler to use the preview namespace or ID defined in the Wrangler config for the remote target. */ shouldUsePreviewId: boolean; + /** + * Use rclone for batch uploading to R2 instead of individual wrangler uploads. + * + * @default false + */ + rcloneBatch?: boolean; }; +async function populateR2IncrementalCacheWithRclone( + options: BuildOptions, + bucket: string, + prefix: string | undefined, + assets: CacheAsset[] +) { + logger.info("\nPopulating R2 incremental cache using rclone batch upload..."); + + // Create a staging dir with proper key paths + const stagingDir = path.join(options.outputDir, ".r2-staging"); + rmSync(stagingDir, { recursive: true, force: true }); + mkdirSync(stagingDir, { recursive: true }); + + for (const { fullPath, key, buildId, isFetch } of assets) { + const cacheKey = computeCacheKey(key, { + prefix, + buildId, + cacheType: isFetch ? "fetch" : "cache", + }); + const destPath = path.join(stagingDir, cacheKey); + mkdirSync(path.dirname(destPath), { recursive: true }); + copyFileSync(fullPath, destPath); + } + + // Use rclone to sync the R2 + const remote = `r2:${bucket}`; + const args = ["copy", stagingDir, remote, "--progress", "--transfers=32", "--checkers=16"]; + + const result = spawnSync("rclone", args, { + stdio: "inherit", + env: process.env, + }); + + if (result.status !== 0) { + throw new Error("rclone copy failed"); + } + + logger.info(`Successfully uploaded ${assets.length} assets to R2`); +} + async function populateR2IncrementalCache( options: BuildOptions, config: WranglerConfig, @@ -151,6 +198,10 @@ async function populateR2IncrementalCache( const assets = getCacheAssets(options); + if (populateCacheOptions.rcloneBatch) { + return populateR2IncrementalCacheWithRclone(options, bucket, prefix, assets); + } + for (const { fullPath, key, buildId, isFetch } of tqdm(assets)) { const cacheKey = computeCacheKey(key, { prefix, @@ -329,7 +380,7 @@ export async function populateCache( */ async function populateCacheCommand( target: "local" | "remote", - args: WithWranglerArgs<{ cacheChunkSize?: number }> + args: WithWranglerArgs<{ cacheChunkSize?: number; rcloneBatch?: boolean }> ) { printHeaders(`populate cache - ${target}`); @@ -344,6 +395,7 @@ async function populateCacheCommand( wranglerConfigPath: args.wranglerConfigPath, cacheChunkSize: args.cacheChunkSize, shouldUsePreviewId: false, + rcloneBatch: args.rcloneBatch, }); } @@ -372,8 +424,14 @@ export function addPopulateCacheCommand(y: T) { } export function withPopulateCacheOptions(args: T) { - return withWranglerOptions(args).options("cacheChunkSize", { - type: "number", - desc: "Number of entries per chunk when populating the cache", - }); + return withWranglerOptions(args) + .options("cacheChunkSize", { + type: "number", + desc: "Number of entries per chunk when populating the cache", + }) + .options("rcloneBatch", { + type: "boolean", + desc: "Use rclone for batch uploading to R2 (faster for large caches)", + default: false, + }); } diff --git a/packages/cloudflare/src/cli/commands/preview.ts b/packages/cloudflare/src/cli/commands/preview.ts index 2b41dd3e9..885585555 100644 --- a/packages/cloudflare/src/cli/commands/preview.ts +++ b/packages/cloudflare/src/cli/commands/preview.ts @@ -17,7 +17,7 @@ import { * @param args */ export async function previewCommand( - args: WithWranglerArgs<{ cacheChunkSize?: number; remote: boolean }> + args: WithWranglerArgs<{ cacheChunkSize?: number; rcloneBatch?: boolean; remote: boolean }> ): Promise { printHeaders("preview"); @@ -32,6 +32,7 @@ export async function previewCommand( wranglerConfigPath: args.wranglerConfigPath, cacheChunkSize: args.cacheChunkSize, shouldUsePreviewId: args.remote, + rcloneBatch: args.rcloneBatch, }); runWrangler(options, ["dev", ...args.wranglerArgs], { logging: "all" }); diff --git a/packages/cloudflare/src/cli/commands/upload.ts b/packages/cloudflare/src/cli/commands/upload.ts index b3eff12bc..35c7403fb 100644 --- a/packages/cloudflare/src/cli/commands/upload.ts +++ b/packages/cloudflare/src/cli/commands/upload.ts @@ -19,7 +19,9 @@ import { * * @param args */ -export async function uploadCommand(args: WithWranglerArgs<{ cacheChunkSize?: number }>): Promise { +export async function uploadCommand( + args: WithWranglerArgs<{ cacheChunkSize?: number; rcloneBatch?: boolean }> +): Promise { printHeaders("upload"); const { config } = await retrieveCompiledConfig(); @@ -40,6 +42,7 @@ export async function uploadCommand(args: WithWranglerArgs<{ cacheChunkSize?: nu wranglerConfigPath: args.wranglerConfigPath, cacheChunkSize: args.cacheChunkSize, shouldUsePreviewId: false, + rcloneBatch: args.rcloneBatch, }); runWrangler( diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fd4fff784..0186b5ef1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1086,6 +1086,9 @@ importers: glob: specifier: 'catalog:' version: 11.0.0 + rclone.js: + specifier: ^0.6.6 + version: 0.6.6 ts-tqdm: specifier: ^0.8.6 version: 0.8.6 @@ -5030,6 +5033,10 @@ packages: engines: {node: '>=0.4.0'} hasBin: true + adm-zip@0.5.16: + resolution: {integrity: sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==} + engines: {node: '>=12.0'} + agent-base@6.0.2: resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} engines: {node: '>= 6.0.0'} @@ -8427,6 +8434,13 @@ packages: resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} hasBin: true + rclone.js@0.6.6: + resolution: {integrity: sha512-Dxh34cab/fNjFq5SSm0fYLNkGzG2cQSBy782UW9WwxJCEiVO4cGXkvaXcNlgv817dK8K8PuQ+NHUqSAMMhWujQ==} + engines: {node: '>=12'} + cpu: [arm, arm64, mips, mipsel, x32, x64] + os: [darwin, freebsd, linux, openbsd, sunos, win32] + hasBin: true + react-dom@18.3.1: resolution: {integrity: sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==} peerDependencies: @@ -14763,6 +14777,8 @@ snapshots: acorn@8.15.0: {} + adm-zip@0.5.16: {} + agent-base@6.0.2: dependencies: debug: 4.4.0 @@ -19102,6 +19118,11 @@ snapshots: minimist: 1.2.8 strip-json-comments: 2.0.1 + rclone.js@0.6.6: + dependencies: + adm-zip: 0.5.16 + mri: 1.2.0 + react-dom@18.3.1(react@18.3.1): dependencies: loose-envify: 1.4.0 From 5aefc2160a065877ae9924ea095be494db61a3b7 Mon Sep 17 00:00:00 2001 From: krzysztof-palka-monogo Date: Fri, 3 Oct 2025 16:54:03 +0200 Subject: [PATCH 2/8] fix(test): remove useless test --- .../src/cli/commands/populate-cache.spec.ts | 22 +------------------ 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/packages/cloudflare/src/cli/commands/populate-cache.spec.ts b/packages/cloudflare/src/cli/commands/populate-cache.spec.ts index b8ee06b7d..29fbcacd3 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.spec.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.spec.ts @@ -5,7 +5,7 @@ import type { BuildOptions } from "@opennextjs/aws/build/helper.js"; import mockFs from "mock-fs"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; -import { getCacheAssets, withPopulateCacheOptions } from "./populate-cache.js"; +import { getCacheAssets } from "./populate-cache.js"; describe("getCacheAssets", () => { beforeAll(() => { @@ -68,23 +68,3 @@ describe("getCacheAssets", () => { `); }); }); - -describe("withPopulateCacheOptions", () => { - test("includes rcloneBatch option with correct defaults", () => { - interface MockYargs { - options: (name: string, config: Record) => MockYargs; - } - - const mockYargs: MockYargs = { - options: (name: string, config: Record) => { - expect(name).toBeDefined(); - expect(config).toBeDefined(); - return mockYargs; - }, - }; - - const result = withPopulateCacheOptions(mockYargs as never); - - expect(result).toBe(mockYargs); - }); -}); From c79d18de1717f6c02d4e8715d819ba2c3fc3165e Mon Sep 17 00:00:00 2001 From: krzysztof-palka-monogo Date: Fri, 3 Oct 2025 17:10:55 +0200 Subject: [PATCH 3/8] test: add tests for populateCache with R2 cache --- .../src/cli/commands/populate-cache.spec.ts | 135 +++++++++++++++++- 1 file changed, 133 insertions(+), 2 deletions(-) diff --git a/packages/cloudflare/src/cli/commands/populate-cache.spec.ts b/packages/cloudflare/src/cli/commands/populate-cache.spec.ts index 29fbcacd3..892329d14 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.spec.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.spec.ts @@ -3,9 +3,9 @@ import path from "node:path"; import type { BuildOptions } from "@opennextjs/aws/build/helper.js"; import mockFs from "mock-fs"; -import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import { afterAll, beforeAll, describe, expect, test, vi } from "vitest"; -import { getCacheAssets } from "./populate-cache.js"; +import { getCacheAssets, populateCache } from "./populate-cache.js"; describe("getCacheAssets", () => { beforeAll(() => { @@ -68,3 +68,134 @@ describe("getCacheAssets", () => { `); }); }); + +vi.mock("../utils/run-wrangler.js", () => ({ + runWrangler: vi.fn(), +})); + +vi.mock("./helpers.js", () => ({ + getEnvFromPlatformProxy: vi.fn(async () => ({})), + quoteShellMeta: vi.fn((s) => s), +})); + +vi.mock("node:child_process", () => ({ + spawnSync: vi.fn(() => ({ status: 0 })), +})); + +describe("populateCache", () => { + describe("R2_CACHE_NAME", () => { + test("calls runWrangler when rcloneBatch is false", async () => { + const { runWrangler } = await import("../utils/run-wrangler.js"); + + const buildOptions: BuildOptions = { + outputDir: "/test/output", + } as BuildOptions; + + const openNextConfig = { + default: { + override: { + incrementalCache: "cf-r2-incremental-cache", + }, + }, + }; + + const wranglerConfig = { + r2_buckets: [ + { + binding: "NEXT_INC_CACHE_R2_BUCKET", + bucket_name: "test-bucket", + }, + ], + }; + + const populateCacheOptions = { + target: "local" as const, + shouldUsePreviewId: false, + rcloneBatch: false, + }; + + vi.mocked(runWrangler).mockClear(); + + mockFs({ + "/test/output": { + cache: { + buildID: { + path: { + to: { + "test.cache": JSON.stringify({ data: "test" }), + }, + }, + }, + }, + }, + }); + + // For this test we do not need whole configuration, just the part that is being used. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + await populateCache(buildOptions, openNextConfig as any, wranglerConfig as any, populateCacheOptions); + + expect(runWrangler).toHaveBeenCalled(); + + mockFs.restore(); + }); + + test("calls spawnSync with rclone when rcloneBatch is true", async () => { + const { spawnSync } = await import("node:child_process"); + + const buildOptions: BuildOptions = { + outputDir: "/test/output", + } as BuildOptions; + + const openNextConfig = { + default: { + override: { + incrementalCache: "cf-r2-incremental-cache", + }, + }, + }; + + const wranglerConfig = { + r2_buckets: [ + { + binding: "NEXT_INC_CACHE_R2_BUCKET", + bucket_name: "test-bucket", + }, + ], + }; + + const populateCacheOptions = { + target: "local" as const, + shouldUsePreviewId: false, + rcloneBatch: true, + }; + + vi.mocked(spawnSync).mockClear(); + + mockFs({ + "/test/output": { + cache: { + buildID: { + path: { + to: { + "test.cache": JSON.stringify({ data: "test" }), + }, + }, + }, + }, + }, + }); + + // For this test we do not need whole configuration, just the part that is being used. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + await populateCache(buildOptions, openNextConfig as any, wranglerConfig as any, populateCacheOptions); + + expect(spawnSync).toHaveBeenCalledWith( + "rclone", + expect.arrayContaining(["copy", expect.any(String), "r2:test-bucket"]), + expect.any(Object) + ); + + mockFs.restore(); + }); + }); +}); From d0c5606d8ab0a7066686437dc392fce06450cc1d Mon Sep 17 00:00:00 2001 From: krzysztof-palka-monogo Date: Mon, 6 Oct 2025 08:37:10 +0200 Subject: [PATCH 4/8] commit to trigger pkg-pr-new action --- packages/cloudflare/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/cloudflare/README.md b/packages/cloudflare/README.md index 8c7eec22f..519aaa383 100644 --- a/packages/cloudflare/README.md +++ b/packages/cloudflare/README.md @@ -60,7 +60,8 @@ Deploy your application to production with the following: ### Batch Cache Population (rclone) -The `--rcloneBatch` flag enables faster R2 cache uploads using rclone batch mode. This flag is supported by the following commands: +The `--rcloneBatch` flag enables faster R2 cache uploads using rclone batch mode. +This flag is supported by the following commands: - `populateCache` - Explicitly populate cache - `deploy` - Deploy and populate cache From 15a874f56b5ef397cb6ed67dcf76222517ddffce Mon Sep 17 00:00:00 2001 From: krzysztof-palka-monogo Date: Mon, 6 Oct 2025 10:26:30 +0200 Subject: [PATCH 5/8] refactor: not expose rclone to user and use batch upload based on env variables --- .changeset/rclone-batch-upload.md | 40 +++-- packages/cloudflare/README.md | 55 ++---- .../cloudflare/src/cli/commands/deploy.ts | 5 +- .../src/cli/commands/populate-cache.ts | 162 +++++++++++++----- .../cloudflare/src/cli/commands/preview.ts | 3 +- .../cloudflare/src/cli/commands/upload.ts | 5 +- 6 files changed, 155 insertions(+), 115 deletions(-) diff --git a/.changeset/rclone-batch-upload.md b/.changeset/rclone-batch-upload.md index 356c87c2f..2b4881ba2 100644 --- a/.changeset/rclone-batch-upload.md +++ b/.changeset/rclone-batch-upload.md @@ -2,33 +2,41 @@ "@opennextjs/cloudflare": minor --- -feature: add --rcloneBatch flag for faster R2 cache uploads +feature: optional batch upload for faster R2 cache population -This adds a new optional `--rcloneBatch` flag that enables batch uploading to R2 using rclone instead of individual wrangler uploads. This significantly improves upload performance for large caches. +This update adds optional batch upload support for R2 cache population, significantly improving upload performance for large caches when enabled via environment variables. -**Supported commands:** +**Key Changes:** + +1. **Optional Batch Upload**: Configure R2 credentials via environment variables to enable faster batch uploads: + + - `R2_ACCESS_KEY_ID` + - `R2_SECRET_ACCESS_KEY` + - `R2_ACCOUNT_ID` + +2. **Automatic Detection**: When credentials are detected, batch upload is automatically used for better performance + +3. **Smart Fallback**: If credentials are not configured, the CLI falls back to standard Wrangler uploads with a helpful message about enabling batch upload for better performance + +**All deployment commands support batch upload:** - `populateCache` - Explicit cache population - `deploy` - Deploy with cache population - `upload` - Upload version with cache population - `preview` - Preview with cache population -**Performance improvements:** +**Performance Benefits (when batch upload is enabled):** -- Creates staging directory with all cache files organized by R2 keys -- Uses rclone's parallel transfer capabilities (32 transfers, 16 checkers) -- Reduces API calls to Cloudflare +- Parallel transfer capabilities (32 concurrent transfers) +- Significantly faster for large caches +- Reduced API calls to Cloudflare **Usage:** ```bash -opennextjs-cloudflare deploy --rcloneBatch -opennextjs-cloudflare populateCache remote --rcloneBatch +# Enable batch upload by setting environment variables (recommended for large caches) +export R2_ACCESS_KEY_ID=your_key +export R2_SECRET_ACCESS_KEY=your_secret +export R2_ACCOUNT_ID=your_account +opennextjs-cloudflare deploy # batch upload automatically used ``` - -**Requirements:** - -- The `rclone.js` package (included as dependency) provides the binary -- An rclone config file at `~/.config/rclone/rclone.conf` with R2 credentials (see README for setup instructions) - -The original wrangler-based upload remains the default behavior for backward compatibility. diff --git a/packages/cloudflare/README.md b/packages/cloudflare/README.md index 519aaa383..12dda8bab 100644 --- a/packages/cloudflare/README.md +++ b/packages/cloudflare/README.md @@ -56,54 +56,21 @@ Deploy your application to production with the following: bun opennextjs-cloudflare build && bun opennextjs-cloudflare deploy ``` -## CLI Options +### Batch Cache Population (Optional, Recommended) -### Batch Cache Population (rclone) - -The `--rcloneBatch` flag enables faster R2 cache uploads using rclone batch mode. -This flag is supported by the following commands: - -- `populateCache` - Explicitly populate cache -- `deploy` - Deploy and populate cache -- `upload` - Upload version and populate cache -- `preview` - Preview and populate cache - -**Usage:** +For improved performance with large caches, you can enable batch upload by providing R2 credentials via environment variables: ```bash -# Standalone cache population -npx opennextjs-cloudflare populateCache local --rcloneBatch -npx opennextjs-cloudflare populateCache remote --rcloneBatch - -# During deployment -npx opennextjs-cloudflare deploy --rcloneBatch - -# During upload -npx opennextjs-cloudflare upload --rcloneBatch - -# During preview -npx opennextjs-cloudflare preview --rcloneBatch +export R2_ACCESS_KEY_ID=your_access_key_id +export R2_SECRET_ACCESS_KEY=your_secret_access_key +export R2_ACCOUNT_ID=your_account_id ``` -**Requirements:** - -1. The `rclone.js` package (included as a dependency) provides the rclone binary automatically -2. An rclone configuration file is required at `~/.config/rclone/rclone.conf` with your R2 credentials - -**rclone Configuration:** - -Create or update `~/.config/rclone/rclone.conf` with your R2 bucket configuration: - -```ini -[r2] -type = s3 -provider = Cloudflare -access_key_id = YOUR_ACCESS_KEY_ID -secret_access_key = YOUR_SECRET_ACCESS_KEY -endpoint = https://YOUR_ACCOUNT_ID.r2.cloudflarestorage.com -acl = private -``` +**Benefits:** -See [Cloudflare's rclone documentation](https://developers.cloudflare.com/r2/examples/rclone/) for more details. +- Significantly faster uploads for large caches using parallel transfers +- Reduced API calls to Cloudflare +- Automatically enabled when credentials are provided -**Default:** `false` (uses standard wrangler-based uploads) +**Fallback:** +If these environment variables are not set, the CLI will use standard Wrangler uploads. Both methods work correctly - batch upload is simply faster for large caches. diff --git a/packages/cloudflare/src/cli/commands/deploy.ts b/packages/cloudflare/src/cli/commands/deploy.ts index 0aaa0332d..6e89d0295 100644 --- a/packages/cloudflare/src/cli/commands/deploy.ts +++ b/packages/cloudflare/src/cli/commands/deploy.ts @@ -19,9 +19,7 @@ import { * * @param args */ -export async function deployCommand( - args: WithWranglerArgs<{ cacheChunkSize?: number; rcloneBatch?: boolean }> -): Promise { +export async function deployCommand(args: WithWranglerArgs<{ cacheChunkSize?: number }>): Promise { printHeaders("deploy"); const { config } = await retrieveCompiledConfig(); @@ -42,7 +40,6 @@ export async function deployCommand( wranglerConfigPath: args.wranglerConfigPath, cacheChunkSize: args.cacheChunkSize, shouldUsePreviewId: false, - rcloneBatch: args.rcloneBatch, }); runWrangler( diff --git a/packages/cloudflare/src/cli/commands/populate-cache.ts b/packages/cloudflare/src/cli/commands/populate-cache.ts index d9ceef1cd..e87b878bd 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.ts @@ -1,5 +1,6 @@ import { spawnSync } from "node:child_process"; import { copyFileSync, cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; import path from "node:path"; import type { BuildOptions } from "@opennextjs/aws/build/helper.js"; @@ -128,52 +129,117 @@ type PopulateCacheOptions = { * Instructs Wrangler to use the preview namespace or ID defined in the Wrangler config for the remote target. */ shouldUsePreviewId: boolean; +}; + +/** + * Check if R2 credentials are available via environment variables for batch upload + */ +function hasR2EnvCredentials(): boolean { + return !!(process.env.R2_ACCESS_KEY_ID && process.env.R2_SECRET_ACCESS_KEY && process.env.R2_ACCOUNT_ID); +} + +/** + * Create a temporary configuration file for batch upload from environment variables + * @returns Path to the temporary config file or null if env vars not available + */ +function createTempRcloneConfig(): string | null { + const accessKey = process.env.R2_ACCESS_KEY_ID; + const secretKey = process.env.R2_SECRET_ACCESS_KEY; + const accountId = process.env.R2_ACCOUNT_ID; + + if (!accessKey || !secretKey || !accountId) { + return null; + } + + const tempDir = tmpdir(); + const tempConfigPath = path.join(tempDir, `rclone-config-${Date.now()}.conf`); + + const configContent = `[r2] +type = s3 +provider = Cloudflare +access_key_id = ${accessKey} +secret_access_key = ${secretKey} +endpoint = https://${accountId}.r2.cloudflarestorage.com +acl = private +`; + /** - * Use rclone for batch uploading to R2 instead of individual wrangler uploads. + * 0o600 is an octal number (the 0o prefix indicates octal in JavaScript) + * that represents Unix file permissions: + * + * - 6 (owner): read (4) + write (2) = readable and writable by the file owner + * - 0 (group): no permissions for the group + * - 0 (others): no permissions for anyone else * - * @default false + * In symbolic notation, this is: rw------- */ - rcloneBatch?: boolean; -}; + writeFileSync(tempConfigPath, configContent, { mode: 0o600 }); + return tempConfigPath; +} -async function populateR2IncrementalCacheWithRclone( +/** + * Populate R2 incremental cache using batch upload for better performance + * Uses parallel transfers to significantly speed up cache population + */ +async function populateR2IncrementalCacheWithBatchUpload( options: BuildOptions, bucket: string, prefix: string | undefined, assets: CacheAsset[] ) { - logger.info("\nPopulating R2 incremental cache using rclone batch upload..."); - - // Create a staging dir with proper key paths - const stagingDir = path.join(options.outputDir, ".r2-staging"); - rmSync(stagingDir, { recursive: true, force: true }); - mkdirSync(stagingDir, { recursive: true }); + logger.info("\nPopulating R2 incremental cache using batch upload..."); - for (const { fullPath, key, buildId, isFetch } of assets) { - const cacheKey = computeCacheKey(key, { - prefix, - buildId, - cacheType: isFetch ? "fetch" : "cache", - }); - const destPath = path.join(stagingDir, cacheKey); - mkdirSync(path.dirname(destPath), { recursive: true }); - copyFileSync(fullPath, destPath); + // Create temporary config from env vars - required for batch upload + const tempConfigPath = createTempRcloneConfig(); + if (!tempConfigPath) { + throw new Error( + "R2 credentials not found. Please set R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, and R2_ACCOUNT_ID environment variables to enable batch upload." + ); } - // Use rclone to sync the R2 - const remote = `r2:${bucket}`; - const args = ["copy", stagingDir, remote, "--progress", "--transfers=32", "--checkers=16"]; + const env = { ...process.env }; + env.RCLONE_CONFIG = tempConfigPath; + logger.info("Using batch upload with R2 credentials from environment variables"); - const result = spawnSync("rclone", args, { - stdio: "inherit", - env: process.env, - }); + try { + // Create a staging dir with proper key paths + const stagingDir = path.join(options.outputDir, ".r2-staging"); + rmSync(stagingDir, { recursive: true, force: true }); + mkdirSync(stagingDir, { recursive: true }); - if (result.status !== 0) { - throw new Error("rclone copy failed"); - } + for (const { fullPath, key, buildId, isFetch } of assets) { + const cacheKey = computeCacheKey(key, { + prefix, + buildId, + cacheType: isFetch ? "fetch" : "cache", + }); + const destPath = path.join(stagingDir, cacheKey); + mkdirSync(path.dirname(destPath), { recursive: true }); + copyFileSync(fullPath, destPath); + } - logger.info(`Successfully uploaded ${assets.length} assets to R2`); + // Use rclone to sync the R2 + const remote = `r2:${bucket}`; + const args = ["copy", stagingDir, remote, "--progress", "--transfers=32", "--checkers=16"]; + + const result = spawnSync("rclone", args, { + stdio: "inherit", + env, + }); + + if (result.status !== 0) { + throw new Error("Batch upload failed"); + } + + logger.info(`Successfully uploaded ${assets.length} assets to R2 using batch upload`); + } finally { + try { + // Cleanup temporary config file + rmSync(tempConfigPath); + } catch { + console.warn(`Failed to remove temporary config at ${tempConfigPath}`); + } + } } async function populateR2IncrementalCache( @@ -198,10 +264,23 @@ async function populateR2IncrementalCache( const assets = getCacheAssets(options); - if (populateCacheOptions.rcloneBatch) { - return populateR2IncrementalCacheWithRclone(options, bucket, prefix, assets); + // Use batch upload if R2 credentials are available (optional but recommended) + const canUseBatchUpload = hasR2EnvCredentials(); + + if (canUseBatchUpload) { + try { + return await populateR2IncrementalCacheWithBatchUpload(options, bucket, prefix, assets); + } catch (error) { + logger.warn(`Batch upload failed: ${error instanceof Error ? error.message : error}`); + logger.info("Falling back to standard uploads..."); + } + } else { + logger.info( + "Using standard cache uploads. For faster performance with large caches, set R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, and R2_ACCOUNT_ID environment variables to enable batch upload." + ); } + // Standard upload fallback (using Wrangler) for (const { fullPath, key, buildId, isFetch } of tqdm(assets)) { const cacheKey = computeCacheKey(key, { prefix, @@ -380,7 +459,7 @@ export async function populateCache( */ async function populateCacheCommand( target: "local" | "remote", - args: WithWranglerArgs<{ cacheChunkSize?: number; rcloneBatch?: boolean }> + args: WithWranglerArgs<{ cacheChunkSize?: number }> ) { printHeaders(`populate cache - ${target}`); @@ -395,7 +474,6 @@ async function populateCacheCommand( wranglerConfigPath: args.wranglerConfigPath, cacheChunkSize: args.cacheChunkSize, shouldUsePreviewId: false, - rcloneBatch: args.rcloneBatch, }); } @@ -424,14 +502,8 @@ export function addPopulateCacheCommand(y: T) { } export function withPopulateCacheOptions(args: T) { - return withWranglerOptions(args) - .options("cacheChunkSize", { - type: "number", - desc: "Number of entries per chunk when populating the cache", - }) - .options("rcloneBatch", { - type: "boolean", - desc: "Use rclone for batch uploading to R2 (faster for large caches)", - default: false, - }); + return withWranglerOptions(args).options("cacheChunkSize", { + type: "number", + desc: "Number of entries per chunk when populating the cache", + }); } diff --git a/packages/cloudflare/src/cli/commands/preview.ts b/packages/cloudflare/src/cli/commands/preview.ts index 885585555..2b41dd3e9 100644 --- a/packages/cloudflare/src/cli/commands/preview.ts +++ b/packages/cloudflare/src/cli/commands/preview.ts @@ -17,7 +17,7 @@ import { * @param args */ export async function previewCommand( - args: WithWranglerArgs<{ cacheChunkSize?: number; rcloneBatch?: boolean; remote: boolean }> + args: WithWranglerArgs<{ cacheChunkSize?: number; remote: boolean }> ): Promise { printHeaders("preview"); @@ -32,7 +32,6 @@ export async function previewCommand( wranglerConfigPath: args.wranglerConfigPath, cacheChunkSize: args.cacheChunkSize, shouldUsePreviewId: args.remote, - rcloneBatch: args.rcloneBatch, }); runWrangler(options, ["dev", ...args.wranglerArgs], { logging: "all" }); diff --git a/packages/cloudflare/src/cli/commands/upload.ts b/packages/cloudflare/src/cli/commands/upload.ts index 35c7403fb..b3eff12bc 100644 --- a/packages/cloudflare/src/cli/commands/upload.ts +++ b/packages/cloudflare/src/cli/commands/upload.ts @@ -19,9 +19,7 @@ import { * * @param args */ -export async function uploadCommand( - args: WithWranglerArgs<{ cacheChunkSize?: number; rcloneBatch?: boolean }> -): Promise { +export async function uploadCommand(args: WithWranglerArgs<{ cacheChunkSize?: number }>): Promise { printHeaders("upload"); const { config } = await retrieveCompiledConfig(); @@ -42,7 +40,6 @@ export async function uploadCommand( wranglerConfigPath: args.wranglerConfigPath, cacheChunkSize: args.cacheChunkSize, shouldUsePreviewId: false, - rcloneBatch: args.rcloneBatch, }); runWrangler( From 0d53eac536e21d4594e323be3b83b83018d625a8 Mon Sep 17 00:00:00 2001 From: krzysztof-palka-monogo Date: Mon, 6 Oct 2025 10:46:01 +0200 Subject: [PATCH 6/8] test: update populateCache tests after implementation changes --- .../src/cli/commands/populate-cache.spec.ts | 170 ++++++++---------- 1 file changed, 79 insertions(+), 91 deletions(-) diff --git a/packages/cloudflare/src/cli/commands/populate-cache.spec.ts b/packages/cloudflare/src/cli/commands/populate-cache.spec.ts index 892329d14..250e11a0e 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.spec.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.spec.ts @@ -1,9 +1,10 @@ +import { spawnSync } from "node:child_process"; import { mkdirSync, writeFileSync } from "node:fs"; import path from "node:path"; import type { BuildOptions } from "@opennextjs/aws/build/helper.js"; import mockFs from "mock-fs"; -import { afterAll, beforeAll, describe, expect, test, vi } from "vitest"; +import { afterAll, afterEach, beforeAll, describe, expect, test, vi } from "vitest"; import { getCacheAssets, populateCache } from "./populate-cache.js"; @@ -83,119 +84,106 @@ vi.mock("node:child_process", () => ({ })); describe("populateCache", () => { - describe("R2_CACHE_NAME", () => { - test("calls runWrangler when rcloneBatch is false", async () => { - const { runWrangler } = await import("../utils/run-wrangler.js"); - - const buildOptions: BuildOptions = { - outputDir: "/test/output", - } as BuildOptions; - - const openNextConfig = { - default: { - override: { - incrementalCache: "cf-r2-incremental-cache", - }, - }, - }; - - const wranglerConfig = { - r2_buckets: [ - { - binding: "NEXT_INC_CACHE_R2_BUCKET", - bucket_name: "test-bucket", - }, - ], - }; + // Test fixtures + const createTestBuildOptions = (): BuildOptions => + ({ + outputDir: "/test/output", + }) as BuildOptions; + + const createTestOpenNextConfig = () => ({ + default: { + override: { + incrementalCache: "cf-r2-incremental-cache", + }, + }, + }); - const populateCacheOptions = { - target: "local" as const, - shouldUsePreviewId: false, - rcloneBatch: false, - }; + const createTestWranglerConfig = () => ({ + r2_buckets: [ + { + binding: "NEXT_INC_CACHE_R2_BUCKET", + bucket_name: "test-bucket", + }, + ], + }); - vi.mocked(runWrangler).mockClear(); + const createTestPopulateCacheOptions = () => ({ + target: "local" as const, + shouldUsePreviewId: false, + }); - mockFs({ - "/test/output": { - cache: { - buildID: { - path: { - to: { - "test.cache": JSON.stringify({ data: "test" }), - }, + const setupMockFileSystem = () => { + mockFs({ + "/test/output": { + cache: { + buildID: { + path: { + to: { + "test.cache": JSON.stringify({ data: "test" }), }, }, }, }, - }); - - // For this test we do not need whole configuration, just the part that is being used. - // eslint-disable-next-line @typescript-eslint/no-explicit-any - await populateCache(buildOptions, openNextConfig as any, wranglerConfig as any, populateCacheOptions); - - expect(runWrangler).toHaveBeenCalled(); + }, + }); + }; + describe("R2 incremental cache", () => { + afterEach(() => { mockFs.restore(); + vi.unstubAllEnvs(); }); - test("calls spawnSync with rclone when rcloneBatch is true", async () => { - const { spawnSync } = await import("node:child_process"); + test("uses standard upload when R2 credentials are not provided", async () => { + const { runWrangler } = await import("../utils/run-wrangler.js"); + + // Ensure no batch upload credentials are set + vi.stubEnv("R2_ACCESS_KEY_ID", undefined); + vi.stubEnv("R2_SECRET_ACCESS_KEY", undefined); + vi.stubEnv("R2_ACCOUNT_ID", undefined); - const buildOptions: BuildOptions = { - outputDir: "/test/output", - } as BuildOptions; + setupMockFileSystem(); + vi.mocked(runWrangler).mockClear(); - const openNextConfig = { - default: { - override: { - incrementalCache: "cf-r2-incremental-cache", - }, - }, - }; + // Test uses partial types for simplicity - full config not needed + await populateCache( + createTestBuildOptions(), + createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + createTestPopulateCacheOptions() + ); - const wranglerConfig = { - r2_buckets: [ - { - binding: "NEXT_INC_CACHE_R2_BUCKET", - bucket_name: "test-bucket", - }, - ], - }; + expect(runWrangler).toHaveBeenCalled(); + expect(spawnSync).not.toHaveBeenCalled(); + }); - const populateCacheOptions = { - target: "local" as const, - shouldUsePreviewId: false, - rcloneBatch: true, - }; + test("uses batch upload with temporary config when R2 credentials are provided", async () => { + // Set R2 credentials to enable batch upload + vi.stubEnv("R2_ACCESS_KEY_ID", "test_access_key"); + vi.stubEnv("R2_SECRET_ACCESS_KEY", "test_secret_key"); + vi.stubEnv("R2_ACCOUNT_ID", "test_account_id"); + setupMockFileSystem(); vi.mocked(spawnSync).mockClear(); - mockFs({ - "/test/output": { - cache: { - buildID: { - path: { - to: { - "test.cache": JSON.stringify({ data: "test" }), - }, - }, - }, - }, - }, - }); - - // For this test we do not need whole configuration, just the part that is being used. - // eslint-disable-next-line @typescript-eslint/no-explicit-any - await populateCache(buildOptions, openNextConfig as any, wranglerConfig as any, populateCacheOptions); + // Test uses partial types for simplicity - full config not needed + await populateCache( + createTestBuildOptions(), + createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + createTestPopulateCacheOptions() + ); + // Verify batch upload was used with correct parameters and temporary config expect(spawnSync).toHaveBeenCalledWith( "rclone", expect.arrayContaining(["copy", expect.any(String), "r2:test-bucket"]), - expect.any(Object) + expect.objectContaining({ + env: expect.objectContaining({ + RCLONE_CONFIG: expect.stringMatching(/rclone-config-\d+\.conf$/), + }), + }) ); - - mockFs.restore(); }); }); }); From 3d181e5bf2001c85f313bb249ec1495e9699f8bb Mon Sep 17 00:00:00 2001 From: krzysztof-palka-monogo Date: Mon, 6 Oct 2025 14:48:09 +0200 Subject: [PATCH 7/8] fix: enhance error handling in populateCache for rclone uploads --- .../src/cli/commands/populate-cache.spec.ts | 63 ++++++++++++++++++- .../src/cli/commands/populate-cache.ts | 18 ++++-- 2 files changed, 75 insertions(+), 6 deletions(-) diff --git a/packages/cloudflare/src/cli/commands/populate-cache.spec.ts b/packages/cloudflare/src/cli/commands/populate-cache.spec.ts index 250e11a0e..94f6ec8f8 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.spec.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.spec.ts @@ -80,7 +80,7 @@ vi.mock("./helpers.js", () => ({ })); vi.mock("node:child_process", () => ({ - spawnSync: vi.fn(() => ({ status: 0 })), + spawnSync: vi.fn(() => ({ status: 0, stderr: Buffer.from("") })), })); describe("populateCache", () => { @@ -177,13 +177,72 @@ describe("populateCache", () => { // Verify batch upload was used with correct parameters and temporary config expect(spawnSync).toHaveBeenCalledWith( "rclone", - expect.arrayContaining(["copy", expect.any(String), "r2:test-bucket"]), + expect.arrayContaining(["copy", expect.any(String), "r2:test-bucket", "--error-on-no-transfer"]), expect.objectContaining({ + stdio: ["inherit", "inherit", "pipe"], env: expect.objectContaining({ RCLONE_CONFIG: expect.stringMatching(/rclone-config-\d+\.conf$/), }), }) ); }); + + test("handles rclone errors with status > 0", async () => { + const { runWrangler } = await import("../utils/run-wrangler.js"); + + // Set R2 credentials + vi.stubEnv("R2_ACCESS_KEY_ID", "test_access_key"); + vi.stubEnv("R2_SECRET_ACCESS_KEY", "test_secret_key"); + vi.stubEnv("R2_ACCOUNT_ID", "test_account_id"); + + setupMockFileSystem(); + + // Mock rclone failure without stderr output + vi.mocked(spawnSync).mockReturnValueOnce({ + status: 7, // Fatal error exit code + stderr: "", // No stderr output + } as any); // eslint-disable-line @typescript-eslint/no-explicit-any + + vi.mocked(runWrangler).mockClear(); + + await populateCache( + createTestBuildOptions(), + createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + createTestPopulateCacheOptions() + ); + + // Should fall back to standard upload when batch upload fails + expect(runWrangler).toHaveBeenCalled(); + }); + }); + + test("handles rclone errors with status = 0 and stderr output", async () => { + const { runWrangler } = await import("../utils/run-wrangler.js"); + + // Set R2 credentials + vi.stubEnv("R2_ACCESS_KEY_ID", "test_access_key"); + vi.stubEnv("R2_SECRET_ACCESS_KEY", "test_secret_key"); + vi.stubEnv("R2_ACCOUNT_ID", "test_account_id"); + + setupMockFileSystem(); + + // Mock rclone error in stderr + vi.mocked(spawnSync).mockReturnValueOnce({ + status: 0, // non-error exit code + stderr: Buffer.from("ERROR : Failed to copy: AccessDenied: Access Denied (403)"), + } as any); // eslint-disable-line @typescript-eslint/no-explicit-any + + vi.mocked(runWrangler).mockClear(); + + await populateCache( + createTestBuildOptions(), + createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + createTestPopulateCacheOptions() + ); + + // Should fall back to standard upload when batch upload fails + expect(runWrangler).toHaveBeenCalled(); }); }); diff --git a/packages/cloudflare/src/cli/commands/populate-cache.ts b/packages/cloudflare/src/cli/commands/populate-cache.ts index e87b878bd..b9f572067 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.ts @@ -220,15 +220,25 @@ async function populateR2IncrementalCacheWithBatchUpload( // Use rclone to sync the R2 const remote = `r2:${bucket}`; - const args = ["copy", stagingDir, remote, "--progress", "--transfers=32", "--checkers=16"]; + const args = [ + "copy", + stagingDir, + remote, + "--progress", + "--transfers=32", + "--checkers=16", + "--error-on-no-transfer", // Fail if no files transferred + ]; const result = spawnSync("rclone", args, { - stdio: "inherit", + stdio: ["inherit", "inherit", "pipe"], // Capture stderr while showing stdout env, }); - if (result.status !== 0) { - throw new Error("Batch upload failed"); + // Check for errors in both exit code and stderr + const stderrOutput = result.stderr?.toString().trim(); + if (result.status !== 0 || stderrOutput) { + throw new Error("Batch upload failed."); } logger.info(`Successfully uploaded ${assets.length} assets to R2 using batch upload`); From 32dc2944fb4ceea82d62e83348fe4b204e05a571 Mon Sep 17 00:00:00 2001 From: krzysztof-palka-monogo Date: Mon, 6 Oct 2025 15:05:49 +0200 Subject: [PATCH 8/8] docs: add links for creating R2 access API tokens in README --- .changeset/rclone-batch-upload.md | 4 ++++ packages/cloudflare/README.md | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.changeset/rclone-batch-upload.md b/.changeset/rclone-batch-upload.md index 2b4881ba2..ccc98e044 100644 --- a/.changeset/rclone-batch-upload.md +++ b/.changeset/rclone-batch-upload.md @@ -40,3 +40,7 @@ export R2_SECRET_ACCESS_KEY=your_secret export R2_ACCOUNT_ID=your_account opennextjs-cloudflare deploy # batch upload automatically used ``` + +**Note:** + +You can follow documentation https://developers.cloudflare.com/r2/api/tokens/ for creating API tokens with appropriate permissions for R2 access. diff --git a/packages/cloudflare/README.md b/packages/cloudflare/README.md index 12dda8bab..7f368603c 100644 --- a/packages/cloudflare/README.md +++ b/packages/cloudflare/README.md @@ -66,6 +66,10 @@ export R2_SECRET_ACCESS_KEY=your_secret_access_key export R2_ACCOUNT_ID=your_account_id ``` +**Note:** + +You can follow documentation https://developers.cloudflare.com/r2/api/tokens/ for creating API tokens with appropriate permissions for R2 access. + **Benefits:** - Significantly faster uploads for large caches using parallel transfers