diff --git a/.changeset/rclone-batch-upload.md b/.changeset/rclone-batch-upload.md new file mode 100644 index 00000000..ccc98e04 --- /dev/null +++ b/.changeset/rclone-batch-upload.md @@ -0,0 +1,46 @@ +--- +"@opennextjs/cloudflare": minor +--- + +feature: optional batch upload for faster R2 cache population + +This update adds optional batch upload support for R2 cache population, significantly improving upload performance for large caches when enabled via environment variables. + +**Key Changes:** + +1. **Optional Batch Upload**: Configure R2 credentials via environment variables to enable faster batch uploads: + + - `R2_ACCESS_KEY_ID` + - `R2_SECRET_ACCESS_KEY` + - `R2_ACCOUNT_ID` + +2. **Automatic Detection**: When credentials are detected, batch upload is automatically used for better performance + +3. **Smart Fallback**: If credentials are not configured, the CLI falls back to standard Wrangler uploads with a helpful message about enabling batch upload for better performance + +**All deployment commands support batch upload:** + +- `populateCache` - Explicit cache population +- `deploy` - Deploy with cache population +- `upload` - Upload version with cache population +- `preview` - Preview with cache population + +**Performance Benefits (when batch upload is enabled):** + +- Parallel transfer capabilities (32 concurrent transfers) +- Significantly faster for large caches +- Reduced API calls to Cloudflare + +**Usage:** + +```bash +# Enable batch upload by setting environment variables (recommended for large caches) +export R2_ACCESS_KEY_ID=your_key +export R2_SECRET_ACCESS_KEY=your_secret +export R2_ACCOUNT_ID=your_account +opennextjs-cloudflare deploy # batch upload automatically used +``` + +**Note:** + +You can follow documentation https://developers.cloudflare.com/r2/api/tokens/ for creating API tokens with appropriate permissions for R2 access. diff --git a/packages/cloudflare/README.md b/packages/cloudflare/README.md index cc1f5e72..7f368603 100644 --- a/packages/cloudflare/README.md +++ b/packages/cloudflare/README.md @@ -55,3 +55,26 @@ Deploy your application to production with the following: # or bun opennextjs-cloudflare build && bun opennextjs-cloudflare deploy ``` + +### Batch Cache Population (Optional, Recommended) + +For improved performance with large caches, you can enable batch upload by providing R2 credentials via environment variables: + +```bash +export R2_ACCESS_KEY_ID=your_access_key_id +export R2_SECRET_ACCESS_KEY=your_secret_access_key +export R2_ACCOUNT_ID=your_account_id +``` + +**Note:** + +You can follow documentation https://developers.cloudflare.com/r2/api/tokens/ for creating API tokens with appropriate permissions for R2 access. + +**Benefits:** + +- Significantly faster uploads for large caches using parallel transfers +- Reduced API calls to Cloudflare +- Automatically enabled when credentials are provided + +**Fallback:** +If these environment variables are not set, the CLI will use standard Wrangler uploads. Both methods work correctly - batch upload is simply faster for large caches. diff --git a/packages/cloudflare/package.json b/packages/cloudflare/package.json index 888110ec..1f4884b7 100644 --- a/packages/cloudflare/package.json +++ b/packages/cloudflare/package.json @@ -57,6 +57,7 @@ "cloudflare": "^4.4.1", "enquirer": "^2.4.1", "glob": "catalog:", + "rclone.js": "^0.6.6", "ts-tqdm": "^0.8.6", "yargs": "catalog:" }, diff --git a/packages/cloudflare/src/cli/commands/populate-cache.spec.ts b/packages/cloudflare/src/cli/commands/populate-cache.spec.ts index 29fbcacd..94f6ec8f 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.spec.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.spec.ts @@ -1,11 +1,12 @@ +import { spawnSync } from "node:child_process"; import { mkdirSync, writeFileSync } from "node:fs"; import path from "node:path"; import type { BuildOptions } from "@opennextjs/aws/build/helper.js"; import mockFs from "mock-fs"; -import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import { afterAll, afterEach, beforeAll, describe, expect, test, vi } from "vitest"; -import { getCacheAssets } from "./populate-cache.js"; +import { getCacheAssets, populateCache } from "./populate-cache.js"; describe("getCacheAssets", () => { beforeAll(() => { @@ -68,3 +69,180 @@ describe("getCacheAssets", () => { `); }); }); + +vi.mock("../utils/run-wrangler.js", () => ({ + runWrangler: vi.fn(), +})); + +vi.mock("./helpers.js", () => ({ + getEnvFromPlatformProxy: vi.fn(async () => ({})), + quoteShellMeta: vi.fn((s) => s), +})); + +vi.mock("node:child_process", () => ({ + spawnSync: vi.fn(() => ({ status: 0, stderr: Buffer.from("") })), +})); + +describe("populateCache", () => { + // Test fixtures + const createTestBuildOptions = (): BuildOptions => + ({ + outputDir: "/test/output", + }) as BuildOptions; + + const createTestOpenNextConfig = () => ({ + default: { + override: { + incrementalCache: "cf-r2-incremental-cache", + }, + }, + }); + + const createTestWranglerConfig = () => ({ + r2_buckets: [ + { + binding: "NEXT_INC_CACHE_R2_BUCKET", + bucket_name: "test-bucket", + }, + ], + }); + + const createTestPopulateCacheOptions = () => ({ + target: "local" as const, + shouldUsePreviewId: false, + }); + + const setupMockFileSystem = () => { + mockFs({ + "/test/output": { + cache: { + buildID: { + path: { + to: { + "test.cache": JSON.stringify({ data: "test" }), + }, + }, + }, + }, + }, + }); + }; + + describe("R2 incremental cache", () => { + afterEach(() => { + mockFs.restore(); + vi.unstubAllEnvs(); + }); + + test("uses standard upload when R2 credentials are not provided", async () => { + const { runWrangler } = await import("../utils/run-wrangler.js"); + + // Ensure no batch upload credentials are set + vi.stubEnv("R2_ACCESS_KEY_ID", undefined); + vi.stubEnv("R2_SECRET_ACCESS_KEY", undefined); + vi.stubEnv("R2_ACCOUNT_ID", undefined); + + setupMockFileSystem(); + vi.mocked(runWrangler).mockClear(); + + // Test uses partial types for simplicity - full config not needed + await populateCache( + createTestBuildOptions(), + createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + createTestPopulateCacheOptions() + ); + + expect(runWrangler).toHaveBeenCalled(); + expect(spawnSync).not.toHaveBeenCalled(); + }); + + test("uses batch upload with temporary config when R2 credentials are provided", async () => { + // Set R2 credentials to enable batch upload + vi.stubEnv("R2_ACCESS_KEY_ID", "test_access_key"); + vi.stubEnv("R2_SECRET_ACCESS_KEY", "test_secret_key"); + vi.stubEnv("R2_ACCOUNT_ID", "test_account_id"); + + setupMockFileSystem(); + vi.mocked(spawnSync).mockClear(); + + // Test uses partial types for simplicity - full config not needed + await populateCache( + createTestBuildOptions(), + createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + createTestPopulateCacheOptions() + ); + + // Verify batch upload was used with correct parameters and temporary config + expect(spawnSync).toHaveBeenCalledWith( + "rclone", + expect.arrayContaining(["copy", expect.any(String), "r2:test-bucket", "--error-on-no-transfer"]), + expect.objectContaining({ + stdio: ["inherit", "inherit", "pipe"], + env: expect.objectContaining({ + RCLONE_CONFIG: expect.stringMatching(/rclone-config-\d+\.conf$/), + }), + }) + ); + }); + + test("handles rclone errors with status > 0", async () => { + const { runWrangler } = await import("../utils/run-wrangler.js"); + + // Set R2 credentials + vi.stubEnv("R2_ACCESS_KEY_ID", "test_access_key"); + vi.stubEnv("R2_SECRET_ACCESS_KEY", "test_secret_key"); + vi.stubEnv("R2_ACCOUNT_ID", "test_account_id"); + + setupMockFileSystem(); + + // Mock rclone failure without stderr output + vi.mocked(spawnSync).mockReturnValueOnce({ + status: 7, // Fatal error exit code + stderr: "", // No stderr output + } as any); // eslint-disable-line @typescript-eslint/no-explicit-any + + vi.mocked(runWrangler).mockClear(); + + await populateCache( + createTestBuildOptions(), + createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + createTestPopulateCacheOptions() + ); + + // Should fall back to standard upload when batch upload fails + expect(runWrangler).toHaveBeenCalled(); + }); + }); + + test("handles rclone errors with status = 0 and stderr output", async () => { + const { runWrangler } = await import("../utils/run-wrangler.js"); + + // Set R2 credentials + vi.stubEnv("R2_ACCESS_KEY_ID", "test_access_key"); + vi.stubEnv("R2_SECRET_ACCESS_KEY", "test_secret_key"); + vi.stubEnv("R2_ACCOUNT_ID", "test_account_id"); + + setupMockFileSystem(); + + // Mock rclone error in stderr + vi.mocked(spawnSync).mockReturnValueOnce({ + status: 0, // non-error exit code + stderr: Buffer.from("ERROR : Failed to copy: AccessDenied: Access Denied (403)"), + } as any); // eslint-disable-line @typescript-eslint/no-explicit-any + + vi.mocked(runWrangler).mockClear(); + + await populateCache( + createTestBuildOptions(), + createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + createTestPopulateCacheOptions() + ); + + // Should fall back to standard upload when batch upload fails + expect(runWrangler).toHaveBeenCalled(); + }); +}); diff --git a/packages/cloudflare/src/cli/commands/populate-cache.ts b/packages/cloudflare/src/cli/commands/populate-cache.ts index 8cc68707..b9f57206 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.ts @@ -1,4 +1,6 @@ -import { cpSync, existsSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { copyFileSync, cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; import path from "node:path"; import type { BuildOptions } from "@opennextjs/aws/build/helper.js"; @@ -129,6 +131,127 @@ type PopulateCacheOptions = { shouldUsePreviewId: boolean; }; +/** + * Check if R2 credentials are available via environment variables for batch upload + */ +function hasR2EnvCredentials(): boolean { + return !!(process.env.R2_ACCESS_KEY_ID && process.env.R2_SECRET_ACCESS_KEY && process.env.R2_ACCOUNT_ID); +} + +/** + * Create a temporary configuration file for batch upload from environment variables + * @returns Path to the temporary config file or null if env vars not available + */ +function createTempRcloneConfig(): string | null { + const accessKey = process.env.R2_ACCESS_KEY_ID; + const secretKey = process.env.R2_SECRET_ACCESS_KEY; + const accountId = process.env.R2_ACCOUNT_ID; + + if (!accessKey || !secretKey || !accountId) { + return null; + } + + const tempDir = tmpdir(); + const tempConfigPath = path.join(tempDir, `rclone-config-${Date.now()}.conf`); + + const configContent = `[r2] +type = s3 +provider = Cloudflare +access_key_id = ${accessKey} +secret_access_key = ${secretKey} +endpoint = https://${accountId}.r2.cloudflarestorage.com +acl = private +`; + + /** + * 0o600 is an octal number (the 0o prefix indicates octal in JavaScript) + * that represents Unix file permissions: + * + * - 6 (owner): read (4) + write (2) = readable and writable by the file owner + * - 0 (group): no permissions for the group + * - 0 (others): no permissions for anyone else + * + * In symbolic notation, this is: rw------- + */ + writeFileSync(tempConfigPath, configContent, { mode: 0o600 }); + return tempConfigPath; +} + +/** + * Populate R2 incremental cache using batch upload for better performance + * Uses parallel transfers to significantly speed up cache population + */ +async function populateR2IncrementalCacheWithBatchUpload( + options: BuildOptions, + bucket: string, + prefix: string | undefined, + assets: CacheAsset[] +) { + logger.info("\nPopulating R2 incremental cache using batch upload..."); + + // Create temporary config from env vars - required for batch upload + const tempConfigPath = createTempRcloneConfig(); + if (!tempConfigPath) { + throw new Error( + "R2 credentials not found. Please set R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, and R2_ACCOUNT_ID environment variables to enable batch upload." + ); + } + + const env = { ...process.env }; + env.RCLONE_CONFIG = tempConfigPath; + logger.info("Using batch upload with R2 credentials from environment variables"); + + try { + // Create a staging dir with proper key paths + const stagingDir = path.join(options.outputDir, ".r2-staging"); + rmSync(stagingDir, { recursive: true, force: true }); + mkdirSync(stagingDir, { recursive: true }); + + for (const { fullPath, key, buildId, isFetch } of assets) { + const cacheKey = computeCacheKey(key, { + prefix, + buildId, + cacheType: isFetch ? "fetch" : "cache", + }); + const destPath = path.join(stagingDir, cacheKey); + mkdirSync(path.dirname(destPath), { recursive: true }); + copyFileSync(fullPath, destPath); + } + + // Use rclone to sync the R2 + const remote = `r2:${bucket}`; + const args = [ + "copy", + stagingDir, + remote, + "--progress", + "--transfers=32", + "--checkers=16", + "--error-on-no-transfer", // Fail if no files transferred + ]; + + const result = spawnSync("rclone", args, { + stdio: ["inherit", "inherit", "pipe"], // Capture stderr while showing stdout + env, + }); + + // Check for errors in both exit code and stderr + const stderrOutput = result.stderr?.toString().trim(); + if (result.status !== 0 || stderrOutput) { + throw new Error("Batch upload failed."); + } + + logger.info(`Successfully uploaded ${assets.length} assets to R2 using batch upload`); + } finally { + try { + // Cleanup temporary config file + rmSync(tempConfigPath); + } catch { + console.warn(`Failed to remove temporary config at ${tempConfigPath}`); + } + } +} + async function populateR2IncrementalCache( options: BuildOptions, config: WranglerConfig, @@ -151,6 +274,23 @@ async function populateR2IncrementalCache( const assets = getCacheAssets(options); + // Use batch upload if R2 credentials are available (optional but recommended) + const canUseBatchUpload = hasR2EnvCredentials(); + + if (canUseBatchUpload) { + try { + return await populateR2IncrementalCacheWithBatchUpload(options, bucket, prefix, assets); + } catch (error) { + logger.warn(`Batch upload failed: ${error instanceof Error ? error.message : error}`); + logger.info("Falling back to standard uploads..."); + } + } else { + logger.info( + "Using standard cache uploads. For faster performance with large caches, set R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, and R2_ACCOUNT_ID environment variables to enable batch upload." + ); + } + + // Standard upload fallback (using Wrangler) for (const { fullPath, key, buildId, isFetch } of tqdm(assets)) { const cacheKey = computeCacheKey(key, { prefix, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fd4fff78..0186b5ef 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1086,6 +1086,9 @@ importers: glob: specifier: 'catalog:' version: 11.0.0 + rclone.js: + specifier: ^0.6.6 + version: 0.6.6 ts-tqdm: specifier: ^0.8.6 version: 0.8.6 @@ -5030,6 +5033,10 @@ packages: engines: {node: '>=0.4.0'} hasBin: true + adm-zip@0.5.16: + resolution: {integrity: sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==} + engines: {node: '>=12.0'} + agent-base@6.0.2: resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} engines: {node: '>= 6.0.0'} @@ -8427,6 +8434,13 @@ packages: resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} hasBin: true + rclone.js@0.6.6: + resolution: {integrity: sha512-Dxh34cab/fNjFq5SSm0fYLNkGzG2cQSBy782UW9WwxJCEiVO4cGXkvaXcNlgv817dK8K8PuQ+NHUqSAMMhWujQ==} + engines: {node: '>=12'} + cpu: [arm, arm64, mips, mipsel, x32, x64] + os: [darwin, freebsd, linux, openbsd, sunos, win32] + hasBin: true + react-dom@18.3.1: resolution: {integrity: sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==} peerDependencies: @@ -14763,6 +14777,8 @@ snapshots: acorn@8.15.0: {} + adm-zip@0.5.16: {} + agent-base@6.0.2: dependencies: debug: 4.4.0 @@ -19102,6 +19118,11 @@ snapshots: minimist: 1.2.8 strip-json-comments: 2.0.1 + rclone.js@0.6.6: + dependencies: + adm-zip: 0.5.16 + mri: 1.2.0 + react-dom@18.3.1(react@18.3.1): dependencies: loose-envify: 1.4.0