Skip to content

Commit b95e7ca

Browse files
vicbvicb
authored
use wrangler r2 bulk put for R2 cache population (#991)
1 parent 7064bcb commit b95e7ca

File tree

8 files changed

+180
-550
lines changed

8 files changed

+180
-550
lines changed

‎.changeset/beige-aliens-tickle.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@opennextjs/cloudflare": minor
3+
---
4+
5+
use `wrangler r2 bulk put` for R2 cache population

‎packages/cloudflare/README.md‎

Lines changed: 0 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -55,30 +55,3 @@ Deploy your application to production with the following:
5555
# or
5656
bun opennextjs-cloudflare build && bun opennextjs-cloudflare deploy
5757
```
58-
59-
### Batch Cache Population (Optional, Recommended)
60-
61-
For improved performance with large caches, you can enable batch upload by providing R2 credentials via .env or environment variables.
62-
63-
Create a `.env` file in your project root (automatically loaded by the CLI):
64-
65-
```bash
66-
R2_ACCESS_KEY_ID=your_access_key_id
67-
R2_SECRET_ACCESS_KEY=your_secret_access_key
68-
CF_ACCOUNT_ID=your_account_id
69-
```
70-
71-
You can also set the environment variables for CI builds.
72-
73-
**Note:**
74-
75-
You can follow documentation https://developers.cloudflare.com/r2/api/tokens/ for creating API tokens with appropriate permissions for R2 access.
76-
77-
**Benefits:**
78-
79-
- Significantly faster uploads for large caches using parallel transfers
80-
- Reduced API calls to Cloudflare
81-
- Automatically enabled when credentials are provided
82-
83-
**Fallback:**
84-
If these environment variables are not set, the CLI will use standard Wrangler uploads. Both methods work correctly - batch upload is simply faster for large caches.

‎packages/cloudflare/package.json‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,11 +55,9 @@
5555
"@ast-grep/napi": "0.40.0",
5656
"@dotenvx/dotenvx": "catalog:",
5757
"@opennextjs/aws": "3.8.5",
58-
"@types/rclone.js": "^0.6.3",
5958
"cloudflare": "^4.4.1",
6059
"enquirer": "^2.4.1",
6160
"glob": "catalog:",
62-
"rclone.js": "^0.6.6",
6361
"ts-tqdm": "^0.8.6",
6462
"yargs": "catalog:"
6563
},

‎packages/cloudflare/src/api/cloudflare-context.ts‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -79,13 +79,8 @@ declare global {
7979
CF_PREVIEW_DOMAIN?: string;
8080
// Should have the `Workers Scripts:Read` permission
8181
CF_WORKERS_SCRIPTS_API_TOKEN?: string;
82-
83-
// Cloudflare account id - needed for skew protection and R2 batch population
82+
// Cloudflare account id - needed for skew protection
8483
CF_ACCOUNT_ID?: string;
85-
86-
// R2 API credentials for batch cache population (optional, enables faster uploads)
87-
R2_ACCESS_KEY_ID?: string;
88-
R2_SECRET_ACCESS_KEY?: string;
8984
}
9085
}
9186

‎packages/cloudflare/src/cli/commands/populate-cache.spec.ts‎

Lines changed: 44 additions & 186 deletions
Original file line numberDiff line numberDiff line change
@@ -78,44 +78,7 @@ vi.mock("./helpers.js", () => ({
7878
quoteShellMeta: vi.fn((s) => s),
7979
}));
8080

81-
// Mock rclone.js promises API to simulate successful copy operations by default
82-
vi.mock("rclone.js", () => ({
83-
default: {
84-
promises: {
85-
copy: vi.fn(() => Promise.resolve("")),
86-
},
87-
},
88-
}));
89-
9081
describe("populateCache", () => {
91-
// Test fixtures
92-
const createTestBuildOptions = (): BuildOptions =>
93-
({
94-
outputDir: "/test/output",
95-
}) as BuildOptions;
96-
97-
const createTestOpenNextConfig = () => ({
98-
default: {
99-
override: {
100-
incrementalCache: "cf-r2-incremental-cache",
101-
},
102-
},
103-
});
104-
105-
const createTestWranglerConfig = () => ({
106-
r2_buckets: [
107-
{
108-
binding: "NEXT_INC_CACHE_R2_BUCKET",
109-
bucket_name: "test-bucket",
110-
},
111-
],
112-
});
113-
114-
const createTestPopulateCacheOptions = () => ({
115-
target: "local" as const,
116-
shouldUsePreviewId: false,
117-
});
118-
11982
const setupMockFileSystem = () => {
12083
mockFs({
12184
"/test/output": {
@@ -132,153 +95,48 @@ describe("populateCache", () => {
13295
});
13396
};
13497

135-
describe("R2 incremental cache", () => {
136-
afterEach(() => {
137-
mockFs.restore();
138-
vi.unstubAllEnvs();
139-
});
140-
141-
test("uses sequential upload for local target (skips batch upload)", async () => {
142-
const { runWrangler } = await import("../utils/run-wrangler.js");
143-
const rcloneModule = (await import("rclone.js")).default;
144-
145-
setupMockFileSystem();
146-
vi.mocked(runWrangler).mockClear();
147-
vi.mocked(rcloneModule.promises.copy).mockClear();
148-
149-
// Test with local target - should skip batch upload even with credentials
150-
await populateCache(
151-
createTestBuildOptions(),
152-
createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
153-
createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
154-
{ target: "local" as const, shouldUsePreviewId: false },
155-
{
156-
R2_ACCESS_KEY_ID: "test_access_key",
157-
R2_SECRET_ACCESS_KEY: "test_secret_key",
158-
CF_ACCOUNT_ID: "test_account_id",
159-
} as any // eslint-disable-line @typescript-eslint/no-explicit-any
160-
);
161-
162-
// Should use sequential upload (runWrangler), not batch upload (rclone.js)
163-
expect(runWrangler).toHaveBeenCalled();
164-
expect(rcloneModule.promises.copy).not.toHaveBeenCalled();
165-
});
166-
167-
test("uses sequential upload when R2 credentials are not provided", async () => {
168-
const { runWrangler } = await import("../utils/run-wrangler.js");
169-
const rcloneModule = (await import("rclone.js")).default;
170-
171-
setupMockFileSystem();
172-
vi.mocked(runWrangler).mockClear();
173-
vi.mocked(rcloneModule.promises.copy).mockClear();
174-
175-
// Test uses partial types for simplicity - full config not needed
176-
// Pass empty envVars to simulate no R2 credentials
177-
await populateCache(
178-
createTestBuildOptions(),
179-
createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
180-
createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
181-
createTestPopulateCacheOptions(),
182-
{} as any // eslint-disable-line @typescript-eslint/no-explicit-any
183-
);
184-
185-
expect(runWrangler).toHaveBeenCalled();
186-
expect(rcloneModule.promises.copy).not.toHaveBeenCalled();
187-
});
188-
189-
test("uses batch upload with temporary config for remote target when R2 credentials are provided", async () => {
190-
const rcloneModule = (await import("rclone.js")).default;
191-
192-
setupMockFileSystem();
193-
vi.mocked(rcloneModule.promises.copy).mockClear();
194-
195-
// Test uses partial types for simplicity - full config not needed
196-
// Pass envVars with R2 credentials and remote target to enable batch upload
197-
await populateCache(
198-
createTestBuildOptions(),
199-
createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
200-
createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
201-
{ target: "remote" as const, shouldUsePreviewId: false },
202-
{
203-
R2_ACCESS_KEY_ID: "test_access_key",
204-
R2_SECRET_ACCESS_KEY: "test_secret_key",
205-
CF_ACCOUNT_ID: "test_account_id",
206-
} as any // eslint-disable-line @typescript-eslint/no-explicit-any
207-
);
208-
209-
// Verify batch upload was used with correct parameters and temporary config
210-
expect(rcloneModule.promises.copy).toHaveBeenCalledWith(
211-
expect.any(String), // staging directory
212-
"r2:test-bucket",
213-
expect.objectContaining({
214-
progress: true,
215-
transfers: 16,
216-
checkers: 8,
217-
env: expect.objectContaining({
218-
RCLONE_CONFIG: expect.stringMatching(/rclone-config-\d+\.conf$/),
219-
}),
220-
})
221-
);
222-
});
223-
224-
test("handles rclone errors with status > 0 for remote target", async () => {
225-
const { runWrangler } = await import("../utils/run-wrangler.js");
226-
const rcloneModule = (await import("rclone.js")).default;
227-
228-
setupMockFileSystem();
229-
230-
// Mock rclone failure - Promise rejection
231-
vi.mocked(rcloneModule.promises.copy).mockRejectedValueOnce(
232-
new Error("rclone copy failed with exit code 7")
233-
);
234-
235-
vi.mocked(runWrangler).mockClear();
236-
237-
// Pass envVars with R2 credentials and remote target to enable batch upload (which will fail)
238-
await populateCache(
239-
createTestBuildOptions(),
240-
createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
241-
createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
242-
{ target: "remote" as const, shouldUsePreviewId: false },
243-
{
244-
R2_ACCESS_KEY_ID: "test_access_key",
245-
R2_SECRET_ACCESS_KEY: "test_secret_key",
246-
CF_ACCOUNT_ID: "test_account_id",
247-
} as any // eslint-disable-line @typescript-eslint/no-explicit-any
248-
);
249-
250-
// Should fall back to sequential upload when batch upload fails
251-
expect(runWrangler).toHaveBeenCalled();
252-
});
253-
254-
test("handles rclone errors with stderr output for remote target", async () => {
255-
const { runWrangler } = await import("../utils/run-wrangler.js");
256-
const rcloneModule = (await import("rclone.js")).default;
257-
258-
setupMockFileSystem();
259-
260-
// Mock rclone error - Promise rejection with stderr message
261-
vi.mocked(rcloneModule.promises.copy).mockRejectedValueOnce(
262-
new Error("ERROR : Failed to copy: AccessDenied: Access Denied (403)")
263-
);
264-
265-
vi.mocked(runWrangler).mockClear();
266-
267-
// Pass envVars with R2 credentials and remote target to enable batch upload (which will fail)
268-
await populateCache(
269-
createTestBuildOptions(),
270-
createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
271-
createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
272-
{ target: "remote" as const, shouldUsePreviewId: false },
273-
{
274-
R2_ACCESS_KEY_ID: "test_access_key",
275-
R2_SECRET_ACCESS_KEY: "test_secret_key",
276-
CF_ACCOUNT_ID: "test_account_id",
277-
} as any // eslint-disable-line @typescript-eslint/no-explicit-any
278-
);
279-
280-
// Should fall back to standard upload when batch upload fails
281-
expect(runWrangler).toHaveBeenCalled();
282-
});
283-
});
98+
describe.each([{ target: "local" as const }, { target: "remote" as const }])(
99+
"R2 incremental cache",
100+
({ target }) => {
101+
afterEach(() => {
102+
mockFs.restore();
103+
});
104+
105+
test(target, async () => {
106+
const { runWrangler } = await import("../utils/run-wrangler.js");
107+
108+
setupMockFileSystem();
109+
vi.mocked(runWrangler).mockClear();
110+
111+
await populateCache(
112+
{
113+
outputDir: "/test/output",
114+
} as BuildOptions,
115+
{
116+
default: {
117+
override: {
118+
incrementalCache: "cf-r2-incremental-cache",
119+
},
120+
},
121+
} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
122+
{
123+
r2_buckets: [
124+
{
125+
binding: "NEXT_INC_CACHE_R2_BUCKET",
126+
bucket_name: "test-bucket",
127+
},
128+
],
129+
} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
130+
{ target, shouldUsePreviewId: false },
131+
{} as any // eslint-disable-line @typescript-eslint/no-explicit-any
132+
);
133+
134+
expect(runWrangler).toHaveBeenCalledWith(
135+
expect.anything(),
136+
expect.arrayContaining(["r2 bulk put", "test-bucket"]),
137+
expect.objectContaining({ target })
138+
);
139+
});
140+
}
141+
);
284142
});

0 commit comments

Comments
 (0)