diff --git a/.vscode/settings.json b/.vscode/settings.json index be72144c746..cedecd94a73 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -6,5 +6,8 @@ "parameters": ["--import", "tsx"] } ], - "typescript.tsdk": "./node_modules/typescript/lib" + "typescript.tsdk": "./node_modules/typescript/lib", + "cSpell.words": [ + "ssec" + ] } diff --git a/packages/file-storage/CHANGELOG.md b/packages/file-storage/CHANGELOG.md index e9955e3454b..b6c1db7a952 100644 --- a/packages/file-storage/CHANGELOG.md +++ b/packages/file-storage/CHANGELOG.md @@ -2,6 +2,11 @@ This is the changelog for [`file-storage`](https://github.com/remix-run/remix/tree/main/packages/file-storage). It follows [semantic versioning](https://semver.org/). +## Unreleased + +- Add support for Cloudflare R2 buckets through `r2-file-storage.ts`. +- Added tests for `r2-file-storage.ts` located in `r2-file-storage.test.ts`. +- Updated `README.MD` inside of `file-storage` with docs and examples pertaining to `R2FileStorage`. ## v0.11.0 (2025-11-05) - Move `@remix-run/lazy-file` to `peerDependencies` diff --git a/packages/file-storage/README.md b/packages/file-storage/README.md index 8bdb478d2e6..d779004ff96 100644 --- a/packages/file-storage/README.md +++ b/packages/file-storage/README.md @@ -77,6 +77,110 @@ class CustomFileStorage implements FileStorage { } ``` +## Cloudflare R2 + +Use the `R2FileStorage` provides an adapter to store and retrieve files from a Cloudflare R2 bucket from Workers. It uses the `FileStorage` interface and types from Cloudflare. + +### Documentation +https://developers.cloudflare.com/r2/api/workers/workers-api-reference/#r2object-definition + +## USAGE + +```ts +import { R2FileStorage } from '@remix-run/file-storage/r2' + +// Pass the bucket that will be used +let storage = new R2FileStorage(env.MY_BUCKET) + +let file = new File(['hello world'], 'hello.txt', { type: 'text/plain' }) +let key = 'user123/hello.txt' + +// To check if R2 has file +await.storage.has(key) + +// To set a file in R2 +await storage.set(key,file) + +// To get a file from R2 +await storage.get(key) + + +// To upload file +let uploadedFile = await storage.put(key, file) + +// response will return a file +return new Response(uploadedFile, { + headers: { + 'Content-Type': uploadedFile.type, + 'Content-Length': String(uploadedFile.size), + } +}) + +// To delete a file +await storage.remove(key) +``` + +### Listing + +```ts +// Keys only +let a = await storage.list({ prefix: 'user123/' }) + +// Include metadata for each file +let b = await storage.list({ prefix: 'user123/', includeMetadata: true }) +// b.files: [{ key, lastModified, name, size, type }, ...] + +// Paginate with cursor +if (b.cursor !== undefined) { + let c = await storage.list({ cursor: b.cursor }) +} +``` + +## Options (R2) + +`R2FileStorage` supports Cloudflare R2 options on `get`, `list` and `set`/`put`. Refer to the Cloudflare documentation to learn what these options are. + +### Documentation +https://developers.cloudflare.com/r2/api/workers/workers-api-reference/#method-specific-types + + +### Examples +```ts +// Conditional + ranged GET +await storage.get(key, { + onlyIf: { + etagMatches: '"abc123"', + uploadedAfter: new Date(Date.now() - 60_000), + }, + range: { offset: 0, length: 1024 }, +}) + +// Checksums, encryption, and metadata on PUT/SET +await storage.set(key, file, { + httpMetadata: { contentType: file.type }, + sha256: new Uint8Array([/* ... */]), + storageCLass: 'InfrequentAccess' + customMetadata: { tag: 'docs' }, +}) +``` + +Notes: +- `set` merges default metadata with your options. Defaults include `httpMetadata.contentType` and `customMetadata` for `name`, `lastModified`, and `size`. Your provided `httpMetadata`/`customMetadata` override defaults if the same fields are set. +- `put` returns a new `File` whose `name` is the key. `get` returns a `File` whose `name` is the original filename (stored in metadata) when available. +- `list({ includeMetadata: true })` returns file metadata populated from R2 `httpMetadata`/`customMetadata`. + +### Environment + +`R2FileStorage` runs in Cloudflare Workers (or compatible environments) where an `R2Bucket` binding is available. For TypeScript, install `@cloudflare/workers-types` and declare your env binding: + +```ts +import type { R2Bucket } from '@cloudflare/workers-types' + +interface Env { + MY_BUCKET: R2Bucket +} +``` + ## Related Packages - [`form-data-parser`](https://github.com/remix-run/remix/tree/main/packages/form-data-parser) - Pairs well with this library for storing `FileUpload` objects received in `multipart/form-data` requests diff --git a/packages/file-storage/package.json b/packages/file-storage/package.json index 6011f151827..1ecac008e8c 100644 --- a/packages/file-storage/package.json +++ b/packages/file-storage/package.json @@ -22,6 +22,7 @@ ".": "./src/index.ts", "./local": "./src/local.ts", "./memory": "./src/memory.ts", + "./r2": "./src/r2.ts", "./package.json": "./package.json" }, "publishConfig": { @@ -38,6 +39,10 @@ "types": "./dist/memory.d.ts", "default": "./dist/memory.js" }, + "./r2": { + "types": "./dist/r2.d.ts", + "default": "./dist/r2.js" + }, "./package.json": "./package.json" } }, @@ -47,6 +52,8 @@ "devDependencies": { "@remix-run/form-data-parser": "workspace:^", "@types/node": "^24.6.0", + "esbuild": "^0.25.10", + "@cloudflare/workers-types": "4.20251014.0", "typescript": "^5.9.3" }, "scripts": { diff --git a/packages/file-storage/src/lib/r2-file-storage.test.ts b/packages/file-storage/src/lib/r2-file-storage.test.ts new file mode 100644 index 00000000000..ff1240d0275 --- /dev/null +++ b/packages/file-storage/src/lib/r2-file-storage.test.ts @@ -0,0 +1,235 @@ +import * as assert from 'node:assert/strict' +import { beforeEach, describe, it } from 'node:test' +import { parseFormData } from '@remix-run/form-data-parser' + +import { R2FileStorage } from './r2-file-storage.ts' +import type { R2Bucket, R2Objects } from '@cloudflare/workers-types' + +class MockR2Bucket implements Partial { + #storage = new Map() + + async get(key: string) { + let stored = this.#storage.get(key) + if (!stored) return null + + return { + key, + body: new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(stored.data)) + controller.close() + } + }), + arrayBuffer: async () => stored.data, + httpMetadata: stored.metadata.httpMetadata, + customMetadata: stored.metadata.customMetadata, + uploaded: stored.metadata.uploaded, + size: stored.data.byteLength, + } as any + } + + async put(key: string, value: ArrayBuffer, options?: any) { + this.#storage.set(key, { + data: value, + metadata: { + httpMetadata: options?.httpMetadata ?? {}, + customMetadata: options?.customMetadata ?? {}, + uploaded: new Date() + } + }) + + return { + key, + size: value.byteLength, + uploaded: new Date(), + httpMetadata: options?.httpMetadata, + customMetadata: options?.customMetadata, + } as any + } + + async delete(key: string) { + this.#storage.delete(key) + } + + async head(key: string) { + let stored = this.#storage.get(key) + if (!stored) return null + return { key, ...stored.metadata } as any + } + + async list(options?: any) { + let keys = Array.from(this.#storage.keys()) + + if (options?.prefix) { + keys = keys.filter(k => k.startsWith(options.prefix)) + } + + keys.sort() + + let startIndex = options?.cursor ? parseInt(options.cursor) : 0 + let limit = options?.limit ?? 1000 + let endIndex = Math.min(startIndex + limit, keys.length) + + let objects = keys.slice(startIndex, endIndex).map(key => { + let stored = this.#storage.get(key)! + let obj: any = { + key, + size: stored.data.byteLength, + uploaded: stored.metadata.uploaded, + } + + if (options?.include?.includes('httpMetadata')) { + obj.httpMetadata = stored.metadata.httpMetadata + } + if (options?.include?.includes('customMetadata')) { + obj.customMetadata = stored.metadata.customMetadata + } + + return obj + }) + + return { + objects, + truncated: endIndex < keys.length, + cursor: endIndex < keys.length && endIndex > startIndex ? endIndex.toString() : undefined, + delimitedPrefixes: [] + } as R2Objects + } +} + +describe('R2FileStorage', () => { + let mockBucket: MockR2Bucket + let storage: R2FileStorage + + beforeEach(() => { + mockBucket = new MockR2Bucket() + storage = new R2FileStorage(mockBucket as unknown as R2Bucket) + }) + + it('stores and retrieves files', async () => { + let lastModified = Date.now() + let file = new File(['Hello, world!'], 'hello.txt', { + type: 'text/plain', + lastModified, + }) + + await storage.set('hello', file) + + assert.ok(await storage.has('hello')) + + let retrieved = await storage.get('hello') + + assert.ok(retrieved) + assert.equal(retrieved.name, 'hello.txt') + assert.equal(retrieved.type, 'text/plain') + assert.equal(retrieved.lastModified, lastModified) + assert.equal(retrieved.size, 13) + + let text = await retrieved.text() + + assert.equal(text, 'Hello, world!') + + await storage.remove('hello') + + assert.ok(!(await storage.has('hello'))) + assert.equal(await storage.get('hello'), null) + }) + + it('lists files with pagination', async () => { + let allKeys = ['a', 'b', 'c', 'd', 'e'] + + await Promise.all( + allKeys.map((key) => + storage.set(key, new File([`Hello ${key}!`], `hello.txt`, { type: 'text/plain' })), + ), + ) + + let { cursor, files } = await storage.list() + assert.equal(cursor, undefined) + assert.equal(files.length, 5) + assert.deepEqual(files.map((f) => f.key).sort(), allKeys) + + let { cursor: cursor1, files: files1 } = await storage.list({ limit: 0 }) + assert.equal(cursor1, undefined) + assert.equal(files1.length, 0) + + let { cursor: cursor2, files: files2 } = await storage.list({ limit: 2 }) + assert.notEqual(cursor2, undefined) + assert.equal(files2.length, 2) + + let { cursor: cursor3, files: files3 } = await storage.list({ cursor: cursor2 }) + assert.equal(cursor3, undefined) + assert.equal(files3.length, 3) + + assert.deepEqual([...files2, ...files3].map((f) => f.key).sort(), allKeys) + }) + + it('lists files by key prefix', async () => { + let allKeys = ['a', 'b', 'b/c', 'c', 'd'] + + await Promise.all( + allKeys.map((key) => + storage.set(key, new File([`Hello ${key}!`], `hello.txt`, { type: 'text/plain' })), + ), + ) + + let { cursor, files } = await storage.list({ prefix: 'b' }) + assert.equal(cursor, undefined) + assert.equal(files.length, 2) + assert.deepEqual(files.map((f) => f.key).sort(), ['b', 'b/c']) + }) + + it('lists files with metadata', async () => { + let allKeys = ['a', 'b', 'c', 'd', 'e'] + + await Promise.all( + allKeys.map((key) => + storage.set(key, new File([`Hello ${key}!`], `hello.txt`, { type: 'text/plain' })), + ), + ) + + let { cursor, files } = await storage.list({ includeMetadata: true }) + assert.equal(cursor, undefined) + assert.equal(files.length, 5) + assert.deepEqual(files.map((f) => f.key).sort(), allKeys) + files.forEach((f) => assert.ok('lastModified' in f)) + files.forEach((f) => assert.ok('name' in f)) + files.forEach((f) => assert.ok('size' in f)) + files.forEach((f) => assert.ok('type' in f)) + }) + + describe('integration with form-data-parser', () => { + it('stores and lists file uploads', async () => { + let boundary = '----WebKitFormBoundary7MA4YWxkTrZu0gW' + let request = new Request('http://example.com', { + method: 'POST', + headers: { + 'Content-Type': `multipart/form-data; boundary=${boundary}`, + }, + body: [ + `--${boundary}`, + 'Content-Disposition: form-data; name="hello"; filename="hello.txt"', + 'Content-Type: text/plain', + '', + 'Hello, world!', + `--${boundary}--`, + ].join('\r\n'), + }) + + await parseFormData(request, async (file) => { + await storage.set('hello', file) + }) + + assert.ok(await storage.has('hello')) + + let { files } = await storage.list({ includeMetadata: true }) + + assert.equal(files.length, 1) + assert.equal(files[0].key, 'hello') + assert.equal(files[0].name, 'hello.txt') + assert.equal(files[0].size, 13) + assert.equal(files[0].type, 'text/plain') + assert.ok(files[0].lastModified) + }) + }) +}) diff --git a/packages/file-storage/src/lib/r2-file-storage.ts b/packages/file-storage/src/lib/r2-file-storage.ts new file mode 100644 index 00000000000..00b89c09183 --- /dev/null +++ b/packages/file-storage/src/lib/r2-file-storage.ts @@ -0,0 +1,107 @@ +import type { FileKey, FileMetadata, FileStorage, ListOptions, ListResult } from './file-storage.ts' +import type { R2Bucket, R2GetOptions, R2ListOptions, R2Object, R2ObjectBody, R2Objects, R2PutOptions } from '@cloudflare/workers-types' + +export class R2FileStorage implements FileStorage { + #r2: R2Bucket + + constructor(r2: R2Bucket) { + this.#r2 = r2 + } + + async get(key: string, options?: R2GetOptions): Promise { + let object = await this.#r2.get(key, options) as R2ObjectBody; + + if (object == null ) { + return null + } + + let fileArray = await object.arrayBuffer() + + return new File([fileArray], object.customMetadata?.name ?? object.key, { + type: object.httpMetadata?.contentType, + lastModified: parseInt(object.customMetadata?.lastModified ?? object.uploaded.getTime().toString()) + }) as File + } + + async put(key: string, file: File, options?: R2PutOptions): Promise { + let fileArray = await file.arrayBuffer() + let object = await this.#r2.put(key, fileArray, { + httpMetadata: { + contentType: file.type + }, + customMetadata: { + lastModified: file.lastModified.toString(), + name: file.name, + size: file.size.toString() + }, + ...options + }) as R2Object + + return new File([fileArray], object.key, { + type: object.httpMetadata?.contentType, + lastModified: object.uploaded.getTime() + }) as File + } + + async remove(key: string): Promise { + await this.#r2.delete(key) + } + + //The cloudflare R2ListOptions type is missing the include to check for presence of metadata or not. So did not include type + async list(options?: T): Promise> { + let r2Options: any = { + limit: options?.limit, + prefix: options?.prefix, + cursor: options?.cursor, + } + + if (options?.includeMetadata) { + r2Options.include = ['httpMetadata', 'customMetadata'] + } + + let objects = await this.#r2.list(r2Options) as R2Objects + + return { + cursor: objects.truncated ? objects.cursor : undefined, + files: objects.objects.map(obj => { + if (options?.includeMetadata) { + return { + key: obj.key, + lastModified: obj.uploaded.getTime(), + name: obj.customMetadata?.name ?? obj.key, + size: obj.size, + type: obj.httpMetadata?.contentType ?? '', + } as FileMetadata + } + return { key: obj.key } as FileKey + }) as any, + } + } + + async has(key: string): Promise { + let object = await this.#r2.head(key) as R2Object | null + if (object == null) { + return false + } + return true + } + + + async set(key: string, file: File, options?: R2PutOptions): Promise { + let fileArray = await file.arrayBuffer() + await this.#r2.put(key, fileArray, { + httpMetadata: { + contentType: file.type + }, + customMetadata: { + lastModified: file.lastModified.toString(), + name: file.name, + size: file.size.toString() + }, + ...options + }) + return; + } + + +} diff --git a/packages/file-storage/src/r2.ts b/packages/file-storage/src/r2.ts new file mode 100644 index 00000000000..14d3d57b144 --- /dev/null +++ b/packages/file-storage/src/r2.ts @@ -0,0 +1 @@ +export { R2FileStorage } from './lib/r2-file-storage.ts' diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5ecaaa72aec..af661537acb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -112,12 +112,18 @@ importers: specifier: workspace:^ version: link:../lazy-file devDependencies: + '@cloudflare/workers-types': + specifier: 4.20251014.0 + version: 4.20251014.0 '@remix-run/form-data-parser': specifier: workspace:^ version: link:../form-data-parser '@types/node': specifier: ^24.6.0 version: 24.6.0 + esbuild: + specifier: ^0.25.10 + version: 0.25.12 typescript: specifier: ^5.9.3 version: 5.9.3