r2 implementation (#289)

* r2 first up

* update undici to current

* fix the way I handle head

* list working; adjustments for cleaner code

* fixed ava

* bucket tests complete

* fixes wrong implementations; adds r2 to miniflare api

* see if windows is mad at larger than size

* see if windows is mad at larger than size

* improve code clarity; improve comments; cleaning

* add startAfter support; setup for range return

* range in R2Object; test cases for range & startAfter

* setup delimitedPrefixes test

* improve get->range validation

* first changeset

* improve storage system

* improve readability

* fix durable-objects shadow-storage & recorder-storage

* exportable _valueToArray; internal list function has object for params

* storage finished; update R2 tests

* get redis tests working

* add proper suffix larger than size support

* simplify range; better stream handling in R2ObjectBody

* remove dump

* throw for durable object shadow storage -> getRange & head

* fix recorder storage

* update delimitedPrefixes

* drop dead code

* simplify limit

* fix ava

Author: CraigglesO

package-lock.json

@@ -44,6 +44,10 @@ export class ShadowStorage extends Storage {
     return count;
   }
 
+  async head(_key: string): Promise<undefined> {
+    throw new Error('"head" is not supported in Durable Object shadow storage');
+  }
+
   async get(key: string): Promise<StoredValue | undefined> {
     return (await this.getMany([key]))[0];
   }
@@ -82,6 +86,12 @@ export class ShadowStorage extends Storage {
     return result;
   }
 
+  async getRange(_key: string): Promise<undefined> {
+    throw new Error(
+      '"getRange" is not supported in Durable Object shadow storage'
+    );
+  }
+
   put(key: string, value: StoredValue): void {
     // Store fresh copy so further mutations from caller aren't stored
     this.copies.set(key, { value: value.value.slice() });

packages/durable-objects/src/shadow.ts

@@ -22,6 +22,7 @@ workers in a sandbox implementing Workers' runtime APIs.
 - 🔑 Variables and Secrets with `.env` Files
 - 📚 Modules Support
 - 📦 KV (with optional persistence)
+- 🪣 R2 (with optional persistence)
 - ✨ Cache (with optional persistence)
 - 📌 Durable Objects (with optional persistence)
 - 🌐 Workers Sites
@@ -128,6 +129,10 @@ KV Options:
  -k, --kv                KV namespace to bind                                                  [array]
      --kv-persist        Persist KV data (to optional path)                           [boolean/string]
 
+R2 Options:
+ -r, --r2                R2 bucket to bind                                                     [array]
+     --r2-persist        Persist R2 data (to optional path)                           [boolean/string]
+
 Durable Objects Options:
  -o, --do                Durable Object to bind                             [array:NAME=CLASS[@MOUNT]]
      --do-persist        Persist Durable Object data (to optional path)               [boolean/string]

packages/miniflare/README.md

@@ -53,6 +53,7 @@
     "@miniflare/html-rewriter": "2.5.0",
     "@miniflare/http-server": "2.5.0",
     "@miniflare/kv": "2.5.0",
+    "@miniflare/r2": "2.5.0",
     "@miniflare/runner-vm": "2.5.0",
     "@miniflare/scheduler": "2.5.0",
     "@miniflare/shared": "2.5.0",

packages/miniflare/package.json

@@ -22,6 +22,7 @@ import {
   startServer,
 } from "@miniflare/http-server";
 import { KVNamespace, KVPlugin } from "@miniflare/kv";
+import { R2Bucket, R2Plugin } from "@miniflare/r2";
 import { VMScriptRunner } from "@miniflare/runner-vm";
 import {
   CronScheduler,
@@ -46,6 +47,7 @@ export const PLUGINS = {
 
   // Storage
   KVPlugin,
+  R2Plugin,
   DurableObjectsPlugin,
   CachePlugin,
   SitesPlugin,
@@ -104,6 +106,12 @@ export class Miniflare extends MiniflareCore<Plugins> {
     return plugin.getNamespace(storage, namespace);
   }
 
+  async getR2Bucket(bucket: string): Promise<R2Bucket> {
+    const plugin = (await this.getPlugins()).R2Plugin;
+    const storage = this.getPluginStorage("R2Plugin");
+    return plugin.getBucket(storage, bucket);
+  }
+
   async getCaches(): Promise<CacheStorage> {
     const plugin = (await this.getPlugins()).CachePlugin;
     return plugin.getCaches();

packages/miniflare/src/api.ts

@@ -0,0 +1,17 @@
+# `@miniflare/r2`
+
+Workers R2 module for [Miniflare](https://github.com/cloudflare/miniflare): a
+fun, full-featured, fully-local simulator for Cloudflare Workers. See
+[🪣 R2](https://miniflare.dev/storage/r2) for more details.
+
+## Example
+
+```js
+import { R2Bucket } from "@miniflare/r2";
+import { MemoryStorage } from "@miniflare/storage-memory";
+
+const r2 = new R2Bucket(new MemoryStorage());
+await r2.put("key", "value");
+const value = await r2.get("key");
+console.log(await value.text()); // "value"
+```

packages/r2/README.md

@@ -0,0 +1,44 @@
+{
+  "name": "@miniflare/r2",
+  "version": "2.5.0",
+  "description": "Workers R2 module for Miniflare: a fun, full-featured, fully-local simulator for Cloudflare Workers",
+  "keywords": [
+    "cloudflare",
+    "workers",
+    "worker",
+    "local",
+    "cloudworker"
+  ],
+  "author": "CraigglesO <[email protected]>",
+  "license": "MIT",
+  "main": "./dist/src/index.js",
+  "types": "./dist/src/index.d.ts",
+  "files": [
+    "dist/src"
+  ],
+  "engines": {
+    "node": ">=16.7"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/cloudflare/miniflare.git",
+    "directory": "packages/r2"
+  },
+  "bugs": {
+    "url": "https://github.com/cloudflare/miniflare/issues"
+  },
+  "homepage": "https://github.com/cloudflare/miniflare/tree/master/packages/r2#readme",
+  "volta": {
+    "extends": "../../package.json"
+  },
+  "dependencies": {
+    "@miniflare/shared": "2.5.0",
+    "undici": "5.5.1"
+  },
+  "devDependencies": {
+    "@miniflare/shared-test": "2.5.0"
+  }
+}