Add validation for custom metadata size

Author: mrbbot

packages/tre/src/plugins/r2/errors.ts

@@ -14,6 +14,7 @@ enum CfCode {
   InternalError = 10001,
   NoSuchObjectKey = 10007,
   EntityTooLarge = 100100,
+  MetadataTooLarge = 10012,
   InvalidObjectName = 10020,
   InvalidMaxKeys = 10022,
   InvalidArgument = 10029,
@@ -109,6 +110,16 @@ export class EntityTooLarge extends R2Error {
   }
 }
 
+export class MetadataTooLarge extends R2Error {
+  constructor() {
+    super(
+      Status.BadRequest,
+      "Your metadata headers exceed the maximum allowed metadata size.",
+      CfCode.MetadataTooLarge
+    );
+  }
+}
+
 export class BadDigest extends R2Error {
   constructor(
     algorithm: "MD5" | "SHA-1" | "SHA-256" | "SHA-384" | "SHA-512",

packages/tre/src/plugins/r2/gateway.ts

@@ -111,6 +111,7 @@ export class R2Gateway {
     const checksums = validate
       .key(key)
       .size(value)
+      .metadataSize(options.customMetadata)
       .condition(meta, options.onlyIf)
       .hash(value, options);
 

packages/tre/src/plugins/r2/validator.ts

@@ -5,14 +5,17 @@ import {
   EntityTooLarge,
   InvalidMaxKeys,
   InvalidObjectName,
+  MetadataTooLarge,
   PreconditionFailed,
 } from "./errors";
 import { R2Object, R2ObjectMetadata } from "./r2Object";
 import { R2Conditional } from "./schemas";
 
 export const MAX_LIST_KEYS = 1_000;
 const MAX_KEY_SIZE = 1024;
-const MAX_VALUE_SIZE = 5 * 1_000 * 1_000 * 1_000 - 5 * 1_000 * 1_000;
+// https://developers.cloudflare.com/r2/platform/limits/
+const MAX_VALUE_SIZE = 5_000_000_000 - 5_000_000; // 5GB - 5MB
+const MAX_METADATA_SIZE = 2048; // 2048B
 
 function identity(ms: number) {
   return ms;
@@ -67,6 +70,14 @@ export type R2Hashes = Record<
   Buffer | undefined
 >;
 
+function serialisedLength(x: string) {
+  //  Adapted from internal R2 gateway implementation
+  for (let i = 0; i < x.length; i++) {
+    if (x.charCodeAt(i) >= 256) return x.length * 2;
+  }
+  return x.length;
+}
+
 export class Validator {
   hash(value: Uint8Array, hashes: R2Hashes): R2StringChecksums {
     const checksums: R2StringChecksums = {};
@@ -95,13 +106,24 @@ export class Validator {
   }
 
   size(value: Uint8Array): Validator {
-    // TODO: should we be validating httpMetadata/customMetadata size too
     if (value.byteLength > MAX_VALUE_SIZE) {
       throw new EntityTooLarge();
     }
     return this;
   }
 
+  metadataSize(customMetadata?: Record<string, string>): Validator {
+    if (customMetadata === undefined) return this;
+    let metadataLength = 0;
+    for (const [key, value] of Object.entries(customMetadata)) {
+      metadataLength += serialisedLength(key) + serialisedLength(value);
+    }
+    if (metadataLength > MAX_METADATA_SIZE) {
+      throw new MetadataTooLarge();
+    }
+    return this;
+  }
+
   key(key: string): Validator {
     const keyLength = Buffer.byteLength(key);
     if (keyLength >= MAX_KEY_SIZE) {

packages/tre/test/plugins/r2/index.spec.ts

@@ -146,13 +146,31 @@ class TestR2Bucket implements R2Bucket {
     options?: R2PutOptions
   ): Promise<R2Object> {
     const url = `http://localhost/${encodeURIComponent(this.ns + key)}`;
+
+    let valueBlob: Blob;
+    if (value === null) {
+      valueBlob = new Blob([]);
+    } else if (value instanceof ArrayBuffer) {
+      valueBlob = new Blob([new Uint8Array(value)]);
+    } else if (ArrayBuffer.isView(value)) {
+      valueBlob = new Blob([viewToArray(value)]);
+    } else if (value instanceof ReadableStream) {
+      // @ts-expect-error `ReadableStream` is an `AsyncIterable`
+      valueBlob = await blob(value);
+    } else {
+      valueBlob = new Blob([value]);
+    }
+
+    // We can't store options in headers as some put() tests include extended
+    // characters in them, and `undici` validates all headers are byte strings,
+    // so use a form data body instead
+    const formData = new FormData();
+    formData.set("options", maybeJsonStringify(options));
+    formData.set("value", valueBlob);
     const res = await this.mf.dispatchFetch(url, {
       method: "PUT",
-      headers: {
-        Accept: "multipart/form-data",
-        "Test-Options": maybeJsonStringify(options),
-      },
-      body: ArrayBuffer.isView(value) ? viewToArray(value) : value,
+      headers: { Accept: "multipart/form-data" },
+      body: formData,
     });
     return deconstructResponse(res);
   }
@@ -376,11 +394,12 @@ const test = miniflareTest<{ BUCKET: R2Bucket }, Context>(
       const options = maybeJsonParse(optionsHeader);
       return constructResponse(await env.BUCKET.get(key, options));
     } else if (method === "PUT") {
-      const optionsHeader = request.headers.get("Test-Options");
-      const options = maybeJsonParse(optionsHeader);
-      return constructResponse(
-        await env.BUCKET.put(key, await request.arrayBuffer(), options)
-      );
+      const formData = await request.formData();
+      const optionsData = formData.get("options");
+      if (typeof optionsData !== "string") throw new TypeError();
+      const options = maybeJsonParse(optionsData);
+      const value = formData.get("value");
+      return constructResponse(await env.BUCKET.put(key, value, options));
     } else if (method === "DELETE") {
       const keys = await request.json<string | string[]>();
       await env.BUCKET.delete(keys);
@@ -796,6 +815,39 @@ test("put: stores only if passes onlyIf", async (t) => {
   const object = await r2.put("no-key", "2", { onlyIf: { etagMatches: etag } });
   t.is(object as R2Object | null, null);
 });
+test("put: validates metadata size", async (t) => {
+  const { r2 } = t.context;
+
+  // TODO(soon): add check for max value size once we have streaming support
+  //  (don't really want to allocate 5GB buffers in tests :sweat_smile:)
+
+  const expectations: ThrowsExpectation = {
+    instanceOf: Error,
+    message:
+      "put: Your metadata headers exceed the maximum allowed metadata size. (10012)",
+  };
+
+  // Check with ASCII characters
+  await r2.put("key", "value", { customMetadata: { key: "x".repeat(2045) } });
+  await t.throwsAsync(
+    r2.put("key", "value", { customMetadata: { key: "x".repeat(2046) } }),
+    expectations
+  );
+  await r2.put("key", "value", { customMetadata: { hi: "x".repeat(2046) } });
+
+  // Check with extended characters: note "🙂" is 2 UTF-16 code units, so
+  // `"🙂".length === 2`, and it requires 4 bytes to store
+  await r2.put("key", "value", { customMetadata: { key: "🙂".repeat(511) } }); // 3 + 4*511 = 2047
+  await r2.put("key", "value", { customMetadata: { key1: "🙂".repeat(511) } }); // 4 + 4*511 = 2048
+  await t.throwsAsync(
+    r2.put("key", "value", { customMetadata: { key12: "🙂".repeat(511) } }), // 5 + 4*511 = 2049
+    expectations
+  );
+  await t.throwsAsync(
+    r2.put("key", "value", { customMetadata: { key: "🙂".repeat(512) } }), // 3 + 4*512 = 2051
+    expectations
+  );
+});
 
 test("delete: deletes existing keys", async (t) => {
   const { r2 } = t.context;