Support SHA-* checksums and return from R2Bucket#{get,head}

Ref: https://community.cloudflare.com/t/2022-9-16-workers-runtime-release-notes/420496
Ref: cloudflare/workerd#103

Author: mrbbot

packages/tre/src/plugins/r2/errors.ts

@@ -14,7 +14,6 @@ enum CfCode {
   InternalError = 10001,
   NoSuchObjectKey = 10007,
   EntityTooLarge = 100100,
-  InvalidDigest = 10014,
   InvalidObjectName = 10020,
   InvalidMaxKeys = 10022,
   InvalidArgument = 10029,
@@ -110,21 +109,21 @@ export class EntityTooLarge extends R2Error {
   }
 }
 
-export class InvalidDigest extends R2Error {
-  constructor() {
-    super(
-      Status.BadRequest,
-      "The Content-MD5 you specified is not valid.",
-      CfCode.InvalidDigest
-    );
-  }
-}
-
 export class BadDigest extends R2Error {
-  constructor() {
+  constructor(
+    algorithm: "MD5" | "SHA-1" | "SHA-256" | "SHA-384" | "SHA-512",
+    provided: Buffer,
+    calculated: Buffer
+  ) {
     super(
       Status.BadRequest,
-      "The Content-MD5 you specified did not match what we received.",
+      [
+        `The ${algorithm} checksum you specified did not match what we received.`,
+        `You provided a ${algorithm} checksum with value: ${provided.toString(
+          "hex"
+        )}`,
+        `Actual ${algorithm} was: ${calculated.toString("hex")}`,
+      ].join("\n"),
       CfCode.BadDigest
     );
   }

packages/tre/src/plugins/r2/gateway.ts

@@ -1,3 +1,4 @@
+import crypto from "crypto";
 import { z } from "zod";
 import { Log } from "../../shared";
 import { RangeStoredValueMeta, Storage } from "../../storage";
@@ -101,28 +102,24 @@ export class R2Gateway {
     value: Uint8Array,
     options: R2PutOptions
   ): Promise<R2Object> {
-    const { customMetadata, md5, httpMetadata } = options;
-
-    const hash = validate.key(key).size(value).md5(value, md5);
+    const checksums = validate.key(key).size(value).hash(value, options);
 
     // build metadata
+    const md5Hash = crypto.createHash("md5").update(value).digest("hex");
     const metadata: R2ObjectMetadata = {
       key,
       size: value.byteLength,
-      etag: hash.toString("hex"),
+      etag: md5Hash,
       version: createVersion(),
-      httpEtag: `"${hash}"`,
+      httpEtag: `"${md5Hash}"`,
       uploaded: Date.now(),
-      httpMetadata: httpMetadata ?? {},
-      customMetadata: customMetadata ?? {},
+      httpMetadata: options.httpMetadata ?? {},
+      customMetadata: options.customMetadata ?? {},
+      checksums,
     };
 
     // Store value with expiration and metadata
-    await this.storage.put<R2ObjectMetadata>(key, {
-      value,
-      metadata,
-    });
-
+    await this.storage.put<R2ObjectMetadata>(key, { value, metadata });
     return new R2Object(metadata);
   }
 

packages/tre/src/plugins/r2/r2Object.ts

@@ -33,6 +33,9 @@ export interface R2ObjectMetadata {
   customMetadata: Record<string, string>;
   // If a GET request was made with a range option, this will be added
   range?: R2Range;
+  // Hashes used to check the received object’s integrity. At most one can be
+  // specified.
+  checksums?: R2StringChecksums;
 }
 
 export interface EncodedMetadata {
@@ -60,6 +63,7 @@ export class R2Object implements R2ObjectMetadata {
   readonly httpMetadata: R2HttpFields;
   readonly customMetadata: Record<string, string>;
   readonly range?: R2Range;
+  readonly checksums: R2StringChecksums;
 
   constructor(metadata: R2ObjectMetadata) {
     this.key = metadata.key;
@@ -71,6 +75,22 @@ export class R2Object implements R2ObjectMetadata {
     this.httpMetadata = metadata.httpMetadata;
     this.customMetadata = metadata.customMetadata;
     this.range = metadata.range;
+
+    // For non-multipart uploads, we always need to store an MD5 hash in
+    // `checksums`, but never explicitly stored one. Luckily, `R2Bucket#put()`
+    // always makes `etag` an MD5 hash.
+    const checksums: R2StringChecksums = { ...metadata.checksums };
+    const etag = metadata.etag;
+    if (etag.length === 32 && HEX_REGEXP.test(etag)) {
+      checksums.md5 = metadata.etag;
+    } else if (etag.length === 24 && BASE64_REGEXP.test(etag)) {
+      // TODO: remove this when we switch underlying storage mechanisms
+      // Previous versions of Miniflare 3 base64 encoded `etag` instead
+      checksums.md5 = Buffer.from(etag, "base64").toString("hex");
+    } else {
+      assert.fail("Expected `etag` to be an MD5 hash");
+    }
+    this.checksums = checksums;
   }
 
   // Format for return to the Workers Runtime
@@ -83,6 +103,13 @@ export class R2Object implements R2ObjectMetadata {
         k,
         v,
       })),
+      checksums: {
+        0: this.checksums.md5,
+        1: this.checksums.sha1,
+        2: this.checksums.sha256,
+        3: this.checksums.sha384,
+        4: this.checksums.sha512,
+      },
     };
   }
 

packages/tre/src/plugins/r2/schemas.ts

@@ -111,10 +111,10 @@ export const R2PutRequestSchema = z
     httpFields: R2HttpFieldsSchema.optional(), // (renamed in transform)
     onlyIf: R2ConditionalSchema.optional(),
     md5: Base64DataSchema.optional(), // (intentionally base64, not hex)  // TODO: make sure we're testing this is base64
-    sha1: HexDataSchema.optional(), // TODO: support
-    sha256: HexDataSchema.optional(), // TODO: support
-    sha384: HexDataSchema.optional(), // TODO: support
-    sha512: HexDataSchema.optional(), // TODO: support
+    sha1: HexDataSchema.optional(),
+    sha256: HexDataSchema.optional(),
+    sha384: HexDataSchema.optional(),
+    sha512: HexDataSchema.optional(),
   })
   .transform((value) => ({
     method: value.method,

packages/tre/src/plugins/r2/validator.ts

@@ -1,4 +1,5 @@
 import crypto from "crypto";
+import { R2StringChecksums } from "@cloudflare/workers-types/experimental";
 import {
   BadDigest,
   EntityTooLarge,
@@ -62,14 +63,37 @@ function testR2Conditional(
 
   return true;
 }
+
+export const R2_HASH_ALGORITHMS = [
+  { name: "MD5", field: "md5" },
+  { name: "SHA-1", field: "sha1" },
+  { name: "SHA-256", field: "sha256" },
+  { name: "SHA-384", field: "sha384" },
+  { name: "SHA-512", field: "sha512" },
+] as const;
+export type R2Hashes = Record<
+  typeof R2_HASH_ALGORITHMS[number]["field"],
+  Buffer | undefined
+>;
+
 export class Validator {
-  md5(value: Uint8Array, md5?: Buffer): Buffer {
-    const md5Hash = crypto.createHash("md5").update(value).digest();
-    if (md5 !== undefined && !md5.equals(md5Hash)) {
-      throw new BadDigest();
+  hash(value: Uint8Array, hashes: R2Hashes): R2StringChecksums {
+    const checksums: R2StringChecksums = {};
+    for (const { name, field } of R2_HASH_ALGORITHMS) {
+      const providedHash = hashes[field];
+      if (providedHash !== undefined) {
+        const computedHash = crypto.createHash(field).update(value).digest();
+        if (!providedHash.equals(computedHash)) {
+          throw new BadDigest(name, providedHash, computedHash);
+        }
+        // Store computed hash to ensure consistent casing in returned checksums
+        // from `R2Object`
+        checksums[field] = computedHash.toString("hex");
+      }
     }
-    return md5Hash;
+    return checksums;
   }
+
   condition(meta: R2Object, onlyIf?: R2Conditional): Validator {
     // test conditional should it exist
     if (!testR2Conditional(onlyIf, meta) || meta?.size === 0) {