feat(chunking): add chunking algorithm

Author: tyler-french

build/bazel/remote/execution/v2/remote_execution.proto

@@ -486,7 +486,7 @@ service ContentAddressableStorage {
   //
   // When blob splitting and splicing is used at the same time, the clients and
   // the server SHOULD agree out-of-band upon a chunking algorithm used by both
-  // parties to benefit from each others chunk data and avoid unnecessary data
+  // parties to benefit from each other's chunk data and avoid unnecessary data
   // duplication.
   //
   // Errors:
@@ -2185,25 +2185,27 @@ message ChunkingFunction {
     // about the chunking algorithm can be made.
     UNKNOWN = 0;
 
-    // This is a variant of the FastCDC chunking algorithm as described in the
-    // 2020 paper by Wen Xia, et al.
-    // See https://ieeexplore.ieee.org/document/9055082 for details.
-    // Reference implementation could be found in the Rust library
-    // https://docs.rs/fastcdc/3.2.1/fastcdc/v2020/index.html
-    // with the gear tables available at
-    // https://github.com/nlfiedler/fastcdc-rs/blob/3.2.1/src/v2020/mod.rs
+    // The FastCDC chunking algorithm as described in the 2020 paper by
+    // Wen Xia, et al. See https://ieeexplore.ieee.org/document/9055082
+    // for details.
+    //
+    // Supported implementations:
+    //   - Rust: https://docs.rs/fastcdc/3.2.1/fastcdc/v2020/index.html
+    //   - Go: https://github.com/buildbuddy-io/fastcdc2020
     //
-    // Server which supports this chunking function MUST advertise the following
+    // Servers which support this chunking function SHOULD advertise the following
     // configuration parameters through the CacheCapabilities message:
-    //   - normalization_level
-    //   - min_chunk_size_bytes
     //   - avg_chunk_size_bytes
-    //   - max_chunk_size_bytes
+    //   - normalization_level
     //   - seed
     //
-    // Client MUST use these advertised parameters to setup the FastCDC chunker.
-    // The remaining parameters, such as mask_s, mask_l can be derived from the
-    // average chunk size parameter.
+    // If these parameters are not set, the client SHOULD use the following
+    // defaults: avg_chunk_size_bytes = 524288 (512 KiB), normalization_level = 2,
+    // seed = 0.
+    //
+    // min_chunk_size and max_chunk_size are derived from avg_chunk_size (1/4x
+    // and 4x respectively). The remaining parameters can be derived from the
+    // average chunk size and normalization level.
     FASTCDC_2020 = 1;
   }
 }
@@ -2335,41 +2337,89 @@ message CacheCapabilities {
 }
 
 // The chunking configuration of the server.
+//
+// This configuration describes how the server will chunk blobs. When calling
+// SpliceBlob, the client SHOULD use these configurations if able, but the
+// server is not expected to verify that it can reliably reproduce the chunking.
+// The server only needs to verify that the chunks concatenate together to form
+// the complete blob.
+//
+// This is primarily the server telling the client how it intends to chunk
+// blobs. The client SHOULD try to match this configuration for optimal
+// deduplication, but is not required to do so.
+//
+// Content-defined chunking is most beneficial for larger blobs where
+// deduplication opportunities are greater, so smaller blobs SHOULD be
+// uploaded without chunking.
 message ChunkingConfiguration {
+  // Parameters for the FastCDC content-defined chunking algorithm.
+  //
+  // Implementations MUST follow the FastCDC 2020 paper by Wen Xia, et al.:
+  // https://ieeexplore.ieee.org/document/9055082
+  //
+  // The fastcdc-rs library (https://docs.rs/fastcdc/3.2.1/fastcdc/v2020/)
+  // is a reference implementation that follows the paper specification.
+  //
+  // Key algorithm components from the paper:
+  //
+  // GEAR table: 256 64-bit integers for the rolling hash, computed as:
+  //   GEAR[i] = high_64_bits(MD5(byte(i))) for i in 0..255
+  //
+  // MASKS table: Bit patterns for chunk boundary detection, derived from
+  // the C reference implementation. The mask selection based on average
+  // chunk size and normalization level should match the paper.
+  //
+  // The minimum and maximum chunk sizes are derived from the average:
+  //   - min_chunk_size = avg_chunk_size_bytes / 4
+  //   - max_chunk_size = avg_chunk_size_bytes * 4
+  //
   // If any of the advertised parameters are not within the expected range,
   // the client SHOULD ignore FastCDC chunking function support.
   message FastCDCParams {
+    // The average (expected) chunk size for the FastCDC chunking algorithm.
+    // The value MUST be between 1 KiB and 1 MiB.
+    // If unset, clients SHOULD use 524288 (512 KiB) as the default.
+    //
+    // The minimum chunk size will be avg_chunk_size_bytes / 4.
+    // The maximum chunk size will be avg_chunk_size_bytes * 4.
+    uint64 avg_chunk_size_bytes = 1;
+
     // The normalization level for the FastCDC chunking algorithm.
     // The value MUST be between 0 and 3.
-    uint32 normalization_level = 1;
-
-    // The minimum chunk size for the FastCDC chunking algorithm.
-    // The value MUST be between 256 bytes and 64 KiB.
-    uint64 min_chunk_size_bytes = 2;
-
-    // The average chunk size for the FastCDC chunking algorithm.
-    // The value MUST be between 1 KiB and 256 KiB.
-    uint64 avg_chunk_size_bytes = 3;
-
-    // The maximum chunk size for the FastCDC chunking algorithm.
-    // The value MUST be between 4 KiB and 4 MiB.
-    uint64 max_chunk_size_bytes = 4;
+    //
+    // Higher normalization levels produce chunks closer to the average size
+    // but may reduce deduplication effectiveness:
+    //   - Level 0: No normalization, widest size distribution
+    //   - Level 1: Fewer chunks outside desired range
+    //   - Level 2: Most chunks match desired size (recommended)
+    //   - Level 3: Nearly all chunks are the desired size
+    //
+    // If unset, clients SHOULD use 2 as the default.
+    uint32 normalization_level = 2;
 
-    // The seed for the FastCDC chunking algorithm.
-    uint32 seed = 5;
+    // The seed for the FastCDC mask generation.
+    // If unset, clients SHOULD use 0 as the default.
+    //
+    // Using a consistent seed ensures deterministic chunking across
+    // different clients, which improves deduplication.
+    uint32 seed = 3;
   }
 
   // A list of chunking algorithms that the server supports for splitting and
   // splicing blobs.
   repeated ChunkingFunction.Value supported_chunking_algorithms = 1;
 
-  // The minimum blob size that should be considered for chunking.
-  // Blobs smaller than this threshold SHOULD be sent as single blobs.
-  // If unset, clients SHOULD use max_cas_blob_size_bytes as the
-  // minimum blob size for chunking.
-  // If both this field and max_cas_blob_size_bytes are unset, clients
-  // MAY chunk blobs of any size.
-  uint64 min_blob_size_for_chunking_bytes = 2;
+  // The size threshold for chunking blobs.
+  // Blobs with size less than or equal to this threshold SHOULD be uploaded
+  // as single blobs without chunking. Blobs with size greater than this
+  // threshold SHOULD be chunked.
+  //
+  // This threshold SHOULD be greater than or equal to the maximum chunk size
+  // produced by the chunking function to avoid re-chunking blobs that would
+  // result in a single chunk.
+  //
+  // If unset, clients SHOULD use 2097152 (2 MiB) as the default.
+  uint64 chunking_threshold_bytes = 2;
 
   // The parameters for the FastCDC chunking algorithm.
   FastCDCParams fastcdc_params = 3;