Merge pull request #14330 from lovesegfault/nix-s3-multipart

feat(libstore): add support for multipart s3 uploads

Author: Ericson2314

doc/manual/rl-next/s3-curl-implementation.md

@@ -1,6 +1,6 @@
 ---
 synopsis: "Improved S3 binary cache support via HTTP"
-prs: [13823, 14026, 14120, 14131, 14135, 14144, 14170, 14190, 14198, 14206, 14209, 14222, 14223, 13752]
+prs: [13752, 13823, 14026, 14120, 14131, 14135, 14144, 14170, 14190, 14198, 14206, 14209, 14222, 14223, 14330, 14333, 14335, 14336, 14337, 14350, 14356, 14357, 14374, 14375, 14376, 14377, 14391, 14393, 14420, 14421]
 issues: [13084, 12671, 11748, 12403]
 ---
 
@@ -18,9 +18,23 @@ improvements:
 The new implementation requires curl >= 7.75.0 and `aws-crt-cpp` for credential
 management.
 
-All existing S3 URL formats and parameters remain supported, with the notable
-exception of multi-part uploads, which are no longer supported.
+All existing S3 URL formats and parameters remain supported, however the store
+settings for configuring multipart uploads have changed:
+
+- **`multipart-upload`** (default: `false`): Enable multipart uploads for large
+  files. When enabled, files exceeding the multipart threshold will be uploaded
+  in multiple parts.
+
+- **`multipart-threshold`** (default: `100 MiB`): Minimum file size for using
+  multipart uploads. Files smaller than this will use regular PUT requests.
+  Only takes effect when `multipart-upload` is enabled.
+
+- **`multipart-chunk-size`** (default: `5 MiB`): Size of each part in multipart
+  uploads. Must be at least 5 MiB (AWS S3 requirement). Larger chunk sizes
+  reduce the number of requests but use more memory.
+
+- **`buffer-size`**: Has been replaced by `multipart-chunk-size` and is now an alias to it.
 
 Note that this change also means Nix now supports S3 binary cache stores even
-if build without `aws-crt-cpp`, but only for public buckets which do not
-require auth.
+if built without `aws-crt-cpp`, but only for public buckets which do not
+require authentication.

src/libstore/include/nix/store/s3-binary-cache-store.hh

@@ -61,6 +61,38 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig
           > addressing instead of virtual host based addressing.
         )"};
 
+    const Setting<bool> multipartUpload{
+        this,
+        false,
+        "multipart-upload",
+        R"(
+          Whether to use multipart uploads for large files. When enabled,
+          files exceeding the multipart threshold will be uploaded in
+          multiple parts, which is required for files larger than 5 GiB and
+          can improve performance and reliability for large uploads.
+        )"};
+
+    const Setting<uint64_t> multipartChunkSize{
+        this,
+        5 * 1024 * 1024,
+        "multipart-chunk-size",
+        R"(
+          The size (in bytes) of each part in multipart uploads. Must be
+          at least 5 MiB (AWS S3 requirement). Larger chunk sizes reduce the
+          number of requests but use more memory. Default is 5 MiB.
+        )",
+        {"buffer-size"}};
+
+    const Setting<uint64_t> multipartThreshold{
+        this,
+        100 * 1024 * 1024,
+        "multipart-threshold",
+        R"(
+          The minimum file size (in bytes) for using multipart uploads.
+          Files smaller than this threshold will use regular PUT requests.
+          Default is 100 MiB. Only takes effect when multipart-upload is enabled.
+        )"};
+
     /**
      * Set of settings that are part of the S3 URI itself.
      * These are needed for region specification and other S3-specific settings.