Skip to content

Commit bf947bf

Browse files
committed
feat(libstore/s3-binary-cache-store): add multipart upload config settings
Add three configuration settings to `S3BinaryCacheStoreConfig` to control multipart upload behavior: - `bool multipart-upload` (default `false`): Enable/disable multipart uploads - `uint64_t multipart-chunk-size` (default 5 MiB): Size of each upload part - `uint64_t multipart-threshold` (default 100 MiB): Minimum file size for multipart The feature is disabled by default.
1 parent 2d83bc6 commit bf947bf

File tree

2 files changed

+55
-0
lines changed

2 files changed

+55
-0
lines changed

src/libstore/include/nix/store/s3-binary-cache-store.hh

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,38 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig
6161
> addressing instead of virtual host based addressing.
6262
)"};
6363

64+
const Setting<bool> multipartUpload{
65+
this,
66+
false,
67+
"multipart-upload",
68+
R"(
69+
Whether to use multipart uploads for large files. When enabled,
70+
files exceeding the multipart threshold will be uploaded in
71+
multiple parts, which is required for files larger than 5 GiB and
72+
can improve performance and reliability for large uploads.
73+
)"};
74+
75+
const Setting<uint64_t> multipartChunkSize{
76+
this,
77+
5 * 1024 * 1024,
78+
"multipart-chunk-size",
79+
R"(
80+
The size (in bytes) of each part in multipart uploads. Must be
81+
at least 5 MiB (AWS S3 requirement). Larger chunk sizes reduce the
82+
number of requests but use more memory. Default is 5 MiB.
83+
)",
84+
{"buffer-size"}};
85+
86+
const Setting<uint64_t> multipartThreshold{
87+
this,
88+
100 * 1024 * 1024,
89+
"multipart-threshold",
90+
R"(
91+
The minimum file size (in bytes) for using multipart uploads.
92+
Files smaller than this threshold will use regular PUT requests.
93+
Default is 100 MiB. Only takes effect when multipart-upload is enabled.
94+
)"};
95+
6496
/**
6597
* Set of settings that are part of the S3 URI itself.
6698
* These are needed for region specification and other S3-specific settings.

src/libstore/s3-binary-cache-store.cc

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ namespace nix {
1515

1616
MakeError(UploadToS3, Error);
1717

18+
static constexpr uint64_t AWS_MIN_PART_SIZE = 5 * 1024 * 1024; // 5MiB
1819
static constexpr uint64_t AWS_MAX_PART_SIZE = 5ULL * 1024 * 1024 * 1024; // 5GiB
1920

2021
class S3BinaryCacheStore : public virtual HttpBinaryCacheStore
@@ -253,6 +254,28 @@ S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig(
253254
cacheUri.query[key] = value;
254255
}
255256
}
257+
258+
if (multipartChunkSize < AWS_MIN_PART_SIZE) {
259+
throw UsageError(
260+
"multipart-chunk-size must be at least %s, got %s",
261+
renderSize(AWS_MIN_PART_SIZE),
262+
renderSize(multipartChunkSize.get()));
263+
}
264+
265+
if (multipartChunkSize > AWS_MAX_PART_SIZE) {
266+
throw UsageError(
267+
"multipart-chunk-size must be at most %s, got %s",
268+
renderSize(AWS_MAX_PART_SIZE),
269+
renderSize(multipartChunkSize.get()));
270+
}
271+
272+
if (multipartUpload && multipartThreshold < multipartChunkSize) {
273+
warn(
274+
"multipart-threshold (%s) is less than multipart-chunk-size (%s), "
275+
"which may result in single-part multipart uploads",
276+
renderSize(multipartThreshold.get()),
277+
renderSize(multipartChunkSize.get()));
278+
}
256279
}
257280

258281
std::string S3BinaryCacheStoreConfig::getHumanReadableURI() const

0 commit comments

Comments
 (0)