Merge pull request #13752 from lovesegfault/curl-based-s3-v2

feat(libstore): curl-based s3

Author: xokdvium

.github/workflows/ci.yml

@@ -68,7 +68,6 @@ jobs:
             primary: true
             stdenv: stdenv
             withAWS: true
-            withCurlS3: false
           # TODO: remove once curl-based-s3 fully lands
           - scenario: on ubuntu (no s3)
             runs-on: ubuntu-24.04
@@ -77,32 +76,20 @@ jobs:
             primary: false
             stdenv: stdenv
             withAWS: false
-            withCurlS3: false
-          # TODO: remove once curl-based-s3 fully lands
-          - scenario: on ubuntu (curl s3)
-            runs-on: ubuntu-24.04
-            os: linux
-            instrumented: false
-            primary: false
-            stdenv: stdenv
-            withAWS: false
-            withCurlS3: true
           - scenario: on macos
             runs-on: macos-14
             os: darwin
             instrumented: false
             primary: true
             stdenv: stdenv
             withAWS: true
-            withCurlS3: false
           - scenario: on ubuntu (with sanitizers / coverage)
             runs-on: ubuntu-24.04
             os: linux
             instrumented: true
             primary: false
             stdenv: clangStdenv
             withAWS: true
-            withCurlS3: false
     name: tests ${{ matrix.scenario }}
     runs-on: ${{ matrix.runs-on }}
     timeout-minutes: 60
@@ -126,15 +113,13 @@ jobs:
         nix build --file ci/gha/tests/wrapper.nix componentTests -L \
           --arg withInstrumentation ${{ matrix.instrumented }} \
           --argstr stdenv "${{ matrix.stdenv }}" \
-          ${{ format('--arg withAWS {0}', matrix.withAWS) }} \
-          ${{ format('--arg withCurlS3 {0}', matrix.withCurlS3) }}
+          ${{ format('--arg withAWS {0}', matrix.withAWS) }}
     - name: Run VM tests
       run: |
         nix build --file ci/gha/tests/wrapper.nix vmTests -L \
           --arg withInstrumentation ${{ matrix.instrumented }} \
           --argstr stdenv "${{ matrix.stdenv }}" \
-          ${{ format('--arg withAWS {0}', matrix.withAWS) }} \
-          ${{ format('--arg withCurlS3 {0}', matrix.withCurlS3) }}
+          ${{ format('--arg withAWS {0}', matrix.withAWS) }}
       if: ${{ matrix.os == 'linux' }}
     - name: Run flake checks and prepare the installer tarball
       run: |
@@ -147,7 +132,6 @@ jobs:
           --arg withInstrumentation ${{ matrix.instrumented }} \
           --argstr stdenv "${{ matrix.stdenv }}" \
           ${{ format('--arg withAWS {0}', matrix.withAWS) }} \
-          ${{ format('--arg withCurlS3 {0}', matrix.withCurlS3) }} \
           --out-link coverage-reports
         cat coverage-reports/index.txt >> $GITHUB_STEP_SUMMARY
       if: ${{ matrix.instrumented }}

ci/gha/tests/default.nix

@@ -13,7 +13,6 @@
   withSanitizers ? false,
   withCoverage ? false,
   withAWS ? null,
-  withCurlS3 ? null,
   ...
 }:
 
@@ -59,10 +58,7 @@ rec {
       nix-expr = prev.nix-expr.override { enableGC = !withSanitizers; };
 
       # Override AWS configuration if specified
-      nix-store = prev.nix-store.override (
-        lib.optionalAttrs (withAWS != null) { inherit withAWS; }
-        // lib.optionalAttrs (withCurlS3 != null) { inherit withCurlS3; }
-      );
+      nix-store = prev.nix-store.override (lib.optionalAttrs (withAWS != null) { inherit withAWS; });
 
       mesonComponentOverrides = lib.composeManyExtensions componentOverrides;
       # Unclear how to make Perl bindings work with a dynamically linked ASAN.
@@ -231,12 +227,7 @@ rec {
 
   vmTests = {
   }
-  # FIXME: when the curlS3 implementation is complete, it should also enable these tests.
   // lib.optionalAttrs (withAWS == true) {
-    # S3 binary cache store test only runs when S3 support is enabled
-    inherit (nixosTests) s3-binary-cache-store;
-  }
-  // lib.optionalAttrs (withCurlS3 == true) {
     # S3 binary cache store test using curl implementation
     inherit (nixosTests) curl-s3-binary-cache-store;
   }

ci/gha/tests/wrapper.nix

@@ -6,14 +6,13 @@
   componentTestsPrefix ? "",
   withInstrumentation ? false,
   withAWS ? null,
-  withCurlS3 ? null,
 }@args:
 import ./. (
   args
   // {
     getStdenv = p: p.${stdenv};
     withSanitizers = withInstrumentation;
     withCoverage = withInstrumentation;
-    inherit withAWS withCurlS3;
+    inherit withAWS;
   }
 )

doc/manual/rl-next/s3-curl-implementation.md

@@ -0,0 +1,26 @@
+---
+synopsis: "Improved S3 binary cache support via HTTP"
+prs: [13823, 14026, 14120, 14131, 14135, 14144, 14170, 14190, 14198, 14206, 14209, 14222, 14223, 13752]
+issues: [13084, 12671, 11748, 12403]
+---
+
+S3 binary cache operations now happen via HTTP, leveraging `libcurl`'s native
+AWS SigV4 authentication instead of the AWS C++ SDK, providing significant
+improvements:
+
+- **Reduced memory usage**: Eliminates memory buffering issues that caused
+  segfaults with large files
+- **Fixed upload reliability**: Resolves AWS SDK chunking errors
+  (`InvalidChunkSizeError`)
+- **Lighter dependencies**: Uses lightweight `aws-crt-cpp` instead of full
+  `aws-cpp-sdk`, reducing build complexity
+
+The new implementation requires curl >= 7.75.0 and `aws-crt-cpp` for credential
+management.
+
+All existing S3 URL formats and parameters remain supported, with the notable
+exception of multi-part uploads, which are no longer supported.
+
+Note that this change also means Nix now supports S3 binary cache stores even
+if build without `aws-crt-cpp`, but only for public buckets which do not
+require auth.

packaging/components.nix

@@ -490,7 +490,7 @@ in
 
       Example:
       ```
-      overrideScope (finalScope: prevScope: { aws-sdk-cpp = null; })
+      overrideScope (finalScope: prevScope: { aws-crt-cpp = null; })
       ```
     */
     overrideScope = f: (scope.overrideScope f).nix-everything;

packaging/dependencies.nix

@@ -16,21 +16,6 @@ in
 scope: {
   inherit stdenv;
 
-  aws-sdk-cpp =
-    (pkgs.aws-sdk-cpp.override {
-      apis = [
-        "identity-management"
-        "s3"
-        "transfer"
-      ];
-      customMemoryManagement = false;
-    }).overrideAttrs
-      {
-        # only a stripped down version is built, which takes a lot less resources
-        # to build, so we don't need a "big-parallel" machine.
-        requiredSystemFeatures = [ ];
-      };
-
   boehmgc =
     (pkgs.boehmgc.override {
       enableLargeConfig = true;

src/libstore-tests/s3-binary-cache-store.cc

@@ -1,27 +1,9 @@
 #include "nix/store/s3-binary-cache-store.hh"
+#include "nix/store/http-binary-cache-store.hh"
+#include "nix/store/filetransfer.hh"
+#include "nix/store/s3-url.hh"
 
-#if NIX_WITH_S3_SUPPORT
-
-#  include <gtest/gtest.h>
-
-namespace nix {
-
-TEST(S3BinaryCacheStore, constructConfig)
-{
-    S3BinaryCacheStoreConfig config{"s3", "foobar", {}};
-
-    EXPECT_EQ(config.bucketName, "foobar");
-}
-
-} // namespace nix
-
-#elif NIX_WITH_CURL_S3
-
-#  include "nix/store/http-binary-cache-store.hh"
-#  include "nix/store/filetransfer.hh"
-#  include "nix/store/s3-url.hh"
-
-#  include <gtest/gtest.h>
+#include <gtest/gtest.h>
 
 namespace nix {
 
@@ -141,5 +123,3 @@ TEST(S3BinaryCacheStore, parameterFiltering)
 }
 
 } // namespace nix
-
-#endif

src/libstore-tests/s3-url.cc

@@ -1,10 +1,8 @@
 #include "nix/store/s3-url.hh"
 #include "nix/util/tests/gmock-matchers.hh"
 
-#if NIX_WITH_S3_SUPPORT || NIX_WITH_CURL_S3
-
-#  include <gtest/gtest.h>
-#  include <gmock/gmock.h>
+#include <gtest/gtest.h>
+#include <gmock/gmock.h>
 
 namespace nix {
 
@@ -228,5 +226,3 @@ INSTANTIATE_TEST_SUITE_P(
     [](const ::testing::TestParamInfo<S3ToHttpsConversionTestCase> & info) { return info.param.description; });
 
 } // namespace nix
-
-#endif

src/libstore/aws-creds.cc

@@ -1,6 +1,6 @@
 #include "nix/store/aws-creds.hh"
 
-#if NIX_WITH_CURL_S3
+#if NIX_WITH_AWS_AUTH
 
 #  include <aws/crt/Types.h>
 #  include "nix/store/s3-url.hh"

src/libstore/builtins/fetchurl.cc

@@ -41,7 +41,7 @@ static void builtinFetchurl(const BuiltinBuilderContext & ctx)
             FileTransferRequest request(VerbatimURL{url});
             request.decompress = false;
 
-#if NIX_WITH_CURL_S3
+#if NIX_WITH_AWS_AUTH
             // Use pre-resolved credentials if available
             if (ctx.awsCredentials && request.uri.scheme() == "s3") {
                 debug("[pid=%d] Using pre-resolved AWS credentials from parent process", getpid());