Merge pull request #14206 from lovesegfault/curl-based-s3-pieces

feat(libstore): add builtin fetchurl S3 credential pre-resolution

Author: xokdvium

src/libstore/builtins/fetchurl.cc

@@ -33,13 +33,26 @@ static void builtinFetchurl(const BuiltinBuilderContext & ctx)
 
     /* Note: have to use a fresh fileTransfer here because we're in
        a forked process. */
+    debug("[pid=%d] builtin:fetchurl creating fresh FileTransfer instance", getpid());
     auto fileTransfer = makeFileTransfer();
 
     auto fetch = [&](const std::string & url) {
         auto source = sinkToSource([&](Sink & sink) {
             FileTransferRequest request(VerbatimURL{url});
             request.decompress = false;
 
+#if NIX_WITH_CURL_S3
+            // Use pre-resolved credentials if available
+            if (ctx.awsCredentials && request.uri.scheme() == "s3") {
+                debug("[pid=%d] Using pre-resolved AWS credentials from parent process", getpid());
+                request.usernameAuth = UsernameAuth{
+                    .username = ctx.awsCredentials->accessKeyId,
+                    .password = ctx.awsCredentials->secretAccessKey,
+                };
+                request.preResolvedAwsSessionToken = ctx.awsCredentials->sessionToken;
+            }
+#endif
+
             auto decompressor = makeDecompressionSink(unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink);
             fileTransfer->download(std::move(request), *decompressor);
             decompressor->finish();

src/libstore/include/nix/store/builtins.hh

@@ -2,6 +2,11 @@
 ///@file
 
 #include "nix/store/derivations.hh"
+#include "nix/store/config.hh"
+
+#if NIX_WITH_CURL_S3
+#  include "nix/store/aws-creds.hh"
+#endif
 
 namespace nix {
 
@@ -12,6 +17,14 @@ struct BuiltinBuilderContext
     std::string netrcData;
     std::string caFileData;
     Path tmpDirInSandbox;
+
+#if NIX_WITH_CURL_S3
+    /**
+     * Pre-resolved AWS credentials for S3 URLs in builtin:fetchurl.
+     * When present, these should be used instead of creating new credential providers.
+     */
+    std::optional<AwsCredentials> awsCredentials;
+#endif
 };
 
 using BuiltinBuilder = std::function<void(const BuiltinBuilderContext &)>;

src/libstore/unix/build/derivation-builder.cc

@@ -46,6 +46,12 @@
 #include "store-config-private.hh"
 #include "build/derivation-check.hh"
 
+#if NIX_WITH_CURL_S3
+#  include "nix/store/aws-creds.hh"
+#  include "nix/store/s3-url.hh"
+#  include "nix/util/url.hh"
+#endif
+
 namespace nix {
 
 struct NotDeterministic : BuildError
@@ -290,6 +296,15 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder
      */
     virtual void startChild();
 
+#if NIX_WITH_CURL_S3
+    /**
+     * Pre-resolve AWS credentials for S3 URLs in builtin:fetchurl.
+     * This should be called before forking to ensure credentials are available in child.
+     * Returns the credentials if successfully resolved, or std::nullopt otherwise.
+     */
+    std::optional<AwsCredentials> preResolveAwsCredentials();
+#endif
+
 private:
 
     /**
@@ -339,10 +354,20 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder
      */
     void writeBuilderFile(const std::string & name, std::string_view contents);
 
+    /**
+     * Arguments passed to runChild().
+     */
+    struct RunChildArgs
+    {
+#if NIX_WITH_CURL_S3
+        std::optional<AwsCredentials> awsCredentials;
+#endif
+    };
+
     /**
      * Run the builder's process.
      */
-    void runChild();
+    void runChild(RunChildArgs args);
 
     /**
      * Move the current process into the chroot, if any. Called early
@@ -920,11 +945,43 @@ void DerivationBuilderImpl::openSlave()
         throw SysError("cannot pipe standard error into log file");
 }
 
+#if NIX_WITH_CURL_S3
+std::optional<AwsCredentials> DerivationBuilderImpl::preResolveAwsCredentials()
+{
+    if (drv.isBuiltin() && drv.builder == "builtin:fetchurl") {
+        auto url = drv.env.find("url");
+        if (url != drv.env.end()) {
+            try {
+                auto parsedUrl = parseURL(url->second);
+                if (parsedUrl.scheme == "s3") {
+                    debug("Pre-resolving AWS credentials for S3 URL in builtin:fetchurl");
+                    auto s3Url = ParsedS3URL::parse(parsedUrl);
+
+                    // Use the preResolveAwsCredentials from aws-creds
+                    auto credentials = nix::preResolveAwsCredentials(s3Url);
+                    debug("Successfully pre-resolved AWS credentials in parent process");
+                    return credentials;
+                }
+            } catch (const std::exception & e) {
+                debug("Error pre-resolving S3 credentials: %s", e.what());
+            }
+        }
+    }
+    return std::nullopt;
+}
+#endif
+
 void DerivationBuilderImpl::startChild()
 {
-    pid = startProcess([&]() {
+    RunChildArgs args{
+#if NIX_WITH_CURL_S3
+        .awsCredentials = preResolveAwsCredentials(),
+#endif
+    };
+
+    pid = startProcess([this, args = std::move(args)]() {
         openSlave();
-        runChild();
+        runChild(std::move(args));
     });
 }
 
@@ -1181,7 +1238,7 @@ void DerivationBuilderImpl::writeBuilderFile(const std::string & name, std::stri
     chownToBuilder(fd.get(), path);
 }
 
-void DerivationBuilderImpl::runChild()
+void DerivationBuilderImpl::runChild(RunChildArgs args)
 {
     /* Warning: in the child we should absolutely not make any SQLite
        calls! */
@@ -1198,6 +1255,9 @@ void DerivationBuilderImpl::runChild()
         BuiltinBuilderContext ctx{
             .drv = drv,
             .tmpDirInSandbox = tmpDirInSandbox(),
+#if NIX_WITH_CURL_S3
+            .awsCredentials = args.awsCredentials,
+#endif
         };
 
         if (drv.isBuiltin() && drv.builder == "builtin:fetchurl") {

src/libstore/unix/build/linux-derivation-builder.cc

@@ -276,6 +276,12 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu
 
     void startChild() override
     {
+        RunChildArgs args{
+#  if NIX_WITH_CURL_S3
+            .awsCredentials = preResolveAwsCredentials(),
+#  endif
+        };
+
         /* Set up private namespaces for the build:
 
            - The PID namespace causes the build to start as PID 1.
@@ -343,7 +349,7 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu
                 if (usingUserNamespace)
                     options.cloneFlags |= CLONE_NEWUSER;
 
-                pid_t child = startProcess([&]() { runChild(); }, options);
+                pid_t child = startProcess([this, args = std::move(args)]() { runChild(std::move(args)); }, options);
 
                 writeFull(sendPid.writeSide.get(), fmt("%d\n", child));
                 _exit(0);