libstore: Put all the AWS credentials logic behind interface class AwsCredentialProvider

This makes it so we don't need to rely on global variables and hacky destructors to
clean up another global variable. Just putting it in the correct order in the class
is more than enough.

Author: xokdvium

src/libstore/aws-creds.cc

@@ -24,50 +24,6 @@ namespace nix {
 
 namespace {
 
-// Global credential provider cache using boost's concurrent map
-// Key: profile name (empty string for default profile)
-using CredentialProviderCache =
-    boost::concurrent_flat_map<std::string, std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider>>;
-
-static CredentialProviderCache credentialProviderCache;
-
-/**
- * Clear all cached credential providers.
- * Called automatically by CrtWrapper destructor during static destruction.
- */
-static void clearAwsCredentialsCache()
-{
-    credentialProviderCache.clear();
-}
-
-static void initAwsCrt()
-{
-    struct CrtWrapper
-    {
-        Aws::Crt::ApiHandle apiHandle;
-
-        CrtWrapper()
-        {
-            apiHandle.InitializeLogging(Aws::Crt::LogLevel::Warn, static_cast<FILE *>(nullptr));
-        }
-
-        ~CrtWrapper()
-        {
-            try {
-                // CRITICAL: Clear credential provider cache BEFORE AWS CRT shuts down
-                // This ensures all providers (which hold references to ClientBootstrap)
-                // are destroyed while AWS CRT is still valid
-                clearAwsCredentialsCache();
-                // Now it's safe for ApiHandle destructor to run
-            } catch (...) {
-                ignoreExceptionInDestructor();
-            }
-        }
-    };
-
-    static CrtWrapper crt;
-}
-
 static AwsCredentials getCredentialsFromProvider(std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> provider)
 {
     if (!provider || !provider->IsValid()) {
@@ -113,7 +69,35 @@ static AwsCredentials getCredentialsFromProvider(std::shared_ptr<Aws::Crt::Auth:
 
 } // anonymous namespace
 
-AwsCredentials getAwsCredentials(const std::string & profile)
+class AwsCredentialProviderImpl : public AwsCredentialProvider
+{
+public:
+    AwsCredentialProviderImpl()
+    {
+        apiHandle.InitializeLogging(Aws::Crt::LogLevel::Warn, static_cast<FILE *>(nullptr));
+    }
+
+    AwsCredentials getCredentialsRaw(const std::string & profile);
+
+    AwsCredentials getCredentials(const ParsedS3URL & url) override
+    {
+        auto profile = url.profile.value_or("");
+        try {
+            return getCredentialsRaw(profile);
+        } catch (AwsAuthError & e) {
+            warn("AWS authentication failed for S3 request %s: %s", url.toHttpsUrl(), e.what());
+            credentialProviderCache.erase(profile);
+            throw;
+        }
+    }
+
+private:
+    Aws::Crt::ApiHandle apiHandle;
+    boost::concurrent_flat_map<std::string, std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider>>
+        credentialProviderCache;
+};
+
+AwsCredentials AwsCredentialProviderImpl::getCredentialsRaw(const std::string & profile)
 {
     // Get or create credential provider with caching
     std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> provider;
@@ -132,8 +116,6 @@ AwsCredentials getAwsCredentials(const std::string & profile)
                 profile.empty() ? "(default)" : profile.c_str());
 
             try {
-                initAwsCrt();
-
                 if (profile.empty()) {
                     Aws::Crt::Auth::CredentialsProviderChainDefaultConfig config;
                     config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
@@ -173,17 +155,15 @@ AwsCredentials getAwsCredentials(const std::string & profile)
     return getCredentialsFromProvider(provider);
 }
 
-void invalidateAwsCredentials(const std::string & profile)
+ref<AwsCredentialProvider> makeAwsCredentialsProvider()
 {
-    credentialProviderCache.erase(profile);
+    return make_ref<AwsCredentialProviderImpl>();
 }
 
-AwsCredentials preResolveAwsCredentials(const ParsedS3URL & s3Url)
+ref<AwsCredentialProvider> getAwsCredentialsProvider()
 {
-    std::string profile = s3Url.profile.value_or("");
-
-    // Get credentials (automatically cached)
-    return getAwsCredentials(profile);
+    static auto instance = makeAwsCredentialsProvider();
+    return instance;
 }
 
 } // namespace nix

src/libstore/filetransfer.cc

@@ -883,22 +883,12 @@ void FileTransferRequest::setupForS3()
     if (usernameAuth) {
         debug("Using pre-resolved AWS credentials from parent process");
         sessionToken = preResolvedAwsSessionToken;
-    } else {
-        std::string profile = parsedS3.profile.value_or("");
-        try {
-            auto creds = getAwsCredentials(profile);
-            usernameAuth = UsernameAuth{
-                .username = creds.accessKeyId,
-                .password = creds.secretAccessKey,
-            };
-            sessionToken = creds.sessionToken;
-        } catch (const AwsAuthError & e) {
-            warn("AWS authentication failed for S3 request %s: %s", uri, e.what());
-            // Invalidate the cached credentials so next request will retry
-            invalidateAwsCredentials(profile);
-            // Continue without authentication - might be a public bucket
-            return;
-        }
+    } else if (auto creds = getAwsCredentialsProvider()->maybeGetCredentials(parsedS3)) {
+        usernameAuth = UsernameAuth{
+            .username = creds->accessKeyId,
+            .password = creds->secretAccessKey,
+        };
+        sessionToken = creds->sessionToken;
     }
     if (sessionToken)
         headers.emplace_back("x-amz-security-token", *sessionToken);

src/libstore/include/nix/store/aws-creds.hh

@@ -5,6 +5,7 @@
 #if NIX_WITH_AWS_AUTH
 
 #  include "nix/store/s3-url.hh"
+#  include "nix/util/ref.hh"
 #  include "nix/util/error.hh"
 
 #  include <memory>
@@ -38,30 +39,39 @@ struct AwsCredentials
  */
 MakeError(AwsAuthError, Error);
 
-/**
- * Get AWS credentials for the given profile.
- * This function automatically caches credential providers to avoid
- * creating multiple providers for the same profile.
- *
- * @param profile The AWS profile name (empty string for default profile)
- * @return AWS credentials
- * @throws AwsAuthError if credentials cannot be resolved
- */
-AwsCredentials getAwsCredentials(const std::string & profile = "");
+class AwsCredentialProvider
+{
+public:
+    /**
+     * Get AWS credentials for the given URL.
+     *
+     * @param url The S3 url to get the credentials for
+     * @return AWS credentials
+     * @throws AwsAuthError if credentials cannot be resolved
+     */
+    virtual AwsCredentials getCredentials(const ParsedS3URL & url) = 0;
+
+    std::optional<AwsCredentials> maybeGetCredentials(const ParsedS3URL & url)
+    {
+        try {
+            return getCredentials(url);
+        } catch (AwsAuthError & e) {
+            return std::nullopt;
+        }
+    }
+
+    virtual ~AwsCredentialProvider() {}
+};
 
 /**
- * Invalidate cached credentials for a profile (e.g., on authentication failure).
- * The next request for this profile will create a new provider.
- *
- * @param profile The AWS profile name to invalidate
+ * Create a new instancee of AwsCredentialProvider.
  */
-void invalidateAwsCredentials(const std::string & profile);
+ref<AwsCredentialProvider> makeAwsCredentialsProvider();
 
 /**
- * Pre-resolve AWS credentials for S3 URLs.
- * Used to cache credentials in parent process before forking.
+ * Get a reference to the global AwsCredentialProvider.
  */
-AwsCredentials preResolveAwsCredentials(const ParsedS3URL & s3Url);
+ref<AwsCredentialProvider> getAwsCredentialsProvider();
 
 } // namespace nix
 #endif

src/libstore/unix/build/derivation-builder.cc

@@ -958,7 +958,7 @@ std::optional<AwsCredentials> DerivationBuilderImpl::preResolveAwsCredentials()
                     auto s3Url = ParsedS3URL::parse(parsedUrl);
 
                     // Use the preResolveAwsCredentials from aws-creds
-                    auto credentials = nix::preResolveAwsCredentials(s3Url);
+                    auto credentials = getAwsCredentialsProvider()->getCredentials(s3Url);
                     debug("Successfully pre-resolved AWS credentials in parent process");
                     return credentials;
                 }