s3: Fallback to anonymous credentials by default

This makes reading public s3 datasets work by default, whereas before
"anonymous" credentials were required.

PiperOrigin-RevId: 745343842
Change-Id: I20f86f1b4ab989f85d2e1afca040c935f46dbb83

Author: laramiel

tensorstore/internal/aws/credentials/common.cc

@@ -54,7 +54,7 @@ AwsCredentialsProvider MakeCache(AwsCredentialsProvider provider) {
 AwsCredentialsProvider MakeDefault(std::string_view profile_name_override) {
   aws_allocator* allocator = GetAwsAllocator();
 
-  struct aws_credentials_provider_chain_default_options options;
+  aws_credentials_provider_chain_default_options options;
   AWS_ZERO_STRUCT(options);
   options.bootstrap = GetAwsClientBootstrap();
   options.tls_ctx = GetAwsTlsCtx();
@@ -67,6 +67,26 @@ AwsCredentialsProvider MakeDefault(std::string_view profile_name_override) {
       aws_credentials_provider_new_chain_default(allocator, &options));
 }
 
+AwsCredentialsProvider MakeDefaultWithAnonymous(
+    std::string_view profile_name_override) {
+  aws_allocator* allocator = GetAwsAllocator();
+
+  auto default_provider = MakeDefault(profile_name_override);
+  auto anonymous_provider = MakeAnonymous();
+
+  aws_credentials_provider* providers[2];
+  AWS_ZERO_ARRAY(providers);
+  providers[0] = default_provider.get();
+  providers[1] = anonymous_provider.get();
+
+  aws_credentials_provider_chain_options options;
+  AWS_ZERO_STRUCT(options);
+  options.provider_count = 2;
+  options.providers = providers;
+
+  return AsProvider(aws_credentials_provider_new_chain(allocator, &options));
+}
+
 AwsCredentialsProvider MakeAnonymous() {
   aws_allocator* allocator = GetAwsAllocator();
 

tensorstore/internal/aws/credentials/common.h

@@ -25,6 +25,11 @@ namespace internal_aws {
 /// Returns a credentials provider that uses the AWS default credentials chain.
 AwsCredentialsProvider MakeDefault(std::string_view profile_name_override);
 
+/// Returns a credentials provider that uses the AWS default credentials chain
+/// with anonymous credentials as a fallback.
+AwsCredentialsProvider MakeDefaultWithAnonymous(
+    std::string_view profile_name_override);
+
 /// Returns anonymous credentials.
 AwsCredentialsProvider MakeAnonymous();
 

tensorstore/internal/aws/credentials/default_credential_provider_test.cc

@@ -40,7 +40,9 @@ using ::tensorstore::internal_aws::AwsCredentialsProvider;
 using ::tensorstore::internal_aws::DisableAwsHttpMocking;
 using ::tensorstore::internal_aws::EnableAwsHttpMocking;
 using ::tensorstore::internal_aws::GetAwsCredentials;
+using ::tensorstore::internal_aws::MakeAnonymous;
 using ::tensorstore::internal_aws::MakeDefault;
+using ::tensorstore::internal_aws::MakeDefaultWithAnonymous;
 
 static constexpr char kAccessKeyId[] = "ASIA1234567890";
 static constexpr char kSecretKey[] = "1234567890abcdef";
@@ -100,13 +102,27 @@ class DefaultCredentialProviderTest : public ::testing::Test {
   absl::Time expiry;
 };
 
-TEST_F(DefaultCredentialProviderTest, Basic) {
+TEST_F(DefaultCredentialProviderTest, Anonymous) {
+  AwsCredentialsProvider provider = MakeAnonymous();
+  auto credentials_future = GetAwsCredentials(provider.get());
+  ASSERT_THAT(credentials_future, IsOk());
+  EXPECT_TRUE(credentials_future.result()->IsAnonymous());
+}
+
+TEST_F(DefaultCredentialProviderTest, DefaultWithoutFallback) {
   AwsCredentialsProvider provider = MakeDefault({});
   auto credentials_future = GetAwsCredentials(provider.get());
   ASSERT_THAT(credentials_future,
               MatchesStatus(absl::StatusCode::kInternal, ".*aws-c-.*"));
 }
 
+TEST_F(DefaultCredentialProviderTest, DefaultWithFallback) {
+  AwsCredentialsProvider provider = MakeDefaultWithAnonymous({});
+  auto credentials_future = GetAwsCredentials(provider.get());
+  ASSERT_THAT(credentials_future, IsOk());
+  EXPECT_TRUE(credentials_future.result()->IsAnonymous());
+}
+
 TEST_F(DefaultCredentialProviderTest, FromEnvironment) {
   SetEnv("AWS_ACCESS_KEY_ID", kAccessKeyId);
   SetEnv("AWS_SECRET_ACCESS_KEY", kSecretKey);

tensorstore/kvstore/s3/aws_credentials_spec.cc

@@ -49,7 +49,7 @@ using Spec = ::tensorstore::internal_kvstore_s3::AwsCredentialsSpec;
 using ::tensorstore::internal_aws::AwsCredentialsProvider;
 using ::tensorstore::internal_aws::MakeAnonymous;
 using ::tensorstore::internal_aws::MakeCache;
-using ::tensorstore::internal_aws::MakeDefault;
+using ::tensorstore::internal_aws::MakeDefaultWithAnonymous;
 using ::tensorstore::internal_aws::MakeEcsRole;
 using ::tensorstore::internal_aws::MakeEnvironment;
 using ::tensorstore::internal_aws::MakeImds;
@@ -152,7 +152,9 @@ Result<AwsCredentialsProvider> MakeAwsCredentialsProvider(const Spec& spec) {
   struct MakeCredentialsVisitor {
     using R = AwsCredentialsProvider;
 
-    R operator()(const Spec::Default& v) { return MakeDefault(v.profile); }
+    R operator()(const Spec::Default& v) {
+      return MakeDefaultWithAnonymous(v.profile);
+    }
     R operator()(const Spec::Anonymous&) { return MakeAnonymous(); }
     R operator()(const Spec::Environment&) { return MakeEnvironment(); }
     R operator()(const Spec::Imds&) { return MakeImds(); }

tensorstore/kvstore/s3/s3_endpoint.cc

@@ -114,6 +114,7 @@ struct ResolveHost {
           default_aws_region,
       });
     }
+    // TODO: Get the message from the response body, if any.
     promise.SetResult(absl::FailedPreconditionError(tensorstore::StrCat(
         "Failed to resolve aws_region for bucket ", QuoteString(bucket))));
   }
@@ -201,21 +202,20 @@ Future<S3EndpointRegion> ResolveEndpointRegion(
   assert(IsValidBucketName(bucket));
 
   // TODO: Handle retries, and sign the request.
-
   if (endpoint.empty()) {
     // Use a HEAD request on bucket.s3.amazonaws.com to acquire the aws_region
     // does not work when there is a . in the bucket name; for now require the
     // aws_region to be set.
     if (!absl::StrContains(bucket, ".")) {
-      std::string url = absl::StrFormat("https://%s.s3.amazonaws.com", bucket);
+      auto request =
+          HttpRequestBuilder(
+              "HEAD", absl::StrFormat("https://%s.s3.amazonaws.com", bucket))
+              .AddHostHeader(host_header)
+              .BuildRequest();
       return PromiseFuturePair<S3EndpointRegion>::Link(
                  ResolveHost<S3VirtualHostFormatter>{
                      std::move(bucket), {}, S3VirtualHostFormatter{}},
-                 transport->IssueRequest(
-                     HttpRequestBuilder("HEAD", std::move(url))
-                         .AddHostHeader(host_header)
-                         .BuildRequest(),
-                     {}))
+                 transport->IssueRequest(std::move(request), {}))
           .future;
     }
 
@@ -224,30 +224,30 @@ Future<S3EndpointRegion> ResolveEndpointRegion(
     // zone. The response will be a 301 request with an 'x-amz-bucket-region'
     // header. We might be able to just do a signed HEAD request against an
     // possibly non-existent file... But try this later.
-    std::string url =
-        absl::StrFormat("https://s3.us-east-1.amazonaws.com/%s", bucket);
+    auto request =
+        HttpRequestBuilder(
+            "HEAD",
+            absl::StrFormat("https://s3.us-east-1.amazonaws.com/%s", bucket))
+            .AddHostHeader(host_header)
+            .BuildRequest();
     return PromiseFuturePair<S3EndpointRegion>::Link(
                ResolveHost<S3PathFormatter>{
                    std::move(bucket), {}, S3PathFormatter{}},
-               transport->IssueRequest(
-                   HttpRequestBuilder("HEAD", std ::move(url))
-                       .AddHostHeader(host_header)
-                       .BuildRequest(),
-                   {}))
+               transport->IssueRequest(std::move(request), {}))
         .future;
   }
 
   // Issue a HEAD request against the endpoint+bucket, which should work for
   // mock S3 backends like localstack or minio.
-  std::string url = absl::StrFormat("%s/%s", endpoint, bucket);
+  auto request =
+      HttpRequestBuilder("HEAD", absl::StrFormat("%s/%s", endpoint, bucket))
+          .AddHostHeader(host_header)
+          .BuildRequest();
   return PromiseFuturePair<S3EndpointRegion>::Link(
              ResolveHost<S3CustomFormatter>{
                  std::move(bucket), "us-east-1",
                  S3CustomFormatter{std::string(endpoint)}},
-             transport->IssueRequest(HttpRequestBuilder("HEAD", std::move(url))
-                                         .AddHostHeader(host_header)
-                                         .BuildRequest(),
-                                     {}))
+             transport->IssueRequest(std::move(request), {}))
       .future;
 }
 

tensorstore/kvstore/s3/schema.yml

@@ -210,6 +210,8 @@ definitions:
     description: |-
       Source credentials using the `default AWS credentials chain
       <https://github.com/aws/aws-sdk-cpp/blob/main/docs/Credentials_Providers.md>`__.
+      If the default credentials chain cannot resolve credentials then anonymous credentials
+      will be used.
     allOf:
     - $ref: Context.aws_credentials
     - type: object