[fix](s3)Use anonymous credentials for S3-compatible storage when credentials are absent (#60443)

…

For S3-compatible object storages such as COS and OBS, authentication
does not fully follow the AWS native credential provider chain. In these
systems, anonymous access is a valid and commonly used mode when no
AK/SK is configured.

However, when `aws_credentials_provider_version` is set to `v2`, BE
currently falls back to the AWS SDK v2 default credential provider chain
if no credentials are explicitly provided. This behavior is AWS-specific
and is not applicable to S3-compatible storage systems like COS and OBS,
which may not support or require the AWS credential resolution chain.

Problem
-------
When `aws_credentials_provider_version = v2` and no AK/SK is configured:
- BE attempts to resolve credentials using the AWS SDK v2 provider chain
- This may lead to unexpected authentication failures or unnecessary
environment dependency
- The behavior is inconsistent with S3-compatible storage expectations

Expected Behavior
-----------------
For S3-compatible object storage:
- If AK/SK is explicitly provided, use the configured credentials
- If no credentials are provided, fall back to anonymous credentials
- Do NOT trigger the AWS SDK v2 default credential provider chain

Author: CalvinKirs

fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java

@@ -18,6 +18,7 @@
 package org.apache.doris.datasource.property.storage;
 
 import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode;
 
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
@@ -108,9 +109,20 @@ private Map<String, String> doBuildS3Configuration(String maxConnections,
         if (StringUtils.isNotBlank(getSessionToken())) {
             s3Props.put("AWS_TOKEN", getSessionToken());
         }
+        String credentialsProviderType = getAwsCredentialsProviderTypeForBackend();
+        if (StringUtils.isNotBlank(credentialsProviderType)) {
+            s3Props.put("AWS_CREDENTIALS_PROVIDER_TYPE", credentialsProviderType);
+        }
         return s3Props;
     }
 
+    protected String getAwsCredentialsProviderTypeForBackend() {
+        if (StringUtils.isBlank(getAccessKey()) && StringUtils.isBlank(getSecretKey())) {
+            return AwsCredentialsProviderMode.ANONYMOUS.name();
+        }
+        return null;
+    }
+
     @Override
     public Map<String, String> getBackendConfigProperties() {
         return generateBackendS3Configuration();

fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java

@@ -291,13 +291,14 @@ public Map<String, String> getBackendConfigProperties() {
         if (StringUtils.isNotBlank(s3ExternalId)) {
             backendProperties.put("AWS_EXTERNAL_ID", s3ExternalId);
         }
-        // Pass credentials provider type to BE
-        if (awsCredentialsProviderMode != null) {
-            backendProperties.put("AWS_CREDENTIALS_PROVIDER_TYPE", awsCredentialsProviderMode.getMode());
-        }
         return backendProperties;
     }
 
+    @Override
+    protected String getAwsCredentialsProviderTypeForBackend() {
+        return awsCredentialsProviderMode == null ? null : awsCredentialsProviderMode.getMode();
+    }
+
     private void convertGlueToS3EndpointIfNeeded() {
         if (this.endpoint.contains("glue")) {
             this.endpoint = "https://s3." + this.region + ".amazonaws.com";
@@ -671,4 +672,3 @@ public static TS3StorageParam getS3TStorageParam(Map<String, String> properties)
     }
 
 }
-

fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java

@@ -173,10 +173,14 @@ public void testAwsCredentialsProvider() throws Exception {
         props.put("cos.endpoint", "cos.ap-beijing.myqcloud.com");
         COSProperties obsStorageProperties = (COSProperties) StorageProperties.createPrimary(props);
         Assertions.assertEquals(AnonymousCredentialsProvider.class, obsStorageProperties.getAwsCredentialsProvider().getClass());
+        Map<String, String> backendProps = obsStorageProperties.getBackendConfigProperties();
+        Assertions.assertEquals("ANONYMOUS", backendProps.get("AWS_CREDENTIALS_PROVIDER_TYPE"));
         props.put("cos.access_key", "myAccessKey");
         props.put("cos.secret_key", "mySecretKey");
         obsStorageProperties = (COSProperties) StorageProperties.createPrimary(props);
         Assertions.assertEquals(StaticCredentialsProvider.class, obsStorageProperties.getAwsCredentialsProvider().getClass());
+        backendProps = obsStorageProperties.getBackendConfigProperties();
+        Assertions.assertNull(backendProps.get("AWS_CREDENTIALS_PROVIDER_TYPE"));
     }
 
     @Test

fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java

@@ -149,10 +149,14 @@ public void testAwsCredentialsProvider() throws Exception {
         props.put("obs.endpoint", "obs.cn-north-4.myhuaweicloud.com");
         OBSProperties obsStorageProperties = (OBSProperties) StorageProperties.createPrimary(props);
         Assertions.assertEquals(AnonymousCredentialsProvider.class, obsStorageProperties.getAwsCredentialsProvider().getClass());
+        Map<String, String> backendProps = obsStorageProperties.getBackendConfigProperties();
+        Assertions.assertEquals("ANONYMOUS", backendProps.get("AWS_CREDENTIALS_PROVIDER_TYPE"));
         props.put("obs.access_key", "myAccessKey");
         props.put("obs.secret_key", "mySecretKey");
         obsStorageProperties = (OBSProperties) StorageProperties.createPrimary(props);
         Assertions.assertEquals(StaticCredentialsProvider.class, obsStorageProperties.getAwsCredentialsProvider().getClass());
+        backendProps = obsStorageProperties.getBackendConfigProperties();
+        Assertions.assertNull(backendProps.get("AWS_CREDENTIALS_PROVIDER_TYPE"));
     }
 
     @Test

fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java

@@ -246,10 +246,14 @@ public void testAwsCredentialsProvider() throws Exception {
         ossProps.put("oss.endpoint", "oss-cn-hangzhou.aliyuncs.com");
         OSSProperties ossStorageProperties = (OSSProperties) StorageProperties.createPrimary(ossProps);
         Assertions.assertEquals(AnonymousCredentialsProvider.class, ossStorageProperties.getAwsCredentialsProvider().getClass());
+        Map<String, String> backendProps = ossStorageProperties.getBackendConfigProperties();
+        Assertions.assertEquals("ANONYMOUS", backendProps.get("AWS_CREDENTIALS_PROVIDER_TYPE"));
         ossProps.put("oss.access_key", "myAccessKey");
         ossProps.put("oss.secret_key", "mySecretKey");
         ossStorageProperties = (OSSProperties) StorageProperties.createPrimary(ossProps);
         Assertions.assertEquals(StaticCredentialsProvider.class, ossStorageProperties.getAwsCredentialsProvider().getClass());
+        backendProps = ossStorageProperties.getBackendConfigProperties();
+        Assertions.assertNull(backendProps.get("AWS_CREDENTIALS_PROVIDER_TYPE"));
     }
 
     @Test

fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java

@@ -149,6 +149,7 @@ public void testToNativeS3Configuration() throws UserException {
         Assertions.assertEquals("88", s3Props.get("AWS_MAX_CONNECTIONS"));
         Assertions.assertEquals("6000", s3Props.get("AWS_CONNECTION_TIMEOUT_MS"));
         Assertions.assertEquals("true", s3Props.get("use_path_style"));
+        Assertions.assertEquals("DEFAULT", s3Props.get("AWS_CREDENTIALS_PROVIDER_TYPE"));
         origProps.remove("use_path_style");
         origProps.remove("s3.connection.maximum");
         origProps.remove("s3.connection.timeout");
@@ -159,6 +160,30 @@ public void testToNativeS3Configuration() throws UserException {
         Assertions.assertEquals("6000", s3Props.get("AWS_CONNECTION_TIMEOUT_MS"));
     }
 
+    @Test
+    public void testBackendCredentialsProviderType() throws UserException {
+        Map<String, String> props = new HashMap<>();
+        props.put("s3.endpoint", "s3.us-west-2.amazonaws.com");
+        props.put("s3.region", "us-west-2");
+
+        String[][] cases = new String[][] {
+                {"default", "DEFAULT"},
+                {"env", "ENV"},
+                {"system_properties", "SYSTEM_PROPERTIES"},
+                {"web_identity", "WEB_IDENTITY"},
+                {"container", "CONTAINER"},
+                {"instance_profile", "INSTANCE_PROFILE"},
+                {"anonymous", "ANONYMOUS"}
+        };
+
+        for (String[] testCase : cases) {
+            props.put("s3.credentials_provider_type", testCase[0]);
+            S3Properties s3Props = (S3Properties) StorageProperties.createPrimary(props);
+            Map<String, String> backendProps = s3Props.getBackendConfigProperties();
+            Assertions.assertEquals(testCase[1], backendProps.get("AWS_CREDENTIALS_PROVIDER_TYPE"));
+        }
+    }
+
 
     @Test
     public void testGetRegion() throws UserException {