Skip to content

Commit 58e3f6d

Browse files
CalvinKirsCalvinKirs
authored
[feat](catalog)Support for Loading Catalog Credentials via AwsCredentialsProviderChain (#58740)
### Changes This update enables Catalogs to automatically load credentials for S3 or S3-compatible storage using AwsCredentialsProviderChain. Users no longer need to explicitly specify AK/SK in Catalog properties. The feature provides stronger support for cloud-native environments such as IRSA, containers, and EC2 Instance Profiles. ↳ All credential resolution is unified through DefaultDorisAwsCredentialsProviderChain, ensuring consistent and secure handling across the project. Key Behavior #### No need to configure AK/SK manually: Credentials can be automatically derived from IRSA, WebIdentity, container metadata, EC2 instance profile, environment variables, system properties, or AWS profile files. ↳ #### Unified credential resolution: All modules use DefaultDorisAwsCredentialsProviderChain to avoid inconsistent custom implementations. ↳ eg ``` CREATE CATALOG `iceberg_fs_catalog` PROPERTIES ( "type" = "iceberg", "iceberg.catalog.type" = "hadoop", "warehouse" = "s3://<bucket>/iceberg/fs/s3/warehouse/", "s3.region"="us-east-1", "s3.endpoint" = "s3.us-east-1.amazonaws.com" ); ```
1 parent 43f2d40 commit 58e3f6d

File tree

11 files changed

+462
-43
lines changed

11 files changed

+462
-43
lines changed

‎fe/fe-core/src/main/java/com/amazonaws/glue/catalog/credentials/ConfigurationAWSCredentialsProvider.java‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,12 @@
2222
import com.amazonaws.auth.AWSCredentialsProvider;
2323
import com.amazonaws.auth.BasicAWSCredentials;
2424
import com.amazonaws.auth.BasicSessionCredentials;
25-
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
2625
import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider;
2726
import com.amazonaws.glue.catalog.util.AWSGlueConfig;
2827
import com.amazonaws.util.StringUtils;
28+
import org.apache.doris.common.Config;
29+
import org.apache.doris.datasource.property.common.AwsCredentialsProviderFactory;
30+
import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode;
2931
import org.apache.hadoop.conf.Configuration;
3032

3133
public class ConfigurationAWSCredentialsProvider implements AWSCredentialsProvider {
@@ -47,9 +49,9 @@ public AWSCredentials getCredentials() {
4749
return (StringUtils.isNullOrEmpty(sessionToken) ? new BasicAWSCredentials(accessKey,
4850
secretKey) : new BasicSessionCredentials(accessKey, secretKey, sessionToken));
4951
}
50-
51-
AWSCredentialsProvider longLivedProvider = new DefaultAWSCredentialsProviderChain();
52-
52+
String credentialsProviderModeString = StringUtils.lowerCase(conf.get(AWSGlueConfig.AWS_CREDENTIALS_PROVIDER_MODE));
53+
AwsCredentialsProviderMode credentialsProviderMode=AwsCredentialsProviderMode.fromString(credentialsProviderModeString);
54+
AWSCredentialsProvider longLivedProvider = AwsCredentialsProviderFactory.createV1(credentialsProviderMode);
5355
if (!StringUtils.isNullOrEmpty(roleArn)) {
5456
STSAssumeRoleSessionCredentialsProvider.Builder builder =
5557
new STSAssumeRoleSessionCredentialsProvider.Builder(roleArn, "local-session")
@@ -61,6 +63,9 @@ public AWSCredentials getCredentials() {
6163
STSAssumeRoleSessionCredentialsProvider provider = builder.build();
6264
return provider.getCredentials();
6365
}
66+
if (Config.aws_credentials_provider_version.equalsIgnoreCase("v2")) {
67+
return longLivedProvider.getCredentials();
68+
}
6469
throw new SdkClientException("Unable to load AWS credentials from any provider in the chain");
6570

6671
}

‎fe/fe-core/src/main/java/com/amazonaws/glue/catalog/util/AWSGlueConfig.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,4 +63,5 @@ private AWSGlueConfig() {
6363
public static final String AWS_GLUE_SESSION_TOKEN = "aws.glue.session-token";
6464
public static final String AWS_GLUE_ROLE_ARN = "aws.glue.role-arn";
6565
public static final String AWS_GLUE_EXTERNAL_ID = "aws.glue.external-id";
66+
public static final String AWS_CREDENTIALS_PROVIDER_MODE = "aws.credentials.provider.mode";
6667
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderFactory.java‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
//
18+
// Copied from
19+
// https://github.com/awslabs/aws-glue-data-catalog-client-for-apache-hive-metastore/blob/branch-3.4.0/
20+
//
21+
22+
package org.apache.doris.datasource.property.common;
23+
24+
25+
import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
26+
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
27+
import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain;
28+
import software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider;
29+
import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider;
30+
import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
31+
import software.amazon.awssdk.auth.credentials.SystemPropertyCredentialsProvider;
32+
import software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider;
33+
34+
import java.util.ArrayList;
35+
import java.util.List;
36+
37+
public final class AwsCredentialsProviderFactory {
38+
39+
private AwsCredentialsProviderFactory() {
40+
}
41+
42+
/* =========================
43+
* AWS SDK V1
44+
* ========================= */
45+
46+
public static com.amazonaws.auth.AWSCredentialsProvider createV1(
47+
AwsCredentialsProviderMode mode) {
48+
49+
switch (mode) {
50+
case ENV:
51+
return new com.amazonaws.auth.EnvironmentVariableCredentialsProvider();
52+
case SYSTEM_PROPERTIES:
53+
return new com.amazonaws.auth.SystemPropertiesCredentialsProvider();
54+
case WEB_IDENTITY:
55+
return com.amazonaws.auth.WebIdentityTokenCredentialsProvider.create();
56+
case CONTAINER:
57+
return new com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper();
58+
case ANONYMOUS:
59+
throw new UnsupportedOperationException(
60+
"AWS SDK V1 does not support anonymous credentials provider.");
61+
case INSTANCE_PROFILE:
62+
return new com.amazonaws.auth.InstanceProfileCredentialsProvider();
63+
case DEFAULT:
64+
return createDefaultV1();
65+
default:
66+
throw new UnsupportedOperationException(
67+
"AWS SDK V1 does not support credentials provider mode: " + mode);
68+
}
69+
}
70+
71+
private static com.amazonaws.auth.AWSCredentialsProvider createDefaultV1() {
72+
List<com.amazonaws.auth.AWSCredentialsProvider> providers = new ArrayList<>();
73+
providers.add(new com.amazonaws.auth.InstanceProfileCredentialsProvider());
74+
//lazy + env
75+
providers.add(com.amazonaws.auth.WebIdentityTokenCredentialsProvider.create());
76+
providers.add(new com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper());
77+
providers.add(new com.amazonaws.auth.EnvironmentVariableCredentialsProvider());
78+
providers.add(new com.amazonaws.auth.SystemPropertiesCredentialsProvider());
79+
return new com.amazonaws.auth.AWSCredentialsProviderChain(
80+
providers.toArray(new com.amazonaws.auth.AWSCredentialsProvider[0]));
81+
}
82+
83+
/* =========================
84+
* AWS SDK V2
85+
* ========================= */
86+
87+
public static AwsCredentialsProvider createV2(
88+
AwsCredentialsProviderMode mode,
89+
boolean includeAnonymousInDefault) {
90+
switch (mode) {
91+
case ENV:
92+
return EnvironmentVariableCredentialsProvider.create();
93+
case SYSTEM_PROPERTIES:
94+
return SystemPropertyCredentialsProvider.create();
95+
case WEB_IDENTITY:
96+
return WebIdentityTokenFileCredentialsProvider.create();
97+
case CONTAINER:
98+
return ContainerCredentialsProvider.create();
99+
case INSTANCE_PROFILE:
100+
return InstanceProfileCredentialsProvider.create();
101+
case ANONYMOUS:
102+
return AnonymousCredentialsProvider.create();
103+
case DEFAULT:
104+
return createDefaultV2(includeAnonymousInDefault);
105+
default:
106+
throw new UnsupportedOperationException(
107+
"AWS SDK V2 does not support credentials provider mode: " + mode);
108+
}
109+
}
110+
111+
private static AwsCredentialsProvider createDefaultV2(
112+
boolean includeAnonymous) {
113+
114+
List<AwsCredentialsProvider> providers = new ArrayList<>();
115+
providers.add(InstanceProfileCredentialsProvider.create());
116+
providers.add(WebIdentityTokenFileCredentialsProvider.create());
117+
providers.add(ContainerCredentialsProvider.create());
118+
providers.add(EnvironmentVariableCredentialsProvider.create());
119+
providers.add(SystemPropertyCredentialsProvider.create());
120+
if (includeAnonymous) {
121+
providers.add(AnonymousCredentialsProvider.create());
122+
}
123+
return AwsCredentialsProviderChain.builder()
124+
.credentialsProviders(providers)
125+
.build();
126+
}
127+
128+
public static String getV2ClassName(AwsCredentialsProviderMode mode, boolean includeAnonymousInDefault) {
129+
switch (mode) {
130+
case ENV:
131+
return EnvironmentVariableCredentialsProvider.class.getName();
132+
case SYSTEM_PROPERTIES:
133+
return SystemPropertyCredentialsProvider.class.getName();
134+
case WEB_IDENTITY:
135+
return WebIdentityTokenFileCredentialsProvider.class.getName();
136+
case CONTAINER:
137+
return ContainerCredentialsProvider.class.getName();
138+
case INSTANCE_PROFILE:
139+
return InstanceProfileCredentialsProvider.class.getName();
140+
case ANONYMOUS:
141+
return AnonymousCredentialsProvider.class.getName();
142+
case DEFAULT:
143+
List<String> providers = new ArrayList<>();
144+
providers.add(EnvironmentVariableCredentialsProvider.class.getName());
145+
providers.add(SystemPropertyCredentialsProvider.class.getName());
146+
providers.add(WebIdentityTokenFileCredentialsProvider.class.getName());
147+
providers.add(ContainerCredentialsProvider.class.getName());
148+
providers.add(InstanceProfileCredentialsProvider.class.getName());
149+
if (includeAnonymousInDefault) {
150+
providers.add(AnonymousCredentialsProvider.class.getName());
151+
}
152+
return String.join("+", providers);
153+
default:
154+
throw new UnsupportedOperationException(
155+
"AWS SDK V2 does not support credentials provider mode: " + mode);
156+
}
157+
}
158+
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderMode.java‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.datasource.property.common;
19+
20+
public enum AwsCredentialsProviderMode {
21+
22+
DEFAULT("DEFAULT"),
23+
24+
ENV("ENV"),
25+
26+
SYSTEM_PROPERTIES("SYSTEM_PROPERTIES"),
27+
28+
WEB_IDENTITY("WEB_IDENTITY"),
29+
30+
CONTAINER("CONTAINER"),
31+
32+
INSTANCE_PROFILE("INSTANCE_PROFILE"),
33+
34+
ANONYMOUS("ANONYMOUS");
35+
36+
private final String mode;
37+
38+
AwsCredentialsProviderMode(String mode) {
39+
this.mode = mode;
40+
}
41+
42+
public String getMode() {
43+
return mode;
44+
}
45+
46+
47+
public static AwsCredentialsProviderMode fromString(String value) {
48+
if (value == null || value.isEmpty()) {
49+
return DEFAULT;
50+
}
51+
52+
String normalized = value.trim().toUpperCase().replace('-', '_');
53+
54+
switch (normalized) {
55+
case "ENV":
56+
return ENV;
57+
case "SYSTEM_PROPERTIES":
58+
return SYSTEM_PROPERTIES;
59+
case "WEB_IDENTITY":
60+
return WEB_IDENTITY;
61+
case "CONTAINER":
62+
return CONTAINER;
63+
case "INSTANCE_PROFILE":
64+
return INSTANCE_PROFILE;
65+
case "ANONYMOUS":
66+
return ANONYMOUS;
67+
case "DEFAULT":
68+
return DEFAULT;
69+
default:
70+
throw new IllegalArgumentException(
71+
"Unsupported AWS credentials provider mode: " + value);
72+
}
73+
}
74+
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,8 +114,6 @@ private ParamRules buildRules() {
114114

115115
return new ParamRules().requireTogether(new String[]{glueAccessKey, glueSecretKey},
116116
"glue.access_key and glue.secret_key must be set together")
117-
.requireAtLeastOne(new String[]{glueAccessKey, glueIAMRole},
118-
"At least one of glue.access_key or glue.role_arn must be set")
119117
.require(glueEndpoint, "glue.endpoint must be set")
120118
.check(() -> StringUtils.isNotBlank(glueEndpoint) && !glueEndpoint.startsWith("https://"),
121119
"glue.endpoint must use https protocol,please set glue.endpoint to https://...");
@@ -137,6 +135,22 @@ private void checkAndInit() {
137135
}
138136
}
139137

138+
/**
139+
* Validate that at least one Glue credential (an access key or an IAM role) is explicitly provided.
140+
*
141+
* Purpose: Some catalog implementations (for example, Iceberg) do not support obtaining credentials
142+
* from the default credential chain (instance metadata, environment variables, etc.). In addition,
143+
* the configuration or UI may only expose two options: {@code glue.access_key} and {@code glue.role_arn}.
144+
* In such cases, at least one of these must be explicitly set. If neither is provided, an
145+
* {@link IllegalArgumentException} is thrown to prompt the user to complete the configuration.
146+
*/
147+
protected void requireExplicitGlueCredentials() {
148+
if (StringUtils.isNotBlank(glueAccessKey) || StringUtils.isNotBlank(glueIAMRole)) {
149+
return;
150+
}
151+
throw new IllegalArgumentException("At least one of glue.access_key or glue.role_arn must be set");
152+
}
153+
140154
private String extractRegionFromEndpoint(Matcher matcher) {
141155
for (int i = 1; i <= matcher.groupCount(); i++) {
142156
String group = matcher.group(i);

‎fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HiveGlueMetaStoreProperties.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package org.apache.doris.datasource.property.metastore;
1919

2020
import org.apache.doris.datasource.property.ConnectorProperty;
21+
import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode;
2122

2223
import com.amazonaws.ClientConfiguration;
2324
import com.amazonaws.glue.catalog.util.AWSGlueConfig;
@@ -75,6 +76,14 @@ public class HiveGlueMetaStoreProperties extends AbstractHiveProperties {
7576
description = "Catalog separator character for AWS Glue.")
7677
protected String awsGlueCatalogSeparator = "";
7778

79+
@ConnectorProperty(names = {"glue.credentials_provider_type"},
80+
required = false,
81+
description = "The credentials provider type of S3. "
82+
+ "Options are: DEFAULT, ASSUME_ROLE, ANONYMOUS, ENVIRONMENT, SYSTEM_PROPERTIES, "
83+
+ "WEB_IDENTITY_TOKEN_FILE, INSTANCE_PROFILE. "
84+
+ "If not set, it will use the default provider chain of AWS SDK.")
85+
protected String awsCredentialsProviderType = AwsCredentialsProviderMode.DEFAULT.name();
86+
7887
// ========== Constructor ==========
7988

8089
/**
@@ -115,6 +124,8 @@ private void initHiveConf() {
115124
setHiveConfPropertiesIfNotNull(hiveConf, AWSGlueConfig.AWS_GLUE_SESSION_TOKEN, baseProperties.glueSessionToken);
116125
setHiveConfPropertiesIfNotNull(hiveConf, AWSGlueConfig.AWS_GLUE_ROLE_ARN, baseProperties.glueIAMRole);
117126
setHiveConfPropertiesIfNotNull(hiveConf, AWSGlueConfig.AWS_GLUE_EXTERNAL_ID, baseProperties.glueExternalId);
127+
setHiveConfPropertiesIfNotNull(hiveConf,
128+
AWSGlueConfig.AWS_CREDENTIALS_PROVIDER_MODE, awsCredentialsProviderType);
118129
}
119130

120131
private static void setHiveConfPropertiesIfNotNull(HiveConf hiveConf, String key, String value) {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ public String getIcebergCatalogType() {
5555
public void initNormalizeAndCheckProps() {
5656
super.initNormalizeAndCheckProps();
5757
glueProperties = AWSGlueMetaStoreBaseProperties.of(origProps);
58+
glueProperties.requireExplicitGlueCredentials();
5859
s3Properties = S3Properties.of(origProps);
5960
}
6061

0 commit comments

Comments
 (0)