Skip to content

Commit 30768d6

Browse files
imsayari404j-sund
imsayari404
authored and
j-sund
committed
Add support for S3 WebIdentity authentication
1 parent 8a00634 commit 30768d6

File tree

7 files changed

+78
-3
lines changed

7 files changed

+78
-3
lines changed

‎presto-docs/src/main/sphinx/connector/hive.rst‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -371,6 +371,13 @@ Property Name Description
371371
``hive.s3.skip-glacier-objects`` Ignore Glacier objects rather than failing the query. This
372372
will skip data that may be expected to be part of the table
373373
or partition. Defaults to ``false``.
374+
375+
``hive.s3.web.identity.auth.enabled`` Enables Web Identity authentication for S3 access. Requires
376+
``hive.s3.iam-role`` to be specified. Additionally, ensure that
377+
the environment variables ``AWS_WEB_IDENTITY_TOKEN_FILE`` and
378+
``AWS_REGION`` are set for proper authentication. Since this
379+
implementation uses AWS SDK 1.x, setting these environment
380+
variables is necessary.
374381
============================================ =================================================================
375382

376383
S3 Credentials

‎presto-hive/src/main/java/com/facebook/presto/hive/s3/HiveS3Config.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,20 @@ public class HiveS3Config
6161
private String s3UserAgentPrefix = "";
6262
private PrestoS3AclType s3AclType = PrestoS3AclType.PRIVATE;
6363
private boolean skipGlacierObjects;
64+
private boolean s3WebIdentityEnabled;
65+
66+
public boolean isS3WebIdentityEnabled()
67+
{
68+
return s3WebIdentityEnabled;
69+
}
70+
71+
@Config("hive.s3.web.identity.auth.enabled")
72+
@ConfigDescription("Enable web identity token authentication for assuming an AWS IAM role")
73+
public HiveS3Config setS3WebIdentityEnabled(boolean s3WebIdentityEnabled)
74+
{
75+
this.s3WebIdentityEnabled = s3WebIdentityEnabled;
76+
return this;
77+
}
6478

6579
public String getS3AwsAccessKey()
6680
{

‎presto-hive/src/main/java/com/facebook/presto/hive/s3/PrestoS3ConfigurationUpdater.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ public class PrestoS3ConfigurationUpdater
5252
private final String userAgentPrefix;
5353
private final PrestoS3AclType aclType;
5454
private boolean skipGlacierObjects;
55+
private final boolean webIdentityEnabled;
5556

5657
@Inject
5758
public PrestoS3ConfigurationUpdater(HiveS3Config config)
@@ -84,6 +85,7 @@ public PrestoS3ConfigurationUpdater(HiveS3Config config)
8485
this.userAgentPrefix = config.getS3UserAgentPrefix();
8586
this.aclType = config.getS3AclType();
8687
this.skipGlacierObjects = config.isSkipGlacierObjects();
88+
this.webIdentityEnabled = config.isS3WebIdentityEnabled();
8789
}
8890

8991
@Override
@@ -124,6 +126,7 @@ public void updateConfiguration(Configuration config)
124126
if (sseKmsKeyId != null) {
125127
config.set(S3_SSE_KMS_KEY_ID, sseKmsKeyId);
126128
}
129+
config.setBoolean(S3_WEB_IDENTITY_ENABLED, webIdentityEnabled);
127130
config.setInt(S3_MAX_CLIENT_RETRIES, maxClientRetries);
128131
config.setInt(S3_MAX_ERROR_RETRIES, maxErrorRetries);
129132
config.set(S3_MAX_BACKOFF_TIME, maxBackoffTime.toString());

‎presto-hive/src/main/java/com/facebook/presto/hive/s3/PrestoS3FileSystem.java‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
2525
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
2626
import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider;
27+
import com.amazonaws.auth.WebIdentityTokenCredentialsProvider;
2728
import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
2829
import com.amazonaws.event.ProgressEvent;
2930
import com.amazonaws.event.ProgressEventType;
@@ -136,6 +137,7 @@
136137
import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_USER_AGENT_PREFIX;
137138
import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_USER_AGENT_SUFFIX;
138139
import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_USE_INSTANCE_CREDENTIALS;
140+
import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_WEB_IDENTITY_ENABLED;
139141
import static com.google.common.base.Preconditions.checkArgument;
140142
import static com.google.common.base.Preconditions.checkPositionIndexes;
141143
import static com.google.common.base.Strings.isNullOrEmpty;
@@ -196,6 +198,7 @@ public class PrestoS3FileSystem
196198
private PrestoS3AclType s3AclType;
197199
private boolean skipGlacierObjects;
198200
private PrestoS3StorageClass s3StorageClass;
201+
private boolean webIdentityEnabled;
199202

200203
@Override
201204
public void initialize(URI uri, Configuration conf)
@@ -237,7 +240,8 @@ public void initialize(URI uri, Configuration conf)
237240
String userAgentPrefix = conf.get(S3_USER_AGENT_PREFIX, defaults.getS3UserAgentPrefix());
238241
this.skipGlacierObjects = conf.getBoolean(S3_SKIP_GLACIER_OBJECTS, defaults.isSkipGlacierObjects());
239242
this.s3StorageClass = conf.getEnum(S3_STORAGE_CLASS, defaults.getS3StorageClass());
240-
243+
this.webIdentityEnabled = conf.getBoolean(S3_WEB_IDENTITY_ENABLED, false);
244+
checkArgument(!(webIdentityEnabled && isNullOrEmpty(s3IamRole)), "Invalid configuration: hive.s3.iam-role must be provided when hive.s3.web.identity.auth.enabled is set to true");
241245
ClientConfiguration configuration = new ClientConfiguration()
242246
.withMaxErrorRetry(maxErrorRetries)
243247
.withProtocol(sslEnabled ? Protocol.HTTPS : Protocol.HTTP)
@@ -854,8 +858,17 @@ private AWSCredentialsProvider createAwsCredentialsProvider(URI uri, Configurati
854858
return InstanceProfileCredentialsProvider.getInstance();
855859
}
856860

857-
if (s3IamRole != null) {
858-
return new STSAssumeRoleSessionCredentialsProvider.Builder(s3IamRole, s3IamRoleSessionName).build();
861+
if (!isNullOrEmpty(s3IamRole)) {
862+
if (webIdentityEnabled) {
863+
log.debug("Using Web Identity Token Credentials Provider.");
864+
WebIdentityTokenCredentialsProvider.Builder providerBuilder = WebIdentityTokenCredentialsProvider.builder()
865+
.roleArn(s3IamRole)
866+
.roleSessionName(s3IamRoleSessionName);
867+
return providerBuilder.build();
868+
}
869+
log.debug("Using STS Assume Role Session Credentials Provider.");
870+
return new STSAssumeRoleSessionCredentialsProvider.Builder(s3IamRole, s3IamRoleSessionName)
871+
.build();
859872
}
860873

861874
String providerClass = conf.get(S3_CREDENTIALS_PROVIDER);

‎presto-hive/src/main/java/com/facebook/presto/hive/s3/S3ConfigurationUpdater.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ public interface S3ConfigurationUpdater
2828
String S3_PIN_CLIENT_TO_CURRENT_REGION = "presto.s3.pin-client-to-current-region";
2929
String S3_USE_INSTANCE_CREDENTIALS = "presto.s3.use-instance-credentials";
3030
String S3_IAM_ROLE = "presto.hive.s3.iam-role";
31+
String S3_WEB_IDENTITY_ENABLED = "presto.hive.s3.web.identity.auth.enabled";
3132
String S3_IAM_ROLE_SESSION_NAME = "presto.hive.s3.iam-role-session-name";
3233
String S3_MULTIPART_MIN_PART_SIZE = "presto.s3.multipart.min-part-size";
3334
String S3_MULTIPART_MIN_FILE_SIZE = "presto.s3.multipart.min-file-size";

‎presto-hive/src/test/java/com/facebook/presto/hive/s3/TestHiveS3Config.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ public void testDefaults()
4141
.setS3PathStyleAccess(false)
4242
.setS3UseInstanceCredentials(false)
4343
.setS3IamRole(null)
44+
.setS3WebIdentityEnabled(false)
4445
.setS3StorageClass(PrestoS3StorageClass.STANDARD)
4546
.setS3IamRoleSessionName("presto-session")
4647
.setS3SslEnabled(true)
@@ -76,6 +77,7 @@ public void testExplicitPropertyMappings()
7677
.put("hive.s3.path-style-access", "true")
7778
.put("hive.s3.use-instance-credentials", "true")
7879
.put("hive.s3.iam-role", "roleArn")
80+
.put("hive.s3.web.identity.auth.enabled", "true")
7981
.put("hive.s3.storage-class", "INTELLIGENT_TIERING")
8082
.put("hive.s3.iam-role-session-name", "roleSessionName")
8183
.put("hive.s3.ssl.enabled", "false")
@@ -108,6 +110,7 @@ public void testExplicitPropertyMappings()
108110
.setS3PathStyleAccess(true)
109111
.setS3UseInstanceCredentials(true)
110112
.setS3IamRole("roleArn")
113+
.setS3WebIdentityEnabled(true)
111114
.setS3StorageClass(PrestoS3StorageClass.INTELLIGENT_TIERING)
112115
.setS3IamRoleSessionName("roleSessionName")
113116
.setS3SslEnabled(false)

‎presto-hive/src/test/java/com/facebook/presto/hive/s3/TestPrestoS3FileSystem.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
2222
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
2323
import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider;
24+
import com.amazonaws.auth.WebIdentityTokenCredentialsProvider;
2425
import com.amazonaws.services.s3.AmazonS3Client;
2526
import com.amazonaws.services.s3.AmazonS3EncryptionClient;
2627
import com.amazonaws.services.s3.S3ClientOptions;
@@ -86,6 +87,7 @@
8687
import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_USER_AGENT_PREFIX;
8788
import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_USER_AGENT_SUFFIX;
8889
import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_USE_INSTANCE_CREDENTIALS;
90+
import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_WEB_IDENTITY_ENABLED;
8991
import static com.google.common.base.Preconditions.checkArgument;
9092
import static com.google.common.io.MoreFiles.deleteRecursively;
9193
import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE;
@@ -98,6 +100,7 @@
98100
import static org.testng.Assert.assertFalse;
99101
import static org.testng.Assert.assertNull;
100102
import static org.testng.Assert.assertTrue;
103+
import static org.testng.Assert.expectThrows;
101104

102105
public class TestPrestoS3FileSystem
103106
{
@@ -147,6 +150,37 @@ public void testEndpointWithPinToCurrentRegionConfiguration()
147150
}
148151
}
149152

153+
@Test
154+
public void testWebIdentityTokenCredentials()
155+
throws Exception
156+
{
157+
Configuration config = new Configuration();
158+
config.set(S3_IAM_ROLE, "role");
159+
config.setBoolean(S3_WEB_IDENTITY_ENABLED, true);
160+
161+
try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
162+
fs.initialize(new URI("s3n://test-bucket/"), config);
163+
assertInstanceOf(getAwsCredentialsProvider(fs), WebIdentityTokenCredentialsProvider.class);
164+
}
165+
}
166+
167+
@Test
168+
public void testWebIdentityEnabledWithoutIamRole()
169+
{
170+
Configuration config = new Configuration();
171+
config.setBoolean(S3_WEB_IDENTITY_ENABLED, true);
172+
// S3_IAM_ROLE is NOT set (invalid case)
173+
174+
IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> {
175+
try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
176+
fs.initialize(new URI("s3n://test-bucket/"), config);
177+
}
178+
});
179+
180+
assertEquals("Invalid configuration: hive.s3.iam-role must be provided when hive.s3.web.identity.auth.enabled is set to true",
181+
exception.getMessage());
182+
}
183+
150184
@Test
151185
public void testAssumeRoleCredentials()
152186
throws Exception

0 commit comments

Comments
 (0)