Skip to content

Commit 9f6431b

Browse files
shyikoshyiko
committed
Support vended credentials beyond loadTable
1 parent 0228ccf commit 9f6431b

File tree

12 files changed

+426
-44
lines changed

12 files changed

+426
-44
lines changed

‎ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,10 @@
3030
import com.altinity.ice.rest.catalog.internal.rest.RESTCatalogAdapter;
3131
import com.altinity.ice.rest.catalog.internal.rest.RESTCatalogAuthorizationHandler;
3232
import com.altinity.ice.rest.catalog.internal.rest.RESTCatalogHandler;
33-
import com.altinity.ice.rest.catalog.internal.rest.RESTCatalogMiddlewareTableAWCredentials;
33+
import com.altinity.ice.rest.catalog.internal.rest.RESTCatalogMiddlewareConfig;
34+
import com.altinity.ice.rest.catalog.internal.rest.RESTCatalogMiddlewareCredentials;
3435
import com.altinity.ice.rest.catalog.internal.rest.RESTCatalogMiddlewareTableConfig;
36+
import com.altinity.ice.rest.catalog.internal.rest.RESTCatalogMiddlewareTableCredentials;
3537
import com.altinity.ice.rest.catalog.internal.rest.RESTCatalogServlet;
3638
import com.fasterxml.jackson.databind.ObjectMapper;
3739
import com.google.common.net.HostAndPort;
@@ -49,6 +51,7 @@
4951
import org.apache.iceberg.CatalogUtil;
5052
import org.apache.iceberg.aws.s3.S3FileIOProperties;
5153
import org.apache.iceberg.catalog.Catalog;
54+
import org.apache.iceberg.relocated.com.google.common.base.Function;
5255
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
5356
import org.eclipse.jetty.server.Server;
5457
import org.eclipse.jetty.server.ServerConnector;
@@ -235,6 +238,10 @@ private static Server createBaseServer(
235238
mux.insertHandler(createAuthorizationHandler(config.bearerTokens(), config));
236239

237240
restCatalogAdapter = new RESTCatalogAdapter(catalog);
241+
var globalConfig = config.toIcebergConfigDefaults();
242+
if (!globalConfig.isEmpty()) {
243+
restCatalogAdapter = new RESTCatalogMiddlewareConfig(restCatalogAdapter, globalConfig);
244+
}
238245
var loadTableConfig = config.toIcebergLoadTableConfig();
239246
if (!loadTableConfig.isEmpty()) {
240247
restCatalogAdapter =
@@ -244,12 +251,17 @@ private static Server createBaseServer(
244251
if (awsAuth) {
245252
Map<String, AwsCredentialsProvider> awsCredentialsProviders =
246253
createAwsCredentialsProviders(config.bearerTokens(), config, icebergConfig);
254+
Function<String, AwsCredentialsProvider> auth = awsCredentialsProviders::get;
247255
restCatalogAdapter =
248-
new RESTCatalogMiddlewareTableAWCredentials(
249-
restCatalogAdapter, awsCredentialsProviders::get);
256+
new RESTCatalogMiddlewareTableCredentials(
257+
new RESTCatalogMiddlewareCredentials(restCatalogAdapter, auth), auth);
250258
}
251259
} else {
252260
restCatalogAdapter = new RESTCatalogAdapter(catalog);
261+
var globalConfig = config.toIcebergConfigDefaults();
262+
if (!globalConfig.isEmpty()) {
263+
restCatalogAdapter = new RESTCatalogMiddlewareConfig(restCatalogAdapter, globalConfig);
264+
}
253265
var loadTableConfig = config.toIcebergLoadTableConfig();
254266
if (!loadTableConfig.isEmpty()) {
255267
restCatalogAdapter =
@@ -258,9 +270,10 @@ private static Server createBaseServer(
258270

259271
if (awsAuth) {
260272
DefaultCredentialsProvider awsCredentialsProvider = DefaultCredentialsProvider.create();
273+
Function<String, AwsCredentialsProvider> auth = uid -> awsCredentialsProvider;
261274
restCatalogAdapter =
262-
new RESTCatalogMiddlewareTableAWCredentials(
263-
restCatalogAdapter, uid -> awsCredentialsProvider);
275+
new RESTCatalogMiddlewareTableCredentials(
276+
new RESTCatalogMiddlewareCredentials(restCatalogAdapter, auth), auth);
264277
}
265278
}
266279

‎ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/config/Config.java‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,18 @@ public static Config load(String configFile) throws IOException {
182182
}
183183

184184
public Map<String, String> toIcebergLoadTableConfig() {
185+
var m = toIcebergConfigDefaults();
186+
for (Map.Entry<String, String> e : loadTableProperties.entrySet()) {
187+
if (e.getValue() != null) {
188+
m.put(e.getKey(), e.getValue());
189+
} else {
190+
m.remove(e.getKey());
191+
}
192+
}
193+
return m;
194+
}
195+
196+
public Map<String, String> toIcebergConfigDefaults() {
185197
var m = new HashMap<String, String>();
186198
String iceIODefault = "ice.io.default.";
187199
if (s3 != null) {
@@ -199,13 +211,6 @@ public Map<String, String> toIcebergLoadTableConfig() {
199211
String region = warehouse.split(":")[3];
200212
m.putIfAbsent(iceIODefault + AwsClientProperties.CLIENT_REGION, region);
201213
}
202-
for (Map.Entry<String, String> e : loadTableProperties.entrySet()) {
203-
if (e.getValue() != null) {
204-
m.put(e.getKey(), e.getValue());
205-
} else {
206-
m.remove(e.getKey());
207-
}
208-
}
209214
return m;
210215
}
211216

‎ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/rest/RESTCatalogMiddlewareConfig.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/*
2+
* Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*/
10+
package com.altinity.ice.rest.catalog.internal.rest;
11+
12+
import com.altinity.ice.rest.catalog.internal.auth.Session;
13+
import java.util.Map;
14+
import org.apache.iceberg.rest.RESTResponse;
15+
import org.apache.iceberg.rest.responses.ConfigResponse;
16+
17+
public class RESTCatalogMiddlewareConfig extends RESTCatalogMiddleware {
18+
19+
private final Map<String, String> defaults;
20+
21+
public RESTCatalogMiddlewareConfig(RESTCatalogHandler next, Map<String, String> defaults) {
22+
super(next);
23+
this.defaults = defaults;
24+
}
25+
26+
@Override
27+
public <T extends RESTResponse> T handle(
28+
Session session,
29+
Route route,
30+
Map<String, String> vars,
31+
Object requestBody,
32+
Class<T> responseType) {
33+
T restResponse = this.next.handle(session, route, vars, requestBody, responseType);
34+
if (restResponse instanceof ConfigResponse configResponse) {
35+
Map<String, String> config = configResponse.defaults();
36+
for (var e : defaults.entrySet()) {
37+
config.putIfAbsent(e.getKey(), e.getValue());
38+
}
39+
}
40+
return restResponse;
41+
}
42+
}

‎ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/rest/RESTCatalogMiddlewareCredentials.java‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
/*
2+
* Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*/
10+
package com.altinity.ice.rest.catalog.internal.rest;
11+
12+
import com.altinity.ice.rest.catalog.internal.auth.Session;
13+
import java.util.HashMap;
14+
import java.util.Map;
15+
import javax.annotation.Nullable;
16+
import org.apache.iceberg.aws.s3.S3FileIOProperties;
17+
import org.apache.iceberg.relocated.com.google.common.base.Function;
18+
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
19+
import org.apache.iceberg.rest.RESTResponse;
20+
import org.apache.iceberg.rest.credentials.ImmutableCredential;
21+
import org.apache.iceberg.rest.responses.ImmutableLoadCredentialsResponse;
22+
import org.apache.iceberg.rest.responses.LoadCredentialsResponse;
23+
import software.amazon.awssdk.auth.credentials.AwsCredentials;
24+
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
25+
import software.amazon.awssdk.identity.spi.AwsSessionCredentialsIdentity;
26+
27+
public class RESTCatalogMiddlewareCredentials extends RESTCatalogMiddleware {
28+
29+
private static final ImmutableLoadCredentialsResponse EMPTY_RESPONSE =
30+
ImmutableLoadCredentialsResponse.builder().build();
31+
private final Function<String, AwsCredentialsProvider> awsCredentialsProvider;
32+
33+
public RESTCatalogMiddlewareCredentials(
34+
RESTCatalogHandler next,
35+
@Nullable Function<String, AwsCredentialsProvider> awsCredentialsProvider) {
36+
super(next);
37+
this.awsCredentialsProvider = awsCredentialsProvider;
38+
}
39+
40+
@Override
41+
public <T extends RESTResponse> T handle(
42+
Session session,
43+
Route route,
44+
Map<String, String> vars,
45+
Object requestBody,
46+
Class<T> responseType) {
47+
if (route == Route.CREDENTIALS) {
48+
LoadCredentialsResponse res;
49+
if (awsCredentialsProvider != null) {
50+
Map<String, String> config = new HashMap<>();
51+
applyCredentials(session, config);
52+
res =
53+
ImmutableLoadCredentialsResponse.builder()
54+
.addCredentials(
55+
ImmutableCredential.builder().prefix("s3").putAllConfig(config).build())
56+
.build();
57+
} else {
58+
res = EMPTY_RESPONSE;
59+
}
60+
return responseType.cast(res);
61+
}
62+
return next.handle(session, route, vars, requestBody, responseType);
63+
}
64+
65+
private void applyCredentials(Session session, Map<String, String> config) {
66+
String providerLookupKey = null;
67+
if (session != null) {
68+
providerLookupKey = session.uid();
69+
}
70+
AwsCredentialsProvider credentialsProvider = awsCredentialsProvider.apply(providerLookupKey);
71+
Preconditions.checkState(credentialsProvider != null); // null here means misconfiguration
72+
AwsCredentials awsCredentials = credentialsProvider.resolveCredentials();
73+
config.put(S3FileIOProperties.ACCESS_KEY_ID, awsCredentials.accessKeyId());
74+
config.put(S3FileIOProperties.SECRET_ACCESS_KEY, awsCredentials.secretAccessKey());
75+
if (awsCredentials instanceof AwsSessionCredentialsIdentity awsSessionCredentials) {
76+
config.put(S3FileIOProperties.SESSION_TOKEN, awsSessionCredentials.sessionToken());
77+
// S3FileIOProperties.SESSION_TOKEN_EXPIRES_AT_MS
78+
awsSessionCredentials
79+
.expirationTime()
80+
.ifPresent(
81+
exp ->
82+
config.put("s3.session-token-expires-at-ms", String.valueOf(exp.toEpochMilli())));
83+
}
84+
}
85+
}

ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/rest/RESTCatalogMiddlewareTableAWCredentials.java renamed to ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/rest/RESTCatalogMiddlewareTableCredentials.java

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,11 @@
2020
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
2121
import software.amazon.awssdk.identity.spi.AwsSessionCredentialsIdentity;
2222

23-
public class RESTCatalogMiddlewareTableAWCredentials extends RESTCatalogMiddleware {
23+
public class RESTCatalogMiddlewareTableCredentials extends RESTCatalogMiddleware {
2424

2525
private final Function<String, AwsCredentialsProvider> awsCredentialsProvider;
2626

27-
public RESTCatalogMiddlewareTableAWCredentials(
27+
public RESTCatalogMiddlewareTableCredentials(
2828
RESTCatalogHandler next, Function<String, AwsCredentialsProvider> awsCredentialsProvider) {
2929
super(next);
3030
this.awsCredentialsProvider = awsCredentialsProvider;
@@ -55,10 +55,17 @@ private void applyCredentials(Session session, Map<String, String> tableConfig)
5555
AwsCredentials awsCredentials = credentialsProvider.resolveCredentials();
5656
tableConfig.put(S3FileIOProperties.ACCESS_KEY_ID, awsCredentials.accessKeyId());
5757
tableConfig.put(S3FileIOProperties.SECRET_ACCESS_KEY, awsCredentials.secretAccessKey());
58-
if (awsCredentials instanceof AwsSessionCredentialsIdentity) {
59-
tableConfig.put(
60-
S3FileIOProperties.SESSION_TOKEN,
61-
((AwsSessionCredentialsIdentity) awsCredentials).sessionToken());
58+
if (awsCredentials instanceof AwsSessionCredentialsIdentity awsSessionCredentials) {
59+
tableConfig.put(S3FileIOProperties.SESSION_TOKEN, awsSessionCredentials.sessionToken());
60+
if (awsSessionCredentials.expirationTime().isPresent()) {
61+
// S3FileIOProperties.SESSION_TOKEN_EXPIRES_AT_MS
62+
awsSessionCredentials
63+
.expirationTime()
64+
.ifPresent(
65+
exp ->
66+
tableConfig.put(
67+
"s3.session-token-expires-at-ms", String.valueOf(exp.toEpochMilli())));
68+
}
6269
}
6370
}
6471
}

‎ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/rest/Route.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import org.apache.iceberg.rest.responses.GetNamespaceResponse;
4141
import org.apache.iceberg.rest.responses.ListNamespacesResponse;
4242
import org.apache.iceberg.rest.responses.ListTablesResponse;
43+
import org.apache.iceberg.rest.responses.LoadCredentialsResponse;
4344
import org.apache.iceberg.rest.responses.LoadTableResponse;
4445
import org.apache.iceberg.rest.responses.LoadViewResponse;
4546
import org.apache.iceberg.rest.responses.OAuthTokenResponse;
@@ -48,6 +49,7 @@
4849

4950
public enum Route {
5051
TOKENS(HTTPRequest.HTTPMethod.POST, "v1/oauth/tokens", null, OAuthTokenResponse.class),
52+
CREDENTIALS(HTTPRequest.HTTPMethod.GET, "v1/credentials", null, LoadCredentialsResponse.class),
5153
CONFIG(HTTPRequest.HTTPMethod.GET, "v1/config", null, ConfigResponse.class),
5254
LIST_NAMESPACES(
5355
HTTPRequest.HTTPMethod.GET, ResourcePaths.V1_NAMESPACES, null, ListNamespacesResponse.class),

‎ice/src/main/java/com/altinity/ice/cli/Main.java‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,11 @@ void createTable(
157157
names = {"-p"},
158158
description = "Create table if not exists")
159159
boolean createTableIfNotExists,
160+
@CommandLine.Option(
161+
names = "--use-vended-credentials",
162+
description =
163+
"Use vended credentials to access input files (instead of credentials from execution environment)")
164+
boolean useVendedCredentials,
160165
@CommandLine.Option(names = {"--s3-region"}) String s3Region,
161166
@CommandLine.Option(
162167
names = {"--s3-no-sign-request"},
@@ -203,6 +208,7 @@ void createTable(
203208
schemaFile,
204209
location,
205210
createTableIfNotExists,
211+
useVendedCredentials,
206212
s3NoSignRequest,
207213
partitions,
208214
sortOrders);
@@ -269,10 +275,10 @@ void insert(
269275
"Add files to catalog without copying them even if files are in different location(s) from table (implies --no-copy)")
270276
boolean forceNoCopy,
271277
@CommandLine.Option(
272-
names = "--force-table-auth",
278+
names = {"--use-vended-credentials", "--force-table-auth" /* deprecated */},
273279
description =
274280
"Use table credentials to access input files (instead of credentials from execution environment)")
275-
boolean forceTableAuth,
281+
boolean useVendedCredentials,
276282
@CommandLine.Option(names = {"--s3-region"}) String s3Region,
277283
@CommandLine.Option(
278284
names = {"--s3-no-sign-request"},
@@ -377,6 +383,7 @@ void insert(
377383
dataFiles[0],
378384
null,
379385
createTableIfNotExists,
386+
useVendedCredentials,
380387
s3NoSignRequest,
381388
partitions,
382389
sortOrders);
@@ -390,7 +397,7 @@ void insert(
390397
.noCommit(noCommit)
391398
.noCopy(noCopy)
392399
.forceNoCopy(forceNoCopy)
393-
.forceTableAuth(forceTableAuth)
400+
.useVendedCredentials(useVendedCredentials)
394401
.s3NoSignRequest(s3NoSignRequest)
395402
.s3CopyObject(s3CopyObject)
396403
.s3MultipartUploadThreadCount(s3MultipartUploadThreadCount)

0 commit comments

Comments
 (0)