Skip to content

Commit d1f75b6

Browse files
gyakamigyakami
authored
Add support for S3 compatible storage (#3188)
* Add support for S3 compatible storage Signed-off-by: Go Yakami <gyakami@lycorp.co.jp> * Update S3 checksum validation configuration parameters for clarity Signed-off-by: Go Yakami <gyakami@lycorp.co.jp> --------- Signed-off-by: Go Yakami <gyakami@lycorp.co.jp>
1 parent 80500d6 commit d1f75b6

File tree

6 files changed

+416
-16
lines changed

6 files changed

+416
-16
lines changed

‎libs/java/server_aws_common/src/main/java/io/athenz/server/aws/common/store/impl/S3ChangeLogStore.java‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,13 @@
1616

1717
package io.athenz.server.aws.common.store.impl;
1818

19+
import com.yahoo.athenz.auth.util.Crypto;
1920
import io.athenz.server.aws.common.utils.Utils;
2021
import software.amazon.awssdk.core.ResponseInputStream;
22+
import software.amazon.awssdk.http.apache.ApacheHttpClient;
2123
import software.amazon.awssdk.regions.Region;
2224
import software.amazon.awssdk.services.s3.S3Client;
25+
import software.amazon.awssdk.services.s3.S3ClientBuilder;
2326
import software.amazon.awssdk.services.s3.model.*;
2427
import com.fasterxml.jackson.databind.DeserializationFeature;
2528
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -31,6 +34,10 @@
3134
import org.slf4j.Logger;
3235
import org.slf4j.LoggerFactory;
3336

37+
import javax.net.ssl.TrustManagerFactory;
38+
import java.net.URI;
39+
import java.security.KeyStore;
40+
import java.security.cert.X509Certificate;
3441
import java.util.*;
3542
import java.util.concurrent.ConcurrentHashMap;
3643
import java.util.concurrent.ExecutorService;
@@ -53,6 +60,8 @@ public class S3ChangeLogStore implements ChangeLogStore {
5360

5461
private static final String NUMBER_OF_THREADS = "athenz.zts.bucket.threads";
5562
private static final String DEFAULT_TIMEOUT_SECONDS = "athenz.zts.bucket.threads.timeout";
63+
private static final String ZTS_PROP_AWS_S3_ENDPOINT = "athenz.zts.aws_s3_endpoint";
64+
private static final String ZTS_PROP_AWS_S3_CA_CERT = "athenz.zts.aws_s3_ca_cert";
5665
private final int nThreads = Integer.parseInt(System.getProperty(NUMBER_OF_THREADS, "10"));
5766
private final int defaultTimeoutSeconds = Integer.parseInt(System.getProperty(DEFAULT_TIMEOUT_SECONDS, "1800"));
5867
protected Map<String, SignedDomain> tempSignedDomainMap = new ConcurrentHashMap<>();
@@ -497,7 +506,38 @@ S3Client getS3Client() {
497506
throw new RuntimeException("S3ChangeLogStore: Couldn't detect AWS region");
498507
}
499508

500-
return S3Client.builder().region(Region.of(awsRegion)).build();
509+
S3ClientBuilder s3ClientBuilder = S3Client.builder().region(Region.of(awsRegion));
510+
511+
// check if we have a custom endpoint
512+
String s3Endpoint = System.getProperty(ZTS_PROP_AWS_S3_ENDPOINT);
513+
if (!StringUtil.isEmpty(s3Endpoint)) {
514+
s3ClientBuilder.endpointOverride(URI.create(s3Endpoint));
515+
}
516+
517+
// check if we have a custom ca cert
518+
String s3CaCert = System.getProperty(ZTS_PROP_AWS_S3_CA_CERT);
519+
if (!StringUtil.isEmpty(s3CaCert)) {
520+
try {
521+
ApacheHttpClient.Builder httpClientBuilder = ApacheHttpClient.builder();
522+
X509Certificate[] certs = Crypto.loadX509Certificates(s3CaCert);
523+
KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
524+
keyStore.load(null, null); // Initialize empty keystore
525+
int i = 0;
526+
for (X509Certificate cert : certs) {
527+
keyStore.setCertificateEntry("custom-ca-" + i++, cert);
528+
}
529+
TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
530+
tmf.init(keyStore);
531+
532+
httpClientBuilder.tlsTrustManagersProvider(tmf::getTrustManagers);
533+
s3ClientBuilder.httpClient(httpClientBuilder.build());
534+
} catch (Exception ex) {
535+
LOGGER.error("S3ChangeLogStore: unable to load custom ca cert: {}", s3CaCert, ex);
536+
throw new RuntimeException("S3ChangeLogStore: unable to load custom ca cert");
537+
}
538+
}
539+
540+
return s3ClientBuilder.build();
501541
}
502542

503543
public ExecutorService getExecutorService() {

‎libs/java/server_aws_common/src/main/java/io/athenz/syncer/aws/common/impl/S3ClientFactory.java‎

Lines changed: 48 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -16,18 +16,26 @@
1616

1717
package io.athenz.syncer.aws.common.impl;
1818

19+
import com.yahoo.athenz.auth.util.Crypto;
1920
import io.athenz.syncer.common.zms.Config;
2021
import org.slf4j.Logger;
2122
import org.slf4j.LoggerFactory;
2223
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
2324
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
2425
import software.amazon.awssdk.http.SdkHttpClient;
2526
import software.amazon.awssdk.services.s3.S3Client;
27+
import software.amazon.awssdk.services.s3.S3ClientBuilder;
2628
import software.amazon.awssdk.http.apache.ApacheHttpClient;
29+
import software.amazon.awssdk.core.checksums.RequestChecksumCalculation;
30+
import software.amazon.awssdk.core.checksums.ResponseChecksumValidation;
2731
import software.amazon.awssdk.regions.Region;
2832
import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain;
2933
import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
3034

35+
import javax.net.ssl.TrustManagerFactory;
36+
import java.net.URI;
37+
import java.security.KeyStore;
38+
import java.security.cert.X509Certificate;
3139
import java.time.Duration;
3240

3341
public class S3ClientFactory {
@@ -71,31 +79,56 @@ public static S3Client getS3Client() throws Exception {
7179
}
7280
}
7381

74-
SdkHttpClient apacheHttpClient = ApacheHttpClient.builder()
82+
ApacheHttpClient.Builder httpClientBuilder = ApacheHttpClient.builder()
7583
.connectionTimeout(Duration.ofMillis(connectionTimeout))
76-
.socketTimeout(Duration.ofMillis(requestTimeout))
77-
.build();
84+
.socketTimeout(Duration.ofMillis(requestTimeout));
85+
86+
final String caCertPath = Config.getInstance().getConfigParam(Config.SYNC_CFG_PARAM_AWS_S3_CA_CERT);
87+
if (!Config.isEmpty(caCertPath)) {
88+
X509Certificate[] certs = Crypto.loadX509Certificates(caCertPath);
89+
KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
90+
keyStore.load(null, null); // Initialize empty keystore
91+
int i = 0;
92+
for (X509Certificate cert : certs) {
93+
keyStore.setCertificateEntry("custom-ca-" + i++, cert);
94+
}
95+
TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
96+
tmf.init(keyStore);
97+
98+
httpClientBuilder.tlsTrustManagersProvider(tmf::getTrustManagers);
99+
}
100+
101+
SdkHttpClient apacheHttpClient = httpClientBuilder.build();
102+
103+
S3ClientBuilder s3ClientBuilder = S3Client.builder()
104+
.httpClient(apacheHttpClient)
105+
.region(getRegion());
106+
107+
// Enable checksum calculation and validation if configured
108+
final String checksumValidation = Config.getInstance().getConfigParam(Config.SYNC_CFG_PARAM_AWS_S3_CHECKSUM_VALIDATION);
109+
if (!Config.isEmpty(checksumValidation) && Boolean.parseBoolean(checksumValidation)) {
110+
s3ClientBuilder
111+
.requestChecksumCalculation(RequestChecksumCalculation.WHEN_REQUIRED)
112+
.responseChecksumValidation(ResponseChecksumValidation.WHEN_REQUIRED);
113+
LOGGER.debug("S3 checksum calculation and validation enabled");
114+
}
115+
116+
final String awsS3Endpoint = Config.getInstance().getConfigParam(Config.SYNC_CFG_PARAM_AWS_S3_ENDPOINT);
117+
if (!Config.isEmpty(awsS3Endpoint)) {
118+
s3ClientBuilder.endpointOverride(URI.create(awsS3Endpoint));
119+
}
78120

79-
S3Client s3client;
80121
final String awsKeyId = Config.getInstance().getConfigParam(Config.SYNC_CFG_PARAM_AWS_KEY_ID);
81122
final String awsAccKey = Config.getInstance().getConfigParam(Config.SYNC_CFG_PARAM_AWS_ACCESS_KEY);
82123
if (!Config.isEmpty(awsKeyId) && !Config.isEmpty(awsAccKey)) {
83124
AwsBasicCredentials awsCreds = AwsBasicCredentials.builder()
84125
.accessKeyId(awsKeyId).secretAccessKey(awsAccKey).build();
85126
StaticCredentialsProvider credentialsProvider = StaticCredentialsProvider.create(awsCreds);
86-
87-
s3client = S3Client.builder()
88-
.credentialsProvider(credentialsProvider)
89-
.httpClient(apacheHttpClient)
90-
.region(getRegion())
91-
.build();
92-
} else {
93-
s3client = S3Client.builder()
94-
.httpClient(apacheHttpClient)
95-
.region(getRegion())
96-
.build();
127+
s3ClientBuilder.credentialsProvider(credentialsProvider);
97128
}
98129

130+
S3Client s3client = s3ClientBuilder.build();
131+
99132
verifyBucketExist(s3client, bucket);
100133

101134
LOGGER.debug("success: using bucket: {}", bucket);

‎libs/java/server_aws_common/src/test/java/io/athenz/server/aws/common/store/impl/S3ChangeLogStoreTest.java‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,17 +21,27 @@
2121
import static org.testng.Assert.*;
2222

2323
import java.io.*;
24+
import java.net.URI;
25+
import java.security.cert.X509Certificate;
2426
import java.util.*;
2527
import java.util.concurrent.TimeUnit;
2628

29+
import com.yahoo.athenz.auth.util.Crypto;
2730
import com.yahoo.athenz.zms.JWSDomain;
2831
import com.yahoo.rdl.Timestamp;
32+
import org.mockito.ArgumentCaptor;
33+
import org.mockito.MockedStatic;
2934
import org.mockito.Mockito;
3035
import org.testng.annotations.BeforeMethod;
3136
import org.testng.annotations.Test;
3237

3338
import software.amazon.awssdk.core.ResponseInputStream;
39+
import software.amazon.awssdk.http.SdkHttpClient;
40+
import software.amazon.awssdk.http.TlsTrustManagersProvider;
41+
import software.amazon.awssdk.http.apache.ApacheHttpClient;
42+
import software.amazon.awssdk.regions.Region;
3443
import software.amazon.awssdk.services.s3.S3Client;
44+
import software.amazon.awssdk.services.s3.S3ClientBuilder;
3545
import software.amazon.awssdk.services.s3.model.*;
3646
import com.yahoo.athenz.zms.DomainData;
3747
import com.yahoo.athenz.zms.SignedDomain;
@@ -641,6 +651,56 @@ public void testGetS3Client() {
641651
assertNotNull(s3Client);
642652
}
643653

654+
@Test
655+
public void testGetS3ClientWithCustomEndpointAndCaCert() throws Exception {
656+
System.setProperty(ZTS_PROP_AWS_BUCKET_NAME, "test-bucket");
657+
System.setProperty(ZTS_PROP_AWS_REGION_NAME, "us-west-2");
658+
System.setProperty("athenz.zts.aws_s3_endpoint", "https://custom.s3.endpoint");
659+
System.setProperty("athenz.zts.aws_s3_ca_cert", "src/test/resources/dummy_ca.pem");
660+
661+
// Mocks
662+
try (MockedStatic<ApacheHttpClient> mockHttpClientStatic = Mockito.mockStatic(ApacheHttpClient.class);
663+
MockedStatic<S3Client> mockS3ClientStatic = Mockito.mockStatic(S3Client.class);
664+
MockedStatic<Crypto> mockCryptoStatic = Mockito.mockStatic(Crypto.class)) {
665+
666+
// Mock Crypto
667+
mockCryptoStatic.when(() -> Crypto.loadX509Certificates(any(String.class))).thenReturn(new X509Certificate[]{mock(X509Certificate.class)});
668+
669+
// Mock ApacheHttpClient builder
670+
ApacheHttpClient.Builder mockHttpBuilder = mock(ApacheHttpClient.Builder.class);
671+
SdkHttpClient mockHttpClient = mock(SdkHttpClient.class);
672+
673+
mockHttpClientStatic.when(ApacheHttpClient::builder).thenReturn(mockHttpBuilder);
674+
when(mockHttpBuilder.tlsTrustManagersProvider(any(TlsTrustManagersProvider.class))).thenReturn(mockHttpBuilder);
675+
when(mockHttpBuilder.build()).thenReturn(mockHttpClient);
676+
677+
// Mock S3Client builder
678+
S3ClientBuilder mockS3Builder = mock(S3ClientBuilder.class);
679+
S3Client mockS3Client = mock(S3Client.class);
680+
681+
mockS3ClientStatic.when(S3Client::builder).thenReturn(mockS3Builder);
682+
when(mockS3Builder.region(any(Region.class))).thenReturn(mockS3Builder);
683+
when(mockS3Builder.endpointOverride(any(URI.class))).thenReturn(mockS3Builder);
684+
when(mockS3Builder.httpClient(any(SdkHttpClient.class))).thenReturn(mockS3Builder);
685+
when(mockS3Builder.build()).thenReturn(mockS3Client);
686+
687+
S3ChangeLogStore store = new S3ChangeLogStore();
688+
S3Client client = store.getS3Client();
689+
assertNotNull(client);
690+
691+
// Verify ApacheHttpClient configured with TrustManager
692+
Mockito.verify(mockHttpBuilder).tlsTrustManagersProvider(any(TlsTrustManagersProvider.class));
693+
694+
// Verify S3Client configured with Endpoint Override
695+
ArgumentCaptor<URI> uriCaptor = ArgumentCaptor.forClass(URI.class);
696+
Mockito.verify(mockS3Builder).endpointOverride(uriCaptor.capture());
697+
assertEquals(uriCaptor.getValue().toString(), "https://custom.s3.endpoint");
698+
} finally {
699+
System.clearProperty("athenz.zts.aws_s3_endpoint");
700+
System.clearProperty("athenz.zts.aws_s3_ca_cert");
701+
}
702+
}
703+
644704
@Test
645705
public void initNoRegionException() {
646706
System.clearProperty(ZTS_PROP_AWS_REGION_NAME);

0 commit comments

Comments
 (0)