Skip to content

Commit 7647219

Browse files
chenkinschenkins
authored
Merge pull request #106 from shift7-ch/94-hybrid-integrationtests
Hybrid integrationtests aka AWS.
2 parents 74188d5 + cc008f6 commit 7647219

29 files changed

+955
-791
lines changed

‎.github/workflows/build.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ jobs:
1515
matrix:
1616
os: [ ubuntu-latest ]
1717
steps:
18+
- name: Docker network prune
19+
run: docker network prune --force
1820
- uses: actions/checkout@v4
1921
- name: Set up JDK 21
2022
uses: actions/setup-java@v4

‎hub/src/main/java/cloud/katta/protocols/hub/serializer/StorageProfileDtoWrapperDeserializer.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66

77
import ch.cyberduck.core.serializer.Deserializer;
88

9+
import cloud.katta.client.model.Protocol;
10+
911
import org.apache.logging.log4j.LogManager;
1012
import org.apache.logging.log4j.Logger;
1113

@@ -46,7 +48,7 @@ public <L> List<L> listForKey(final String key) {
4648
properties.add(String.format("3.storage.class.options=%s", dto.getStorageClass().name()));
4749
properties.add(String.format("3.storage.class=%s", dto.getStorageClass().name()));
4850
}
49-
if(dto.getStsEndpoint() != null) {
51+
if(dto.getProtocol() == Protocol.S3_STS) {
5052
properties.add(String.format("%s=%s", S3AssumeRoleProtocol.OAUTH_TOKENEXCHANGE, true));
5153
properties.add(String.format("%s=%s", S3AssumeRoleProtocol.S3_ASSUMEROLE_ROLEARN, dto.getStsRoleArn()));
5254
if(dto.getStsRoleArn2() != null) {

‎hub/src/main/java/cloud/katta/protocols/s3/S3AssumeRoleSession.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
import ch.cyberduck.core.OAuthTokens;
1010
import ch.cyberduck.core.aws.CustomClientConfiguration;
1111
import ch.cyberduck.core.exception.LoginCanceledException;
12+
import ch.cyberduck.core.http.CustomServiceUnavailableRetryStrategy;
13+
import ch.cyberduck.core.http.ExecutionCountServiceUnavailableRetryStrategy;
1214
import ch.cyberduck.core.oauth.OAuth2AuthorizationService;
1315
import ch.cyberduck.core.oauth.OAuth2RequestInterceptor;
1416
import ch.cyberduck.core.preferences.HostPreferences;
@@ -92,7 +94,9 @@ protected String getWebIdentityToken(final OAuthTokens oauth) {
9294
}
9395
log.debug("Register interceptor {}", sts);
9496
configuration.addInterceptorLast(sts);
95-
configuration.setServiceUnavailableRetryStrategy(new S3AuthenticationResponseInterceptor(this, sts));
97+
final S3AuthenticationResponseInterceptor interceptor = new S3AuthenticationResponseInterceptor(this, sts);
98+
configuration.setServiceUnavailableRetryStrategy(new CustomServiceUnavailableRetryStrategy(host,
99+
new ExecutionCountServiceUnavailableRetryStrategy(interceptor)));
96100
return sts;
97101
}
98102
return super.configureCredentialsStrategy(proxy, configuration, prompt);

‎hub/src/main/java/cloud/katta/protocols/s3/STSChainedAssumeRoleRequestInterceptor.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -56,11 +56,6 @@ public TemporaryAccessTokens authorize(final OAuthTokens oauth) throws Backgroun
5656
return this.authorize(oauth, super.authorize(oauth));
5757
}
5858

59-
@Override
60-
public TemporaryAccessTokens refresh(final OAuthTokens oauth) throws BackgroundException {
61-
return this.authorize(oauth, super.refresh(oauth));
62-
}
63-
6459
/**
6560
* Assume role with previously obtained temporary access token
6661
*

‎hub/src/main/java/cloud/katta/workflows/CreateVaultService.java‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import cloud.katta.client.api.UsersResourceApi;
3838
import cloud.katta.client.api.VaultResourceApi;
3939
import cloud.katta.client.model.CreateS3STSBucketDto;
40+
import cloud.katta.client.model.Protocol;
4041
import cloud.katta.client.model.UserDto;
4142
import cloud.katta.client.model.VaultDto;
4243
import cloud.katta.crypto.UserKeys;
@@ -94,7 +95,7 @@ public void createVault(final UserKeys userKeys, final StorageProfileDtoWrapper
9495
final UvfMetadataPayload metadataPayload = UvfMetadataPayload.create()
9596
.withStorage(new VaultMetadataJWEBackendDto()
9697
.provider(storageProfileWrapper.getId().toString())
97-
.defaultPath(storageProfileWrapper.getStsEndpoint() != null ? storageProfileWrapper.getBucketPrefix() + vaultModel.vaultId() : vaultModel.bucketName())
98+
.defaultPath(storageProfileWrapper.getProtocol() == Protocol.S3_STS ? storageProfileWrapper.getBucketPrefix() + vaultModel.vaultId() : vaultModel.bucketName())
9899
.nickname(vaultModel.vaultName())
99100
.username(vaultModel.accessKeyId())
100101
.password(vaultModel.secretKey()))
@@ -131,7 +132,7 @@ public void createVault(final UserKeys userKeys, final StorageProfileDtoWrapper
131132
final OAuthTokens tokens = keychain.findOAuthTokens(hubSession.getHost());
132133
final Host bookmark = new VaultServiceImpl(vaultResource, storageProfileResource).getStorageBackend(ProtocolFactory.get(),
133134
configResource.apiConfigGet(), vaultDto.getId(), metadataPayload.storage(), tokens);
134-
if(storageProfileWrapper.getStsEndpoint() == null) {
135+
if(storageProfileWrapper.getProtocol() == Protocol.S3) {
135136
// permanent: template upload into existing bucket from client (not backend)
136137
templateUploadService.uploadTemplate(bookmark, metadataPayload, storageDto, hashedRootDirId);
137138
}
@@ -143,6 +144,7 @@ public void createVault(final UserKeys userKeys, final StorageProfileDtoWrapper
143144
vaultDto.getId().toString(),
144145
storageProfileWrapper.getStsEndpoint(),
145146
String.format("%s%s", storageProfileWrapper.getBucketPrefix(), vaultDto.getId()),
147+
vaultModel.region(),
146148
storageProfileWrapper.getBucketAcceleration()
147149
);
148150
log.debug("Create STS bucket {} for vault {}", storageDto, vaultDto);
@@ -191,12 +193,12 @@ void uploadTemplate(final Host bookmark, final UvfMetadataPayload metadataPayloa
191193
static class STSInlinePolicyService {
192194
static STSInlinePolicyService disabled = new STSInlinePolicyService() {
193195
@Override
194-
TemporaryAccessTokens getSTSTokensFromAccessTokenWithCreateBucketInlinePolicy(final String token, final String roleArn, final String roleSessionName, final String stsEndpoint, final String bucketName, final Boolean bucketAcceleration) {
196+
TemporaryAccessTokens getSTSTokensFromAccessTokenWithCreateBucketInlinePolicy(final String token, final String roleArn, final String roleSessionName, final String stsEndpoint, final String bucketName, final String region, final Boolean bucketAcceleration) {
195197
return new TemporaryAccessTokens(null);
196198
}
197199
};
198200

199-
TemporaryAccessTokens getSTSTokensFromAccessTokenWithCreateBucketInlinePolicy(final String token, final String roleArn, final String roleSessionName, final String stsEndpoint, final String bucketName, final Boolean bucketAcceleration) throws IOException {
201+
TemporaryAccessTokens getSTSTokensFromAccessTokenWithCreateBucketInlinePolicy(final String token, final String roleArn, final String roleSessionName, final String stsEndpoint, final String bucketName, final String region, final Boolean bucketAcceleration) throws IOException {
200202
log.debug("Get STS tokens from {} to pass to backend {} with role {} and session name {}", token, stsEndpoint, roleArn, roleSessionName);
201203

202204
final AssumeRoleWithWebIdentityRequest request = new AssumeRoleWithWebIdentityRequest();
@@ -214,9 +216,13 @@ TemporaryAccessTokens getSTSTokensFromAccessTokenWithCreateBucketInlinePolicy(fi
214216

215217
AWSSecurityTokenServiceClientBuilder serviceBuild = AWSSecurityTokenServiceClientBuilder
216218
.standard();
219+
// Exactly only one of Region or EndpointConfiguration may be set.
217220
if(stsEndpoint != null) {
218221
serviceBuild.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(stsEndpoint, null));
219222
}
223+
else {
224+
serviceBuild.withRegion(region);
225+
}
220226
final AWSSecurityTokenService service = serviceBuild
221227
.withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials()))
222228
.build();

‎hub/src/main/java/cloud/katta/workflows/VaultServiceImpl.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ public Host getStorageBackend(final ProtocolFactory protocols, final ConfigDto c
9494
switch(storageProfile.getProtocol()) {
9595
case S3:
9696
case S3_STS:
97-
final Profile profile = new Profile(protocols.forType(Protocol.Type.s3), new StorageProfileDtoWrapperDeserializer(
97+
final Profile profile = new Profile(protocols.forType(protocols.find(ProtocolFactory.BUNDLED_PROFILE_PREDICATE), Protocol.Type.s3), new StorageProfileDtoWrapperDeserializer(
9898
new HubConfigDtoDeserializer(configDto), storageProfile));
9999
log.debug("Register storage profile {}", profile);
100100
protocols.register(profile);
@@ -106,6 +106,10 @@ public Host getStorageBackend(final ProtocolFactory protocols, final ConfigDto c
106106
final String provider = vaultMetadata.getProvider();
107107
log.debug("Lookup provider {} from vault metadata", provider);
108108
final Protocol protocol = protocols.forName(provider);
109+
if((protocol.getOAuthTokenUrl() != null) && (!protocol.getOAuthTokenUrl().equals(configDto.getKeycloakTokenEndpoint()))) {
110+
// this may happen if the storage profile ID is deployed to two different hubs
111+
throw new AccessException(String.format("Expected keycloak endpoint %s, found %s.", configDto.getKeycloakTokenEndpoint(), protocol.getOAuthTokenUrl()));
112+
}
109113
final Host bookmark = new Host(protocol);
110114
log.debug("Configure bookmark for vault {}", vaultMetadata);
111115
bookmark.setNickname(vaultMetadata.getNickname());

‎hub/src/test/java/cloud/katta/client/model/ObjectMapperTest.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,9 @@ public void testAWSStatic() throws IOException {
2626

2727
final StorageProfileS3Dto awsStaticProfile = mapper.readValue(this.getClass().getResourceAsStream("/setup/aws_static/aws_static_profile.json"), StorageProfileS3Dto.class);
2828
assertEquals(Protocol.S3, awsStaticProfile.getProtocol());
29-
assertNull(awsStaticProfile.getScheme());
29+
assertEquals("https", awsStaticProfile.getScheme());
3030
assertNull(awsStaticProfile.getHostname());
31-
assertNull(awsStaticProfile.getPort());
31+
assertEquals(443, awsStaticProfile.getPort());
3232
// default STANDARD from backend
3333
assertEquals(S3STORAGECLASSES.STANDARD, awsStaticProfile.getStorageClass());
3434
assertFalse(awsStaticProfile.getArchived());
@@ -42,14 +42,14 @@ public void testAWSSTS() throws IOException {
4242

4343

4444
final StorageProfileS3STSDto awsSTSProfile = mapper.readValue(this.getClass().getResourceAsStream("/setup/aws_sts/aws_sts_profile.json"), StorageProfileS3STSDto.class);
45-
assertEquals("cipherduck", awsSTSProfile.getBucketPrefix());
45+
assertEquals("katta", awsSTSProfile.getBucketPrefix());
4646
assertEquals("eu-west-1", awsSTSProfile.getRegion());
4747
assertEquals(Arrays.asList("eu-west-1", "eu-west-2", "eu-west-3"), awsSTSProfile.getRegions());
4848
assertFalse(awsSTSProfile.getWithPathStyleAccessEnabled());
49-
assertEquals("arn:aws:iam::930717317329:role/cipherduck-createbucket", awsSTSProfile.getStsRoleArnHub());
50-
assertEquals("arn:aws:iam::930717317329:role/cipherduck-createbucket", awsSTSProfile.getStsRoleArnClient());
51-
assertEquals("arn:aws:iam::930717317329:role/cipherduck_chain_01", awsSTSProfile.getStsRoleArn());
52-
assertEquals("arn:aws:iam::930717317329:role/cipherduck_chain_02", awsSTSProfile.getStsRoleArn2());
49+
assertEquals("arn:aws:iam::430118840017:role/testing.katta.cloud-kc-realms-chipotle-createbucket", awsSTSProfile.getStsRoleArnHub());
50+
assertEquals("arn:aws:iam::430118840017:role/testing.katta.cloud-kc-realms-chipotle-createbucket", awsSTSProfile.getStsRoleArnClient());
51+
assertEquals("arn:aws:iam::430118840017:role/testing.katta.cloud-kc-realms-chipotle-sts-chain-01", awsSTSProfile.getStsRoleArn());
52+
assertEquals("arn:aws:iam::430118840017:role/testing.katta.cloud-kc-realms-chipotle-sts-chain-02", awsSTSProfile.getStsRoleArn2());
5353
assertEquals(Protocol.S3_STS, awsSTSProfile.getProtocol());
5454
assertNull(awsSTSProfile.getScheme());
5555
assertNull(awsSTSProfile.getHostname());
@@ -80,7 +80,7 @@ public void testMinioSTS() throws IOException {
8080
final ObjectMapper mapper = new ObjectMapper();
8181
mapper.registerModule(new JsonNullableModule());
8282
final StorageProfileS3STSDto minioSTSProfile = mapper.readValue(this.getClass().getResourceAsStream("/setup/minio_sts/minio_sts_profile.json"), StorageProfileS3STSDto.class);
83-
assertEquals("cipherduck", minioSTSProfile.getBucketPrefix());
83+
assertEquals("katta", minioSTSProfile.getBucketPrefix());
8484
assertEquals("eu-central-1", minioSTSProfile.getRegion());
8585
assertEquals(Arrays.asList("eu-west-1", "eu-west-2", "eu-west-3", "eu-north-1", "eu-south-1", "eu-south-2", "eu-central-1", "eu-central-2"), minioSTSProfile.getRegions());
8686
assertTrue(minioSTSProfile.getWithPathStyleAccessEnabled());

‎hub/src/test/java/cloud/katta/core/AbstractHubSynchronizeTest.java‎

Lines changed: 41 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,7 @@
44

55
package cloud.katta.core;
66

7-
import ch.cyberduck.core.AlphanumericRandomStringService;
8-
import ch.cyberduck.core.AttributedList;
9-
import ch.cyberduck.core.DisabledConnectionCallback;
10-
import ch.cyberduck.core.DisabledListProgressListener;
11-
import ch.cyberduck.core.ListService;
12-
import ch.cyberduck.core.OAuthTokens;
13-
import ch.cyberduck.core.Path;
14-
import ch.cyberduck.core.Session;
15-
import ch.cyberduck.core.SimplePathPredicate;
7+
import ch.cyberduck.core.*;
168
import ch.cyberduck.core.exception.AccessDeniedException;
179
import ch.cyberduck.core.exception.BackgroundException;
1810
import ch.cyberduck.core.exception.NotfoundException;
@@ -27,11 +19,12 @@
2719
import ch.cyberduck.core.features.Vault;
2820
import ch.cyberduck.core.features.Write;
2921
import ch.cyberduck.core.io.StatusOutputStream;
22+
import ch.cyberduck.core.proxy.DisabledProxyFinder;
23+
import ch.cyberduck.core.s3.S3Session;
3024
import ch.cyberduck.core.transfer.Transfer;
3125
import ch.cyberduck.core.transfer.TransferItem;
3226
import ch.cyberduck.core.transfer.TransferStatus;
33-
34-
import cloud.katta.client.api.UsersResourceApi;
27+
import ch.cyberduck.core.worker.DeleteWorker;
3528

3629
import org.apache.commons.io.IOUtils;
3730
import org.apache.commons.lang3.RandomUtils;
@@ -53,16 +46,23 @@
5346
import java.util.EnumSet;
5447
import java.util.List;
5548
import java.util.UUID;
49+
import java.util.stream.Collectors;
5650

5751
import cloud.katta.client.ApiClient;
5852
import cloud.katta.client.ApiException;
53+
import cloud.katta.client.api.ConfigResourceApi;
5954
import cloud.katta.client.api.StorageProfileResourceApi;
55+
import cloud.katta.client.api.UsersResourceApi;
56+
import cloud.katta.client.api.VaultResourceApi;
57+
import cloud.katta.client.model.ConfigDto;
58+
import cloud.katta.client.model.Protocol;
6059
import cloud.katta.client.model.S3SERVERSIDEENCRYPTION;
6160
import cloud.katta.client.model.S3STORAGECLASSES;
6261
import cloud.katta.client.model.StorageProfileDto;
6362
import cloud.katta.client.model.StorageProfileS3Dto;
6463
import cloud.katta.client.model.StorageProfileS3STSDto;
6564
import cloud.katta.crypto.UserKeys;
65+
import cloud.katta.crypto.uvf.VaultMetadataJWEBackendDto;
6666
import cloud.katta.model.StorageProfileDtoWrapper;
6767
import cloud.katta.protocols.hub.HubSession;
6868
import cloud.katta.protocols.hub.HubVaultRegistry;
@@ -72,6 +72,7 @@
7272
import cloud.katta.workflows.CreateVaultService;
7373
import cloud.katta.workflows.DeviceKeysServiceImpl;
7474
import cloud.katta.workflows.UserKeysServiceImpl;
75+
import cloud.katta.workflows.VaultServiceImpl;
7576
import com.fasterxml.jackson.annotation.JsonInclude;
7677
import com.fasterxml.jackson.databind.ObjectMapper;
7778

@@ -228,6 +229,30 @@ public void test03AddVault(final HubTestConfig config) throws Exception {
228229
log.info("Creating vault in {}", hubSession);
229230
final UUID vaultId = UUID.randomUUID();
230231

232+
233+
if(storageProfileWrapper.getProtocol() == Protocol.S3) {
234+
// empty bucket
235+
final HostPasswordStore keychain = PasswordStoreFactory.get();
236+
237+
final OAuthTokens tokens = keychain.findOAuthTokens(hubSession.getHost());
238+
final Host bookmark = new VaultServiceImpl(new VaultResourceApi(hubSession.getClient()), new StorageProfileResourceApi(hubSession.getClient()))
239+
.getStorageBackend(
240+
ProtocolFactory.get(),
241+
new ConfigResourceApi(hubSession.getClient()).apiConfigGet(), vaultId, new VaultMetadataJWEBackendDto()
242+
.provider(storageProfileWrapper.getId().toString())
243+
.defaultPath(config.vault.bucketName)
244+
.nickname(config.vault.bucketName)
245+
.username(config.vault.username)
246+
.password(config.vault.password), tokens);
247+
final S3Session session = new S3Session(bookmark);
248+
session.open(new DisabledProxyFinder(), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback());
249+
session.login(new DisabledLoginCallback(), new DisabledCancelCallback());
250+
new DeleteWorker(new DisabledLoginCallback(),
251+
session.getFeature(ListService.class).list(new Path("/" + config.vault.bucketName, EnumSet.of(AbstractPath.Type.directory)), new DisabledListProgressListener()).toStream().filter(f -> session.getFeature(Delete.class).isSupported(f)).collect(Collectors.toList()),
252+
new DisabledListProgressListener()).run(session);
253+
session.close();
254+
}
255+
231256
final UserKeys userKeys = new UserKeysServiceImpl(hubSession).getUserKeys(hubSession.getHost(), hubSession.getMe(),
232257
new DeviceKeysServiceImpl().getDeviceKeys(hubSession.getHost()));
233258
new CreateVaultService(hubSession).createVault(userKeys, storageProfileWrapper, new CreateVaultService.CreateVaultModel(
@@ -237,7 +262,7 @@ public void test03AddVault(final HubTestConfig config) throws Exception {
237262
final AttributedList<Path> vaults = hubSession.getFeature(ListService.class).list(Home.ROOT, new DisabledListProgressListener());
238263
assertFalse(vaults.isEmpty());
239264

240-
final Path bucket = new Path(storageProfileWrapper.getStsEndpoint() != null ? storageProfileWrapper.getBucketPrefix() + vaultId : config.vault.bucketName,
265+
final Path bucket = new Path(storageProfileWrapper.getProtocol() == Protocol.S3_STS ? storageProfileWrapper.getBucketPrefix() + vaultId : config.vault.bucketName,
241266
EnumSet.of(Path.Type.volume, Path.Type.directory));
242267
final HubVaultRegistry vaultRegistry = hubSession.getRegistry();
243268
{
@@ -326,9 +351,11 @@ public void test04SetupCode(final HubTestConfig config) throws Exception {
326351
assertEquals(StringUtils.EMPTY, hubSession.getHost().getCredentials().getPassword());
327352
final ListService feature = hubSession.getFeature(ListService.class);
328353
final AttributedList<Path> vaults = feature.list(Home.ROOT, new DisabledListProgressListener());
329-
assertEquals(2, vaults.size());
354+
final ConfigDto configDto = new ConfigResourceApi(hubSession.getClient()).apiConfigGet();
355+
final int expectedNumberOfVaults = configDto.getKeycloakTokenEndpoint().contains("localhost") ? 2 : 4;
356+
assertEquals(expectedNumberOfVaults, vaults.size());
330357
assertEquals(vaults, feature.list(Home.ROOT, new DisabledListProgressListener()));
331-
for(Path vault : vaults) {
358+
for(final Path vault : vaults) {
332359
assertTrue(hubSession.getFeature(Find.class).find(vault));
333360
}
334361
new UsersResourceApi(hubSession.getClient()).apiUsersMeGet(true);

0 commit comments

Comments
 (0)