From 9956d53de9f7a8c93222736e2f73aa0ff2c233e9 Mon Sep 17 00:00:00 2001 From: Tuan Pham <103537251+phantumcode@users.noreply.github.com> Date: Thu, 15 Aug 2024 15:17:57 -0500 Subject: [PATCH 1/7] feat(storage): update AWSS3StoragePlugin to support multiple buckets (#2895) --- .../storage/s3/AWSS3StoragePlugin.java | 152 +++++++++++++++--- .../AWSS3StorageGetPresignedUrlOptions.java | 8 +- .../storage/s3/AWSS3StoragePluginTest.kt | 85 ++++++++++ core/api/core.api | 29 ++++ .../core/configuration/AmplifyOutputsData.kt | 18 ++- .../amplifyframework/storage/BucketInfo.kt | 17 ++ .../storage/InvalidStorageBucketException.kt | 25 +++ .../amplifyframework/storage/StorageBucket.kt | 32 ++++ .../storage/options/StorageGetUrlOptions.java | 12 +- .../storage/options/StorageOptions.java | 42 +++++ .../configuration/AmplifyOutputsDataTest.kt | 45 ++++++ .../AmplifyOutputsDataBuilder.kt | 10 ++ 12 files changed, 445 insertions(+), 30 deletions(-) create mode 100644 core/src/main/java/com/amplifyframework/storage/BucketInfo.kt create mode 100644 core/src/main/java/com/amplifyframework/storage/InvalidStorageBucketException.kt create mode 100644 core/src/main/java/com/amplifyframework/storage/StorageBucket.kt diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/AWSS3StoragePlugin.java b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/AWSS3StoragePlugin.java index 9375a14d9b..26403ccb70 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/AWSS3StoragePlugin.java +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/AWSS3StoragePlugin.java @@ -18,6 +18,7 @@ import android.annotation.SuppressLint; import android.content.Context; import androidx.annotation.NonNull; +import androidx.annotation.Nullable; import androidx.annotation.OptIn; import androidx.annotation.VisibleForTesting; @@ -28,7 +29,12 @@ import com.amplifyframework.core.Consumer; import com.amplifyframework.core.NoOpConsumer; import com.amplifyframework.core.configuration.AmplifyOutputsData; +import com.amplifyframework.storage.BucketInfo; +import com.amplifyframework.storage.InvalidStorageBucketException; +import com.amplifyframework.storage.OutputsStorageBucket; +import com.amplifyframework.storage.ResolvedStorageBucket; import com.amplifyframework.storage.StorageAccessLevel; +import com.amplifyframework.storage.StorageBucket; import com.amplifyframework.storage.StorageException; import com.amplifyframework.storage.StoragePath; import com.amplifyframework.storage.StoragePlugin; @@ -95,6 +101,9 @@ import java.io.File; import java.io.InputStream; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -121,11 +130,16 @@ public final class AWSS3StoragePlugin extends StoragePlugin { private final ExecutorService executorService; private final AuthCredentialsProvider authCredentialsProvider; private final AWSS3StoragePluginConfiguration awsS3StoragePluginConfiguration; - private AWSS3StorageService storageService; + private AWSS3StorageService defaultStorageService; @SuppressWarnings("deprecation") private StorageAccessLevel defaultAccessLevel; private int defaultUrlExpiration; + private Map awsS3StorageServicesByBucketName = new HashMap<>(); + private Context context; + @SuppressLint("UnsafeOptInUsageError") + private List configuredBuckets; + /** * Constructs the AWS S3 Storage Plugin initializing the executor service. */ @@ -194,6 +208,7 @@ public String getPluginKey() { return AWS_S3_STORAGE_PLUGIN_KEY; } + @SuppressLint("UnsafeOptInUsageError") @Override @SuppressWarnings("deprecation") public void configure( @@ -237,7 +252,8 @@ public void configure( ); } - configure(context, region, bucket); + BucketInfo bucketInfo = new BucketInfo(bucket, region); + configure(context, region, (ResolvedStorageBucket) StorageBucket.fromBucketInfo(bucketInfo)); } @Override @@ -252,17 +268,26 @@ public void configure(@NonNull AmplifyOutputsData configuration, @NonNull Contex ); } - configure(context, storage.getAwsRegion(), storage.getBucketName()); + this.configuredBuckets = storage.getBuckets(); + BucketInfo bucketInfo = new BucketInfo(storage.getBucketName(), storage.getAwsRegion()); + configure(context, storage.getAwsRegion(), (ResolvedStorageBucket) StorageBucket.fromBucketInfo(bucketInfo)); } @SuppressWarnings("deprecation") + @SuppressLint("UnsafeOptInUsageError") private void configure( - @NonNull Context context, - @NonNull String region, - @NonNull String bucket + @NonNull Context context, + @NonNull String region, + @NonNull ResolvedStorageBucket bucket ) throws StorageException { try { - this.storageService = (AWSS3StorageService) storageServiceFactory.create(context, region, bucket); + this.context = context; + this.defaultStorageService = (AWSS3StorageService) storageServiceFactory.create( + context, + region, + bucket.getBucketInfo().getName()); + this.awsS3StorageServicesByBucketName.clear(); + this.awsS3StorageServicesByBucketName.put(bucket.getBucketInfo().getName(), this.defaultStorageService); } catch (RuntimeException exception) { throw new StorageException( "Failed to create storage service.", @@ -280,7 +305,7 @@ private void configure( @NonNull @Override public S3Client getEscapeHatch() { - return storageService.getClient(); + return defaultStorageService.getClient(); } @NonNull @@ -334,6 +359,18 @@ public StorageGetUrlOperation getUrl( validateObjectExistence ); + AWSS3StorageService storageService = defaultStorageService; + try { + storageService = getStorageService(options.getBucket()); + } catch (InvalidStorageBucketException exception) { + onError.accept( + new StorageException( + "Unable to find bucket from name in Amplify Outputs.", + exception, + "Ensure the bucket name used is available in Amplify Outputs.") + ); + } + AWSS3StorageGetPresignedUrlOperation operation = new AWSS3StorageGetPresignedUrlOperation( storageService, @@ -369,6 +406,18 @@ public StorageGetUrlOperation getUrl( validateObjectExistence ); + AWSS3StorageService storageService = defaultStorageService; + try { + storageService = getStorageService(options.getBucket()); + } catch (InvalidStorageBucketException exception) { + onError.accept( + new StorageException( + "Unable to find bucket from name in Amplify Outputs.", + exception, + "Ensure the bucket name used is available in Amplify Outputs.") + ); + } + AWSS3StoragePathGetPresignedUrlOperation operation = new AWSS3StoragePathGetPresignedUrlOperation( storageService, @@ -457,7 +506,7 @@ public StorageDownloadFileOperation downloadFile( ); AWSS3StorageDownloadFileOperation operation = new AWSS3StorageDownloadFileOperation( - storageService, + defaultStorageService, executorService, authCredentialsProvider, request, @@ -493,7 +542,7 @@ public StorageDownloadFileOperation downloadFile( AWSS3StoragePathDownloadFileOperation operation = new AWSS3StoragePathDownloadFileOperation( request, - storageService, + defaultStorageService, executorService, authCredentialsProvider, onProgress, @@ -584,7 +633,7 @@ public StorageUploadFileOperation uploadFile( ); AWSS3StorageUploadFileOperation operation = new AWSS3StorageUploadFileOperation( - storageService, + defaultStorageService, executorService, authCredentialsProvider, request, @@ -623,7 +672,7 @@ public StorageUploadFileOperation uploadFile( AWSS3StoragePathUploadFileOperation operation = new AWSS3StoragePathUploadFileOperation( request, - storageService, + defaultStorageService, executorService, authCredentialsProvider, onProgress, @@ -712,7 +761,7 @@ public StorageUploadInputStreamOperation uploadInputStream( ); AWSS3StorageUploadInputStreamOperation operation = new AWSS3StorageUploadInputStreamOperation( - storageService, + defaultStorageService, executorService, authCredentialsProvider, awsS3StoragePluginConfiguration, @@ -752,7 +801,7 @@ public StorageUploadInputStreamOperation uploadInputStream( AWSS3StoragePathUploadInputStreamOperation operation = new AWSS3StoragePathUploadInputStreamOperation( request, - storageService, + defaultStorageService, executorService, authCredentialsProvider, onProgress, @@ -804,7 +853,7 @@ public StorageRemoveOperation remove( AWSS3StorageRemoveOperation operation = new AWSS3StorageRemoveOperation( - storageService, + defaultStorageService, executorService, authCredentialsProvider, request, @@ -829,7 +878,7 @@ public StorageRemoveOperation remove( AWSS3StoragePathRemoveOperation operation = new AWSS3StoragePathRemoveOperation( - storageService, + defaultStorageService, executorService, authCredentialsProvider, request, @@ -849,12 +898,12 @@ public void getTransfer( @NonNull Consumer onError) { executorService.submit(() -> { try { - TransferRecord transferRecord = storageService.getTransfer(transferId); + TransferRecord transferRecord = defaultStorageService.getTransfer(transferId); if (transferRecord != null) { TransferObserver transferObserver = new TransferObserver( transferRecord.getId(), - storageService.getTransferManager().getTransferStatusUpdater(), + defaultStorageService.getTransferManager().getTransferStatusUpdater(), transferRecord.getBucketName(), transferRecord.getKey(), transferRecord.getFile(), @@ -867,7 +916,7 @@ public void getTransfer( AWSS3StorageUploadInputStreamOperation operation = new AWSS3StorageUploadInputStreamOperation( transferId, - storageService, + defaultStorageService, executorService, authCredentialsProvider, awsS3StoragePluginConfiguration, @@ -878,7 +927,7 @@ public void getTransfer( AWSS3StorageUploadFileOperation operation = new AWSS3StorageUploadFileOperation( transferId, - storageService, + defaultStorageService, executorService, authCredentialsProvider, awsS3StoragePluginConfiguration, @@ -892,7 +941,7 @@ public void getTransfer( downloadFileOperation = new AWSS3StorageDownloadFileOperation( transferId, new File(transferRecord.getFile()), - storageService, + defaultStorageService, executorService, authCredentialsProvider, awsS3StoragePluginConfiguration, @@ -960,7 +1009,7 @@ public StorageListOperation list(@NonNull String path, AWSS3StorageListOperation operation = new AWSS3StorageListOperation( - storageService, + defaultStorageService, executorService, authCredentialsProvider, request, @@ -989,7 +1038,7 @@ public StorageListOperation list( AWSS3StoragePathListOperation operation = new AWSS3StoragePathListOperation( - storageService, + defaultStorageService, executorService, authCredentialsProvider, request, @@ -1001,6 +1050,63 @@ public StorageListOperation list( return operation; } + @SuppressLint("UnsafeOptInUsageError") + @VisibleForTesting + @NonNull + AWSS3StorageService getStorageService(@Nullable StorageBucket bucket) throws InvalidStorageBucketException { + if (bucket == null) { + return defaultStorageService; + } + + if (bucket instanceof OutputsStorageBucket) { + AWSS3StorageService service = getAWSS3StorageService((OutputsStorageBucket) bucket); + if (service == null) { + throw new InvalidStorageBucketException(); + } else { + return service; + } + } + + if (bucket instanceof ResolvedStorageBucket) { + return getAWSS3StorageService((ResolvedStorageBucket) bucket); + } + + return defaultStorageService; + } + + @SuppressLint("UnsafeOptInUsageError") + private AWSS3StorageService getAWSS3StorageService(OutputsStorageBucket outputsStorageBucket) { + if (configuredBuckets != null && !configuredBuckets.isEmpty()) { + String name = outputsStorageBucket.getName(); + for (AmplifyOutputsData.StorageBucket configuredBucket : configuredBuckets) { + if (configuredBucket.getName().equals(name)) { + String bucketName = configuredBucket.getBucketName(); + AWSS3StorageService service = awsS3StorageServicesByBucketName.get(bucketName); + if (service == null) { + String region = configuredBucket.getAwsRegion(); + service = (AWSS3StorageService) storageServiceFactory.create(context, region, bucketName); + awsS3StorageServicesByBucketName.put(bucketName, service); + } + + return service; + } + } + } + return null; + } + + @SuppressLint("UnsafeOptInUsageError") + private AWSS3StorageService getAWSS3StorageService(ResolvedStorageBucket resolvedStorageBucket) { + String bucketName = resolvedStorageBucket.getBucketInfo().getName(); + AWSS3StorageService service = awsS3StorageServicesByBucketName.get(bucketName); + if (service == null) { + String region = resolvedStorageBucket.getBucketInfo().getRegion(); + service = (AWSS3StorageService) storageServiceFactory.create(context, region, bucketName); + awsS3StorageServicesByBucketName.put(bucketName, service); + } + return service; + } + /** * Holds the keys for the various configuration properties for this plugin. */ diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/options/AWSS3StorageGetPresignedUrlOptions.java b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/options/AWSS3StorageGetPresignedUrlOptions.java index 5c204235fd..754c52874b 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/options/AWSS3StorageGetPresignedUrlOptions.java +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/options/AWSS3StorageGetPresignedUrlOptions.java @@ -63,7 +63,8 @@ public static Builder from(@NonNull AWSS3StorageGetPresignedUrlOptions options) .targetIdentityId(options.getTargetIdentityId()) .expires(options.getExpires()) .setValidateObjectExistence(options.getValidateObjectExistence()) - .expires(options.getExpires()); + .expires(options.getExpires()) + .bucket(options.getBucket()); } /** @@ -106,6 +107,7 @@ public boolean equals(Object obj) { return ObjectsCompat.equals(getAccessLevel(), that.getAccessLevel()) && ObjectsCompat.equals(getTargetIdentityId(), that.getTargetIdentityId()) && ObjectsCompat.equals(getExpires(), that.getExpires()) && + ObjectsCompat.equals(getBucket(), that.getBucket()) && ObjectsCompat.equals(getValidateObjectExistence(), that.getValidateObjectExistence()); } } @@ -117,7 +119,8 @@ public int hashCode() { getAccessLevel(), getTargetIdentityId(), getExpires(), - getValidateObjectExistence() + getValidateObjectExistence(), + getBucket() ); } @@ -130,6 +133,7 @@ public String toString() { ", targetIdentityId=" + getTargetIdentityId() + ", expires=" + getExpires() + ", validateObjectExistence=" + getValidateObjectExistence() + + ", bucket=" + getBucket() + '}'; } diff --git a/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/AWSS3StoragePluginTest.kt b/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/AWSS3StoragePluginTest.kt index ebcbdd9fd4..e5e13cfe40 100644 --- a/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/AWSS3StoragePluginTest.kt +++ b/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/AWSS3StoragePluginTest.kt @@ -15,11 +15,15 @@ package com.amplifyframework.storage.s3 +import com.amplifyframework.storage.BucketInfo +import com.amplifyframework.storage.InvalidStorageBucketException +import com.amplifyframework.storage.StorageBucket import com.amplifyframework.storage.StorageException import com.amplifyframework.storage.s3.service.AWSS3StorageService import com.amplifyframework.storage.s3.service.StorageService import com.amplifyframework.testutils.configuration.amplifyOutputsData import io.kotest.assertions.throwables.shouldThrow +import io.kotest.matchers.shouldNotBe import io.mockk.every import io.mockk.mockk import io.mockk.verify @@ -63,4 +67,85 @@ class AWSS3StoragePluginTest { plugin.configure(data, mockk()) } } + + @Test + fun `getStorageService returns default storage service if bucket is null`() { + val data = amplifyOutputsData { + storage { + awsRegion = "test-region" + bucketName = "test-bucket" + buckets { + awsRegion = "test-region" + bucketName = "test-bucket" + name = "test=name" + } + } + } + + plugin.configure(data, mockk()) + val service = plugin.getStorageService(null) + service shouldNotBe null + } + + @Test + fun `get AWSS3StorageService from BucketInfo`() { + val data = amplifyOutputsData { + storage { + awsRegion = "test-region" + bucketName = "test-bucket" + buckets { + awsRegion = "test-region" + bucketName = "test-bucket" + name = "test=name" + } + } + } + + plugin.configure(data, mockk()) + val bucketInfo = BucketInfo("test-bucket", "test-region") + val bucket = StorageBucket.fromBucketInfo(bucketInfo) + val service = plugin.getStorageService(bucket) + service shouldNotBe null + } + + @Test + fun `get AWSS3StorageService from AmplifyOutputs`() { + val data = amplifyOutputsData { + storage { + awsRegion = "test-region" + bucketName = "test-bucket" + buckets { + awsRegion = "test-region" + bucketName = "test-bucket" + name = "test=name" + } + } + } + + plugin.configure(data, mockk()) + val bucket = StorageBucket.fromOutputs("test=name") + val service = plugin.getStorageService(bucket) + service shouldNotBe null + } + + @Test + fun `getStorageService throws InvalidStorageBucketException`() { + val data = amplifyOutputsData { + storage { + awsRegion = "test-region" + bucketName = "test-bucket" + buckets { + awsRegion = "test-region" + bucketName = "test-bucket" + name = "test=name" + } + } + } + + plugin.configure(data, mockk()) + val bucket = StorageBucket.fromOutputs("myBucket") + shouldThrow { + plugin.getStorageService(bucket) + } + } } diff --git a/core/api/core.api b/core/api/core.api index 4beafbf12c..d1b72287fe 100644 --- a/core/api/core.api +++ b/core/api/core.api @@ -3930,6 +3930,19 @@ public final class com/amplifyframework/predictions/result/TranslateTextResult$B public fun translatedText (Ljava/lang/String;)Lcom/amplifyframework/predictions/result/TranslateTextResult$Builder; } +public final class com/amplifyframework/storage/BucketInfo { + public fun (Ljava/lang/String;Ljava/lang/String;)V + public final fun component1 ()Ljava/lang/String; + public final fun component2 ()Ljava/lang/String; + public final fun copy (Ljava/lang/String;Ljava/lang/String;)Lcom/amplifyframework/storage/BucketInfo; + public static synthetic fun copy$default (Lcom/amplifyframework/storage/BucketInfo;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)Lcom/amplifyframework/storage/BucketInfo; + public fun equals (Ljava/lang/Object;)Z + public final fun getName ()Ljava/lang/String; + public final fun getRegion ()Ljava/lang/String; + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + public final class com/amplifyframework/storage/IdentityIdProvidedStoragePath : com/amplifyframework/storage/StoragePath { public final fun copy (Lkotlin/jvm/functions/Function1;)Lcom/amplifyframework/storage/IdentityIdProvidedStoragePath; public static synthetic fun copy$default (Lcom/amplifyframework/storage/IdentityIdProvidedStoragePath;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lcom/amplifyframework/storage/IdentityIdProvidedStoragePath; @@ -3938,6 +3951,10 @@ public final class com/amplifyframework/storage/IdentityIdProvidedStoragePath : public fun toString ()Ljava/lang/String; } +public final class com/amplifyframework/storage/InvalidStorageBucketException : com/amplifyframework/AmplifyException { + public fun ()V +} + public final class com/amplifyframework/storage/ObjectMetadata { public static final field CACHE_CONTROL Ljava/lang/String; public static final field CONTENT_DISPOSITION Ljava/lang/String; @@ -3999,6 +4016,18 @@ public final class com/amplifyframework/storage/StorageAccessLevel : java/lang/E public static fun values ()[Lcom/amplifyframework/storage/StorageAccessLevel; } +public abstract class com/amplifyframework/storage/StorageBucket { + public static final field Companion Lcom/amplifyframework/storage/StorageBucket$Companion; + public fun ()V + public static final fun fromBucketInfo (Lcom/amplifyframework/storage/BucketInfo;)Lcom/amplifyframework/storage/StorageBucket; + public static final fun fromOutputs (Ljava/lang/String;)Lcom/amplifyframework/storage/StorageBucket; +} + +public final class com/amplifyframework/storage/StorageBucket$Companion { + public final fun fromBucketInfo (Lcom/amplifyframework/storage/BucketInfo;)Lcom/amplifyframework/storage/StorageBucket; + public final fun fromOutputs (Ljava/lang/String;)Lcom/amplifyframework/storage/StorageBucket; +} + public final class com/amplifyframework/storage/StorageCategory : com/amplifyframework/core/category/Category, com/amplifyframework/storage/StorageCategoryBehavior { public fun ()V public fun downloadFile (Lcom/amplifyframework/storage/StoragePath;Ljava/io/File;Lcom/amplifyframework/core/Consumer;Lcom/amplifyframework/core/Consumer;)Lcom/amplifyframework/storage/operation/StorageDownloadFileOperation; diff --git a/core/src/main/java/com/amplifyframework/core/configuration/AmplifyOutputsData.kt b/core/src/main/java/com/amplifyframework/core/configuration/AmplifyOutputsData.kt index 0dae51b792..ac76d9634e 100644 --- a/core/src/main/java/com/amplifyframework/core/configuration/AmplifyOutputsData.kt +++ b/core/src/main/java/com/amplifyframework/core/configuration/AmplifyOutputsData.kt @@ -187,6 +187,14 @@ interface AmplifyOutputsData { interface Storage { val awsRegion: String val bucketName: String + val buckets: List + } + + @InternalAmplifyApi + interface StorageBucket { + val name: String + val awsRegion: String + val bucketName: String } @InternalAmplifyApi @@ -353,9 +361,17 @@ internal data class AmplifyOutputsDataImpl( @Serializable data class Storage( override val awsRegion: String, - override val bucketName: String + override val bucketName: String, + override val buckets: List = emptyList() ) : AmplifyOutputsData.Storage + @Serializable + data class StorageBucket( + override val name: String, + override val awsRegion: String, + override val bucketName: String + ) : AmplifyOutputsData.StorageBucket + @Serializable data class AmazonLocationServiceConfig( override val style: String diff --git a/core/src/main/java/com/amplifyframework/storage/BucketInfo.kt b/core/src/main/java/com/amplifyframework/storage/BucketInfo.kt new file mode 100644 index 0000000000..ca3da36064 --- /dev/null +++ b/core/src/main/java/com/amplifyframework/storage/BucketInfo.kt @@ -0,0 +1,17 @@ +/* + * Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amplifyframework.storage + +data class BucketInfo(val name: String, val region: String) diff --git a/core/src/main/java/com/amplifyframework/storage/InvalidStorageBucketException.kt b/core/src/main/java/com/amplifyframework/storage/InvalidStorageBucketException.kt new file mode 100644 index 0000000000..4117be8b98 --- /dev/null +++ b/core/src/main/java/com/amplifyframework/storage/InvalidStorageBucketException.kt @@ -0,0 +1,25 @@ +/* + * Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amplifyframework.storage + +import com.amplifyframework.AmplifyException + +/** + * Exception thrown when an invalid StorageBucket is specified. + */ +class InvalidStorageBucketException internal constructor( + message: String = "Unable to find bucket from name in Amplify Outputs.", + recoverySuggestion: String = "Ensure the bucket name used is available in Amplify Outputs." +) : AmplifyException(message, recoverySuggestion) diff --git a/core/src/main/java/com/amplifyframework/storage/StorageBucket.kt b/core/src/main/java/com/amplifyframework/storage/StorageBucket.kt new file mode 100644 index 0000000000..60f6e5c306 --- /dev/null +++ b/core/src/main/java/com/amplifyframework/storage/StorageBucket.kt @@ -0,0 +1,32 @@ +/* + * Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amplifyframework.storage + +import com.amplifyframework.annotations.InternalAmplifyApi + +abstract class StorageBucket { + companion object { + @JvmStatic + fun fromOutputs(name: String): StorageBucket = OutputsStorageBucket(name) + @JvmStatic + fun fromBucketInfo(bucketInfo: BucketInfo): StorageBucket = ResolvedStorageBucket(bucketInfo) + } +} + +@InternalAmplifyApi +data class OutputsStorageBucket internal constructor(val name: String) : StorageBucket() + +@InternalAmplifyApi +data class ResolvedStorageBucket internal constructor(val bucketInfo: BucketInfo) : StorageBucket() diff --git a/core/src/main/java/com/amplifyframework/storage/options/StorageGetUrlOptions.java b/core/src/main/java/com/amplifyframework/storage/options/StorageGetUrlOptions.java index ddb3be33d4..27ef4f4243 100644 --- a/core/src/main/java/com/amplifyframework/storage/options/StorageGetUrlOptions.java +++ b/core/src/main/java/com/amplifyframework/storage/options/StorageGetUrlOptions.java @@ -32,7 +32,7 @@ public class StorageGetUrlOptions extends StorageOptions { */ @SuppressWarnings("deprecation") protected StorageGetUrlOptions(final Builder builder) { - super(builder.getAccessLevel(), builder.getTargetIdentityId()); + super(builder.getAccessLevel(), builder.getTargetIdentityId(), builder.getBucket()); this.expires = builder.getExpires(); } @@ -69,9 +69,10 @@ public static Builder builder() { @SuppressWarnings("deprecation") public static Builder from(@NonNull StorageGetUrlOptions options) { return builder() - .accessLevel(options.getAccessLevel()) - .targetIdentityId(options.getTargetIdentityId()) - .expires(options.getExpires()); + .accessLevel(options.getAccessLevel()) + .targetIdentityId(options.getTargetIdentityId()) + .bucket(options.getBucket()) + .expires(options.getExpires()); } /** @@ -97,6 +98,7 @@ public boolean equals(Object obj) { StorageGetUrlOptions that = (StorageGetUrlOptions) obj; return ObjectsCompat.equals(getAccessLevel(), that.getAccessLevel()) && ObjectsCompat.equals(getTargetIdentityId(), that.getTargetIdentityId()) && + ObjectsCompat.equals(getBucket(), that.getBucket()) && ObjectsCompat.equals(getExpires(), that.getExpires()); } } @@ -110,6 +112,7 @@ public int hashCode() { return ObjectsCompat.hash( getAccessLevel(), getTargetIdentityId(), + getBucket(), getExpires() ); } @@ -124,6 +127,7 @@ public String toString() { return "StorageGetUrlOptions {" + "accessLevel=" + getAccessLevel() + ", targetIdentityId=" + getTargetIdentityId() + + ", bucket=" + getBucket() + ", expires=" + getExpires() + '}'; } diff --git a/core/src/main/java/com/amplifyframework/storage/options/StorageOptions.java b/core/src/main/java/com/amplifyframework/storage/options/StorageOptions.java index 1be0c44314..95f111b5cb 100644 --- a/core/src/main/java/com/amplifyframework/storage/options/StorageOptions.java +++ b/core/src/main/java/com/amplifyframework/storage/options/StorageOptions.java @@ -19,6 +19,7 @@ import androidx.annotation.Nullable; import com.amplifyframework.storage.StorageAccessLevel; +import com.amplifyframework.storage.StorageBucket; import com.amplifyframework.storage.StoragePath; /** @@ -31,11 +32,23 @@ abstract class StorageOptions { private final StorageAccessLevel accessLevel; private final String targetIdentityId; + private final StorageBucket bucket; + @SuppressWarnings("deprecation") StorageOptions(StorageAccessLevel accessLevel, String targetIdentityId) { this.accessLevel = accessLevel; this.targetIdentityId = targetIdentityId; + this.bucket = null; + } + + @SuppressWarnings("deprecation") + StorageOptions(StorageAccessLevel accessLevel, + String targetIdentityId, + StorageBucket bucket) { + this.accessLevel = accessLevel; + this.targetIdentityId = targetIdentityId; + this.bucket = bucket; } /** @@ -61,6 +74,15 @@ public final String getTargetIdentityId() { return targetIdentityId; } + /** + * Gets the storage bucket. + * @return storage bucket + */ + @Nullable + public final StorageBucket getBucket() { + return bucket; + } + /** * Builds storage options. */ @@ -69,6 +91,7 @@ abstract static class Builder { @SuppressWarnings("deprecation") private StorageAccessLevel accessLevel; private String targetIdentityId; + private StorageBucket bucket; /** * Configures the storage access level to set on new @@ -99,6 +122,16 @@ public final B targetIdentityId(@Nullable String targetIdentityId) { return (B) this; } + /** + * Configure the storage bucket that will be used on newly built StorageOptions. + * @param bucket Storage bucket for new StorageOptions instances + * @return Current Builder instance, for fluent method chaining + */ + public final B bucket(StorageBucket bucket) { + this.bucket = bucket; + return (B) this; + } + @SuppressWarnings("deprecation") @Deprecated @Nullable @@ -112,6 +145,15 @@ public final String getTargetIdentityId() { return targetIdentityId; } + /** + * Gets the storage bucket. + * @return storage bucket + */ + @Nullable + public final StorageBucket getBucket() { + return bucket; + } + /** * Constructs and returns a new immutable instance of the * StorageOptions, using the configurations that diff --git a/core/src/test/java/com/amplifyframework/core/configuration/AmplifyOutputsDataTest.kt b/core/src/test/java/com/amplifyframework/core/configuration/AmplifyOutputsDataTest.kt index a570ed8617..8bbfc8228c 100644 --- a/core/src/test/java/com/amplifyframework/core/configuration/AmplifyOutputsDataTest.kt +++ b/core/src/test/java/com/amplifyframework/core/configuration/AmplifyOutputsDataTest.kt @@ -253,6 +253,49 @@ class AmplifyOutputsDataTest { outputs.storage?.run { awsRegion shouldBe "us-east-1" bucketName shouldBe "myBucket" + buckets.size shouldBe 0 + } + } + + @Test + fun `parses multi-bucket storage configuration`() { + val json = createJson( + Keys.storage to mapOf( + Keys.region to "us-east-1", + Keys.bucket to "myBucket", + Keys.buckets to listOf( + mapOf( + Keys.region to "us-east-1", + Keys.bucket to "myBucket", + Keys.name to "name1" + ), + mapOf( + Keys.region to "us-east-2", + Keys.bucket to "myBucket2", + Keys.name to "name2" + ) + ) + ) + ) + + val outputs = AmplifyOutputsData.deserialize(json) + + outputs.storage.shouldNotBeNull() + outputs.storage?.run { + awsRegion shouldBe "us-east-1" + bucketName shouldBe "myBucket" + buckets.size shouldBe 2 + buckets[0].apply { + name shouldBe "name1" + awsRegion shouldBe "us-east-1" + bucketName shouldBe "myBucket" + } + + buckets[1].apply { + name shouldBe "name2" + awsRegion shouldBe "us-east-2" + bucketName shouldBe "myBucket2" + } } } @@ -361,6 +404,8 @@ class AmplifyOutputsDataTest { // Storage const val storage = "storage" const val bucket = "bucket_name" + const val buckets = "buckets" + const val name = "name" // Custom const val custom = "custom" diff --git a/testutils/src/main/java/com/amplifyframework/testutils/configuration/AmplifyOutputsDataBuilder.kt b/testutils/src/main/java/com/amplifyframework/testutils/configuration/AmplifyOutputsDataBuilder.kt index eada856d39..f799ccf69f 100644 --- a/testutils/src/main/java/com/amplifyframework/testutils/configuration/AmplifyOutputsDataBuilder.kt +++ b/testutils/src/main/java/com/amplifyframework/testutils/configuration/AmplifyOutputsDataBuilder.kt @@ -163,4 +163,14 @@ class NotificationsBuilder : AmplifyOutputsData.Notifications { class StorageBuilder : AmplifyOutputsData.Storage { override var awsRegion: String = "us-east-1" override var bucketName: String = "bucket-name" + override var buckets: MutableList = mutableListOf() + fun buckets(func: StorageBucketBuilder.() -> Unit) { + buckets += StorageBucketBuilder().apply(func) + } +} + +class StorageBucketBuilder : AmplifyOutputsData.StorageBucket { + override var awsRegion: String = "us-east-1" + override var bucketName: String = "bucket-name" + override var name: String = "test-name" } From 44ad870c28fd0d78e91ec269b8b6bf62844e1187 Mon Sep 17 00:00:00 2001 From: Tuan Pham <103537251+phantumcode@users.noreply.github.com> Date: Thu, 22 Aug 2024 10:22:10 -0500 Subject: [PATCH 2/7] feat(storage): update storage API options to include bucket (#2896) --- .../storage/s3/AWSS3StoragePlugin.java | 97 ++++++++++++++----- .../AWSS3StorageDownloadFileOptions.java | 14 ++- .../s3/options/AWSS3StorageListOptions.java | 12 ++- .../s3/options/AWSS3StorageRemoveOptions.java | 12 ++- .../AWSS3StorageUploadFileOptions.java | 18 ++-- .../AWSS3StorageUploadInputStreamOptions.java | 10 +- .../storage/s3/AWSS3StoragePluginTest.kt | 6 +- .../options/StorageDownloadFileOptions.java | 20 ++-- .../storage/options/StorageListOptions.java | 14 ++- .../options/StoragePagedListOptions.java | 2 +- .../storage/options/StorageRemoveOptions.java | 14 ++- .../options/StorageUploadFileOptions.java | 16 +-- .../StorageUploadInputStreamOptions.java | 16 +-- .../storage/options/StorageUploadOptions.java | 9 +- 14 files changed, 177 insertions(+), 83 deletions(-) diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/AWSS3StoragePlugin.java b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/AWSS3StoragePlugin.java index 26403ccb70..01fde9ba41 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/AWSS3StoragePlugin.java +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/AWSS3StoragePlugin.java @@ -362,13 +362,8 @@ public StorageGetUrlOperation getUrl( AWSS3StorageService storageService = defaultStorageService; try { storageService = getStorageService(options.getBucket()); - } catch (InvalidStorageBucketException exception) { - onError.accept( - new StorageException( - "Unable to find bucket from name in Amplify Outputs.", - exception, - "Ensure the bucket name used is available in Amplify Outputs.") - ); + } catch (StorageException exception) { + onError.accept(exception); } AWSS3StorageGetPresignedUrlOperation operation = @@ -409,13 +404,8 @@ public StorageGetUrlOperation getUrl( AWSS3StorageService storageService = defaultStorageService; try { storageService = getStorageService(options.getBucket()); - } catch (InvalidStorageBucketException exception) { - onError.accept( - new StorageException( - "Unable to find bucket from name in Amplify Outputs.", - exception, - "Ensure the bucket name used is available in Amplify Outputs.") - ); + } catch (StorageException exception) { + onError.accept(exception); } AWSS3StoragePathGetPresignedUrlOperation operation = @@ -505,8 +495,15 @@ public StorageDownloadFileOperation downloadFile( useAccelerateEndpoint ); + AWSS3StorageService storageService = defaultStorageService; + try { + storageService = getStorageService(options.getBucket()); + } catch (StorageException exception) { + onError.accept(exception); + } + AWSS3StorageDownloadFileOperation operation = new AWSS3StorageDownloadFileOperation( - defaultStorageService, + storageService, executorService, authCredentialsProvider, request, @@ -540,9 +537,16 @@ public StorageDownloadFileOperation downloadFile( useAccelerateEndpoint ); + AWSS3StorageService storageService = defaultStorageService; + try { + storageService = getStorageService(options.getBucket()); + } catch (StorageException exception) { + onError.accept(exception); + } + AWSS3StoragePathDownloadFileOperation operation = new AWSS3StoragePathDownloadFileOperation( request, - defaultStorageService, + storageService, executorService, authCredentialsProvider, onProgress, @@ -632,8 +636,15 @@ public StorageUploadFileOperation uploadFile( useAccelerateEndpoint ); + AWSS3StorageService storageService = defaultStorageService; + try { + storageService = getStorageService(options.getBucket()); + } catch (StorageException exception) { + onError.accept(exception); + } + AWSS3StorageUploadFileOperation operation = new AWSS3StorageUploadFileOperation( - defaultStorageService, + storageService, executorService, authCredentialsProvider, request, @@ -670,9 +681,16 @@ public StorageUploadFileOperation uploadFile( useAccelerateEndpoint ); + AWSS3StorageService storageService = defaultStorageService; + try { + storageService = getStorageService(options.getBucket()); + } catch (StorageException exception) { + onError.accept(exception); + } + AWSS3StoragePathUploadFileOperation operation = new AWSS3StoragePathUploadFileOperation( request, - defaultStorageService, + storageService, executorService, authCredentialsProvider, onProgress, @@ -760,8 +778,15 @@ public StorageUploadInputStreamOperation uploadInputStream( useAccelerateEndpoint ); + AWSS3StorageService storageService = defaultStorageService; + try { + storageService = getStorageService(options.getBucket()); + } catch (StorageException exception) { + onError.accept(exception); + } + AWSS3StorageUploadInputStreamOperation operation = new AWSS3StorageUploadInputStreamOperation( - defaultStorageService, + storageService, executorService, authCredentialsProvider, awsS3StoragePluginConfiguration, @@ -798,10 +823,17 @@ public StorageUploadInputStreamOperation uploadInputStream( useAccelerateEndpoint ); + AWSS3StorageService storageService = defaultStorageService; + try { + storageService = getStorageService(options.getBucket()); + } catch (StorageException exception) { + onError.accept(exception); + } + AWSS3StoragePathUploadInputStreamOperation operation = new AWSS3StoragePathUploadInputStreamOperation( request, - defaultStorageService, + storageService, executorService, authCredentialsProvider, onProgress, @@ -851,9 +883,16 @@ public StorageRemoveOperation remove( options.getTargetIdentityId() ); + AWSS3StorageService storageService = defaultStorageService; + try { + storageService = getStorageService(options.getBucket()); + } catch (StorageException exception) { + onError.accept(exception); + } + AWSS3StorageRemoveOperation operation = new AWSS3StorageRemoveOperation( - defaultStorageService, + storageService, executorService, authCredentialsProvider, request, @@ -876,9 +915,16 @@ public StorageRemoveOperation remove( ) { AWSS3StoragePathRemoveRequest request = new AWSS3StoragePathRemoveRequest(path); + AWSS3StorageService storageService = defaultStorageService; + try { + storageService = getStorageService(options.getBucket()); + } catch (StorageException exception) { + onError.accept(exception); + } + AWSS3StoragePathRemoveOperation operation = new AWSS3StoragePathRemoveOperation( - defaultStorageService, + storageService, executorService, authCredentialsProvider, request, @@ -1053,7 +1099,7 @@ public StorageListOperation list( @SuppressLint("UnsafeOptInUsageError") @VisibleForTesting @NonNull - AWSS3StorageService getStorageService(@Nullable StorageBucket bucket) throws InvalidStorageBucketException { + AWSS3StorageService getStorageService(@Nullable StorageBucket bucket) throws StorageException { if (bucket == null) { return defaultStorageService; } @@ -1061,7 +1107,10 @@ AWSS3StorageService getStorageService(@Nullable StorageBucket bucket) throws Inv if (bucket instanceof OutputsStorageBucket) { AWSS3StorageService service = getAWSS3StorageService((OutputsStorageBucket) bucket); if (service == null) { - throw new InvalidStorageBucketException(); + throw new StorageException( + "Unable to find bucket from name in Amplify Outputs.", + new InvalidStorageBucketException(), + "Ensure the bucket name used is available in Amplify Outputs."); } else { return service; } diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/options/AWSS3StorageDownloadFileOptions.java b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/options/AWSS3StorageDownloadFileOptions.java index e9d7f024eb..416df30011 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/options/AWSS3StorageDownloadFileOptions.java +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/options/AWSS3StorageDownloadFileOptions.java @@ -57,9 +57,10 @@ public static Builder builder() { @SuppressWarnings("deprecation") public static Builder from(@NonNull final AWSS3StorageDownloadFileOptions options) { return builder() - .accessLevel(options.getAccessLevel()) - .targetIdentityId(options.getTargetIdentityId()) - .setUseAccelerateEndpoint(options.useAccelerateEndpoint()); + .accessLevel(options.getAccessLevel()) + .targetIdentityId(options.getTargetIdentityId()) + .setUseAccelerateEndpoint(options.useAccelerateEndpoint()) + .bucket(options.getBucket()); } /** @@ -90,7 +91,8 @@ public boolean equals(Object obj) { } else { AWSS3StorageDownloadFileOptions that = (AWSS3StorageDownloadFileOptions) obj; return ObjectsCompat.equals(getAccessLevel(), that.getAccessLevel()) && - ObjectsCompat.equals(getTargetIdentityId(), that.getTargetIdentityId()); + ObjectsCompat.equals(getTargetIdentityId(), that.getTargetIdentityId()) && + ObjectsCompat.equals(getBucket(), that.getBucket()); } } @@ -99,7 +101,8 @@ public boolean equals(Object obj) { public int hashCode() { return ObjectsCompat.hash( getAccessLevel(), - getTargetIdentityId() + getTargetIdentityId(), + getBucket() ); } @@ -111,6 +114,7 @@ public String toString() { "accessLevel=" + getAccessLevel() + ", targetIdentityId=" + getTargetIdentityId() + ", useAccelerationMode=" + useAccelerateEndpoint() + + ", bucket=" + getBucket() + '}'; } diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/options/AWSS3StorageListOptions.java b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/options/AWSS3StorageListOptions.java index d6335b85d5..b314ba292f 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/options/AWSS3StorageListOptions.java +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/options/AWSS3StorageListOptions.java @@ -55,8 +55,9 @@ public static Builder builder() { @SuppressWarnings("deprecation") public static Builder from(@NonNull final AWSS3StorageListOptions options) { return builder() - .accessLevel(options.getAccessLevel()) - .targetIdentityId(options.getTargetIdentityId()); + .accessLevel(options.getAccessLevel()) + .targetIdentityId(options.getTargetIdentityId()) + .bucket(options.getBucket()); } /** @@ -78,7 +79,8 @@ public boolean equals(Object obj) { } else { AWSS3StorageListOptions that = (AWSS3StorageListOptions) obj; return ObjectsCompat.equals(getAccessLevel(), that.getAccessLevel()) && - ObjectsCompat.equals(getTargetIdentityId(), that.getTargetIdentityId()); + ObjectsCompat.equals(getTargetIdentityId(), that.getTargetIdentityId()) && + ObjectsCompat.equals(getBucket(), that.getBucket()); } } @@ -87,7 +89,8 @@ public boolean equals(Object obj) { public int hashCode() { return ObjectsCompat.hash( getAccessLevel(), - getTargetIdentityId() + getTargetIdentityId(), + getBucket() ); } @@ -98,6 +101,7 @@ public String toString() { return "AWSS3StorageListOptions {" + "accessLevel=" + getAccessLevel() + ", targetIdentityId=" + getTargetIdentityId() + + ", bucket=" + getBucket() + '}'; } diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/options/AWSS3StorageRemoveOptions.java b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/options/AWSS3StorageRemoveOptions.java index 244abe1f88..b74f08b752 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/options/AWSS3StorageRemoveOptions.java +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/options/AWSS3StorageRemoveOptions.java @@ -56,8 +56,9 @@ public static Builder builder() { @SuppressWarnings("deprecation") public static Builder from(@NonNull final AWSS3StorageRemoveOptions options) { return builder() - .accessLevel(options.getAccessLevel()) - .targetIdentityId(options.getTargetIdentityId()); + .accessLevel(options.getAccessLevel()) + .targetIdentityId(options.getTargetIdentityId()) + .bucket(options.getBucket()); } /** @@ -79,7 +80,8 @@ public boolean equals(Object obj) { } else { AWSS3StorageRemoveOptions that = (AWSS3StorageRemoveOptions) obj; return ObjectsCompat.equals(getAccessLevel(), that.getAccessLevel()) && - ObjectsCompat.equals(getTargetIdentityId(), that.getTargetIdentityId()); + ObjectsCompat.equals(getTargetIdentityId(), that.getTargetIdentityId()) && + ObjectsCompat.equals(getBucket(), that.getBucket()); } } @@ -88,7 +90,8 @@ public boolean equals(Object obj) { public int hashCode() { return ObjectsCompat.hash( getAccessLevel(), - getTargetIdentityId() + getTargetIdentityId(), + getBucket() ); } @@ -99,6 +102,7 @@ public String toString() { return "AWSS3StorageRemoveOptions {" + "accessLevel=" + getAccessLevel() + ", targetIdentityId=" + getTargetIdentityId() + + ", bucket=" + getBucket() + '}'; } diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/options/AWSS3StorageUploadFileOptions.java b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/options/AWSS3StorageUploadFileOptions.java index 39dfdcaf44..00ce8ff610 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/options/AWSS3StorageUploadFileOptions.java +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/options/AWSS3StorageUploadFileOptions.java @@ -71,11 +71,12 @@ public static Builder builder() { @SuppressWarnings("deprecation") public static Builder from(@NonNull final AWSS3StorageUploadFileOptions options) { return builder() - .accessLevel(options.getAccessLevel()) - .targetIdentityId(options.getTargetIdentityId()) - .contentType(options.getContentType()) - .serverSideEncryption(options.getServerSideEncryption()) - .metadata(options.getMetadata()); + .accessLevel(options.getAccessLevel()) + .targetIdentityId(options.getTargetIdentityId()) + .contentType(options.getContentType()) + .serverSideEncryption(options.getServerSideEncryption()) + .metadata(options.getMetadata()) + .bucket(options.getBucket()); } /** @@ -109,7 +110,8 @@ public boolean equals(Object obj) { ObjectsCompat.equals(getTargetIdentityId(), that.getTargetIdentityId()) && ObjectsCompat.equals(getContentType(), that.getContentType()) && ObjectsCompat.equals(getServerSideEncryption(), that.getServerSideEncryption()) && - ObjectsCompat.equals(getMetadata(), that.getMetadata()); + ObjectsCompat.equals(getMetadata(), that.getMetadata()) && + ObjectsCompat.equals(getBucket(), that.getBucket()); } } @@ -121,7 +123,8 @@ public int hashCode() { getTargetIdentityId(), getContentType(), getServerSideEncryption(), - getMetadata() + getMetadata(), + getBucket() ); } @@ -135,6 +138,7 @@ public String toString() { ", contentType=" + getContentType() + ", serverSideEncryption=" + getServerSideEncryption().getName() + ", metadata=" + getMetadata() + + ", bucket=" + getBucket() + '}'; } diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/options/AWSS3StorageUploadInputStreamOptions.java b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/options/AWSS3StorageUploadInputStreamOptions.java index e14296a2cc..3bf4f6a368 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/options/AWSS3StorageUploadInputStreamOptions.java +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/options/AWSS3StorageUploadInputStreamOptions.java @@ -75,7 +75,8 @@ public static Builder from(@NonNull final AWSS3StorageUploadInputStreamOptions o .targetIdentityId(options.getTargetIdentityId()) .contentType(options.getContentType()) .serverSideEncryption(options.getServerSideEncryption()) - .metadata(options.getMetadata()); + .metadata(options.getMetadata()) + .bucket(options.getBucket()); } /** @@ -109,7 +110,8 @@ public boolean equals(Object obj) { ObjectsCompat.equals(getTargetIdentityId(), that.getTargetIdentityId()) && ObjectsCompat.equals(getContentType(), that.getContentType()) && ObjectsCompat.equals(getServerSideEncryption(), that.getServerSideEncryption()) && - ObjectsCompat.equals(getMetadata(), that.getMetadata()); + ObjectsCompat.equals(getMetadata(), that.getMetadata()) && + ObjectsCompat.equals(getBucket(), that.getBucket()); } } @@ -121,7 +123,8 @@ public int hashCode() { getTargetIdentityId(), getContentType(), getServerSideEncryption(), - getMetadata() + getMetadata(), + getBucket() ); } @@ -135,6 +138,7 @@ public String toString() { ", contentType=" + getContentType() + ", serverSideEncryption=" + getServerSideEncryption().getName() + ", metadata=" + getMetadata() + + ", bucket=" + getBucket() + '}'; } diff --git a/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/AWSS3StoragePluginTest.kt b/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/AWSS3StoragePluginTest.kt index e5e13cfe40..a13158f1bf 100644 --- a/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/AWSS3StoragePluginTest.kt +++ b/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/AWSS3StoragePluginTest.kt @@ -24,6 +24,7 @@ import com.amplifyframework.storage.s3.service.StorageService import com.amplifyframework.testutils.configuration.amplifyOutputsData import io.kotest.assertions.throwables.shouldThrow import io.kotest.matchers.shouldNotBe +import io.kotest.matchers.throwable.shouldHaveCauseOfType import io.mockk.every import io.mockk.mockk import io.mockk.verify @@ -129,7 +130,7 @@ class AWSS3StoragePluginTest { } @Test - fun `getStorageService throws InvalidStorageBucketException`() { + fun `getStorageService throws StorageException`() { val data = amplifyOutputsData { storage { awsRegion = "test-region" @@ -144,8 +145,9 @@ class AWSS3StoragePluginTest { plugin.configure(data, mockk()) val bucket = StorageBucket.fromOutputs("myBucket") - shouldThrow { + val exception = shouldThrow { plugin.getStorageService(bucket) } + exception.shouldHaveCauseOfType() } } diff --git a/core/src/main/java/com/amplifyframework/storage/options/StorageDownloadFileOptions.java b/core/src/main/java/com/amplifyframework/storage/options/StorageDownloadFileOptions.java index 846991f968..18e8b22a46 100644 --- a/core/src/main/java/com/amplifyframework/storage/options/StorageDownloadFileOptions.java +++ b/core/src/main/java/com/amplifyframework/storage/options/StorageDownloadFileOptions.java @@ -30,7 +30,7 @@ public class StorageDownloadFileOptions extends StorageOptions { */ @SuppressWarnings("deprecation") protected StorageDownloadFileOptions(final Builder builder) { - super(builder.getAccessLevel(), builder.getTargetIdentityId()); + super(builder.getAccessLevel(), builder.getTargetIdentityId(), builder.getBucket()); } /** @@ -59,8 +59,9 @@ public static Builder builder() { @SuppressWarnings("deprecation") public static Builder from(@NonNull final StorageDownloadFileOptions options) { return builder() - .accessLevel(options.getAccessLevel()) - .targetIdentityId(options.getTargetIdentityId()); + .accessLevel(options.getAccessLevel()) + .targetIdentityId(options.getTargetIdentityId()) + .bucket(options.getBucket()); } /** @@ -85,7 +86,8 @@ public boolean equals(Object obj) { } else { StorageDownloadFileOptions that = (StorageDownloadFileOptions) obj; return ObjectsCompat.equals(getAccessLevel(), that.getAccessLevel()) && - ObjectsCompat.equals(getTargetIdentityId(), that.getTargetIdentityId()); + ObjectsCompat.equals(getTargetIdentityId(), that.getTargetIdentityId()) && + ObjectsCompat.equals(getBucket(), that.getBucket()); } } @@ -97,7 +99,8 @@ public boolean equals(Object obj) { public int hashCode() { return ObjectsCompat.hash( getAccessLevel(), - getTargetIdentityId() + getTargetIdentityId(), + getBucket() ); } @@ -109,9 +112,10 @@ public int hashCode() { @SuppressWarnings("deprecation") public String toString() { return "StorageDownloadFileOptions {" + - "accessLevel=" + getAccessLevel() + - ", targetIdentityId=" + getTargetIdentityId() + - '}'; + "accessLevel=" + getAccessLevel() + + ", targetIdentityId=" + getTargetIdentityId() + + ", bucket=" + getBucket() + + '}'; } /** diff --git a/core/src/main/java/com/amplifyframework/storage/options/StorageListOptions.java b/core/src/main/java/com/amplifyframework/storage/options/StorageListOptions.java index 966704e5c3..ebd9ae7330 100644 --- a/core/src/main/java/com/amplifyframework/storage/options/StorageListOptions.java +++ b/core/src/main/java/com/amplifyframework/storage/options/StorageListOptions.java @@ -31,7 +31,7 @@ public class StorageListOptions extends StorageOptions { */ @SuppressWarnings("deprecation") protected StorageListOptions(final Builder builder) { - super(builder.getAccessLevel(), builder.getTargetIdentityId()); + super(builder.getAccessLevel(), builder.getTargetIdentityId(), builder.getBucket()); } /** @@ -58,8 +58,9 @@ public static Builder builder() { @SuppressWarnings("deprecation") public static Builder from(@NonNull final StorageListOptions options) { return builder() - .accessLevel(options.getAccessLevel()) - .targetIdentityId(options.getTargetIdentityId()); + .accessLevel(options.getAccessLevel()) + .targetIdentityId(options.getTargetIdentityId()) + .bucket(options.getBucket()); } /** @@ -85,7 +86,8 @@ public boolean equals(Object obj) { } else { StorageListOptions that = (StorageListOptions) obj; return ObjectsCompat.equals(getAccessLevel(), that.getAccessLevel()) && - ObjectsCompat.equals(getTargetIdentityId(), that.getTargetIdentityId()); + ObjectsCompat.equals(getTargetIdentityId(), that.getTargetIdentityId()) && + ObjectsCompat.equals(getBucket(), that.getBucket()); } } @@ -97,7 +99,8 @@ public boolean equals(Object obj) { public int hashCode() { return ObjectsCompat.hash( getAccessLevel(), - getTargetIdentityId() + getTargetIdentityId(), + getBucket() ); } @@ -111,6 +114,7 @@ public String toString() { return "StorageListOptions {" + "accessLevel=" + getAccessLevel() + ", targetIdentityId=" + getTargetIdentityId() + + ", bucket=" + getBucket() + '}'; } diff --git a/core/src/main/java/com/amplifyframework/storage/options/StoragePagedListOptions.java b/core/src/main/java/com/amplifyframework/storage/options/StoragePagedListOptions.java index 515f92f259..5609978163 100644 --- a/core/src/main/java/com/amplifyframework/storage/options/StoragePagedListOptions.java +++ b/core/src/main/java/com/amplifyframework/storage/options/StoragePagedListOptions.java @@ -33,7 +33,7 @@ public class StoragePagedListOptions extends StorageOptions { */ @SuppressWarnings("deprecation") protected StoragePagedListOptions(Builder builder) { - super(builder.getAccessLevel(), builder.getTargetIdentityId()); + super(builder.getAccessLevel(), builder.getTargetIdentityId(), builder.getBucket()); pageSize = builder.pageSize; nextToken = builder.nextToken; subpathStrategy = builder.subpathStrategy; diff --git a/core/src/main/java/com/amplifyframework/storage/options/StorageRemoveOptions.java b/core/src/main/java/com/amplifyframework/storage/options/StorageRemoveOptions.java index 24c3adc7a7..f2f9ef1711 100644 --- a/core/src/main/java/com/amplifyframework/storage/options/StorageRemoveOptions.java +++ b/core/src/main/java/com/amplifyframework/storage/options/StorageRemoveOptions.java @@ -31,7 +31,7 @@ public class StorageRemoveOptions extends StorageOptions { */ @SuppressWarnings("deprecation") protected StorageRemoveOptions(final Builder builder) { - super(builder.getAccessLevel(), builder.getTargetIdentityId()); + super(builder.getAccessLevel(), builder.getTargetIdentityId(), builder.getBucket()); } /** @@ -60,8 +60,9 @@ public static Builder builder() { @SuppressWarnings("deprecation") public static Builder from(@NonNull final StorageRemoveOptions options) { return builder() - .accessLevel(options.getAccessLevel()) - .targetIdentityId(options.getTargetIdentityId()); + .accessLevel(options.getAccessLevel()) + .targetIdentityId(options.getTargetIdentityId()) + .bucket(options.getBucket()); } /** @@ -86,7 +87,8 @@ public boolean equals(Object obj) { } else { StorageRemoveOptions that = (StorageRemoveOptions) obj; return ObjectsCompat.equals(getAccessLevel(), that.getAccessLevel()) && - ObjectsCompat.equals(getTargetIdentityId(), that.getTargetIdentityId()); + ObjectsCompat.equals(getTargetIdentityId(), that.getTargetIdentityId()) && + ObjectsCompat.equals(getBucket(), that.getBucket()); } } @@ -98,7 +100,8 @@ public boolean equals(Object obj) { public int hashCode() { return ObjectsCompat.hash( getAccessLevel(), - getTargetIdentityId() + getTargetIdentityId(), + getBucket() ); } @@ -112,6 +115,7 @@ public String toString() { return "StorageRemoveOptions {" + "accessLevel=" + getAccessLevel() + ", targetIdentityId=" + getTargetIdentityId() + + ", bucket=" + getBucket() + '}'; } diff --git a/core/src/main/java/com/amplifyframework/storage/options/StorageUploadFileOptions.java b/core/src/main/java/com/amplifyframework/storage/options/StorageUploadFileOptions.java index 18dbcca58e..1a17ab3ccb 100644 --- a/core/src/main/java/com/amplifyframework/storage/options/StorageUploadFileOptions.java +++ b/core/src/main/java/com/amplifyframework/storage/options/StorageUploadFileOptions.java @@ -58,10 +58,11 @@ public static Builder builder() { @SuppressWarnings("deprecation") public static Builder from(@NonNull final StorageUploadFileOptions options) { return builder() - .accessLevel(options.getAccessLevel()) - .targetIdentityId(options.getTargetIdentityId()) - .contentType(options.getContentType()) - .metadata(options.getMetadata()); + .accessLevel(options.getAccessLevel()) + .targetIdentityId(options.getTargetIdentityId()) + .contentType(options.getContentType()) + .metadata(options.getMetadata()) + .bucket(options.getBucket()); } /** @@ -88,7 +89,8 @@ public boolean equals(Object obj) { return ObjectsCompat.equals(getAccessLevel(), that.getAccessLevel()) && ObjectsCompat.equals(getTargetIdentityId(), that.getTargetIdentityId()) && ObjectsCompat.equals(getContentType(), that.getContentType()) && - ObjectsCompat.equals(getMetadata(), that.getMetadata()); + ObjectsCompat.equals(getMetadata(), that.getMetadata()) && + ObjectsCompat.equals(getBucket(), that.getBucket()); } } @@ -102,7 +104,8 @@ public int hashCode() { getAccessLevel(), getTargetIdentityId(), getContentType(), - getMetadata() + getMetadata(), + getBucket() ); } @@ -118,6 +121,7 @@ public String toString() { ", targetIdentityId=" + getTargetIdentityId() + ", contentType=" + getContentType() + ", metadata=" + getMetadata() + + ", bucket=" + getBucket() + '}'; } diff --git a/core/src/main/java/com/amplifyframework/storage/options/StorageUploadInputStreamOptions.java b/core/src/main/java/com/amplifyframework/storage/options/StorageUploadInputStreamOptions.java index ba46234975..9e5c2dac57 100644 --- a/core/src/main/java/com/amplifyframework/storage/options/StorageUploadInputStreamOptions.java +++ b/core/src/main/java/com/amplifyframework/storage/options/StorageUploadInputStreamOptions.java @@ -58,10 +58,11 @@ public static Builder builder() { @SuppressWarnings("deprecation") public static Builder from(@NonNull final StorageUploadInputStreamOptions options) { return builder() - .accessLevel(options.getAccessLevel()) - .targetIdentityId(options.getTargetIdentityId()) - .contentType(options.getContentType()) - .metadata(options.getMetadata()); + .accessLevel(options.getAccessLevel()) + .targetIdentityId(options.getTargetIdentityId()) + .contentType(options.getContentType()) + .metadata(options.getMetadata()) + .bucket(options.getBucket()); } /** @@ -88,7 +89,8 @@ public boolean equals(Object obj) { return ObjectsCompat.equals(getAccessLevel(), that.getAccessLevel()) && ObjectsCompat.equals(getTargetIdentityId(), that.getTargetIdentityId()) && ObjectsCompat.equals(getContentType(), that.getContentType()) && - ObjectsCompat.equals(getMetadata(), that.getMetadata()); + ObjectsCompat.equals(getMetadata(), that.getMetadata()) && + ObjectsCompat.equals(getBucket(), that.getBucket()); } } @@ -102,7 +104,8 @@ public int hashCode() { getAccessLevel(), getTargetIdentityId(), getContentType(), - getMetadata() + getMetadata(), + getBucket() ); } @@ -118,6 +121,7 @@ public String toString() { ", targetIdentityId=" + getTargetIdentityId() + ", contentType=" + getContentType() + ", metadata=" + getMetadata() + + ", bucket=" + getBucket() + '}'; } diff --git a/core/src/main/java/com/amplifyframework/storage/options/StorageUploadOptions.java b/core/src/main/java/com/amplifyframework/storage/options/StorageUploadOptions.java index 36dcf2a221..9a4144ac93 100644 --- a/core/src/main/java/com/amplifyframework/storage/options/StorageUploadOptions.java +++ b/core/src/main/java/com/amplifyframework/storage/options/StorageUploadOptions.java @@ -41,7 +41,7 @@ public abstract class StorageUploadOptions extends StorageOptions { @SuppressWarnings("deprecation") protected , O extends StorageUploadOptions> StorageUploadOptions(final Builder builder) { - super(builder.getAccessLevel(), builder.getTargetIdentityId()); + super(builder.getAccessLevel(), builder.getTargetIdentityId(), builder.getBucket()); this.contentType = builder.getContentType(); this.metadata = builder.getMetadata(); } @@ -79,7 +79,8 @@ public boolean equals(Object obj) { return ObjectsCompat.equals(getAccessLevel(), that.getAccessLevel()) && ObjectsCompat.equals(getTargetIdentityId(), that.getTargetIdentityId()) && ObjectsCompat.equals(getContentType(), that.getContentType()) && - ObjectsCompat.equals(getMetadata(), that.getMetadata()); + ObjectsCompat.equals(getMetadata(), that.getMetadata()) && + ObjectsCompat.equals(getBucket(), that.getBucket()); } } @@ -93,7 +94,8 @@ public int hashCode() { getAccessLevel(), getTargetIdentityId(), getContentType(), - getMetadata() + getMetadata(), + getBucket() ); } @@ -109,6 +111,7 @@ public String toString() { ", targetIdentityId=" + getTargetIdentityId() + ", contentType=" + getContentType() + ", metadata=" + getMetadata() + + ", bucket=" + getBucket() + '}'; } From 5125555495d397d31f32a14ca63d2102fbce3a2a Mon Sep 17 00:00:00 2001 From: Tuan Pham <103537251+phantumcode@users.noreply.github.com> Date: Tue, 27 Aug 2024 12:13:47 -0500 Subject: [PATCH 3/7] feat(storage): update multipart operations to support multiple buckets/regions (#2899) --- .../storage/s3/transfer/TransferDBTest.kt | 9 ++ .../storage/s3/AWSS3StoragePlugin.java | 141 ++++++++++-------- .../storage/s3/TransferOperations.kt | 1 + .../storage/s3/service/AWSS3StorageService.kt | 34 ++++- .../service/AWSS3StorageServiceContainer.kt | 88 +++++++++++ .../S3StorageTransferClientProvider.kt | 35 +++++ .../transfer/StorageTransferClientProvider.kt | 22 +++ .../storage/s3/transfer/TransferDB.kt | 15 +- .../storage/s3/transfer/TransferDBHelper.kt | 2 +- .../storage/s3/transfer/TransferManager.kt | 14 +- .../storage/s3/transfer/TransferObserver.kt | 1 + .../storage/s3/transfer/TransferRecord.kt | 3 + .../storage/s3/transfer/TransferTable.kt | 17 ++- .../storage/s3/transfer/UploadOptions.kt | 1 + .../worker/AbortMultiPartUploadWorker.kt | 4 +- .../worker/CompleteMultiPartUploadWorker.kt | 4 +- .../s3/transfer/worker/DownloadWorker.kt | 4 +- .../InitiateMultiPartUploadTransferWorker.kt | 4 +- .../worker/PartUploadTransferWorker.kt | 4 +- .../transfer/worker/SinglePartUploadWorker.kt | 4 +- .../transfer/worker/TransferWorkerFactory.kt | 15 +- .../AWSS3StorageServiceContainerTest.kt | 121 +++++++++++++++ .../storage/s3/AWSS3StoragePluginTest.kt | 7 +- .../storage/s3/StorageComponentTest.java | 5 +- .../worker/AbortMultiPartUploadWorkerTest.kt | 30 +++- .../s3/transfer/worker/DownloadWorkerTest.kt | 10 +- ...itiateMultiPartUploadTransferWorkerTest.kt | 10 +- 27 files changed, 501 insertions(+), 104 deletions(-) create mode 100644 aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/service/AWSS3StorageServiceContainer.kt create mode 100644 aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/S3StorageTransferClientProvider.kt create mode 100644 aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/StorageTransferClientProvider.kt create mode 100644 aws-storage-s3/src/test/java/com/amplifyframework/storage/AWSS3StorageServiceContainerTest.kt diff --git a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/transfer/TransferDBTest.kt b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/transfer/TransferDBTest.kt index 199428831b..1922c896a1 100644 --- a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/transfer/TransferDBTest.kt +++ b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/transfer/TransferDBTest.kt @@ -31,6 +31,7 @@ import org.junit.Test open class TransferDBTest { private val bucketName = "bucket_name" + private val region = "us-east-1" private val fileKey = "file_key" private lateinit var transferDB: TransferDB private lateinit var tempFile: File @@ -55,6 +56,7 @@ open class TransferDBTest { transferId, TransferType.UPLOAD, bucketName, + region, fileKey, tempFile, null, @@ -67,6 +69,7 @@ open class TransferDBTest { Assert.assertEquals(tempFile, File(this.file)) Assert.assertEquals(fileKey, this.key) Assert.assertEquals(bucketName, this.bucketName) + Assert.assertEquals(region, this.region) } ?: Assert.fail("InsertedRecord is null") } @@ -76,6 +79,7 @@ open class TransferDBTest { val uri = transferDB.insertMultipartUploadRecord( uploadID, bucketName, + region, fileKey, tempFile, 1L, @@ -91,6 +95,7 @@ open class TransferDBTest { Assert.assertEquals(fileKey, this.key) Assert.assertEquals(bucketName, this.bucketName) Assert.assertEquals(uploadID, this.multipartId) + Assert.assertEquals(region, this.region) } ?: Assert.fail("InsertedRecord is null") } @@ -104,6 +109,7 @@ open class TransferDBTest { contentValues[0] = transferDB.generateContentValuesForMultiPartUpload( key, bucketName, + region, key, tempFile, 0L, @@ -137,6 +143,7 @@ open class TransferDBTest { contentValues[0] = transferDB.generateContentValuesForMultiPartUpload( key, bucketName, + region, key, tempFile, 0L, @@ -151,6 +158,7 @@ open class TransferDBTest { contentValues[1] = transferDB.generateContentValuesForMultiPartUpload( key, bucketName, + region, key, tempFile, 0L, @@ -165,6 +173,7 @@ open class TransferDBTest { contentValues[2] = transferDB.generateContentValuesForMultiPartUpload( key, bucketName, + region, key, tempFile, 0L, diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/AWSS3StoragePlugin.java b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/AWSS3StoragePlugin.java index 01fde9ba41..46316bef0c 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/AWSS3StoragePlugin.java +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/AWSS3StoragePlugin.java @@ -90,7 +90,9 @@ import com.amplifyframework.storage.s3.request.AWSS3StorageRemoveRequest; import com.amplifyframework.storage.s3.request.AWSS3StorageUploadRequest; import com.amplifyframework.storage.s3.service.AWSS3StorageService; -import com.amplifyframework.storage.s3.service.StorageService; +import com.amplifyframework.storage.s3.service.AWSS3StorageServiceContainer; +import com.amplifyframework.storage.s3.transfer.S3StorageTransferClientProvider; +import com.amplifyframework.storage.s3.transfer.StorageTransferClientProvider; import com.amplifyframework.storage.s3.transfer.TransferObserver; import com.amplifyframework.storage.s3.transfer.TransferRecord; import com.amplifyframework.storage.s3.transfer.TransferStatusUpdater; @@ -101,9 +103,7 @@ import java.io.File; import java.io.InputStream; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -126,20 +126,33 @@ public final class AWSS3StoragePlugin extends StoragePlugin { private static final int DEFAULT_URL_EXPIRATION_DAYS = 7; - private final StorageService.Factory storageServiceFactory; + private final AWSS3StorageService.Factory storageServiceFactory; private final ExecutorService executorService; - private final AuthCredentialsProvider authCredentialsProvider; + private AuthCredentialsProvider authCredentialsProvider; private final AWSS3StoragePluginConfiguration awsS3StoragePluginConfiguration; private AWSS3StorageService defaultStorageService; @SuppressWarnings("deprecation") private StorageAccessLevel defaultAccessLevel; private int defaultUrlExpiration; - private Map awsS3StorageServicesByBucketName = new HashMap<>(); - private Context context; + private AWSS3StorageServiceContainer awss3StorageServiceContainer; @SuppressLint("UnsafeOptInUsageError") private List configuredBuckets; + @SuppressLint("UnsafeOptInUsageError") + private StorageTransferClientProvider clientProvider + = new S3StorageTransferClientProvider((region, bucketName) -> { + if (region != null && bucketName != null) { + StorageBucket bucket = StorageBucket.fromBucketInfo(new BucketInfo(bucketName, region)); + return awss3StorageServiceContainer.get((ResolvedStorageBucket) bucket).getClient(); + } + + if (region != null) { + return S3StorageTransferClientProvider.getS3Client(region, authCredentialsProvider); + } + return defaultStorageService.getClient(); + }); + /** * Constructs the AWS S3 Storage Plugin initializing the executor service. */ @@ -162,13 +175,14 @@ public AWSS3StoragePlugin(AWSS3StoragePluginConfiguration awsS3StoragePluginConf @VisibleForTesting AWSS3StoragePlugin(AuthCredentialsProvider authCredentialsProvider) { - this((context, region, bucket) -> + this((context, region, bucket, clientProvider) -> new AWSS3StorageService( context, region, bucket, authCredentialsProvider, - AWS_S3_STORAGE_PLUGIN_KEY + AWS_S3_STORAGE_PLUGIN_KEY, + clientProvider ), authCredentialsProvider, new AWSS3StoragePluginConfiguration.Builder().build()); @@ -177,13 +191,15 @@ public AWSS3StoragePlugin(AWSS3StoragePluginConfiguration awsS3StoragePluginConf @VisibleForTesting AWSS3StoragePlugin(AuthCredentialsProvider authCredentialsProvider, AWSS3StoragePluginConfiguration awss3StoragePluginConfiguration) { - this((context, region, bucket) -> + + this((context, region, bucket, clientProvider) -> new AWSS3StorageService( context, region, bucket, authCredentialsProvider, - AWS_S3_STORAGE_PLUGIN_KEY + AWS_S3_STORAGE_PLUGIN_KEY, + clientProvider ), authCredentialsProvider, awss3StoragePluginConfiguration); @@ -191,7 +207,7 @@ public AWSS3StoragePlugin(AWSS3StoragePluginConfiguration awsS3StoragePluginConf @VisibleForTesting AWSS3StoragePlugin( - StorageService.Factory storageServiceFactory, + AWSS3StorageService.Factory storageServiceFactory, AuthCredentialsProvider authCredentialsProvider, AWSS3StoragePluginConfiguration awss3StoragePluginConfiguration ) { @@ -281,13 +297,15 @@ private void configure( @NonNull ResolvedStorageBucket bucket ) throws StorageException { try { - this.context = context; - this.defaultStorageService = (AWSS3StorageService) storageServiceFactory.create( + this.defaultStorageService = storageServiceFactory.create( context, region, - bucket.getBucketInfo().getName()); - this.awsS3StorageServicesByBucketName.clear(); - this.awsS3StorageServicesByBucketName.put(bucket.getBucketInfo().getName(), this.defaultStorageService); + bucket.getBucketInfo().getName(), + clientProvider); + this.awss3StorageServiceContainer = new AWSS3StorageServiceContainer( + context, storageServiceFactory, + (S3StorageTransferClientProvider) clientProvider); + this.awss3StorageServiceContainer.put(bucket.getBucketInfo().getName(), this.defaultStorageService); } catch (RuntimeException exception) { throw new StorageException( "Failed to create storage service.", @@ -935,7 +953,8 @@ public StorageRemoveOperation remove( return operation; } - + + @SuppressLint("UnsafeOptInUsageError") @Override @SuppressWarnings("deprecation") public void getTransfer( @@ -951,18 +970,23 @@ public void getTransfer( transferRecord.getId(), defaultStorageService.getTransferManager().getTransferStatusUpdater(), transferRecord.getBucketName(), + transferRecord.getRegion(), transferRecord.getKey(), transferRecord.getFile(), null, transferRecord.getState() != null ? transferRecord.getState() : TransferState.UNKNOWN); TransferType transferType = transferRecord.getType(); + + AWSS3StorageService storageService + = getAwss3StorageServiceFromTransferRecord(onError, transferRecord); + switch (Objects.requireNonNull(transferType)) { case UPLOAD: if (transferRecord.getFile().startsWith(TransferStatusUpdater.TEMP_FILE_PREFIX)) { AWSS3StorageUploadInputStreamOperation operation = new AWSS3StorageUploadInputStreamOperation( transferId, - defaultStorageService, + storageService, executorService, authCredentialsProvider, awsS3StoragePluginConfiguration, @@ -973,7 +997,7 @@ public void getTransfer( AWSS3StorageUploadFileOperation operation = new AWSS3StorageUploadFileOperation( transferId, - defaultStorageService, + storageService, executorService, authCredentialsProvider, awsS3StoragePluginConfiguration, @@ -987,7 +1011,7 @@ public void getTransfer( downloadFileOperation = new AWSS3StorageDownloadFileOperation( transferId, new File(transferRecord.getFile()), - defaultStorageService, + storageService, executorService, authCredentialsProvider, awsS3StoragePluginConfiguration, @@ -1009,6 +1033,25 @@ public void getTransfer( }); } + private AWSS3StorageService getAwss3StorageServiceFromTransferRecord( + @NonNull Consumer onError, + TransferRecord transferRecord + ) { + AWSS3StorageService storageService = defaultStorageService; + if (transferRecord.getRegion() != null && transferRecord.getBucketName() != null) { + try { + BucketInfo bucketInfo = new BucketInfo( + transferRecord.getBucketName(), + transferRecord.getRegion()); + StorageBucket bucket = StorageBucket.fromBucketInfo(bucketInfo); + storageService = getStorageService(bucket); + } catch (StorageException exception) { + onError.accept(exception); + } + } + return storageService; + } + @NonNull @SuppressWarnings("deprecation") @Override @@ -1105,55 +1148,27 @@ AWSS3StorageService getStorageService(@Nullable StorageBucket bucket) throws Sto } if (bucket instanceof OutputsStorageBucket) { - AWSS3StorageService service = getAWSS3StorageService((OutputsStorageBucket) bucket); - if (service == null) { - throw new StorageException( - "Unable to find bucket from name in Amplify Outputs.", - new InvalidStorageBucketException(), - "Ensure the bucket name used is available in Amplify Outputs."); - } else { - return service; - } - } - - if (bucket instanceof ResolvedStorageBucket) { - return getAWSS3StorageService((ResolvedStorageBucket) bucket); - } - - return defaultStorageService; - } - - @SuppressLint("UnsafeOptInUsageError") - private AWSS3StorageService getAWSS3StorageService(OutputsStorageBucket outputsStorageBucket) { - if (configuredBuckets != null && !configuredBuckets.isEmpty()) { - String name = outputsStorageBucket.getName(); - for (AmplifyOutputsData.StorageBucket configuredBucket : configuredBuckets) { - if (configuredBucket.getName().equals(name)) { - String bucketName = configuredBucket.getBucketName(); - AWSS3StorageService service = awsS3StorageServicesByBucketName.get(bucketName); - if (service == null) { + if (configuredBuckets != null && !configuredBuckets.isEmpty()) { + String name = ((OutputsStorageBucket) bucket).getName(); + for (AmplifyOutputsData.StorageBucket configuredBucket : configuredBuckets) { + if (configuredBucket.getName().equals(name)) { + String bucketName = configuredBucket.getBucketName(); String region = configuredBucket.getAwsRegion(); - service = (AWSS3StorageService) storageServiceFactory.create(context, region, bucketName); - awsS3StorageServicesByBucketName.put(bucketName, service); + return awss3StorageServiceContainer.get(bucketName, region); } - - return service; } } + throw new StorageException( + "Unable to find bucket from name in Amplify Outputs.", + new InvalidStorageBucketException(), + "Ensure the bucket name used is available in Amplify Outputs."); } - return null; - } - @SuppressLint("UnsafeOptInUsageError") - private AWSS3StorageService getAWSS3StorageService(ResolvedStorageBucket resolvedStorageBucket) { - String bucketName = resolvedStorageBucket.getBucketInfo().getName(); - AWSS3StorageService service = awsS3StorageServicesByBucketName.get(bucketName); - if (service == null) { - String region = resolvedStorageBucket.getBucketInfo().getRegion(); - service = (AWSS3StorageService) storageServiceFactory.create(context, region, bucketName); - awsS3StorageServicesByBucketName.put(bucketName, service); + if (bucket instanceof ResolvedStorageBucket) { + return awss3StorageServiceContainer.get((ResolvedStorageBucket) bucket); } - return service; + + return defaultStorageService; } /** diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/TransferOperations.kt b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/TransferOperations.kt index 47cd602c41..99d15ea9d8 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/TransferOperations.kt +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/TransferOperations.kt @@ -70,6 +70,7 @@ internal object TransferOperations { transferRecord.id, transferStatusUpdater, transferRecord.bucketName, + transferRecord.region, transferRecord.key, transferRecord.file, listener diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/service/AWSS3StorageService.kt b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/service/AWSS3StorageService.kt index 7d9eef745d..eb7e573c2f 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/service/AWSS3StorageService.kt +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/service/AWSS3StorageService.kt @@ -32,6 +32,8 @@ import com.amplifyframework.storage.StorageItem import com.amplifyframework.storage.options.SubpathStrategy import com.amplifyframework.storage.options.SubpathStrategy.Exclude import com.amplifyframework.storage.result.StorageListResult +import com.amplifyframework.storage.s3.transfer.S3StorageTransferClientProvider +import com.amplifyframework.storage.s3.transfer.StorageTransferClientProvider import com.amplifyframework.storage.s3.transfer.TransferManager import com.amplifyframework.storage.s3.transfer.TransferObserver import com.amplifyframework.storage.s3.transfer.TransferRecord @@ -46,7 +48,6 @@ import java.util.Date import kotlin.time.Duration.Companion.seconds import kotlin.time.ExperimentalTime import kotlinx.coroutines.runBlocking - /** * A representation of an S3 backend service endpoint. */ @@ -55,16 +56,14 @@ internal class AWSS3StorageService( private val awsRegion: String, private val s3BucketName: String, private val authCredentialsProvider: AuthCredentialsProvider, - private val awsS3StoragePluginKey: String + private val awsS3StoragePluginKey: String, + private val clientProvider: StorageTransferClientProvider ) : StorageService { - private var s3Client: S3Client = S3Client { - region = awsRegion - credentialsProvider = authCredentialsProvider - } + private var s3Client: S3Client = S3StorageTransferClientProvider.getS3Client(awsRegion, authCredentialsProvider) val transferManager: TransferManager = - TransferManager(context, s3Client, awsS3StoragePluginKey) + TransferManager(context, clientProvider, awsS3StoragePluginKey) /** * Generate pre-signed URL for an object. @@ -130,6 +129,7 @@ internal class AWSS3StorageService( return transferManager.download( transferId, s3BucketName, + awsRegion, serviceKey, file, useAccelerateEndpoint = useAccelerateEndpoint @@ -153,6 +153,7 @@ internal class AWSS3StorageService( return transferManager.upload( transferId, s3BucketName, + awsRegion, serviceKey, file, metadata, @@ -175,7 +176,7 @@ internal class AWSS3StorageService( metadata: ObjectMetadata, useAccelerateEndpoint: Boolean ): TransferObserver { - val uploadOptions = UploadOptions(s3BucketName, metadata) + val uploadOptions = UploadOptions(s3BucketName, awsRegion, metadata) return transferManager.upload(transferId, serviceKey, inputStream, uploadOptions, useAccelerateEndpoint) } @@ -420,4 +421,21 @@ internal class AWSS3StorageService( fun getClient(): S3Client { return s3Client } + + interface Factory { + /** + * Factory interface to instantiate [StorageService] object. + * + * @param context Android context + * @param region S3 bucket region + * @param bucketName Name of the bucket where the items are stored + * @return An instantiated storage service instance + */ + fun create( + context: Context, + region: String, + bucketName: String, + clientProvider: StorageTransferClientProvider + ): AWSS3StorageService + } } diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/service/AWSS3StorageServiceContainer.kt b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/service/AWSS3StorageServiceContainer.kt new file mode 100644 index 0000000000..f22131df2f --- /dev/null +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/service/AWSS3StorageServiceContainer.kt @@ -0,0 +1,88 @@ +/* + * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amplifyframework.storage.s3.service + +import android.content.Context +import com.amplifyframework.storage.ResolvedStorageBucket +import com.amplifyframework.storage.s3.transfer.S3StorageTransferClientProvider +import com.amplifyframework.storage.s3.transfer.StorageTransferClientProvider +import java.util.concurrent.ConcurrentHashMap + +/** + * A container that stores a list of AWSS3StorageService based on the bucket name associated with the service. + * repository. + */ +internal class AWSS3StorageServiceContainer( + private val context: Context, + private val storageServiceFactory: AWSS3StorageService.Factory, + private val clientProvider: StorageTransferClientProvider, + private val awsS3StorageServicesByBucketName: ConcurrentHashMap +) { + constructor( + context: Context, + storageServiceFactory: AWSS3StorageService.Factory, + clientProvider: S3StorageTransferClientProvider + ) : this(context, storageServiceFactory, clientProvider, ConcurrentHashMap()) + + private val lock = Any() + + /** + * Stores a instance of AWSS3StorageService + * + * @param bucketName the bucket name + * @param service the AWSS3StorageService instance + */ + fun put(bucketName: String, service: AWSS3StorageService) { + synchronized(lock) { + awsS3StorageServicesByBucketName.put(bucketName, service) + } + } + + /** + * Get an AWSS3StorageSErvice instance based on a ResolvedStorageBucket + * @param resolvedStorageBucket An instance of ResolvedStorageBucket with bucket info + * @return An AWSS3StorageService instance associated with the ResolvedStorageBucket + */ + fun get(resolvedStorageBucket: ResolvedStorageBucket): AWSS3StorageService { + synchronized(lock) { + val bucketName: String = resolvedStorageBucket.bucketInfo.name + var service = awsS3StorageServicesByBucketName.get(bucketName) + if (service == null) { + val region: String = resolvedStorageBucket.bucketInfo.region + service = storageServiceFactory.create(context, region, bucketName, clientProvider) + awsS3StorageServicesByBucketName[bucketName] = service + } + return service + } + } + + /** + * Get an AWSS3StorageSErvice instance based on a bucket name and region + * @param bucketName the bucket name associated with the AWSS3StorageService + * @param bucketName the region to associate with a new AWSS3StorageService instance if one doesn't exist + * @return An AWSS3StorageService instance associated with the ResolvedStorageBucket + */ + fun get(bucketName: String, region: String): AWSS3StorageService { + synchronized(lock) { + var service = awsS3StorageServicesByBucketName[bucketName] + if (service == null) { + service = storageServiceFactory.create(context, region, bucketName, clientProvider) + awsS3StorageServicesByBucketName[bucketName] = service + } + + return service + } + } +} diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/S3StorageTransferClientProvider.kt b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/S3StorageTransferClientProvider.kt new file mode 100644 index 0000000000..49d976f7b3 --- /dev/null +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/S3StorageTransferClientProvider.kt @@ -0,0 +1,35 @@ +/* + * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amplifyframework.storage.s3.transfer + +import aws.sdk.kotlin.services.s3.S3Client +import com.amplifyframework.auth.AuthCredentialsProvider + +internal class S3StorageTransferClientProvider( + private val createS3Client: (region: String?, bucketName: String?) -> S3Client +) : StorageTransferClientProvider { + companion object { + @JvmStatic + fun getS3Client(region: String, authCredentialsProvider: AuthCredentialsProvider): S3Client { + return S3Client { + this.region = region + this.credentialsProvider = authCredentialsProvider + } + } + } + override fun getStorageTransferClient(region: String?, bucketName: String?): S3Client { + return createS3Client(region, bucketName) + } +} diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/StorageTransferClientProvider.kt b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/StorageTransferClientProvider.kt new file mode 100644 index 0000000000..523c0dcf3e --- /dev/null +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/StorageTransferClientProvider.kt @@ -0,0 +1,22 @@ +/* + * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amplifyframework.storage.s3.transfer + +import aws.sdk.kotlin.services.s3.S3Client +import com.amplifyframework.annotations.InternalApiWarning + +internal interface StorageTransferClientProvider { + fun getStorageTransferClient(region: String?, bucketName: String?): S3Client +} diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferDB.kt b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferDB.kt index 6413cf7c45..9c2949654b 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferDB.kt +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferDB.kt @@ -79,6 +79,7 @@ internal class TransferDB private constructor(context: Context) { fun insertMultipartUploadRecord( transferId: String, bucket: String, + region: String, key: String, file: File, fileOffset: Long, @@ -90,7 +91,7 @@ internal class TransferDB private constructor(context: Context) { ): Uri { val values: ContentValues = generateContentValuesForMultiPartUpload( transferId, - bucket, key, file, + bucket, region, key, file, fileOffset, partNumber, uploadId, bytesTotal, isLastPart, ObjectMetadata(), null, useAccelerateEndpoint @@ -114,6 +115,7 @@ internal class TransferDB private constructor(context: Context) { transferId: String, type: TransferType, bucket: String, + region: String, key: String, file: File?, cannedAcl: ObjectCannedAcl? = null, @@ -124,6 +126,7 @@ internal class TransferDB private constructor(context: Context) { transferId, type, bucket, + region, key, file, metadata, @@ -398,7 +401,7 @@ internal class TransferDB private constructor(context: Context) { */ fun getTransferRecordById(id: Int): TransferRecord? { var transferRecord: TransferRecord? = null - var c: Cursor? = null + var c: Cursor? try { c = queryTransferById(id) c?.use { @@ -415,7 +418,7 @@ internal class TransferDB private constructor(context: Context) { fun getTransferByTransferId(transferId: String): TransferRecord? { var transferRecord: TransferRecord? = null - var c: Cursor? = null + var c: Cursor? try { c = transferDBHelper.query(getTransferRecordIdUri(transferId)) c.use { @@ -560,6 +563,7 @@ internal class TransferDB private constructor(context: Context) { transferId: String, type: TransferType, bucket: String, + region: String, key: String, file: File, metadata: ObjectMetadata?, @@ -570,6 +574,7 @@ internal class TransferDB private constructor(context: Context) { transferId, type, bucket, + region, key, file, metadata, @@ -602,6 +607,7 @@ internal class TransferDB private constructor(context: Context) { fun generateContentValuesForMultiPartUpload( transferId: String, bucket: String?, + region: String?, key: String?, file: File, fileOffset: Long, @@ -618,6 +624,7 @@ internal class TransferDB private constructor(context: Context) { values.put(TransferTable.COLUMN_TYPE, TransferType.UPLOAD.toString()) values.put(TransferTable.COLUMN_STATE, TransferState.WAITING.toString()) values.put(TransferTable.COLUMN_BUCKET_NAME, bucket) + values.put(TransferTable.COLUMN_REGION, region) values.put(TransferTable.COLUMN_KEY, key) values.put(TransferTable.COLUMN_FILE, file.absolutePath) values.put(TransferTable.COLUMN_BYTES_CURRENT, 0L) @@ -723,6 +730,7 @@ internal class TransferDB private constructor(context: Context) { transferId: String, type: TransferType, bucket: String, + region: String, key: String, file: File?, metadata: ObjectMetadata?, @@ -734,6 +742,7 @@ internal class TransferDB private constructor(context: Context) { values.put(TransferTable.COLUMN_TYPE, type.toString()) values.put(TransferTable.COLUMN_STATE, TransferState.WAITING.toString()) values.put(TransferTable.COLUMN_BUCKET_NAME, bucket) + values.put(TransferTable.COLUMN_REGION, region) values.put(TransferTable.COLUMN_KEY, key) values.put(TransferTable.COLUMN_FILE, file?.absolutePath) values.put(TransferTable.COLUMN_BYTES_CURRENT, 0L) diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferDBHelper.kt b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferDBHelper.kt index 211ea745e5..4403d51df4 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferDBHelper.kt +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferDBHelper.kt @@ -48,7 +48,7 @@ internal class TransferDBHelper(private val context: Context) : SQLiteOpenHelper // This represents the latest database version. // Update this when the database is being upgraded. - private const val DATABASE_VERSION = 9 + private const val DATABASE_VERSION = 10 private const val BASE_PATH = "transfers" private const val TRANSFERS = 10 private const val TRANSFER_ID = 20 diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferManager.kt b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferManager.kt index 8a4bd9cb30..119aeeafb9 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferManager.kt +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferManager.kt @@ -45,7 +45,7 @@ import kotlin.math.min */ internal class TransferManager( context: Context, - s3: S3Client, + clientProvider: StorageTransferClientProvider, private val pluginKey: String, private val workManager: WorkManager = WorkManager.getInstance(context) ) { @@ -71,7 +71,7 @@ internal class TransferManager( init { RouterWorker.workerFactories[pluginKey] = TransferWorkerFactory( transferDB, - s3, + clientProvider, transferStatusUpdater ) } @@ -93,6 +93,7 @@ internal class TransferManager( fun upload( transferId: String, bucket: String, + region: String, key: String, file: File, metadata: ObjectMetadata, @@ -101,12 +102,13 @@ internal class TransferManager( useAccelerateEndpoint: Boolean = false ): TransferObserver { val transferRecordId = if (shouldUploadInMultipart(file)) { - createMultipartUploadRecords(transferId, bucket, key, file, metadata, cannedAcl, useAccelerateEndpoint) + createMultipartUploadRecords(transferId, bucket, region, key, file, metadata, cannedAcl, useAccelerateEndpoint) } else { val uri = transferDB.insertSingleTransferRecord( transferId, TransferType.UPLOAD, bucket, + region, key, file, cannedAcl, @@ -147,6 +149,7 @@ internal class TransferManager( return upload( transferId, options.bucket, + options.region, key, file, options.objectMetadata, @@ -160,6 +163,7 @@ internal class TransferManager( fun download( transferId: String, bucket: String, + region: String, key: String, file: File, listener: TransferListener? = null, @@ -172,6 +176,7 @@ internal class TransferManager( transferId, TransferType.DOWNLOAD, bucket, + region, key, file, useAccelerateEndpoint = useAccelerateEndpoint @@ -246,6 +251,7 @@ internal class TransferManager( private fun createMultipartUploadRecords( transferId: String, bucket: String, + region: String, key: String, file: File, metadata: ObjectMetadata, @@ -263,6 +269,7 @@ internal class TransferManager( contentValues[0] = transferDB.generateContentValuesForMultiPartUpload( transferId, bucket, + region, key, file, fileOffset, @@ -279,6 +286,7 @@ internal class TransferManager( contentValues[partNum] = transferDB.generateContentValuesForMultiPartUpload( UUID.randomUUID().toString(), bucket, + region, key, file, fileOffset, diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferObserver.kt b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferObserver.kt index 021e06a5b8..b6147f5cfe 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferObserver.kt +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferObserver.kt @@ -24,6 +24,7 @@ internal data class TransferObserver @JvmOverloads constructor( val id: Int, private val transferStatusUpdater: TransferStatusUpdater, val bucket: String? = null, + val region: String? = null, val key: String? = null, val filePath: String? = null, private val listener: TransferListener? = null, diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferRecord.kt b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferRecord.kt index c32a85c7ce..73d219b97d 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferRecord.kt +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferRecord.kt @@ -36,6 +36,7 @@ internal data class TransferRecord( var type: TransferType? = null, var state: TransferState? = null, var bucketName: String? = null, + var region: String? = null, var key: String? = null, var versionId: String? = null, var file: String = "", @@ -80,6 +81,8 @@ internal data class TransferRecord( ) this.bucketName = c.getString(c.getColumnIndexOrThrow(TransferTable.COLUMN_BUCKET_NAME)) + this.region = + c.getString(c.getColumnIndexOrThrow(TransferTable.COLUMN_REGION)) this.key = c.getString(c.getColumnIndexOrThrow(TransferTable.COLUMN_KEY)) this.versionId = c.getString(c.getColumnIndexOrThrow(TransferTable.COLUMN_VERSION_ID)) diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferTable.kt b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferTable.kt index 85b515062d..d7417c4937 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferTable.kt +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferTable.kt @@ -134,6 +134,8 @@ internal class TransferTable { const val COLUMN_USE_ACCELERATE_ENDPOINT = "useAccelerateEndpoint" + const val COLUMN_REGION = "region" + private const val TABLE_VERSION_2 = 2 private const val TABLE_VERSION_3 = 3 private const val TABLE_VERSION_4 = 4 @@ -142,8 +144,13 @@ internal class TransferTable { private const val TABLE_VERSION_7 = 7 private const val TABLE_VERSION_8 = 8 private const val TABLE_VERSION_9 = 9 + private const val TABLE_VERSION_10 = 10 - // Database creation SQL statement + // **** DO NOT UPDATE *** + // Database creation SQL statement for TABLE_VERSION 1 + // The current database migration implementation assumes that the original version 1 is always created + // and then incrementally upgrades from the original version 1 to latest version. + // instead of of upgrading from the last/previous version to the latest version. const val DATABASE_CREATE = "create table $TABLE_TRANSFER (" + "$COLUMN_ID integer primary key autoincrement, " + "$COLUMN_MAIN_UPLOAD_ID integer, " + @@ -219,6 +226,9 @@ internal class TransferTable { if (TABLE_VERSION_9 in (oldVersion + 1)..newVersion) { addVersion9Columns(database) } + if (TABLE_VERSION_10 in (oldVersion + 1)..newVersion) { + addVersion10Columns(database) + } database.setTransactionSuccessful() database.endTransaction() } @@ -296,5 +306,10 @@ internal class TransferTable { "DEFAULT 0;" database.execSQL(addConnectionType) } + + private fun addVersion10Columns(database: SQLiteDatabase) { + val addRegion = "ALTER TABLE $TABLE_TRANSFER ADD COLUMN $COLUMN_REGION text " + "DEFAULT null;" + database.execSQL(addRegion) + } } } diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/UploadOptions.kt b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/UploadOptions.kt index f68453c757..635bc7eb7f 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/UploadOptions.kt +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/UploadOptions.kt @@ -24,6 +24,7 @@ import com.amplifyframework.storage.ObjectMetadata internal data class UploadOptions @JvmOverloads constructor( val bucket: String, + val region: String, val objectMetadata: ObjectMetadata = ObjectMetadata(), val cannedAcl: ObjectCannedAcl? = null, val transferListener: TransferListener? = null diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/AbortMultiPartUploadWorker.kt b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/AbortMultiPartUploadWorker.kt index 30aa2d20b9..3f9cb5efb9 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/AbortMultiPartUploadWorker.kt +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/AbortMultiPartUploadWorker.kt @@ -20,6 +20,7 @@ import aws.sdk.kotlin.services.s3.S3Client import aws.sdk.kotlin.services.s3.abortMultipartUpload import aws.sdk.kotlin.services.s3.withConfig import com.amplifyframework.storage.TransferState +import com.amplifyframework.storage.s3.transfer.StorageTransferClientProvider import com.amplifyframework.storage.s3.transfer.TransferDB import com.amplifyframework.storage.s3.transfer.TransferStatusUpdater @@ -27,7 +28,7 @@ import com.amplifyframework.storage.s3.transfer.TransferStatusUpdater * Worker to abort pending multipart upload **/ internal class AbortMultiPartUploadWorker( - private val s3: S3Client, + private val clientProvider: StorageTransferClientProvider, private val transferDB: TransferDB, private val transferStatusUpdater: TransferStatusUpdater, context: Context, @@ -35,6 +36,7 @@ internal class AbortMultiPartUploadWorker( ) : BaseTransferWorker(transferStatusUpdater, transferDB, context, workerParameters) { override suspend fun performWork(): Result { + val s3: S3Client = clientProvider.getStorageTransferClient(transferRecord.region, transferRecord.bucketName) return s3.withConfig { enableAccelerate = transferRecord.useAccelerateEndpoint == 1 }.abortMultipartUpload { diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/CompleteMultiPartUploadWorker.kt b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/CompleteMultiPartUploadWorker.kt index 8b48014a87..d7cadd5dd2 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/CompleteMultiPartUploadWorker.kt +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/CompleteMultiPartUploadWorker.kt @@ -20,6 +20,7 @@ import aws.sdk.kotlin.services.s3.S3Client import aws.sdk.kotlin.services.s3.completeMultipartUpload import aws.sdk.kotlin.services.s3.model.CompletedMultipartUpload import aws.sdk.kotlin.services.s3.withConfig +import com.amplifyframework.storage.s3.transfer.StorageTransferClientProvider import com.amplifyframework.storage.s3.transfer.TransferDB import com.amplifyframework.storage.s3.transfer.TransferStatusUpdater @@ -27,7 +28,7 @@ import com.amplifyframework.storage.s3.transfer.TransferStatusUpdater * Worker to complete multipart upload **/ internal class CompleteMultiPartUploadWorker( - private val s3: S3Client, + private val clientProvider: StorageTransferClientProvider, private val transferDB: TransferDB, private val transferStatusUpdater: TransferStatusUpdater, context: Context, @@ -36,6 +37,7 @@ internal class CompleteMultiPartUploadWorker( override suspend fun performWork(): Result { val completedParts = transferDB.queryPartETagsOfUpload(transferRecord.id) + val s3: S3Client = clientProvider.getStorageTransferClient(transferRecord.region, transferRecord.bucketName) return s3.withConfig { enableAccelerate = transferRecord.useAccelerateEndpoint == 1 }.completeMultipartUpload { diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/DownloadWorker.kt b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/DownloadWorker.kt index 87a9db5612..9af6f8d70d 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/DownloadWorker.kt +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/DownloadWorker.kt @@ -26,6 +26,7 @@ import aws.smithy.kotlin.runtime.io.SdkSource import aws.smithy.kotlin.runtime.io.buffer import com.amplifyframework.storage.s3.transfer.DownloadProgressListener import com.amplifyframework.storage.s3.transfer.DownloadProgressListenerInterceptor +import com.amplifyframework.storage.s3.transfer.StorageTransferClientProvider import com.amplifyframework.storage.s3.transfer.TransferDB import com.amplifyframework.storage.s3.transfer.TransferStatusUpdater import java.io.BufferedOutputStream @@ -40,7 +41,7 @@ import kotlinx.coroutines.withContext * Worker to perform download file task. */ internal class DownloadWorker( - private val s3: S3Client, + private val clientProvider: StorageTransferClientProvider, private val transferDB: TransferDB, private val transferStatusUpdater: TransferStatusUpdater, context: Context, @@ -50,6 +51,7 @@ internal class DownloadWorker( private lateinit var downloadProgressListener: DownloadProgressListener private val defaultBufferSize = 8192L override suspend fun performWork(): Result { + val s3: S3Client = clientProvider.getStorageTransferClient(transferRecord.region, transferRecord.bucketName) s3.withConfig { enableAccelerate = transferRecord.useAccelerateEndpoint == 1 } diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/InitiateMultiPartUploadTransferWorker.kt b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/InitiateMultiPartUploadTransferWorker.kt index ec7be7cb35..b3c013b4bc 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/InitiateMultiPartUploadTransferWorker.kt +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/InitiateMultiPartUploadTransferWorker.kt @@ -21,6 +21,7 @@ import aws.sdk.kotlin.services.s3.S3Client import aws.sdk.kotlin.services.s3.createMultipartUpload import aws.sdk.kotlin.services.s3.withConfig import com.amplifyframework.storage.TransferState +import com.amplifyframework.storage.s3.transfer.StorageTransferClientProvider import com.amplifyframework.storage.s3.transfer.TransferDB import com.amplifyframework.storage.s3.transfer.TransferStatusUpdater @@ -28,7 +29,7 @@ import com.amplifyframework.storage.s3.transfer.TransferStatusUpdater * Worker to initiate multipart upload **/ internal class InitiateMultiPartUploadTransferWorker( - private val s3: S3Client, + private val clientProvider: StorageTransferClientProvider, private val transferDB: TransferDB, private val transferStatusUpdater: TransferStatusUpdater, context: Context, @@ -36,6 +37,7 @@ internal class InitiateMultiPartUploadTransferWorker( ) : BaseTransferWorker(transferStatusUpdater, transferDB, context, workerParameters) { override suspend fun performWork(): Result { + val s3: S3Client = clientProvider.getStorageTransferClient(transferRecord.region, transferRecord.bucketName) transferStatusUpdater.updateTransferState(transferRecord.id, TransferState.IN_PROGRESS) val putObjectRequest = createPutObjectRequest(transferRecord, null) return s3.withConfig { diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/PartUploadTransferWorker.kt b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/PartUploadTransferWorker.kt index d145786b6d..b7a0f6760d 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/PartUploadTransferWorker.kt +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/PartUploadTransferWorker.kt @@ -22,6 +22,7 @@ import aws.sdk.kotlin.services.s3.withConfig import aws.smithy.kotlin.runtime.content.asByteStream import com.amplifyframework.storage.TransferState import com.amplifyframework.storage.s3.transfer.PartUploadProgressListener +import com.amplifyframework.storage.s3.transfer.StorageTransferClientProvider import com.amplifyframework.storage.s3.transfer.TransferDB import com.amplifyframework.storage.s3.transfer.TransferStatusUpdater import com.amplifyframework.storage.s3.transfer.UploadProgressListenerInterceptor @@ -33,7 +34,7 @@ import kotlinx.coroutines.isActive * Worker to upload a part for multipart upload **/ internal class PartUploadTransferWorker( - private val s3: S3Client, + private val clientProvider: StorageTransferClientProvider, private val transferDB: TransferDB, private val transferStatusUpdater: TransferStatusUpdater, context: Context, @@ -51,6 +52,7 @@ internal class PartUploadTransferWorker( transferStatusUpdater.updateTransferState(transferRecord.mainUploadId, TransferState.IN_PROGRESS) multiPartUploadId = inputData.keyValueMap[MULTI_PART_UPLOAD_ID] as String partUploadProgressListener = PartUploadProgressListener(transferRecord, transferStatusUpdater) + val s3: S3Client = clientProvider.getStorageTransferClient(transferRecord.region, transferRecord.bucketName) return s3.withConfig { interceptors += UploadProgressListenerInterceptor(partUploadProgressListener) enableAccelerate = transferRecord.useAccelerateEndpoint == 1 diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/SinglePartUploadWorker.kt b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/SinglePartUploadWorker.kt index 21de0db80c..515c36befc 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/SinglePartUploadWorker.kt +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/SinglePartUploadWorker.kt @@ -22,13 +22,14 @@ import android.content.Context import androidx.work.WorkerParameters import aws.sdk.kotlin.services.s3.S3Client import aws.sdk.kotlin.services.s3.withConfig +import com.amplifyframework.storage.s3.transfer.StorageTransferClientProvider import com.amplifyframework.storage.s3.transfer.TransferDB import com.amplifyframework.storage.s3.transfer.TransferStatusUpdater import com.amplifyframework.storage.s3.transfer.UploadProgressListener import com.amplifyframework.storage.s3.transfer.UploadProgressListenerInterceptor internal class SinglePartUploadWorker( - private val s3: S3Client, + private val clientProvider: StorageTransferClientProvider, private val transferDB: TransferDB, private val transferStatusUpdater: TransferStatusUpdater, context: Context, @@ -40,6 +41,7 @@ internal class SinglePartUploadWorker( override suspend fun performWork(): Result { uploadProgressListener = UploadProgressListener(transferRecord, transferStatusUpdater) val putObjectRequest = createPutObjectRequest(transferRecord, uploadProgressListener) + val s3: S3Client = clientProvider.getStorageTransferClient(transferRecord.region, transferRecord.bucketName) return s3.withConfig { interceptors += UploadProgressListenerInterceptor(uploadProgressListener) enableAccelerate = transferRecord.useAccelerateEndpoint == 1 diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/TransferWorkerFactory.kt b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/TransferWorkerFactory.kt index 88c0dc19f4..f491d1ee95 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/TransferWorkerFactory.kt +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/TransferWorkerFactory.kt @@ -18,6 +18,7 @@ import android.content.Context import androidx.work.WorkerFactory import androidx.work.WorkerParameters import aws.sdk.kotlin.services.s3.S3Client +import com.amplifyframework.storage.s3.transfer.StorageTransferClientProvider import com.amplifyframework.storage.s3.transfer.TransferDB import com.amplifyframework.storage.s3.transfer.TransferStatusUpdater @@ -26,7 +27,7 @@ import com.amplifyframework.storage.s3.transfer.TransferStatusUpdater **/ internal class TransferWorkerFactory( private val transferDB: TransferDB, - private val s3: S3Client, + private val clientProvider: StorageTransferClientProvider, private val transferStatusUpdater: TransferStatusUpdater ) : WorkerFactory() { override fun createWorker( @@ -37,7 +38,7 @@ internal class TransferWorkerFactory( when (workerClassName) { DownloadWorker::class.java.name -> return DownloadWorker( - s3, + clientProvider, transferDB, transferStatusUpdater, appContext, @@ -45,7 +46,7 @@ internal class TransferWorkerFactory( ) SinglePartUploadWorker::class.java.name -> return SinglePartUploadWorker( - s3, + clientProvider, transferDB, transferStatusUpdater, appContext, @@ -53,7 +54,7 @@ internal class TransferWorkerFactory( ) InitiateMultiPartUploadTransferWorker::class.java.name -> return InitiateMultiPartUploadTransferWorker( - s3, + clientProvider, transferDB, transferStatusUpdater, appContext, @@ -61,7 +62,7 @@ internal class TransferWorkerFactory( ) PartUploadTransferWorker::class.java.name -> return PartUploadTransferWorker( - s3, + clientProvider, transferDB, transferStatusUpdater, appContext, @@ -69,7 +70,7 @@ internal class TransferWorkerFactory( ) CompleteMultiPartUploadWorker::class.java.name -> return CompleteMultiPartUploadWorker( - s3, + clientProvider, transferDB, transferStatusUpdater, appContext, @@ -77,7 +78,7 @@ internal class TransferWorkerFactory( ) AbortMultiPartUploadWorker::class.java.name -> return AbortMultiPartUploadWorker( - s3, + clientProvider, transferDB, transferStatusUpdater, appContext, diff --git a/aws-storage-s3/src/test/java/com/amplifyframework/storage/AWSS3StorageServiceContainerTest.kt b/aws-storage-s3/src/test/java/com/amplifyframework/storage/AWSS3StorageServiceContainerTest.kt new file mode 100644 index 0000000000..3119dab168 --- /dev/null +++ b/aws-storage-s3/src/test/java/com/amplifyframework/storage/AWSS3StorageServiceContainerTest.kt @@ -0,0 +1,121 @@ +/* + * Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amplifyframework.storage.s3 + +import android.content.Context +import com.amplifyframework.storage.BucketInfo +import com.amplifyframework.storage.ResolvedStorageBucket +import com.amplifyframework.storage.StorageBucket +import com.amplifyframework.storage.s3.service.AWSS3StorageService +import com.amplifyframework.storage.s3.service.AWSS3StorageServiceContainer +import com.amplifyframework.storage.s3.transfer.StorageTransferClientProvider +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import io.mockk.every +import io.mockk.mockk +import java.util.concurrent.ConcurrentHashMap +import org.junit.Before +import org.junit.Test + +class AWSS3StorageServiceContainerTest { + + private val storageServiceFactory = mockk { + every { create(any(), any(), any(), any()) } returns mockk() + } + private val context = mockk() + private val clientProvider = mockk() + private val bucketName = "testBucket" + private val region = "us-east-1" + + private lateinit var serviceContainerHashMap: ConcurrentHashMap + private lateinit var serviceContainer: AWSS3StorageServiceContainer + @Before + fun setUp() { + serviceContainerHashMap = ConcurrentHashMap() + serviceContainer = AWSS3StorageServiceContainer( + context, + storageServiceFactory, + clientProvider, + serviceContainerHashMap + ) + } + + @Test + fun `put default AWSS3Service in container`() { + val service = storageServiceFactory.create(context, region, bucketName, clientProvider) + serviceContainer.put(bucketName, service) + + serviceContainerHashMap.size shouldBe 1 + serviceContainerHashMap[bucketName] shouldNotBe null + } + + @Test + fun `get non-existent AWSS3Service in container with ResolvedStorageBucket creates new AWSService`() { + val bucketInfo = BucketInfo(bucketName, region) + val bucket: ResolvedStorageBucket = StorageBucket.fromBucketInfo(bucketInfo) as ResolvedStorageBucket + + val service = serviceContainer.get(bucket) + + service shouldNotBe null + serviceContainerHashMap.size shouldBe 1 + serviceContainerHashMap[bucketName] shouldNotBe null + serviceContainerHashMap[bucketName] shouldBe service + } + + @Test + fun `get WSS3Service in container multiple times with ResolvedStorageBucket creates only one service`() { + val bucketInfo = BucketInfo(bucketName, region) + val bucket: ResolvedStorageBucket = StorageBucket.fromBucketInfo(bucketInfo) as ResolvedStorageBucket + + val service = serviceContainer.get(bucket) + val service2 = serviceContainer.get(bucket) + + service shouldNotBe null + service2 shouldNotBe null + service shouldBe service2 + + serviceContainerHashMap.size shouldBe 1 + serviceContainerHashMap[bucketName] shouldNotBe null + serviceContainerHashMap[bucketName] shouldBe service + serviceContainerHashMap[bucketName] shouldBe service2 + } + + @Test + fun `get non-existent AWSS3Service in container with bucket name and region creates new AWSService`() { + val service = serviceContainer.get(bucketName, region) + + service shouldNotBe null + serviceContainerHashMap.size shouldBe 1 + serviceContainerHashMap[bucketName] shouldNotBe null + serviceContainerHashMap[bucketName] shouldBe service + } + + @Test + fun `get WSS3Service in container multiple times with bucket name and region creates only one service`() { + + val service = serviceContainer.get(bucketName, region) + val service2 = serviceContainer.get(bucketName, region) + + service shouldNotBe null + service2 shouldNotBe null + service shouldBe service2 + + serviceContainerHashMap.size shouldBe 1 + serviceContainerHashMap[bucketName] shouldNotBe null + serviceContainerHashMap[bucketName] shouldBe service + serviceContainerHashMap[bucketName] shouldBe service2 + } +} diff --git a/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/AWSS3StoragePluginTest.kt b/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/AWSS3StoragePluginTest.kt index a13158f1bf..5380af91b6 100644 --- a/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/AWSS3StoragePluginTest.kt +++ b/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/AWSS3StoragePluginTest.kt @@ -20,7 +20,6 @@ import com.amplifyframework.storage.InvalidStorageBucketException import com.amplifyframework.storage.StorageBucket import com.amplifyframework.storage.StorageException import com.amplifyframework.storage.s3.service.AWSS3StorageService -import com.amplifyframework.storage.s3.service.StorageService import com.amplifyframework.testutils.configuration.amplifyOutputsData import io.kotest.assertions.throwables.shouldThrow import io.kotest.matchers.shouldNotBe @@ -32,8 +31,8 @@ import org.junit.Test class AWSS3StoragePluginTest { - private val storageServiceFactory = mockk { - every { create(any(), any(), any()) } returns mockk() + private val storageServiceFactory = mockk { + every { create(any(), any(), any(), any()) } returns mockk() } private val plugin = AWSS3StoragePlugin( @@ -54,7 +53,7 @@ class AWSS3StoragePluginTest { plugin.configure(data, mockk()) verify { - storageServiceFactory.create(any(), "test-region", "test-bucket") + storageServiceFactory.create(any(), "test-region", "test-bucket", any()) } } diff --git a/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/StorageComponentTest.java b/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/StorageComponentTest.java index c1ac9ca541..3e238acced 100644 --- a/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/StorageComponentTest.java +++ b/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/StorageComponentTest.java @@ -32,6 +32,7 @@ import com.amplifyframework.storage.s3.configuration.AWSS3StoragePluginConfiguration; import com.amplifyframework.storage.s3.service.AWSS3StorageService; import com.amplifyframework.storage.s3.service.StorageService; +import com.amplifyframework.storage.s3.transfer.StorageTransferClientProvider; import com.amplifyframework.storage.s3.transfer.TransferListener; import com.amplifyframework.storage.s3.transfer.TransferObserver; import com.amplifyframework.testutils.Await; @@ -76,6 +77,7 @@ public final class StorageComponentTest { private StorageCategory storage; private StorageService storageService; + private StorageTransferClientProvider clientProvider; /** * Sets up Storage category by registering a mock AWSS3StoragePlugin @@ -88,7 +90,8 @@ public final class StorageComponentTest { public void setup() throws AmplifyException { this.storage = new StorageCategory(); this.storageService = mock(AWSS3StorageService.class); - StorageService.Factory storageServiceFactory = (context, region, bucket) -> storageService; + AWSS3StorageService.Factory storageServiceFactory + = (context, region, bucket, clientProvider) -> (AWSS3StorageService) storageService; AuthCredentialsProvider cognitoAuthProvider = mock(AuthCredentialsProvider.class); doReturn(RandomString.string()).when(cognitoAuthProvider).getIdentityId(null); this.storage.addPlugin(new AWSS3StoragePlugin(storageServiceFactory, diff --git a/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/transfer/worker/AbortMultiPartUploadWorkerTest.kt b/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/transfer/worker/AbortMultiPartUploadWorkerTest.kt index c3097d71c0..5772881318 100644 --- a/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/transfer/worker/AbortMultiPartUploadWorkerTest.kt +++ b/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/transfer/worker/AbortMultiPartUploadWorkerTest.kt @@ -24,6 +24,8 @@ import aws.sdk.kotlin.services.s3.model.AbortMultipartUploadRequest import aws.sdk.kotlin.services.s3.model.AbortMultipartUploadResponse import aws.sdk.kotlin.services.s3.withConfig import com.amplifyframework.storage.TransferState +import com.amplifyframework.storage.s3.transfer.S3StorageTransferClientProvider +import com.amplifyframework.storage.s3.transfer.StorageTransferClientProvider import com.amplifyframework.storage.s3.transfer.TransferDB import com.amplifyframework.storage.s3.transfer.TransferRecord import com.amplifyframework.storage.s3.transfer.TransferStatusUpdater @@ -52,6 +54,7 @@ internal class AbortMultiPartUploadWorkerTest { private lateinit var transferDB: TransferDB private lateinit var transferStatusUpdater: TransferStatusUpdater private lateinit var workerParameters: WorkerParameters + private lateinit var clientProvider: StorageTransferClientProvider @Before fun setup() { @@ -59,6 +62,7 @@ internal class AbortMultiPartUploadWorkerTest { context = ApplicationProvider.getApplicationContext() workerParameters = mockk(WorkerParameters::class.java.name) s3Client = spyk(recordPrivateCalls = true) + clientProvider = mockk(S3StorageTransferClientProvider::class.java.name) mockkStatic(S3Client::withConfig) transferDB = mockk(TransferDB::class.java.name) transferStatusUpdater = mockk(TransferStatusUpdater::class.java.name) @@ -66,6 +70,7 @@ internal class AbortMultiPartUploadWorkerTest { every { workerParameters.runAttemptCount }.answers { 1 } every { workerParameters.taskExecutor }.answers { ImmediateTaskExecutor() } every { any().withConfig(any()) }.answers { s3Client } + every { clientProvider.getStorageTransferClient(any(), any()) }.answers { s3Client } } @After @@ -95,13 +100,20 @@ internal class AbortMultiPartUploadWorkerTest { every { transferDB.getTransferRecordById(any()) }.answers { transferRecord } every { transferStatusUpdater.updateTransferState(any(), any()) }.answers { } - val worker = AbortMultiPartUploadWorker(s3Client, transferDB, transferStatusUpdater, context, workerParameters) + val worker = AbortMultiPartUploadWorker( + clientProvider, + transferDB, + transferStatusUpdater, + context, + workerParameters + ) val result = worker.doWork() val expectedResult = ListenableWorker.Result.success(workDataOf(BaseTransferWorker.OUTPUT_TRANSFER_RECORD_ID to 1)) verify(exactly = 1) { transferStatusUpdater.updateTransferState(1, TransferState.FAILED) } verify(exactly = 1) { any().withConfig(any()) } + verify(exactly = 1) { clientProvider.getStorageTransferClient(any(), any()) } assertEquals(expectedResult, result) } @@ -128,7 +140,13 @@ internal class AbortMultiPartUploadWorkerTest { every { transferDB.getTransferRecordById(any()) }.answers { transferRecord } every { transferStatusUpdater.updateTransferState(any(), any()) }.answers { } - val worker = AbortMultiPartUploadWorker(s3Client, transferDB, transferStatusUpdater, context, workerParameters) + val worker = AbortMultiPartUploadWorker( + clientProvider, + transferDB, + transferStatusUpdater, + context, + workerParameters + ) val result = worker.doWork() val expectedResult = @@ -157,7 +175,13 @@ internal class AbortMultiPartUploadWorkerTest { every { transferStatusUpdater.updateTransferState(any(), any()) }.answers { } every { transferStatusUpdater.updateOnError(any(), any()) }.answers { } - val worker = AbortMultiPartUploadWorker(s3Client, transferDB, transferStatusUpdater, context, workerParameters) + val worker = AbortMultiPartUploadWorker( + clientProvider, + transferDB, + transferStatusUpdater, + context, + workerParameters + ) val result = worker.doWork() val expectedResult = diff --git a/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/transfer/worker/DownloadWorkerTest.kt b/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/transfer/worker/DownloadWorkerTest.kt index 9d7ad8114c..446015b0cc 100644 --- a/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/transfer/worker/DownloadWorkerTest.kt +++ b/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/transfer/worker/DownloadWorkerTest.kt @@ -26,6 +26,8 @@ import aws.smithy.kotlin.runtime.content.ByteStream import aws.smithy.kotlin.runtime.content.fromFile import com.amplifyframework.storage.TransferState import com.amplifyframework.storage.s3.transfer.DownloadProgressListenerInterceptor +import com.amplifyframework.storage.s3.transfer.S3StorageTransferClientProvider +import com.amplifyframework.storage.s3.transfer.StorageTransferClientProvider import com.amplifyframework.storage.s3.transfer.TransferDB import com.amplifyframework.storage.s3.transfer.TransferRecord import com.amplifyframework.storage.s3.transfer.TransferStatusUpdater @@ -56,12 +58,14 @@ internal class DownloadWorkerTest { private lateinit var transferStatusUpdater: TransferStatusUpdater private lateinit var workerParameters: WorkerParameters private lateinit var downloadInterceptor: DownloadProgressListenerInterceptor + private lateinit var clientProvider: StorageTransferClientProvider @Before fun setup() { context = ApplicationProvider.getApplicationContext() workerParameters = mockk(WorkerParameters::class.java.name) s3Client = mockk(relaxed = true, relaxUnitFun = true) + clientProvider = mockk(S3StorageTransferClientProvider::class.java.name) mockkStatic(S3Client::withConfig) downloadInterceptor = mockk(relaxed = true, relaxUnitFun = true) transferDB = mockk(TransferDB::class.java.name) @@ -70,6 +74,7 @@ internal class DownloadWorkerTest { every { workerParameters.runAttemptCount }.answers { 1 } every { workerParameters.taskExecutor }.answers { ImmediateTaskExecutor() } every { s3Client.withConfig(any()) } returns s3Client + every { clientProvider.getStorageTransferClient(any(), any())}.answers { s3Client } } @After @@ -102,10 +107,11 @@ internal class DownloadWorkerTest { every { transferStatusUpdater.updateProgress(1, any(), any(), true, false) }.answers { } every { transferStatusUpdater.updateProgress(1, any(), any(), true, true) }.answers { } - val worker = DownloadWorker(s3Client, transferDB, transferStatusUpdater, context, workerParameters) + val worker = DownloadWorker(clientProvider, transferDB, transferStatusUpdater, context, workerParameters) val result = worker.doWork() verify(atLeast = 1) { transferStatusUpdater.updateProgress(1, 10 * 1024 * 1024, 10 * 1024 * 1024, true, true) } + verify(exactly = 1) { clientProvider.getStorageTransferClient(any(), any()) } val expectedResult = ListenableWorker.Result.success(workDataOf(BaseTransferWorker.OUTPUT_TRANSFER_RECORD_ID to 1)) assertEquals(expectedResult, result) @@ -131,7 +137,7 @@ internal class DownloadWorkerTest { every { transferStatusUpdater.updateTransferState(1, TransferState.FAILED) }.answers { } every { transferStatusUpdater.updateOnError(1, any()) }.answers { } - val worker = DownloadWorker(s3Client, transferDB, transferStatusUpdater, context, workerParameters) + val worker = DownloadWorker(clientProvider, transferDB, transferStatusUpdater, context, workerParameters) val result = worker.doWork() verify(exactly = 0) { diff --git a/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/transfer/worker/InitiateMultiPartUploadTransferWorkerTest.kt b/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/transfer/worker/InitiateMultiPartUploadTransferWorkerTest.kt index ea423d392e..852f660a95 100644 --- a/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/transfer/worker/InitiateMultiPartUploadTransferWorkerTest.kt +++ b/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/transfer/worker/InitiateMultiPartUploadTransferWorkerTest.kt @@ -23,6 +23,8 @@ import aws.sdk.kotlin.services.s3.S3Client import aws.sdk.kotlin.services.s3.model.CreateMultipartUploadResponse import aws.sdk.kotlin.services.s3.withConfig import com.amplifyframework.storage.TransferState +import com.amplifyframework.storage.s3.transfer.S3StorageTransferClientProvider +import com.amplifyframework.storage.s3.transfer.StorageTransferClientProvider import com.amplifyframework.storage.s3.transfer.TransferDB import com.amplifyframework.storage.s3.transfer.TransferRecord import com.amplifyframework.storage.s3.transfer.TransferStatusUpdater @@ -51,12 +53,14 @@ internal class InitiateMultiPartUploadTransferWorkerTest { private lateinit var transferDB: TransferDB private lateinit var transferStatusUpdater: TransferStatusUpdater private lateinit var workerParameters: WorkerParameters + private lateinit var clientProvider: StorageTransferClientProvider @Before fun setup() { context = ApplicationProvider.getApplicationContext() workerParameters = mockk(WorkerParameters::class.java.name) s3Client = mockk(relaxed = true) + clientProvider = mockk(S3StorageTransferClientProvider::class.java.name) mockkStatic(S3Client::withConfig) transferDB = mockk(TransferDB::class.java.name) transferStatusUpdater = mockk(TransferStatusUpdater::class.java.name) @@ -64,6 +68,7 @@ internal class InitiateMultiPartUploadTransferWorkerTest { every { workerParameters.runAttemptCount }.answers { 1 } every { workerParameters.taskExecutor }.answers { ImmediateTaskExecutor() } every { s3Client.withConfig(any()) } returns s3Client + every { clientProvider.getStorageTransferClient(any(), any())}.answers { s3Client } } @After @@ -89,7 +94,7 @@ internal class InitiateMultiPartUploadTransferWorkerTest { every { transferStatusUpdater.updateMultipartId(1, "upload_id") }.answers { } every { transferStatusUpdater.updateTransferState(any(), TransferState.IN_PROGRESS) }.answers { } val worker = InitiateMultiPartUploadTransferWorker( - s3Client, + clientProvider, transferDB, transferStatusUpdater, context, @@ -97,6 +102,7 @@ internal class InitiateMultiPartUploadTransferWorkerTest { ) val result = worker.doWork() verify(exactly = 1) { transferStatusUpdater.updateMultipartId(1, "upload_id") } + verify(exactly = 1) { clientProvider.getStorageTransferClient(any(), any()) } val output = workDataOf( BaseTransferWorker.MULTI_PART_UPLOAD_ID to "upload_id", BaseTransferWorker.TRANSFER_RECORD_ID to 1 @@ -124,7 +130,7 @@ internal class InitiateMultiPartUploadTransferWorkerTest { every { transferStatusUpdater.updateTransferState(any(), any()) }.answers { } val worker = InitiateMultiPartUploadTransferWorker( - s3Client, + clientProvider, transferDB, transferStatusUpdater, context, From 36e10d8aa1fb2666db32fb89192cdf8bc91098fa Mon Sep 17 00:00:00 2001 From: Tuan Pham <103537251+phantumcode@users.noreply.github.com> Date: Wed, 4 Sep 2024 09:49:01 -0500 Subject: [PATCH 4/7] chore: update BucketInfo member name (#2906) --- .../com/amplifyframework/storage/s3/AWSS3StoragePlugin.java | 4 ++-- .../storage/s3/service/AWSS3StorageServiceContainer.kt | 2 +- core/src/main/java/com/amplifyframework/storage/BucketInfo.kt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/AWSS3StoragePlugin.java b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/AWSS3StoragePlugin.java index 46316bef0c..48dce2ee26 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/AWSS3StoragePlugin.java +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/AWSS3StoragePlugin.java @@ -300,12 +300,12 @@ private void configure( this.defaultStorageService = storageServiceFactory.create( context, region, - bucket.getBucketInfo().getName(), + bucket.getBucketInfo().getBucketName(), clientProvider); this.awss3StorageServiceContainer = new AWSS3StorageServiceContainer( context, storageServiceFactory, (S3StorageTransferClientProvider) clientProvider); - this.awss3StorageServiceContainer.put(bucket.getBucketInfo().getName(), this.defaultStorageService); + this.awss3StorageServiceContainer.put(bucket.getBucketInfo().getBucketName(), this.defaultStorageService); } catch (RuntimeException exception) { throw new StorageException( "Failed to create storage service.", diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/service/AWSS3StorageServiceContainer.kt b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/service/AWSS3StorageServiceContainer.kt index f22131df2f..972c7bcdf3 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/service/AWSS3StorageServiceContainer.kt +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/service/AWSS3StorageServiceContainer.kt @@ -57,7 +57,7 @@ internal class AWSS3StorageServiceContainer( */ fun get(resolvedStorageBucket: ResolvedStorageBucket): AWSS3StorageService { synchronized(lock) { - val bucketName: String = resolvedStorageBucket.bucketInfo.name + val bucketName: String = resolvedStorageBucket.bucketInfo.bucketName var service = awsS3StorageServicesByBucketName.get(bucketName) if (service == null) { val region: String = resolvedStorageBucket.bucketInfo.region diff --git a/core/src/main/java/com/amplifyframework/storage/BucketInfo.kt b/core/src/main/java/com/amplifyframework/storage/BucketInfo.kt index ca3da36064..f6ff639a69 100644 --- a/core/src/main/java/com/amplifyframework/storage/BucketInfo.kt +++ b/core/src/main/java/com/amplifyframework/storage/BucketInfo.kt @@ -14,4 +14,4 @@ */ package com.amplifyframework.storage -data class BucketInfo(val name: String, val region: String) +data class BucketInfo(val bucketName: String, val region: String) From 41c67d3b78d8839d22974d88fa6bf79425462212 Mon Sep 17 00:00:00 2001 From: Tuan Pham <103537251+phantumcode@users.noreply.github.com> Date: Wed, 4 Sep 2024 12:30:11 -0500 Subject: [PATCH 5/7] chore(storage): add integration tests for multi-bucket support (#2900) --- .../AWSS3StorageDownloadAccessLevelTest.java | 46 +++ .../storage/s3/AWSS3StorageDownloadTest.java | 20 +- .../s3/AWSS3StorageListAccessLevelTest.java | 53 ++++ .../s3/AWSS3StorageMultiBucketDownloadTest.kt | 281 ++++++++++++++++++ .../s3/AWSS3StorageMultiBucketGetUrlTest.kt | 105 +++++++ .../s3/AWSS3StorageMultiBucketListTest.kt | 139 +++++++++ .../s3/AWSS3StorageMultiBucketRemoveTest.kt | 101 +++++++ .../s3/AWSS3StorageMultiBucketUploadTest.kt | 280 +++++++++++++++++ .../s3/AWSS3StoragePathDownloadTest.kt | 16 + .../storage/s3/AWSS3StoragePathGetUrlTest.kt | 8 + .../storage/s3/AWSS3StoragePathListTest.kt | 12 + .../storage/s3/AWSS3StoragePathUploadTest.kt | 47 +-- .../s3/AWSS3StorageUploadAccessLevelTest.java | 13 +- .../storage/s3/AWSS3StorageUploadTest.java | 27 +- .../storage/s3/TestStorageCategory.java | 13 +- 15 files changed, 1123 insertions(+), 38 deletions(-) create mode 100644 aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageMultiBucketDownloadTest.kt create mode 100644 aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageMultiBucketGetUrlTest.kt create mode 100644 aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageMultiBucketListTest.kt create mode 100644 aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageMultiBucketRemoveTest.kt create mode 100644 aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageMultiBucketUploadTest.kt diff --git a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageDownloadAccessLevelTest.java b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageDownloadAccessLevelTest.java index d087ae96c5..195a59c420 100644 --- a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageDownloadAccessLevelTest.java +++ b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageDownloadAccessLevelTest.java @@ -23,6 +23,7 @@ import com.amplifyframework.storage.StorageCategory; import com.amplifyframework.storage.StorageException; import com.amplifyframework.storage.options.StorageDownloadFileOptions; +import com.amplifyframework.storage.options.StorageRemoveOptions; import com.amplifyframework.storage.options.StorageUploadFileOptions; import com.amplifyframework.storage.s3.UserCredentials.Credential; import com.amplifyframework.storage.s3.UserCredentials.IdentityIdSource; @@ -33,6 +34,7 @@ import com.amplifyframework.testutils.sync.SynchronousAuth; import com.amplifyframework.testutils.sync.SynchronousStorage; +import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -85,6 +87,15 @@ public static void setUpOnce() throws Exception { uploadTestFile(); } + /** + * Clean up test resources from test suite. + * @throws Exception from failure to remove test resources. + */ + @AfterClass + public static void tearDownOnce() throws Exception { + removeUploadedTestFiles(); + } + /** * Signs out by default and sets up download file destination. * @@ -263,4 +274,39 @@ private static void uploadTestFile() throws Exception { .build(); storage.uploadFile(key, uploadFile, options); } + + private static void removeUploadedTestFiles() throws Exception { + final String key = UPLOAD_NAME; + + synchronousAuth.signOut(); + synchronousAuth.signIn(userOne.getUsername(), userOne.getPassword()); + + StorageRemoveOptions options; + options = StorageRemoveOptions.builder() + .accessLevel(StorageAccessLevel.PUBLIC) + .build(); + storage.remove(key, options); + + options = StorageRemoveOptions.builder() + .accessLevel(StorageAccessLevel.PROTECTED) + .build(); + storage.remove(key, options); + + options = StorageRemoveOptions.builder() + .accessLevel(StorageAccessLevel.PRIVATE) + .build(); + storage.remove(key, options); + + // Upload as user two + synchronousAuth.signOut(); + synchronousAuth.signIn(userTwo.getUsername(), userTwo.getPassword()); + options = StorageRemoveOptions.builder() + .accessLevel(StorageAccessLevel.PROTECTED) + .build(); + storage.remove(key, options); + options = StorageRemoveOptions.builder() + .accessLevel(StorageAccessLevel.PRIVATE) + .build(); + storage.remove(key, options); + } } diff --git a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageDownloadTest.java b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageDownloadTest.java index c84e88bdf7..166f7bea23 100644 --- a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageDownloadTest.java +++ b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageDownloadTest.java @@ -17,7 +17,6 @@ import android.content.Context; -import com.amplifyframework.auth.AuthPlugin; import com.amplifyframework.auth.cognito.AWSCognitoAuthPlugin; import com.amplifyframework.core.Amplify; import com.amplifyframework.core.async.Cancelable; @@ -31,6 +30,7 @@ import com.amplifyframework.storage.TransferState; import com.amplifyframework.storage.operation.StorageDownloadFileOperation; import com.amplifyframework.storage.options.StorageDownloadFileOptions; +import com.amplifyframework.storage.options.StorageRemoveOptions; import com.amplifyframework.storage.options.StorageUploadFileOptions; import com.amplifyframework.storage.s3.options.AWSS3StorageDownloadFileOptions; import com.amplifyframework.storage.s3.test.R; @@ -41,6 +41,7 @@ import com.amplifyframework.testutils.sync.SynchronousStorage; import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -89,7 +90,7 @@ public final class AWSS3StorageDownloadTest { public static void setUpOnce() throws Exception { Context context = getApplicationContext(); WorkmanagerTestUtils.INSTANCE.initializeWorkmanagerTestUtil(context); - SynchronousAuth.delegatingToCognito(context, (AuthPlugin) new AWSCognitoAuthPlugin()); + SynchronousAuth.delegatingToCognito(context, new AWSCognitoAuthPlugin()); // Get a handle to storage storageCategory = TestStorageCategory.create(context, R.raw.amplifyconfiguration); @@ -112,6 +113,21 @@ public static void setUpOnce() throws Exception { synchronousStorage.uploadFile(key, smallFile, uploadOptions); } + /** + * Clean up test resources from test suite. + * @throws Exception from failure to remove test resources. + */ + @AfterClass + public static void tearDownOnce() throws Exception { + StorageRemoveOptions options = StorageRemoveOptions + .builder() + .accessLevel(TESTING_ACCESS_LEVEL) + .build(); + + synchronousStorage.remove(SMALL_FILE_NAME, options); + synchronousStorage.remove(LARGE_FILE_NAME, options); + } + /** * Sets up the options to use for transfer. * diff --git a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageListAccessLevelTest.java b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageListAccessLevelTest.java index 8e042666f0..ab93c9ebd7 100644 --- a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageListAccessLevelTest.java +++ b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageListAccessLevelTest.java @@ -17,6 +17,7 @@ import android.content.Context; +import com.amplifyframework.auth.AuthException; import com.amplifyframework.auth.AuthPlugin; import com.amplifyframework.auth.cognito.AWSCognitoAuthPlugin; import com.amplifyframework.storage.StorageAccessLevel; @@ -25,6 +26,7 @@ import com.amplifyframework.storage.StorageItem; import com.amplifyframework.storage.options.StorageListOptions; import com.amplifyframework.storage.options.StoragePagedListOptions; +import com.amplifyframework.storage.options.StorageRemoveOptions; import com.amplifyframework.storage.options.StorageUploadFileOptions; import com.amplifyframework.storage.result.StorageListResult; import com.amplifyframework.storage.s3.UserCredentials.IdentityIdSource; @@ -35,6 +37,7 @@ import com.amplifyframework.testutils.sync.SynchronousAuth; import com.amplifyframework.testutils.sync.SynchronousStorage; +import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -91,6 +94,16 @@ public static void setUpOnce() throws Exception { uploadTestFiles(); } + /** + * Remove upload test files from test suite. + * + * @throws Exception from failure to remove test resources. + */ + @AfterClass + public static void tearDownOnce() throws Exception { + removeUploadedTestFiles(); + } + /** * Signs out by default. * @@ -326,4 +339,44 @@ private static void uploadMultipleTestFiles() throws Exception { // Upload as user one synchronousAuth.signOut(); } + + private static void removeUploadedTestFiles() throws AuthException, StorageException { + // remove PUBLIC test files + synchronousAuth.signOut(); + synchronousAuth.signIn(userOne.getUsername(), userOne.getPassword()); + StorageRemoveOptions options = StorageRemoveOptions.builder() + .accessLevel(StorageAccessLevel.PUBLIC) + .build(); + for (int i = 0; i < 10; i++) { + storage.remove(pagedUploadKeyPrefix + i, options); + } + storage.remove(uploadKey, options); + + // remove PROTECTED test files + options = StorageRemoveOptions.builder() + .accessLevel(StorageAccessLevel.PROTECTED) + .build(); + storage.remove(uploadKey, options); + + // remove PRIVATE test files + options = StorageRemoveOptions.builder() + .accessLevel(StorageAccessLevel.PRIVATE) + .build(); + storage.remove(uploadKey, options); + + synchronousAuth.signOut(); + synchronousAuth.signIn(userTwo.getUsername(), userTwo.getPassword()); + options = StorageRemoveOptions.builder() + .accessLevel(StorageAccessLevel.PROTECTED) + .build(); + storage.remove(uploadKey, options); + + // remove PRIVATE test files + options = StorageRemoveOptions.builder() + .accessLevel(StorageAccessLevel.PRIVATE) + .build(); + storage.remove(uploadKey, options); + + synchronousAuth.signOut(); + } } diff --git a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageMultiBucketDownloadTest.kt b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageMultiBucketDownloadTest.kt new file mode 100644 index 0000000000..225ed650ba --- /dev/null +++ b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageMultiBucketDownloadTest.kt @@ -0,0 +1,281 @@ +/* + * Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amplifyframework.storage.s3 + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.amplifyframework.auth.cognito.AWSCognitoAuthPlugin +import com.amplifyframework.core.Amplify +import com.amplifyframework.core.async.Cancelable +import com.amplifyframework.core.async.Resumable +import com.amplifyframework.hub.HubChannel +import com.amplifyframework.hub.HubEvent +import com.amplifyframework.hub.SubscriptionToken +import com.amplifyframework.storage.BucketInfo +import com.amplifyframework.storage.StorageBucket +import com.amplifyframework.storage.StorageCategory +import com.amplifyframework.storage.StorageChannelEventName +import com.amplifyframework.storage.StorageException +import com.amplifyframework.storage.StoragePath +import com.amplifyframework.storage.TransferState +import com.amplifyframework.storage.TransferState.Companion.getState +import com.amplifyframework.storage.operation.StorageDownloadFileOperation +import com.amplifyframework.storage.options.StorageDownloadFileOptions +import com.amplifyframework.storage.options.StorageRemoveOptions +import com.amplifyframework.storage.options.StorageUploadFileOptions +import com.amplifyframework.storage.s3.test.R +import com.amplifyframework.storage.s3.util.WorkmanagerTestUtils.initializeWorkmanagerTestUtil +import com.amplifyframework.testutils.FileAssert +import com.amplifyframework.testutils.random.RandomTempFile +import com.amplifyframework.testutils.sync.SynchronousAuth +import com.amplifyframework.testutils.sync.SynchronousStorage +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.matchers.shouldBe +import java.io.File +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicReference +import org.junit.After +import org.junit.AfterClass +import org.junit.BeforeClass +import org.junit.Test + +/** + * Instrumentation test for operational work on download. + */ +class AWSS3StorageMultiBucketDownloadTest { + private val downloadFile: File = RandomTempFile() + private val options = StorageDownloadFileOptions.builder().bucket(TestStorageCategory.getStorageBucket()).build() + // Create a set to remember all the subscriptions + private val subscriptions = mutableSetOf() + companion object { + private val EXTENDED_TIMEOUT_MS = TimeUnit.SECONDS.toMillis(60) + private const val LARGE_FILE_SIZE = 10 * 1024 * 1024L // 10 MB + private const val SMALL_FILE_SIZE = 100L + private val LARGE_FILE_NAME = "large-${System.currentTimeMillis()}" + private val LARGE_FILE_PATH = StoragePath.fromString("public/$LARGE_FILE_NAME") + private val SMALL_FILE_NAME = "small-${System.currentTimeMillis()}" + private val SMALL_FILE_PATH = StoragePath.fromString("public/$SMALL_FILE_NAME") + + lateinit var storageCategory: StorageCategory + lateinit var synchronousStorage: SynchronousStorage + lateinit var synchronousAuth: SynchronousAuth + lateinit var largeFile: File + lateinit var smallFile: File + + /** + * Initialize mobile client and configure the storage. + * Upload the test files ahead of time. + */ + @JvmStatic + @BeforeClass + fun setUpOnce() { + val context = ApplicationProvider.getApplicationContext() + initializeWorkmanagerTestUtil(context) + synchronousAuth = SynchronousAuth.delegatingToCognito(context, AWSCognitoAuthPlugin()) + + // Get a handle to storage + storageCategory = TestStorageCategory.create(context, R.raw.amplifyconfiguration) + synchronousStorage = SynchronousStorage.delegatingTo(storageCategory) + + val uploadOptions = StorageUploadFileOptions.defaultInstance() + + // Upload large test file + largeFile = RandomTempFile(LARGE_FILE_NAME, LARGE_FILE_SIZE) + synchronousStorage.uploadFile(LARGE_FILE_PATH, largeFile, uploadOptions, EXTENDED_TIMEOUT_MS) + + // Upload small test file + smallFile = RandomTempFile(SMALL_FILE_NAME, SMALL_FILE_SIZE) + synchronousStorage.uploadFile(SMALL_FILE_PATH, smallFile, uploadOptions) + } + + @JvmStatic + @AfterClass + fun tearDownOnce() { + synchronousStorage.remove(LARGE_FILE_PATH, StorageRemoveOptions.defaultInstance()) + synchronousStorage.remove(SMALL_FILE_PATH, StorageRemoveOptions.defaultInstance()) + } + } + + /** + * Unsubscribe from everything after each test. + */ + @After + fun tearDown() { + for (token in subscriptions) { + Amplify.Hub.unsubscribe(token) + } + } + + @Test + fun testDownloadSmallFile() { + synchronousStorage.downloadFile(SMALL_FILE_PATH, downloadFile, options) + FileAssert.assertEquals(smallFile, downloadFile) + } + + @Test + fun testDownloadLargeFile() { + synchronousStorage.downloadFile( + LARGE_FILE_PATH, + downloadFile, + options, + EXTENDED_TIMEOUT_MS + ) + FileAssert.assertEquals(largeFile, downloadFile) + } + + @Test + fun testDownloadFileIsCancelable() { + val canceled = CountDownLatch(1) + val opContainer = AtomicReference() + val errorContainer = AtomicReference() + + // Listen to Hub events for cancel + val cancelToken = Amplify.Hub.subscribe(HubChannel.STORAGE) { hubEvent: HubEvent<*> -> + if (StorageChannelEventName.DOWNLOAD_STATE.toString() == hubEvent.name) { + val state = getState(hubEvent.data as String) + if (TransferState.CANCELED == state) { + canceled.countDown() + } + } + } + subscriptions.add(cancelToken) + + // Begin downloading a large file + val op = storageCategory.downloadFile( + LARGE_FILE_PATH, + downloadFile, + options, + { + if (it.currentBytes > 0 && canceled.count > 0) { + opContainer.get().cancel() + } + }, + { errorContainer.set(RuntimeException("Download completed without canceling.")) }, + { newValue -> errorContainer.set(newValue) } + ) + opContainer.set(op) + + // Assert that the required conditions have been met + canceled.await(EXTENDED_TIMEOUT_MS, TimeUnit.MILLISECONDS) shouldBe true + errorContainer.get() shouldBe null + } + + @Test + fun testDownloadFileIsResumable() { + val completed = CountDownLatch(1) + val resumed = CountDownLatch(1) + val opContainer = AtomicReference() + val errorContainer = AtomicReference() + + // Listen to Hub events to resume when operation has been paused + val resumeToken = Amplify.Hub.subscribe(HubChannel.STORAGE) { hubEvent: HubEvent<*> -> + if (StorageChannelEventName.DOWNLOAD_STATE.toString() == hubEvent.name) { + val state = getState(hubEvent.data as String) + if (TransferState.PAUSED == state) { + opContainer.get().resume() + resumed.countDown() + } + } + } + subscriptions.add(resumeToken) + + // Begin downloading a large file + val op = storageCategory.downloadFile( + LARGE_FILE_PATH, + downloadFile, + options, + { + if (it.currentBytes > 0 && resumed.count > 0) { + opContainer.get().pause() + } + }, + { completed.countDown() }, + { errorContainer.set(it) } + ) + opContainer.set(op) + + // Assert that all the required conditions have been met + resumed.await(EXTENDED_TIMEOUT_MS, TimeUnit.MILLISECONDS) shouldBe true + completed.await(EXTENDED_TIMEOUT_MS, TimeUnit.MILLISECONDS) shouldBe true + errorContainer.get() shouldBe null + FileAssert.assertEquals(largeFile, downloadFile) + } + + @Test + fun testGetTransferOnPause() { + val completed = CountDownLatch(1) + val resumed = CountDownLatch(1) + val opContainer = AtomicReference>() + val transferId = AtomicReference() + val errorContainer = AtomicReference() + // Listen to Hub events to resume when operation has been paused + val resumeToken = Amplify.Hub.subscribe(HubChannel.STORAGE) { hubEvent: HubEvent<*> -> + if (StorageChannelEventName.DOWNLOAD_STATE.toString() == hubEvent.name) { + val state = getState(hubEvent.data as String) + if (TransferState.PAUSED == state) { + opContainer.get().clearAllListeners() + storageCategory.getTransfer( + transferId.get(), + { + val getOp = it as StorageDownloadFileOperation<*> + getOp.resume() + resumed.countDown() + getOp.setOnSuccess { completed.countDown() } + }, + { errorContainer.set(it) } + ) + } + } + } + subscriptions.add(resumeToken) + + // Begin downloading a large file + val op = storageCategory.downloadFile( + LARGE_FILE_PATH, + downloadFile, + options, + { + if (it.currentBytes > 0 && resumed.count > 0) { + opContainer.get().pause() + } + }, + { }, + { errorContainer.set(it) } + ) + + opContainer.set(op) + transferId.set(op.transferId) + + // Assert that all the required conditions have been met + resumed.await(EXTENDED_TIMEOUT_MS, TimeUnit.MILLISECONDS) shouldBe true + completed.await(EXTENDED_TIMEOUT_MS, TimeUnit.MILLISECONDS) shouldBe true + errorContainer.get() shouldBe null + FileAssert.assertEquals(largeFile, downloadFile) + } + + @Test + fun testDownloadFromInvalidBucket() { + val bucketName = "amplify-android-storage-integration-test-123xyz" + val region = "us-east-1" + val bucketInfo = BucketInfo(bucketName, region) + val storageBucket = StorageBucket.fromBucketInfo(bucketInfo) + val option = StorageDownloadFileOptions.builder().bucket(storageBucket).build() + + shouldThrow { + synchronousStorage.downloadFile(SMALL_FILE_PATH, downloadFile, option) + } + } +} diff --git a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageMultiBucketGetUrlTest.kt b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageMultiBucketGetUrlTest.kt new file mode 100644 index 0000000000..c804a48a8c --- /dev/null +++ b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageMultiBucketGetUrlTest.kt @@ -0,0 +1,105 @@ +/* + * Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amplifyframework.storage.s3 + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.amplifyframework.auth.cognito.AWSCognitoAuthPlugin +import com.amplifyframework.storage.BucketInfo +import com.amplifyframework.storage.StorageBucket +import com.amplifyframework.storage.StorageCategory +import com.amplifyframework.storage.StoragePath +import com.amplifyframework.storage.options.StorageGetUrlOptions +import com.amplifyframework.storage.options.StorageRemoveOptions +import com.amplifyframework.storage.options.StorageUploadFileOptions +import com.amplifyframework.storage.s3.test.R +import com.amplifyframework.storage.s3.util.WorkmanagerTestUtils.initializeWorkmanagerTestUtil +import com.amplifyframework.testutils.random.RandomTempFile +import com.amplifyframework.testutils.sync.SynchronousAuth +import com.amplifyframework.testutils.sync.SynchronousStorage +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import io.kotest.matchers.string.shouldContain +import java.io.File +import org.junit.AfterClass +import org.junit.BeforeClass +import org.junit.Test + +/** + * Instrumentation test for operational work on download. + */ +class AWSS3StorageMultiBucketGetUrlTest { + private companion object { + const val SMALL_FILE_SIZE = 100L + val SMALL_FILE_NAME = "small-${System.currentTimeMillis()}" + val SMALL_FILE_PATH = StoragePath.fromString("public/$SMALL_FILE_NAME") + + lateinit var storageCategory: StorageCategory + lateinit var synchronousStorage: SynchronousStorage + lateinit var smallFile: File + + /** + * Initialize mobile client and configure the storage. + * Upload the test files ahead of time. + */ + @JvmStatic + @BeforeClass + fun setUpOnce() { + val context = ApplicationProvider.getApplicationContext() + initializeWorkmanagerTestUtil(context) + + SynchronousAuth.delegatingToCognito(context, AWSCognitoAuthPlugin()) + + storageCategory = TestStorageCategory.create(context, R.raw.amplifyconfiguration) + synchronousStorage = SynchronousStorage.delegatingTo(storageCategory) + + // Upload small test file + smallFile = RandomTempFile(SMALL_FILE_NAME, SMALL_FILE_SIZE) + synchronousStorage.uploadFile(SMALL_FILE_PATH, smallFile, StorageUploadFileOptions.defaultInstance()) + } + @JvmStatic + @AfterClass + fun tearDownOnce() { + synchronousStorage.remove(SMALL_FILE_PATH, StorageRemoveOptions.defaultInstance()) + } + } + + @Test + fun testGetUrl() { + val result = synchronousStorage.getUrl( + SMALL_FILE_PATH, + StorageGetUrlOptions.builder().expires(30).bucket(TestStorageCategory.getStorageBucket()).build() + ) + + result.url.path shouldBe "/public/$SMALL_FILE_NAME" + result.url.query shouldContain "X-Amz-Expires=30" + } + + @Test + fun testGetUrlFromInvalidBucket() { + val bucketName = "amplify-android-storage-integration-test-123xyz" + val region = "us-east-1" + val bucketInfo = BucketInfo(bucketName, region) + val invalidBucket = StorageBucket.fromBucketInfo(bucketInfo) + val result = synchronousStorage.getUrl( + SMALL_FILE_PATH, + StorageGetUrlOptions.builder().expires(30).bucket(invalidBucket).build() + ) + + result shouldNotBe null + result.url.host shouldContain bucketName + result.url.host shouldContain region + } +} diff --git a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageMultiBucketListTest.kt b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageMultiBucketListTest.kt new file mode 100644 index 0000000000..785b465c77 --- /dev/null +++ b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageMultiBucketListTest.kt @@ -0,0 +1,139 @@ +/* + * Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amplifyframework.storage.s3 + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.amplifyframework.auth.cognito.AWSCognitoAuthPlugin +import com.amplifyframework.storage.BucketInfo +import com.amplifyframework.storage.StorageBucket +import com.amplifyframework.storage.StorageCategory +import com.amplifyframework.storage.StorageException +import com.amplifyframework.storage.StoragePath +import com.amplifyframework.storage.options.StoragePagedListOptions +import com.amplifyframework.storage.options.StorageRemoveOptions +import com.amplifyframework.storage.options.StorageUploadFileOptions +import com.amplifyframework.storage.s3.test.R +import com.amplifyframework.storage.s3.util.WorkmanagerTestUtils.initializeWorkmanagerTestUtil +import com.amplifyframework.testutils.random.RandomTempFile +import com.amplifyframework.testutils.sync.SynchronousAuth +import com.amplifyframework.testutils.sync.SynchronousStorage +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.matchers.shouldBe +import java.io.File +import java.util.concurrent.TimeUnit +import org.junit.AfterClass +import org.junit.BeforeClass +import org.junit.Test + +/** + * Instrumentation test for operational work on download. + */ +class AWSS3StorageMultiBucketListTest { + companion object { + private val EXTENDED_TIMEOUT_MS = TimeUnit.SECONDS.toMillis(60) + private const val LARGE_FILE_SIZE = 10 * 1024 * 1024L // 10 MB + private const val SMALL_FILE_SIZE = 100L + private val TEST_DIR_NAME = System.currentTimeMillis().toString() + private val LARGE_FILE_NAME = "large-${System.currentTimeMillis()}" + private val LARGE_FILE_STRING_PATH = "public/$TEST_DIR_NAME/$LARGE_FILE_NAME" + private val LARGE_FILE_PATH = StoragePath.fromString(LARGE_FILE_STRING_PATH) + private val SMALL_FILE_NAME = "small-${System.currentTimeMillis()}" + private val SMALL_FILE_STRING_PATH = "public/$TEST_DIR_NAME/$SMALL_FILE_NAME" + private val SMALL_FILE_PATH = StoragePath.fromString(SMALL_FILE_STRING_PATH) + + lateinit var storageCategory: StorageCategory + lateinit var synchronousStorage: SynchronousStorage + lateinit var synchronousAuth: SynchronousAuth + lateinit var largeFile: File + lateinit var smallFile: File + + /** + * Initialize mobile client and configure the storage. + * Upload the test files ahead of time. + */ + @JvmStatic + @BeforeClass + fun setUpOnce() { + val context = ApplicationProvider.getApplicationContext() + initializeWorkmanagerTestUtil(context) + synchronousAuth = SynchronousAuth.delegatingToCognito(context, AWSCognitoAuthPlugin()) + + // Get a handle to storage + storageCategory = TestStorageCategory.create(context, R.raw.amplifyconfiguration) + synchronousStorage = SynchronousStorage.delegatingTo(storageCategory) + + val uploadOptions = StorageUploadFileOptions.defaultInstance() + + // Upload large test file + largeFile = RandomTempFile(LARGE_FILE_NAME, LARGE_FILE_SIZE) + synchronousStorage.uploadFile(LARGE_FILE_PATH, largeFile, uploadOptions, EXTENDED_TIMEOUT_MS) + + // Upload small test file + smallFile = RandomTempFile(SMALL_FILE_NAME, SMALL_FILE_SIZE) + synchronousStorage.uploadFile(SMALL_FILE_PATH, smallFile, uploadOptions) + } + + @JvmStatic + @AfterClass + fun tearDownOnce() { + synchronousStorage.remove(SMALL_FILE_PATH, StorageRemoveOptions.defaultInstance()) + synchronousStorage.remove(LARGE_FILE_PATH, StorageRemoveOptions.defaultInstance()) + } + } + + @Test + fun testListFromBucket() { + val public = StoragePath.fromString("public/$TEST_DIR_NAME") + + val option = StoragePagedListOptions + .builder() + .bucket(TestStorageCategory.getStorageBucket()) + .setPageSize(10) + .build() + + val result = synchronousStorage.list(public, option) + + result.items.apply { + size shouldBe 2 + first { it.path == LARGE_FILE_STRING_PATH }.apply { + path shouldBe LARGE_FILE_STRING_PATH + size shouldBe LARGE_FILE_SIZE + } + first { it.path == SMALL_FILE_STRING_PATH }.apply { + path shouldBe SMALL_FILE_STRING_PATH + size shouldBe SMALL_FILE_SIZE + } + } + } + + @Test + fun testListFromInvalidBucket() { + val bucketName = "amplify-android-storage-integration-test-123xyz" + val region = "us-east-1" + val bucketInfo = BucketInfo(bucketName, region) + val public = StoragePath.fromString("public/$TEST_DIR_NAME") + val storageBucket = StorageBucket.fromBucketInfo(bucketInfo) + val option = StoragePagedListOptions + .builder() + .bucket(storageBucket) + .setPageSize(10) + .build() + + shouldThrow { + synchronousStorage.list(public, option) + } + } +} diff --git a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageMultiBucketRemoveTest.kt b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageMultiBucketRemoveTest.kt new file mode 100644 index 0000000000..115ac0c7e6 --- /dev/null +++ b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageMultiBucketRemoveTest.kt @@ -0,0 +1,101 @@ +/* + * Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amplifyframework.storage.s3 + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.amplifyframework.auth.cognito.AWSCognitoAuthPlugin +import com.amplifyframework.storage.BucketInfo +import com.amplifyframework.storage.StorageBucket +import com.amplifyframework.storage.StorageCategory +import com.amplifyframework.storage.StorageException +import com.amplifyframework.storage.options.StorageDownloadFileOptions +import com.amplifyframework.storage.options.StorageRemoveOptions +import com.amplifyframework.storage.options.StorageUploadFileOptions +import com.amplifyframework.storage.s3.test.R +import com.amplifyframework.storage.s3.util.WorkmanagerTestUtils.initializeWorkmanagerTestUtil +import com.amplifyframework.testutils.random.RandomTempFile +import com.amplifyframework.testutils.sync.SynchronousAuth +import com.amplifyframework.testutils.sync.SynchronousStorage +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.matchers.shouldBe +import java.io.File +import org.junit.BeforeClass +import org.junit.Test + +/** + * Instrumentation test for operational work on remove. + */ +class AWSS3StorageMultiBucketRemoveTest { + // Create a file to download to + private val downloadFile: File = RandomTempFile() + + private companion object { + const val SMALL_FILE_SIZE = 100L + val SMALL_FILE_NAME = "small-${System.currentTimeMillis()}" + + lateinit var storageCategory: StorageCategory + lateinit var synchronousStorage: SynchronousStorage + lateinit var smallFile: File + + /** + * Initialize mobile client and configure the storage. + * Upload the test files ahead of time. + */ + @JvmStatic + @BeforeClass + fun setUpOnce() { + val context = ApplicationProvider.getApplicationContext() + initializeWorkmanagerTestUtil(context) + + SynchronousAuth.delegatingToCognito(context, AWSCognitoAuthPlugin()) + + storageCategory = TestStorageCategory.create(context, R.raw.amplifyconfiguration) + synchronousStorage = SynchronousStorage.delegatingTo(storageCategory) + + // Upload small test file + smallFile = RandomTempFile(SMALL_FILE_NAME, SMALL_FILE_SIZE) + synchronousStorage.uploadFile(SMALL_FILE_NAME, smallFile, StorageUploadFileOptions.defaultInstance()) + } + } + + @Test + fun testRemove() { + val options = StorageRemoveOptions.builder().bucket(TestStorageCategory.getStorageBucket()).build() + val result = synchronousStorage.remove(SMALL_FILE_NAME, options) + + result.path shouldBe "public/$SMALL_FILE_NAME" + shouldThrow { + synchronousStorage.downloadFile( + SMALL_FILE_NAME, + downloadFile, + StorageDownloadFileOptions.defaultInstance() + ) + } + } + + @Test + fun testRemoveFromInvalidBucket() { + val bucketName = "amplify-android-storage-integration-test-123xyz" + val region = "us-east-1" + val bucketInfo = BucketInfo(bucketName, region) + val storageBucket = StorageBucket.fromBucketInfo(bucketInfo) + val option = StorageRemoveOptions.builder().bucket(storageBucket).build() + + shouldThrow { + synchronousStorage.remove(SMALL_FILE_NAME, option) + } + } +} diff --git a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageMultiBucketUploadTest.kt b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageMultiBucketUploadTest.kt new file mode 100644 index 0000000000..fc90f8f8cc --- /dev/null +++ b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageMultiBucketUploadTest.kt @@ -0,0 +1,280 @@ +/* + * Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amplifyframework.storage.s3 + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.amplifyframework.auth.cognito.AWSCognitoAuthPlugin +import com.amplifyframework.core.Amplify +import com.amplifyframework.core.async.Cancelable +import com.amplifyframework.core.async.Resumable +import com.amplifyframework.hub.HubChannel +import com.amplifyframework.hub.HubEvent +import com.amplifyframework.hub.SubscriptionToken +import com.amplifyframework.storage.BucketInfo +import com.amplifyframework.storage.StorageBucket +import com.amplifyframework.storage.StorageCategory +import com.amplifyframework.storage.StorageChannelEventName +import com.amplifyframework.storage.StorageException +import com.amplifyframework.storage.StoragePath +import com.amplifyframework.storage.TransferState +import com.amplifyframework.storage.TransferState.Companion.getState +import com.amplifyframework.storage.operation.StorageUploadFileOperation +import com.amplifyframework.storage.options.StorageRemoveOptions +import com.amplifyframework.storage.options.StorageUploadFileOptions +import com.amplifyframework.storage.s3.options.AWSS3StorageUploadFileOptions +import com.amplifyframework.storage.s3.test.R +import com.amplifyframework.storage.s3.util.WorkmanagerTestUtils.initializeWorkmanagerTestUtil +import com.amplifyframework.testutils.random.RandomTempFile +import com.amplifyframework.testutils.sync.SynchronousAuth +import com.amplifyframework.testutils.sync.SynchronousStorage +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.matchers.shouldBe +import java.io.File +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicReference +import org.junit.After +import org.junit.BeforeClass +import org.junit.Test + +/** + * Instrumentation test for operational work on upload. + */ +class AWSS3StorageMultiBucketUploadTest { + private val defaultFileOptions = StorageUploadFileOptions + .builder() + .bucket(TestStorageCategory.getStorageBucket()) + .build() + + // Create a set to remember all the subscriptions + private val subscriptions = mutableSetOf() + private lateinit var storagePath: StoragePath + companion object { + private const val LARGE_FILE_SIZE = 10 * 1024 * 1024L // 10 MB + private const val SMALL_FILE_SIZE = 100L + private val EXTENDED_TIMEOUT_MS = TimeUnit.SECONDS.toMillis(60) + + lateinit var storageCategory: StorageCategory + lateinit var synchronousStorage: SynchronousStorage + lateinit var synchronousAuth: SynchronousAuth + + /** + * Initialize mobile client and configure the storage. + * + */ + @JvmStatic + @BeforeClass + fun setUpOnce() { + val context = ApplicationProvider.getApplicationContext() + initializeWorkmanagerTestUtil(context) + synchronousAuth = SynchronousAuth.delegatingToCognito(context, AWSCognitoAuthPlugin()) + + // Get a handle to storage + storageCategory = TestStorageCategory.create(context, R.raw.amplifyconfiguration) + synchronousStorage = SynchronousStorage.delegatingTo(storageCategory) + } + } + + /** + * Unsubscribe from everything after each test. + * Remove test file + */ + @After + fun tearDown() { + // Unsubscribe from everything + for (token in subscriptions) { + Amplify.Hub.unsubscribe(token) + } + try { + synchronousStorage.remove(storagePath, StorageRemoveOptions.defaultInstance()) + } catch (ex: StorageException) { + // in some cases, access denied exception made occur here + } + } + + @Test + fun testUploadSmallFile() { + val uploadFile: File = RandomTempFile(SMALL_FILE_SIZE) + storagePath = StoragePath.fromString("public/${uploadFile.name}") + synchronousStorage.uploadFile(storagePath, uploadFile, defaultFileOptions) + } + + @Test + fun testUploadLargeFile() { + val uploadFile: File = RandomTempFile(LARGE_FILE_SIZE) + storagePath = StoragePath.fromString("public/${uploadFile.name}") + val options = AWSS3StorageUploadFileOptions.builder().setUseAccelerateEndpoint(true).build() + synchronousStorage.uploadFile(storagePath, uploadFile, options, EXTENDED_TIMEOUT_MS) + } + + @Test + fun testUploadFileIsCancelable() { + val canceled = CountDownLatch(1) + val opContainer = AtomicReference() + val errorContainer = AtomicReference() + + // Create a file large enough that transfer won't finish before being canceled + val uploadFile: File = RandomTempFile(LARGE_FILE_SIZE) + storagePath = StoragePath.fromString("public/${uploadFile.name}") + + // Listen to Hub events for cancel + val cancelToken = Amplify.Hub.subscribe(HubChannel.STORAGE) { hubEvent: HubEvent<*> -> + if (StorageChannelEventName.UPLOAD_STATE.toString() == hubEvent.name) { + val state = getState(hubEvent.data as String) + if (TransferState.CANCELED == state) { + canceled.countDown() + } + } + } + subscriptions.add(cancelToken) + + // Begin uploading a large file + val op = storageCategory.uploadFile( + storagePath, + uploadFile, + defaultFileOptions, + { + if (it.currentBytes > 0) { + opContainer.get().cancel() + } + }, + { errorContainer.set(RuntimeException("Upload completed without canceling.")) }, + { errorContainer.set(it) } + ) + + opContainer.set(op) + + // Assert that the required conditions have been met + canceled.await(EXTENDED_TIMEOUT_MS, TimeUnit.MILLISECONDS) shouldBe true + errorContainer.get() shouldBe null + } + + @Test + fun testUploadFileIsResumable() { + val completed = CountDownLatch(1) + val resumed = CountDownLatch(1) + val opContainer = AtomicReference() + val errorContainer = AtomicReference() + + // Create a file large enough that transfer won't finish before being paused + val uploadFile: File = RandomTempFile(LARGE_FILE_SIZE) + storagePath = StoragePath.fromString("public/${uploadFile.name}") + + // Listen to Hub events to resume when operation has been paused + val resumeToken = Amplify.Hub.subscribe(HubChannel.STORAGE) { hubEvent: HubEvent<*> -> + if (StorageChannelEventName.UPLOAD_STATE.toString() == hubEvent.name) { + val state = getState(hubEvent.data as String) + if (TransferState.PAUSED == state) { + opContainer.get().resume() + resumed.countDown() + } + } + } + subscriptions.add(resumeToken) + + // Begin uploading a large file + val op = storageCategory.uploadFile( + storagePath, + uploadFile, + defaultFileOptions, + { + if (it.currentBytes > 0 && resumed.count > 0) { + opContainer.get().pause() + } + }, + { completed.countDown() }, + { errorContainer.set(it) } + ) + + opContainer.set(op) + + // Assert that all the required conditions have been met + resumed.await(EXTENDED_TIMEOUT_MS, TimeUnit.MILLISECONDS) shouldBe true + completed.await(EXTENDED_TIMEOUT_MS, TimeUnit.MILLISECONDS) shouldBe true + errorContainer.get()shouldBe null + } + + @Test + fun testUploadFileGetTransferOnPause() { + val completed = CountDownLatch(1) + val resumed = CountDownLatch(1) + val transferId = AtomicReference() + val opContainer = AtomicReference>() + val errorContainer = AtomicReference() + + // Create a file large enough that transfer won't finish before being paused + val uploadFile: File = RandomTempFile(LARGE_FILE_SIZE) + storagePath = StoragePath.fromString("public/${uploadFile.name}") + + // Listen to Hub events to resume when operation has been paused + val resumeToken = Amplify.Hub.subscribe(HubChannel.STORAGE) { hubEvent: HubEvent<*> -> + if (StorageChannelEventName.UPLOAD_STATE.toString() == hubEvent.name) { + val state = getState(hubEvent.data as String) + if (TransferState.PAUSED == state) { + opContainer.get().clearAllListeners() + storageCategory.getTransfer( + transferId.get(), + { + val getOp = it as StorageUploadFileOperation + getOp.resume() + resumed.countDown() + getOp.setOnSuccess { completed.countDown() } + }, + { errorContainer.set(it) } + ) + } + } + } + subscriptions.add(resumeToken) + + // Begin uploading a large file + val op = storageCategory.uploadFile( + storagePath, + uploadFile, + defaultFileOptions, + { + if (it.currentBytes > 0 && resumed.count > 0) { + opContainer.get().pause() + } + }, + { }, + { errorContainer.set(it) } + ) + + opContainer.set(op) + transferId.set(op.transferId) + + // Assert that all the required conditions have been met + resumed.await(EXTENDED_TIMEOUT_MS, TimeUnit.MILLISECONDS) shouldBe true + completed.await(EXTENDED_TIMEOUT_MS, TimeUnit.MILLISECONDS) shouldBe true + errorContainer.get() shouldBe null + } + + @Test + fun testUploadFromInvalidBucket() { + val bucketName = "amplify-android-storage-integration-test-123xyz" + val region = "us-east-1" + val bucketInfo = BucketInfo(bucketName, region) + val storageBucket = StorageBucket.fromBucketInfo(bucketInfo) + val option = StorageUploadFileOptions.builder().bucket(storageBucket).build() + val uploadFile: File = RandomTempFile(SMALL_FILE_SIZE) + storagePath = StoragePath.fromString("public/${uploadFile.name}") + + shouldThrow { + synchronousStorage.uploadFile(storagePath, uploadFile, option) + } + } +} diff --git a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StoragePathDownloadTest.kt b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StoragePathDownloadTest.kt index 52a6cea603..bcd58efc04 100644 --- a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StoragePathDownloadTest.kt +++ b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StoragePathDownloadTest.kt @@ -31,6 +31,7 @@ import com.amplifyframework.storage.TransferState import com.amplifyframework.storage.TransferState.Companion.getState import com.amplifyframework.storage.operation.StorageDownloadFileOperation import com.amplifyframework.storage.options.StorageDownloadFileOptions +import com.amplifyframework.storage.options.StorageRemoveOptions import com.amplifyframework.storage.options.StorageUploadFileOptions import com.amplifyframework.storage.s3.options.AWSS3StorageDownloadFileOptions import com.amplifyframework.storage.s3.test.R @@ -44,6 +45,7 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicReference import org.junit.After +import org.junit.AfterClass import org.junit.Assert.assertNull import org.junit.Assert.assertTrue import org.junit.BeforeClass @@ -136,6 +138,20 @@ class AWSS3StoragePathDownloadTest { synchronousAuth.signOut() } + + @JvmStatic + @AfterClass + fun tearDownOnce() { + synchronousAuth.signOut() + synchronousAuth.signIn(userOne.username, userOne.password) + + synchronousStorage.remove(LARGE_FILE_PATH, StorageRemoveOptions.defaultInstance()) + synchronousStorage.remove(SMALL_FILE_PATH, StorageRemoveOptions.defaultInstance()) + synchronousStorage.remove(userOnePrivateStoragePath, StorageRemoveOptions.defaultInstance()) + synchronousStorage.remove(userOneProtectedStoragePath, StorageRemoveOptions.defaultInstance()) + + synchronousAuth.signOut() + } } /** diff --git a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StoragePathGetUrlTest.kt b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StoragePathGetUrlTest.kt index ab250b466c..8d2cc92fc0 100644 --- a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StoragePathGetUrlTest.kt +++ b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StoragePathGetUrlTest.kt @@ -22,6 +22,7 @@ import com.amplifyframework.storage.StorageCategory import com.amplifyframework.storage.StorageException import com.amplifyframework.storage.StoragePath import com.amplifyframework.storage.options.StorageGetUrlOptions +import com.amplifyframework.storage.options.StorageRemoveOptions import com.amplifyframework.storage.options.StorageUploadFileOptions import com.amplifyframework.storage.s3.options.AWSS3StorageGetPresignedUrlOptions import com.amplifyframework.storage.s3.test.R @@ -30,6 +31,7 @@ import com.amplifyframework.testutils.random.RandomTempFile import com.amplifyframework.testutils.sync.SynchronousAuth import com.amplifyframework.testutils.sync.SynchronousStorage import java.io.File +import org.junit.AfterClass import org.junit.Assert.assertEquals import org.junit.Assert.assertThrows import org.junit.Assert.assertTrue @@ -71,6 +73,12 @@ class AWSS3StoragePathGetUrlTest { smallFile = RandomTempFile(SMALL_FILE_NAME, SMALL_FILE_SIZE) synchronousStorage.uploadFile(SMALL_FILE_PATH, smallFile, StorageUploadFileOptions.defaultInstance()) } + + @JvmStatic + @AfterClass + fun tearDownOnce() { + synchronousStorage.remove(SMALL_FILE_PATH, StorageRemoveOptions.defaultInstance()) + } } @Test diff --git a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StoragePathListTest.kt b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StoragePathListTest.kt index f903536c7b..2a4e1244ff 100644 --- a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StoragePathListTest.kt +++ b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StoragePathListTest.kt @@ -20,6 +20,7 @@ import com.amplifyframework.auth.cognito.AWSCognitoAuthPlugin import com.amplifyframework.storage.StorageCategory import com.amplifyframework.storage.StorageException import com.amplifyframework.storage.StoragePath +import com.amplifyframework.storage.options.StorageRemoveOptions import com.amplifyframework.storage.options.StorageUploadFileOptions import com.amplifyframework.storage.s3.options.AWSS3StoragePagedListOptions import com.amplifyframework.storage.s3.test.R @@ -30,6 +31,7 @@ import com.amplifyframework.testutils.sync.SynchronousStorage import java.io.File import java.util.concurrent.TimeUnit import org.junit.After +import org.junit.AfterClass import org.junit.Assert.assertEquals import org.junit.Assert.assertTrue import org.junit.BeforeClass @@ -107,6 +109,16 @@ class AWSS3StoragePathListTest { synchronousAuth.signOut() } + + @JvmStatic + @AfterClass + fun tearDownOnce() { + synchronousAuth.signIn(userOne.username, userOne.password) + synchronousStorage.remove(SMALL_FILE_PATH, StorageRemoveOptions.defaultInstance()) + synchronousStorage.remove(LARGE_FILE_PATH, StorageRemoveOptions.defaultInstance()) + synchronousStorage.remove(userOnePrivateFileStoragePath, StorageRemoveOptions.defaultInstance()) + synchronousAuth.signOut() + } } /** diff --git a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StoragePathUploadTest.kt b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StoragePathUploadTest.kt index b926597e0a..a68a6a3aaf 100644 --- a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StoragePathUploadTest.kt +++ b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StoragePathUploadTest.kt @@ -32,6 +32,7 @@ import com.amplifyframework.storage.TransferState.Companion.getState import com.amplifyframework.storage.operation.StorageUploadFileOperation import com.amplifyframework.storage.operation.StorageUploadInputStreamOperation import com.amplifyframework.storage.operation.StorageUploadOperation +import com.amplifyframework.storage.options.StorageRemoveOptions import com.amplifyframework.storage.options.StorageUploadFileOptions import com.amplifyframework.storage.options.StorageUploadInputStreamOptions import com.amplifyframework.storage.s3.UserCredentials.Credential @@ -61,7 +62,7 @@ class AWSS3StoragePathUploadTest { // Create a set to remember all the subscriptions private val subscriptions = mutableSetOf() - + private lateinit var storagePath: StoragePath companion object { private const val LARGE_FILE_SIZE = 10 * 1024 * 1024L // 10 MB private const val SMALL_FILE_SIZE = 100L @@ -97,6 +98,7 @@ class AWSS3StoragePathUploadTest { /** * Unsubscribe from everything after each test. + * Remove test file */ @After fun tearDown() { @@ -104,27 +106,32 @@ class AWSS3StoragePathUploadTest { for (token in subscriptions) { Amplify.Hub.unsubscribe(token) } + try { + synchronousStorage.remove(storagePath, StorageRemoveOptions.defaultInstance()) + } catch (ex: StorageException) { + // in some cases, access denied exception made occur here + } synchronousAuth.signOut() } @Test fun testUploadSmallFile() { val uploadFile: File = RandomTempFile(SMALL_FILE_SIZE) - val storagePath = StoragePath.fromString("public/${uploadFile.name}") + storagePath = StoragePath.fromString("public/${uploadFile.name}") synchronousStorage.uploadFile(storagePath, uploadFile, defaultFileOptions) } @Test fun testUploadSmallFileStream() { val uploadFile: File = RandomTempFile(SMALL_FILE_SIZE) - val storagePath = StoragePath.fromString("public/${uploadFile.name}") + storagePath = StoragePath.fromString("public/${uploadFile.name}") synchronousStorage.uploadInputStream(storagePath, FileInputStream(uploadFile), defaultInputStreamOptions) } @Test fun testUploadLargeFile() { val uploadFile: File = RandomTempFile(LARGE_FILE_SIZE) - val storagePath = StoragePath.fromString("public/${uploadFile.name}") + storagePath = StoragePath.fromString("public/${uploadFile.name}") val options = AWSS3StorageUploadFileOptions.builder().setUseAccelerateEndpoint(true).build() synchronousStorage.uploadFile(storagePath, uploadFile, options, EXTENDED_TIMEOUT_MS) } @@ -137,7 +144,7 @@ class AWSS3StoragePathUploadTest { // Create a file large enough that transfer won't finish before being canceled val uploadFile: File = RandomTempFile(LARGE_FILE_SIZE) - val storagePath = StoragePath.fromString("public/${uploadFile.name}") + storagePath = StoragePath.fromString("public/${uploadFile.name}") // Listen to Hub events for cancel val cancelToken = Amplify.Hub.subscribe(HubChannel.STORAGE) { hubEvent: HubEvent<*> -> @@ -180,7 +187,7 @@ class AWSS3StoragePathUploadTest { // Create a file large enough that transfer won't finish before being paused val uploadFile: File = RandomTempFile(LARGE_FILE_SIZE) - val storagePath = StoragePath.fromString("public/${uploadFile.name}") + storagePath = StoragePath.fromString("public/${uploadFile.name}") // Listen to Hub events to resume when operation has been paused val resumeToken = Amplify.Hub.subscribe(HubChannel.STORAGE) { hubEvent: HubEvent<*> -> @@ -226,7 +233,7 @@ class AWSS3StoragePathUploadTest { // Create a file large enough that transfer won't finish before being paused val uploadFile: File = RandomTempFile(LARGE_FILE_SIZE) - val storagePath = StoragePath.fromString("public/${uploadFile.name}") + storagePath = StoragePath.fromString("public/${uploadFile.name}") // Listen to Hub events to resume when operation has been paused val resumeToken = Amplify.Hub.subscribe(HubChannel.STORAGE) { hubEvent: HubEvent<*> -> @@ -282,7 +289,7 @@ class AWSS3StoragePathUploadTest { // Create a file large enough that transfer won't finish before being paused val uploadFile: File = RandomTempFile(LARGE_FILE_SIZE) - val storagePath = StoragePath.fromString("public/${uploadFile.name}") + storagePath = StoragePath.fromString("public/${uploadFile.name}") // Listen to Hub events to resume when operation has been paused val resumeToken = Amplify.Hub.subscribe(HubChannel.STORAGE) { hubEvent: HubEvent<*> -> @@ -331,7 +338,7 @@ class AWSS3StoragePathUploadTest { @Test fun testUploadSmallFileWithAccelerationEnabled() { val uploadFile: File = RandomTempFile(SMALL_FILE_SIZE) - val storagePath = StoragePath.fromString("public/${uploadFile.name}") + storagePath = StoragePath.fromString("public/${uploadFile.name}") val awss3StorageUploadFileOptions = AWSS3StorageUploadFileOptions.builder().setUseAccelerateEndpoint(true).build() @@ -345,7 +352,7 @@ class AWSS3StoragePathUploadTest { @Test fun testUploadLargeFileWithAccelerationEnabled() { val uploadFile: File = RandomTempFile(LARGE_FILE_SIZE) - val storagePath = StoragePath.fromString("public/${uploadFile.name}") + storagePath = StoragePath.fromString("public/${uploadFile.name}") val awss3StorageUploadFileOptions = AWSS3StorageUploadFileOptions.builder().setUseAccelerateEndpoint(true).build() synchronousStorage.uploadFile( @@ -358,7 +365,7 @@ class AWSS3StoragePathUploadTest { @Test(expected = StorageException::class) fun testUploadUnauthenticatedProtectedAccess() { val uploadFile: File = RandomTempFile(SMALL_FILE_SIZE) - val storagePath = StoragePath.fromString("protected/${userOne.identityId}/${uploadFile.name}") + storagePath = StoragePath.fromString("protected/${userOne.identityId}/${uploadFile.name}") synchronousStorage.uploadFile(storagePath, uploadFile, defaultFileOptions) } @@ -366,7 +373,7 @@ class AWSS3StoragePathUploadTest { @Test(expected = StorageException::class) fun testUploadInputStreamUnauthenticatedProtectedAccess() { val uploadFile: File = RandomTempFile(SMALL_FILE_SIZE) - val storagePath = StoragePath.fromString("protected/${userOne.identityId}/${uploadFile.name}") + storagePath = StoragePath.fromString("protected/${userOne.identityId}/${uploadFile.name}") synchronousStorage.uploadInputStream(storagePath, FileInputStream(uploadFile), defaultInputStreamOptions) } @@ -376,7 +383,7 @@ class AWSS3StoragePathUploadTest { synchronousAuth.signIn(userOne.username, userOne.password) val uploadFile: File = RandomTempFile(SMALL_FILE_SIZE) - val storagePath = StoragePath.fromString("protected/${userOne.identityId}/${uploadFile.name}") + storagePath = StoragePath.fromString("protected/${userOne.identityId}/${uploadFile.name}") synchronousStorage.uploadFile(storagePath, uploadFile, defaultFileOptions) } @@ -386,7 +393,7 @@ class AWSS3StoragePathUploadTest { synchronousAuth.signIn(userOne.username, userOne.password) val uploadFile: File = RandomTempFile(SMALL_FILE_SIZE) - val storagePath = StoragePath.fromString("protected/${userOne.identityId}/${uploadFile.name}") + storagePath = StoragePath.fromString("protected/${userOne.identityId}/${uploadFile.name}") synchronousStorage.uploadInputStream(storagePath, FileInputStream(uploadFile), defaultInputStreamOptions) } @@ -396,7 +403,7 @@ class AWSS3StoragePathUploadTest { synchronousAuth.signIn(userOne.username, userOne.password) val uploadFile: File = RandomTempFile(SMALL_FILE_SIZE) - val storagePath = StoragePath.fromString("private/${userOne.identityId}/${uploadFile.name}") + storagePath = StoragePath.fromString("private/${userOne.identityId}/${uploadFile.name}") synchronousStorage.uploadFile(storagePath, uploadFile, defaultFileOptions) } @@ -406,7 +413,7 @@ class AWSS3StoragePathUploadTest { synchronousAuth.signIn(userOne.username, userOne.password) val uploadFile: File = RandomTempFile(SMALL_FILE_SIZE) - val storagePath = StoragePath.fromString("private/${userOne.identityId}/${uploadFile.name}") + storagePath = StoragePath.fromString("private/${userOne.identityId}/${uploadFile.name}") synchronousStorage.uploadInputStream(storagePath, FileInputStream(uploadFile), defaultInputStreamOptions) } @@ -416,7 +423,7 @@ class AWSS3StoragePathUploadTest { synchronousAuth.signIn(userOne.username, userOne.password) val uploadFile: File = RandomTempFile(SMALL_FILE_SIZE) - val storagePath = StoragePath.fromString("protected/${userTwo.identityId}/${uploadFile.name}") + storagePath = StoragePath.fromString("protected/${userTwo.identityId}/${uploadFile.name}") synchronousStorage.uploadFile(storagePath, uploadFile, defaultFileOptions) } @@ -426,7 +433,7 @@ class AWSS3StoragePathUploadTest { synchronousAuth.signIn(userOne.username, userOne.password) val uploadFile: File = RandomTempFile(SMALL_FILE_SIZE) - val storagePath = StoragePath.fromString("protected/${userTwo.identityId}/${uploadFile.name}") + storagePath = StoragePath.fromString("protected/${userTwo.identityId}/${uploadFile.name}") synchronousStorage.uploadInputStream(storagePath, FileInputStream(uploadFile), defaultInputStreamOptions) } @@ -436,7 +443,7 @@ class AWSS3StoragePathUploadTest { synchronousAuth.signIn(userOne.username, userOne.password) val uploadFile: File = RandomTempFile(SMALL_FILE_SIZE) - val storagePath = StoragePath.fromString("private/${userTwo.identityId}/${uploadFile.name}") + storagePath = StoragePath.fromString("private/${userTwo.identityId}/${uploadFile.name}") synchronousStorage.uploadFile(storagePath, uploadFile, defaultFileOptions) } @@ -446,7 +453,7 @@ class AWSS3StoragePathUploadTest { synchronousAuth.signIn(userOne.username, userOne.password) val uploadFile: File = RandomTempFile(SMALL_FILE_SIZE) - val storagePath = StoragePath.fromString("private/${userTwo.identityId}/${uploadFile.name}") + storagePath = StoragePath.fromString("private/${userTwo.identityId}/${uploadFile.name}") synchronousStorage.uploadInputStream(storagePath, FileInputStream(uploadFile), defaultInputStreamOptions) } diff --git a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageUploadAccessLevelTest.java b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageUploadAccessLevelTest.java index 00dac7cef2..c88a5a5538 100644 --- a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageUploadAccessLevelTest.java +++ b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageUploadAccessLevelTest.java @@ -18,11 +18,11 @@ import android.content.Context; import com.amplifyframework.AmplifyException; -import com.amplifyframework.auth.AuthException; import com.amplifyframework.auth.cognito.AWSCognitoAuthPlugin; import com.amplifyframework.storage.StorageAccessLevel; import com.amplifyframework.storage.StorageCategory; import com.amplifyframework.storage.StorageException; +import com.amplifyframework.storage.options.StorageRemoveOptions; import com.amplifyframework.storage.options.StorageUploadFileOptions; import com.amplifyframework.storage.s3.UserCredentials.Credential; import com.amplifyframework.storage.s3.test.R; @@ -110,11 +110,16 @@ public void setUp() throws Exception { } /** - * Sign out after each test. - * @throws AuthException if error encountered while signing out + * Remove test file and sign out after each test. + * @throws Exception if error encountered while signing out */ @After - public void tearDown() throws AuthException { + public void tearDown() throws Exception { + synchronousAuth.signOut(); + synchronousAuth.signIn(userOne.getUsername(), userOne.getPassword()); + StorageAccessLevel accessLevel = uploadOptions.getAccessLevel(); + StorageRemoveOptions options = StorageRemoveOptions.builder().accessLevel(accessLevel).build(); + storage.remove(remoteKey, options); synchronousAuth.signOut(); } diff --git a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageUploadTest.java b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageUploadTest.java index 57319da053..020e7ff73f 100644 --- a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageUploadTest.java +++ b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageUploadTest.java @@ -31,6 +31,7 @@ import com.amplifyframework.storage.TransferState; import com.amplifyframework.storage.operation.StorageUploadFileOperation; import com.amplifyframework.storage.operation.StorageUploadInputStreamOperation; +import com.amplifyframework.storage.options.StorageRemoveOptions; import com.amplifyframework.storage.options.StorageUploadFileOptions; import com.amplifyframework.storage.options.StorageUploadInputStreamOptions; import com.amplifyframework.storage.s3.options.AWSS3StorageUploadFileOptions; @@ -73,6 +74,7 @@ public final class AWSS3StorageUploadTest { private StorageUploadFileOptions options; private Set subscriptions; + private File uploadFile; /** * Initialize mobile client and configure the storage. @@ -106,14 +108,17 @@ public void setUp() { } /** - * Unsubscribe from everything after each test. + * Clean up resources and unsubscribe from everything after each test. + * @throws Exception when failure to remove test resources. */ @After - public void tearDown() { + public void tearDown() throws Exception { // Unsubscribe from everything for (SubscriptionToken token : subscriptions) { Amplify.Hub.unsubscribe(token); } + + synchronousStorage.remove(uploadFile.getName(), StorageRemoveOptions.defaultInstance()); } /** @@ -123,7 +128,7 @@ public void tearDown() { */ @Test public void testUploadSmallFile() throws Exception { - File uploadFile = new RandomTempFile(SMALL_FILE_SIZE); + uploadFile = new RandomTempFile(SMALL_FILE_SIZE); String fileName = uploadFile.getName(); synchronousStorage.uploadFile(fileName, uploadFile, options); } @@ -135,7 +140,7 @@ public void testUploadSmallFile() throws Exception { */ @Test public void testUploadSmallFileStream() throws Exception { - File uploadFile = new RandomTempFile(SMALL_FILE_SIZE); + uploadFile = new RandomTempFile(SMALL_FILE_SIZE); String fileName = uploadFile.getName(); StorageUploadInputStreamOptions options = StorageUploadInputStreamOptions.builder() .accessLevel(TESTING_ACCESS_LEVEL) @@ -150,7 +155,7 @@ public void testUploadSmallFileStream() throws Exception { */ @Test public void testUploadLargeFile() throws Exception { - File uploadFile = new RandomTempFile(LARGE_FILE_SIZE); + uploadFile = new RandomTempFile(LARGE_FILE_SIZE); String fileName = uploadFile.getName(); AWSS3StorageUploadFileOptions options = AWSS3StorageUploadFileOptions.builder().setUseAccelerateEndpoint(true).build(); @@ -172,7 +177,7 @@ public void testUploadFileIsCancelable() throws Exception { final AtomicReference errorContainer = new AtomicReference<>(); // Create a file large enough that transfer won't finish before being canceled - File uploadFile = new RandomTempFile(LARGE_FILE_SIZE); + uploadFile = new RandomTempFile(LARGE_FILE_SIZE); // Listen to Hub events for cancel SubscriptionToken cancelToken = Amplify.Hub.subscribe(HubChannel.STORAGE, hubEvent -> { @@ -222,7 +227,7 @@ public void testUploadFileIsResumable() throws Exception { final AtomicReference errorContainer = new AtomicReference<>(); // Create a file large enough that transfer won't finish before being paused - File uploadFile = new RandomTempFile(LARGE_FILE_SIZE); + uploadFile = new RandomTempFile(LARGE_FILE_SIZE); // Listen to Hub events to resume when operation has been paused SubscriptionToken resumeToken = Amplify.Hub.subscribe(HubChannel.STORAGE, hubEvent -> { @@ -275,7 +280,7 @@ public void testUploadFileGetTransferOnPause() throws Exception { final AtomicReference errorContainer = new AtomicReference<>(); // Create a file large enough that transfer won't finish before being paused - File uploadFile = new RandomTempFile(LARGE_FILE_SIZE); + uploadFile = new RandomTempFile(LARGE_FILE_SIZE); // Listen to Hub events to resume when operation has been paused SubscriptionToken resumeToken = Amplify.Hub.subscribe(HubChannel.STORAGE, hubEvent -> { @@ -338,7 +343,7 @@ public void testUploadInputStreamGetTransferOnPause() throws Exception { final AtomicReference errorContainer = new AtomicReference<>(); // Create a file large enough that transfer won't finish before being paused - File uploadFile = new RandomTempFile(LARGE_FILE_SIZE); + uploadFile = new RandomTempFile(LARGE_FILE_SIZE); // Listen to Hub events to resume when operation has been paused SubscriptionToken resumeToken = Amplify.Hub.subscribe(HubChannel.STORAGE, hubEvent -> { @@ -393,7 +398,7 @@ public void testUploadInputStreamGetTransferOnPause() throws Exception { */ @Test public void testUploadSmallFileWithAccelerationEnabled() throws Exception { - File uploadFile = new RandomTempFile(SMALL_FILE_SIZE); + uploadFile = new RandomTempFile(SMALL_FILE_SIZE); String fileName = uploadFile.getName(); AWSS3StorageUploadFileOptions awss3StorageUploadFileOptions = AWSS3StorageUploadFileOptions.builder().setUseAccelerateEndpoint(true).build(); @@ -408,7 +413,7 @@ public void testUploadSmallFileWithAccelerationEnabled() throws Exception { */ @Test public void testUploadLargeFileWithAccelerationEnabled() throws Exception { - File uploadFile = new RandomTempFile(LARGE_FILE_SIZE); + uploadFile = new RandomTempFile(LARGE_FILE_SIZE); String fileName = uploadFile.getName(); AWSS3StorageUploadFileOptions awss3StorageUploadFileOptions = AWSS3StorageUploadFileOptions.builder().setUseAccelerateEndpoint(true).build(); diff --git a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/TestStorageCategory.java b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/TestStorageCategory.java index e4af8e4543..bb11808ed8 100644 --- a/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/TestStorageCategory.java +++ b/aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/TestStorageCategory.java @@ -23,6 +23,8 @@ import com.amplifyframework.core.AmplifyConfiguration; import com.amplifyframework.core.category.CategoryConfiguration; import com.amplifyframework.core.category.CategoryType; +import com.amplifyframework.storage.BucketInfo; +import com.amplifyframework.storage.StorageBucket; import com.amplifyframework.storage.StorageCategory; import java.util.Objects; @@ -31,8 +33,17 @@ * Factory for creating {@link StorageCategory} instance suitable for test. */ final class TestStorageCategory { + private TestStorageCategory() {} + static StorageBucket getStorageBucket() { + return StorageBucket.fromBucketInfo( + new BucketInfo( + "amplify-android-storage-integration-test", + "us-west-2") + ); + } + /** * Creates an instance of {@link StorageCategory} using the provided configuration resource. * @param context Android Context @@ -46,7 +57,7 @@ static StorageCategory create(@NonNull Context context, @RawRes int resourceId) try { storageCategory.addPlugin(new AWSS3StoragePlugin()); CategoryConfiguration storageConfiguration = AmplifyConfiguration.fromConfigFile(context, resourceId) - .forCategoryType(CategoryType.STORAGE); + .forCategoryType(CategoryType.STORAGE); storageCategory.configure(storageConfiguration, context); // storageCategory.initialize(context); // Doesn't do anything right now. } catch (AmplifyException initializationFailure) { From 05b21e63732b2496f74f272c298004660dab5c4d Mon Sep 17 00:00:00 2001 From: Tuan Pham Date: Wed, 4 Sep 2024 15:34:34 -0500 Subject: [PATCH 6/7] chore: resolve lint and api diff errors --- .../s3/transfer/StorageTransferClientProvider.kt | 1 - .../storage/s3/transfer/TransferManager.kt | 12 ++++++++++-- .../s3/transfer/worker/TransferWorkerFactory.kt | 1 - .../storage/s3/transfer/worker/DownloadWorkerTest.kt | 2 +- .../InitiateMultiPartUploadTransferWorkerTest.kt | 2 +- core/api/core.api | 2 +- 6 files changed, 13 insertions(+), 7 deletions(-) diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/StorageTransferClientProvider.kt b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/StorageTransferClientProvider.kt index 523c0dcf3e..45a5a02230 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/StorageTransferClientProvider.kt +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/StorageTransferClientProvider.kt @@ -15,7 +15,6 @@ package com.amplifyframework.storage.s3.transfer import aws.sdk.kotlin.services.s3.S3Client -import com.amplifyframework.annotations.InternalApiWarning internal interface StorageTransferClientProvider { fun getStorageTransferClient(region: String?, bucketName: String?): S3Client diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferManager.kt b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferManager.kt index 119aeeafb9..03d6ce2619 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferManager.kt +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/TransferManager.kt @@ -20,7 +20,6 @@ import android.content.Context import android.os.Handler import android.os.Looper import androidx.work.WorkManager -import aws.sdk.kotlin.services.s3.S3Client import aws.sdk.kotlin.services.s3.model.ObjectCannedAcl import com.amplifyframework.core.Amplify import com.amplifyframework.core.category.CategoryType @@ -102,7 +101,16 @@ internal class TransferManager( useAccelerateEndpoint: Boolean = false ): TransferObserver { val transferRecordId = if (shouldUploadInMultipart(file)) { - createMultipartUploadRecords(transferId, bucket, region, key, file, metadata, cannedAcl, useAccelerateEndpoint) + createMultipartUploadRecords( + transferId, + bucket, + region, + key, + file, + metadata, + cannedAcl, + useAccelerateEndpoint + ) } else { val uri = transferDB.insertSingleTransferRecord( transferId, diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/TransferWorkerFactory.kt b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/TransferWorkerFactory.kt index f491d1ee95..d814473643 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/TransferWorkerFactory.kt +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/transfer/worker/TransferWorkerFactory.kt @@ -17,7 +17,6 @@ package com.amplifyframework.storage.s3.transfer.worker import android.content.Context import androidx.work.WorkerFactory import androidx.work.WorkerParameters -import aws.sdk.kotlin.services.s3.S3Client import com.amplifyframework.storage.s3.transfer.StorageTransferClientProvider import com.amplifyframework.storage.s3.transfer.TransferDB import com.amplifyframework.storage.s3.transfer.TransferStatusUpdater diff --git a/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/transfer/worker/DownloadWorkerTest.kt b/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/transfer/worker/DownloadWorkerTest.kt index 446015b0cc..7f4bb1072c 100644 --- a/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/transfer/worker/DownloadWorkerTest.kt +++ b/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/transfer/worker/DownloadWorkerTest.kt @@ -74,7 +74,7 @@ internal class DownloadWorkerTest { every { workerParameters.runAttemptCount }.answers { 1 } every { workerParameters.taskExecutor }.answers { ImmediateTaskExecutor() } every { s3Client.withConfig(any()) } returns s3Client - every { clientProvider.getStorageTransferClient(any(), any())}.answers { s3Client } + every { clientProvider.getStorageTransferClient(any(), any()) }.answers { s3Client } } @After diff --git a/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/transfer/worker/InitiateMultiPartUploadTransferWorkerTest.kt b/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/transfer/worker/InitiateMultiPartUploadTransferWorkerTest.kt index 852f660a95..43d2560330 100644 --- a/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/transfer/worker/InitiateMultiPartUploadTransferWorkerTest.kt +++ b/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/transfer/worker/InitiateMultiPartUploadTransferWorkerTest.kt @@ -68,7 +68,7 @@ internal class InitiateMultiPartUploadTransferWorkerTest { every { workerParameters.runAttemptCount }.answers { 1 } every { workerParameters.taskExecutor }.answers { ImmediateTaskExecutor() } every { s3Client.withConfig(any()) } returns s3Client - every { clientProvider.getStorageTransferClient(any(), any())}.answers { s3Client } + every { clientProvider.getStorageTransferClient(any(), any()) }.answers { s3Client } } @After diff --git a/core/api/core.api b/core/api/core.api index d1b72287fe..790095cd96 100644 --- a/core/api/core.api +++ b/core/api/core.api @@ -3937,7 +3937,7 @@ public final class com/amplifyframework/storage/BucketInfo { public final fun copy (Ljava/lang/String;Ljava/lang/String;)Lcom/amplifyframework/storage/BucketInfo; public static synthetic fun copy$default (Lcom/amplifyframework/storage/BucketInfo;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)Lcom/amplifyframework/storage/BucketInfo; public fun equals (Ljava/lang/Object;)Z - public final fun getName ()Ljava/lang/String; + public final fun getBucketName ()Ljava/lang/String; public final fun getRegion ()Ljava/lang/String; public fun hashCode ()I public fun toString ()Ljava/lang/String; From 2cbe597baa74d3383f409307064a4239908e9f94 Mon Sep 17 00:00:00 2001 From: Tuan Pham <103537251+phantumcode@users.noreply.github.com> Date: Thu, 5 Sep 2024 14:21:47 -0500 Subject: [PATCH 7/7] chore: fix storage list with multiple buckets (#2908) --- .../storage/s3/AWSS3StoragePlugin.java | 168 +++++++++--------- .../storage/s3/AWSS3StoragePluginTest.kt | 45 +++++ 2 files changed, 129 insertions(+), 84 deletions(-) diff --git a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/AWSS3StoragePlugin.java b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/AWSS3StoragePlugin.java index 48dce2ee26..46eef5e653 100644 --- a/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/AWSS3StoragePlugin.java +++ b/aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/AWSS3StoragePlugin.java @@ -28,6 +28,7 @@ import com.amplifyframework.auth.CognitoCredentialsProvider; import com.amplifyframework.core.Consumer; import com.amplifyframework.core.NoOpConsumer; +import com.amplifyframework.core.async.AmplifyOperation; import com.amplifyframework.core.configuration.AmplifyOutputsData; import com.amplifyframework.storage.BucketInfo; import com.amplifyframework.storage.InvalidStorageBucketException; @@ -377,23 +378,19 @@ public StorageGetUrlOperation getUrl( validateObjectExistence ); - AWSS3StorageService storageService = defaultStorageService; - try { - storageService = getStorageService(options.getBucket()); - } catch (StorageException exception) { - onError.accept(exception); - } + GetStorageServiceResult result = getStorageServiceResult(options.getBucket()); AWSS3StorageGetPresignedUrlOperation operation = new AWSS3StorageGetPresignedUrlOperation( - storageService, + result.storageService, executorService, authCredentialsProvider, request, awsS3StoragePluginConfiguration, onSuccess, onError); - operation.start(); + + handleGetStorageServiceResult(onError, result, operation); return operation; } @@ -419,22 +416,18 @@ public StorageGetUrlOperation getUrl( validateObjectExistence ); - AWSS3StorageService storageService = defaultStorageService; - try { - storageService = getStorageService(options.getBucket()); - } catch (StorageException exception) { - onError.accept(exception); - } + GetStorageServiceResult result = getStorageServiceResult(options.getBucket()); AWSS3StoragePathGetPresignedUrlOperation operation = new AWSS3StoragePathGetPresignedUrlOperation( - storageService, + result.storageService, executorService, authCredentialsProvider, request, onSuccess, onError); - operation.start(); + + handleGetStorageServiceResult(onError, result, operation); return operation; } @@ -513,15 +506,10 @@ public StorageDownloadFileOperation downloadFile( useAccelerateEndpoint ); - AWSS3StorageService storageService = defaultStorageService; - try { - storageService = getStorageService(options.getBucket()); - } catch (StorageException exception) { - onError.accept(exception); - } + GetStorageServiceResult result = getStorageServiceResult(options.getBucket()); AWSS3StorageDownloadFileOperation operation = new AWSS3StorageDownloadFileOperation( - storageService, + result.storageService, executorService, authCredentialsProvider, request, @@ -530,7 +518,8 @@ public StorageDownloadFileOperation downloadFile( onSuccess, onError ); - operation.start(); + + handleGetStorageServiceResult(onError, result, operation); return operation; } @@ -555,23 +544,19 @@ public StorageDownloadFileOperation downloadFile( useAccelerateEndpoint ); - AWSS3StorageService storageService = defaultStorageService; - try { - storageService = getStorageService(options.getBucket()); - } catch (StorageException exception) { - onError.accept(exception); - } + GetStorageServiceResult result = getStorageServiceResult(options.getBucket()); AWSS3StoragePathDownloadFileOperation operation = new AWSS3StoragePathDownloadFileOperation( request, - storageService, + result.storageService, executorService, authCredentialsProvider, onProgress, onSuccess, onError ); - operation.start(); + + handleGetStorageServiceResult(onError, result, operation); return operation; } @@ -654,15 +639,10 @@ public StorageUploadFileOperation uploadFile( useAccelerateEndpoint ); - AWSS3StorageService storageService = defaultStorageService; - try { - storageService = getStorageService(options.getBucket()); - } catch (StorageException exception) { - onError.accept(exception); - } + GetStorageServiceResult result = getStorageServiceResult(options.getBucket()); AWSS3StorageUploadFileOperation operation = new AWSS3StorageUploadFileOperation( - storageService, + result.storageService, executorService, authCredentialsProvider, request, @@ -671,7 +651,8 @@ public StorageUploadFileOperation uploadFile( onSuccess, onError ); - operation.start(); + + handleGetStorageServiceResult(onError, result, operation); return operation; } @@ -699,23 +680,19 @@ public StorageUploadFileOperation uploadFile( useAccelerateEndpoint ); - AWSS3StorageService storageService = defaultStorageService; - try { - storageService = getStorageService(options.getBucket()); - } catch (StorageException exception) { - onError.accept(exception); - } + GetStorageServiceResult result = getStorageServiceResult(options.getBucket()); AWSS3StoragePathUploadFileOperation operation = new AWSS3StoragePathUploadFileOperation( request, - storageService, + result.storageService, executorService, authCredentialsProvider, onProgress, onSuccess, onError ); - operation.start(); + + handleGetStorageServiceResult(onError, result, operation); return operation; } @@ -796,15 +773,10 @@ public StorageUploadInputStreamOperation uploadInputStream( useAccelerateEndpoint ); - AWSS3StorageService storageService = defaultStorageService; - try { - storageService = getStorageService(options.getBucket()); - } catch (StorageException exception) { - onError.accept(exception); - } + GetStorageServiceResult result = getStorageServiceResult(options.getBucket()); AWSS3StorageUploadInputStreamOperation operation = new AWSS3StorageUploadInputStreamOperation( - storageService, + result.storageService, executorService, authCredentialsProvider, awsS3StoragePluginConfiguration, @@ -813,7 +785,8 @@ public StorageUploadInputStreamOperation uploadInputStream( onSuccess, onError ); - operation.start(); + + handleGetStorageServiceResult(onError, result, operation); return operation; } @@ -841,24 +814,20 @@ public StorageUploadInputStreamOperation uploadInputStream( useAccelerateEndpoint ); - AWSS3StorageService storageService = defaultStorageService; - try { - storageService = getStorageService(options.getBucket()); - } catch (StorageException exception) { - onError.accept(exception); - } + GetStorageServiceResult result = getStorageServiceResult(options.getBucket()); AWSS3StoragePathUploadInputStreamOperation operation = new AWSS3StoragePathUploadInputStreamOperation( request, - storageService, + result.storageService, executorService, authCredentialsProvider, onProgress, onSuccess, onError ); - operation.start(); + + handleGetStorageServiceResult(onError, result, operation); return operation; } @@ -901,16 +870,11 @@ public StorageRemoveOperation remove( options.getTargetIdentityId() ); - AWSS3StorageService storageService = defaultStorageService; - try { - storageService = getStorageService(options.getBucket()); - } catch (StorageException exception) { - onError.accept(exception); - } + GetStorageServiceResult result = getStorageServiceResult(options.getBucket()); AWSS3StorageRemoveOperation operation = new AWSS3StorageRemoveOperation( - storageService, + result.storageService, executorService, authCredentialsProvider, request, @@ -918,7 +882,7 @@ public StorageRemoveOperation remove( onSuccess, onError); - operation.start(); + handleGetStorageServiceResult(onError, result, operation); return operation; } @@ -933,23 +897,18 @@ public StorageRemoveOperation remove( ) { AWSS3StoragePathRemoveRequest request = new AWSS3StoragePathRemoveRequest(path); - AWSS3StorageService storageService = defaultStorageService; - try { - storageService = getStorageService(options.getBucket()); - } catch (StorageException exception) { - onError.accept(exception); - } + GetStorageServiceResult result = getStorageServiceResult(options.getBucket()); AWSS3StoragePathRemoveOperation operation = new AWSS3StoragePathRemoveOperation( - storageService, + result.storageService, executorService, authCredentialsProvider, request, onSuccess, onError); - operation.start(); + handleGetStorageServiceResult(onError, result, operation); return operation; } @@ -1096,9 +1055,11 @@ public StorageListOperation list(@NonNull String path, options.getNextToken(), options.getSubpathStrategy()); + GetStorageServiceResult result = getStorageServiceResult(options.getBucket()); + AWSS3StorageListOperation operation = new AWSS3StorageListOperation( - defaultStorageService, + result.storageService, executorService, authCredentialsProvider, request, @@ -1106,7 +1067,7 @@ public StorageListOperation list(@NonNull String path, onSuccess, onError); - operation.start(); + handleGetStorageServiceResult(onError, result, operation); return operation; } @@ -1125,20 +1086,47 @@ public StorageListOperation list( options.getNextToken(), options.getSubpathStrategy()); + GetStorageServiceResult result = getStorageServiceResult(options.getBucket()); + AWSS3StoragePathListOperation operation = new AWSS3StoragePathListOperation( - defaultStorageService, + result.storageService, executorService, authCredentialsProvider, request, onSuccess, onError); - operation.start(); + handleGetStorageServiceResult(onError, result, operation); return operation; } + private static void handleGetStorageServiceResult( + @NonNull Consumer onError, + GetStorageServiceResult result, + AmplifyOperation operation + ) { + if (result.storageException == null) { + operation.start(); + } else { + onError.accept(result.storageException); + } + } + + @VisibleForTesting + @NonNull + GetStorageServiceResult getStorageServiceResult(@Nullable StorageBucket bucket) { + StorageException storageException = null; + AWSS3StorageService storageService = defaultStorageService; + try { + storageService = getStorageService(bucket); + } catch (StorageException exception) { + storageException = exception; + } + return new GetStorageServiceResult(storageService, storageException); + } + @SuppressLint("UnsafeOptInUsageError") @VisibleForTesting @NonNull @@ -1209,4 +1197,16 @@ public String getConfigurationKey() { return configurationKey; } } + + @VisibleForTesting + @SuppressWarnings("checkstyle:VisibilityModifier") + static class GetStorageServiceResult { + final AWSS3StorageService storageService; + final StorageException storageException; + + GetStorageServiceResult(AWSS3StorageService storageService, StorageException exception) { + this.storageService = storageService; + this.storageException = exception; + } + } } diff --git a/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/AWSS3StoragePluginTest.kt b/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/AWSS3StoragePluginTest.kt index 5380af91b6..2d51cecd60 100644 --- a/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/AWSS3StoragePluginTest.kt +++ b/aws-storage-s3/src/test/java/com/amplifyframework/storage/s3/AWSS3StoragePluginTest.kt @@ -22,6 +22,7 @@ import com.amplifyframework.storage.StorageException import com.amplifyframework.storage.s3.service.AWSS3StorageService import com.amplifyframework.testutils.configuration.amplifyOutputsData import io.kotest.assertions.throwables.shouldThrow +import io.kotest.matchers.shouldBe import io.kotest.matchers.shouldNotBe import io.kotest.matchers.throwable.shouldHaveCauseOfType import io.mockk.every @@ -149,4 +150,48 @@ class AWSS3StoragePluginTest { } exception.shouldHaveCauseOfType() } + + @Test + fun `getStorageServiceResult returns result without exception`() { + val data = amplifyOutputsData { + storage { + awsRegion = "test-region" + bucketName = "test-bucket" + buckets { + awsRegion = "test-region" + bucketName = "test-bucket" + name = "test-name" + } + } + } + + plugin.configure(data, mockk()) + val bucket = StorageBucket.fromOutputs("test-name") + val result = plugin.getStorageServiceResult(bucket) + val service = result.storageService + val exception = result.storageException + service shouldNotBe null + exception shouldBe null + } + + @Test + fun `getStorageServiceResult returns result with exception`() { + val data = amplifyOutputsData { + storage { + awsRegion = "test-region" + bucketName = "test-bucket" + buckets { + awsRegion = "test-region" + bucketName = "test-bucket" + name = "test=name" + } + } + } + + plugin.configure(data, mockk()) + val bucket = StorageBucket.fromOutputs("myBucket") + val exception = plugin.getStorageServiceResult(bucket).storageException + exception shouldNotBe null + exception.shouldHaveCauseOfType() + } }