Skip to content

Commit 0425e51

Browse files
sebalandsebaland
authored
feat(Storage): Implementing support for multiple buckets (#3817)
1 parent f012197 commit 0425e51

21 files changed

+723
-60
lines changed

‎Amplify/Core/Configuration/AmplifyOutputsData.swift‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,25 @@ public struct AmplifyOutputsData: Codable {
176176
public struct Storage: Codable {
177177
public let awsRegion: AWSRegion
178178
public let bucketName: String
179+
public let buckets: [Bucket]?
180+
181+
@_spi(InternalAmplifyConfiguration)
182+
public struct Bucket: Codable {
183+
public let name: String
184+
public let bucketName: String
185+
public let awsRegion: AWSRegion
186+
}
187+
188+
// Internal init used for testing
189+
init(
190+
awsRegion: AWSRegion,
191+
bucketName: String,
192+
buckets: [Bucket]? = nil
193+
) {
194+
self.awsRegion = awsRegion
195+
self.bucketName = bucketName
196+
self.buckets = buckets
197+
}
179198
}
180199

181200
@_spi(InternalAmplifyConfiguration)

‎AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/AWSS3StoragePlugin+AsyncClientBehavior.swift‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ extension AWSS3StoragePlugin {
2727
let prefix = try await prefixResolver.resolvePrefix(for: options.accessLevel,
2828
targetIdentityId: options.targetIdentityId)
2929
let serviceKey = prefix + request.key
30+
31+
let storageService = try storageService(for: options.bucket)
3032
if let pluginOptions = options.pluginOptions as? AWSStorageGetURLOptions, pluginOptions.validateObjectExistence {
3133
try await storageService.validateObjectExistence(serviceKey: serviceKey)
3234
}
@@ -51,6 +53,7 @@ extension AWSS3StoragePlugin {
5153
) async throws -> URL {
5254
let options = options ?? StorageGetURLRequest.Options()
5355
let request = StorageGetURLRequest(path: path, options: options)
56+
let storageService = try storageService(for: options.bucket)
5457
let task = AWSS3StorageGetURLTask(
5558
request,
5659
storageBehaviour: storageService)
@@ -65,7 +68,7 @@ extension AWSS3StoragePlugin {
6568
let request = StorageDownloadDataRequest(path: path, options: options)
6669
let operation = AWSS3StorageDownloadDataOperation(request,
6770
storageConfiguration: storageConfiguration,
68-
storageService: storageService,
71+
storageServiceProvider: storageServiceProvider(for: options.bucket),
6972
authService: authService)
7073
let taskAdapter = AmplifyInProcessReportingOperationTaskAdapter(operation: operation)
7174
queue.addOperation(operation)
@@ -82,7 +85,7 @@ extension AWSS3StoragePlugin {
8285
let request = StorageDownloadDataRequest(key: key, options: options)
8386
let operation = AWSS3StorageDownloadDataOperation(request,
8487
storageConfiguration: storageConfiguration,
85-
storageService: storageService,
88+
storageServiceProvider: storageServiceProvider(for: options.bucket),
8689
authService: authService)
8790
let taskAdapter = AmplifyInProcessReportingOperationTaskAdapter(operation: operation)
8891
queue.addOperation(operation)
@@ -100,7 +103,7 @@ extension AWSS3StoragePlugin {
100103
let request = StorageDownloadFileRequest(key: key, local: local, options: options)
101104
let operation = AWSS3StorageDownloadFileOperation(request,
102105
storageConfiguration: storageConfiguration,
103-
storageService: storageService,
106+
storageServiceProvider: storageServiceProvider(for: options.bucket),
104107
authService: authService)
105108
let taskAdapter = AmplifyInProcessReportingOperationTaskAdapter(operation: operation)
106109
queue.addOperation(operation)
@@ -118,7 +121,7 @@ extension AWSS3StoragePlugin {
118121
let request = StorageDownloadFileRequest(path: path, local: local, options: options)
119122
let operation = AWSS3StorageDownloadFileOperation(request,
120123
storageConfiguration: storageConfiguration,
121-
storageService: storageService,
124+
storageServiceProvider: storageServiceProvider(for: options.bucket),
122125
authService: authService)
123126
let taskAdapter = AmplifyInProcessReportingOperationTaskAdapter(operation: operation)
124127
queue.addOperation(operation)
@@ -136,7 +139,7 @@ extension AWSS3StoragePlugin {
136139
let request = StorageUploadDataRequest(key: key, data: data, options: options)
137140
let operation = AWSS3StorageUploadDataOperation(request,
138141
storageConfiguration: storageConfiguration,
139-
storageService: storageService,
142+
storageServiceProvider: storageServiceProvider(for: options.bucket),
140143
authService: authService)
141144
let taskAdapter = AmplifyInProcessReportingOperationTaskAdapter(operation: operation)
142145
queue.addOperation(operation)
@@ -154,7 +157,7 @@ extension AWSS3StoragePlugin {
154157
let request = StorageUploadDataRequest(path: path, data: data, options: options)
155158
let operation = AWSS3StorageUploadDataOperation(request,
156159
storageConfiguration: storageConfiguration,
157-
storageService: storageService,
160+
storageServiceProvider: storageServiceProvider(for: options.bucket),
158161
authService: authService)
159162
let taskAdapter = AmplifyInProcessReportingOperationTaskAdapter(operation: operation)
160163
queue.addOperation(operation)
@@ -172,7 +175,7 @@ extension AWSS3StoragePlugin {
172175
let request = StorageUploadFileRequest(key: key, local: local, options: options)
173176
let operation = AWSS3StorageUploadFileOperation(request,
174177
storageConfiguration: storageConfiguration,
175-
storageService: storageService,
178+
storageServiceProvider: storageServiceProvider(for: options.bucket),
176179
authService: authService)
177180
let taskAdapter = AmplifyInProcessReportingOperationTaskAdapter(operation: operation)
178181
queue.addOperation(operation)
@@ -190,7 +193,7 @@ extension AWSS3StoragePlugin {
190193
let request = StorageUploadFileRequest(path: path, local: local, options: options)
191194
let operation = AWSS3StorageUploadFileOperation(request,
192195
storageConfiguration: storageConfiguration,
193-
storageService: storageService,
196+
storageServiceProvider: storageServiceProvider(for: options.bucket),
194197
authService: authService)
195198
let taskAdapter = AmplifyInProcessReportingOperationTaskAdapter(operation: operation)
196199
queue.addOperation(operation)
@@ -205,6 +208,7 @@ extension AWSS3StoragePlugin {
205208
) async throws -> String {
206209
let options = options ?? StorageRemoveRequest.Options()
207210
let request = StorageRemoveRequest(key: key, options: options)
211+
let storageService = try storageService(for: options.bucket)
208212
let operation = AWSS3StorageRemoveOperation(request,
209213
storageConfiguration: storageConfiguration,
210214
storageService: storageService,
@@ -222,6 +226,7 @@ extension AWSS3StoragePlugin {
222226
) async throws -> String {
223227
let options = options ?? StorageRemoveRequest.Options()
224228
let request = StorageRemoveRequest(path: path, options: options)
229+
let storageService = try storageService(for: options.bucket)
225230
let task = AWSS3StorageRemoveTask(
226231
request,
227232
storageConfiguration: storageConfiguration,
@@ -233,6 +238,7 @@ extension AWSS3StoragePlugin {
233238
options: StorageListRequest.Options? = nil
234239
) async throws -> StorageListResult {
235240
let options = options ?? StorageListRequest.Options()
241+
let storageService = try storageService(for: options.bucket)
236242
let prefixResolver = storageConfiguration.prefixResolver ?? StorageAccessLevelAwarePrefixResolver(authService: authService)
237243
let prefix = try await prefixResolver.resolvePrefix(for: options.accessLevel, targetIdentityId: options.targetIdentityId)
238244
let result = try await storageService.list(prefix: prefix, options: options)
@@ -250,6 +256,7 @@ extension AWSS3StoragePlugin {
250256
) async throws -> StorageListResult {
251257
let options = options ?? StorageListRequest.Options()
252258
let request = StorageListRequest(path: path, options: options)
259+
let storageService = try storageService(for: options.bucket)
253260
let task = AWSS3StorageListObjectsTask(
254261
request,
255262
storageConfiguration: storageConfiguration,

‎AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/AWSS3StoragePlugin+ClientBehavior.swift‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,6 @@ extension AWSS3StoragePlugin {
1919
///
2020
/// - Tag: AWSS3StoragePlugin.getEscapeHatch
2121
public func getEscapeHatch() -> S3Client {
22-
return storageService.getEscapeHatch()
22+
return defaultStorageService.getEscapeHatch()
2323
}
2424
}

‎AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/AWSS3StoragePlugin+Configure.swift‎

Lines changed: 61 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import Foundation
99
@_spi(InternalAmplifyConfiguration) import Amplify
1010
import AWSPluginsCore
11+
import InternalAmplifyCredentials
1112

1213
extension AWSS3StoragePlugin {
1314

@@ -27,6 +28,7 @@ extension AWSS3StoragePlugin {
2728
let configClosures: ConfigurationClosures
2829
if let config = configuration as? AmplifyOutputsData {
2930
configClosures = try retrieveConfiguration(config)
31+
additionalBucketsByName = retrieveAdditionalBucketsByName(from: config.storage)
3032
} else if let config = configuration as? JSONValue {
3133
configClosures = try retrieveConfiguration(config)
3234
} else {
@@ -38,15 +40,24 @@ extension AWSS3StoragePlugin {
3840
do {
3941
let authService = AWSAuthService()
4042
let defaultAccessLevel = try configClosures.retrieveDefaultAccessLevel()
41-
let storageService = try AWSS3StorageService(authService: authService,
42-
region: configClosures.retrieveRegion(),
43-
bucket: configClosures.retrieveBucket(),
44-
httpClientEngineProxy: self.httpClientEngineProxy)
45-
storageService.urlRequestDelegate = self.urlRequestDelegate
46-
47-
configure(storageService: storageService,
48-
authService: authService,
49-
defaultAccessLevel: defaultAccessLevel)
43+
let defaultBucket: ResolvedStorageBucket = try .fromBucketInfo(
44+
.init(
45+
bucketName: configClosures.retrieveBucket(),
46+
region: configClosures.retrieveRegion()
47+
)
48+
)
49+
50+
let storageService = try createStorageService(
51+
authService: authService,
52+
bucketInfo: defaultBucket.bucketInfo
53+
)
54+
55+
configure(
56+
defaultBucket: defaultBucket,
57+
storageService: storageService,
58+
authService: authService,
59+
defaultAccessLevel: defaultAccessLevel
60+
)
5061
} catch let storageError as StorageError {
5162
throw storageError
5263
} catch {
@@ -67,18 +78,38 @@ extension AWSS3StoragePlugin {
6778
/// Called from the configure method which implements the Plugin protocol. Useful for testing by passing in mocks.
6879
///
6980
/// - Parameters:
70-
/// - storageService: The S3 storage service object.
81+
/// - defaultBucket: The bucket to be used for all API calls by default.
82+
/// - storageService: The S3 storage service object associated with the default bucket
7183
/// - authService: The authentication service object.
7284
/// - defaultAccessLevel: The access level to be used for all API calls by default.
7385
/// - queue: The queue which operations are stored and dispatched for asychronous processing.
74-
func configure(storageService: AWSS3StorageServiceBehavior,
75-
authService: AWSAuthServiceBehavior,
76-
defaultAccessLevel: StorageAccessLevel,
77-
queue: OperationQueue = OperationQueue()) {
78-
self.storageService = storageService
86+
func configure(
87+
defaultBucket: ResolvedStorageBucket,
88+
storageService: AWSS3StorageServiceBehavior,
89+
authService: AWSAuthCredentialsProviderBehavior,
90+
defaultAccessLevel: StorageAccessLevel,
91+
queue: OperationQueue = OperationQueue()
92+
) {
93+
self.defaultBucket = defaultBucket
7994
self.authService = authService
8095
self.queue = queue
8196
self.defaultAccessLevel = defaultAccessLevel
97+
self.storageServicesByBucket[defaultBucket.bucketInfo.bucketName] = storageService
98+
}
99+
100+
/// Creates a new AWSS3StorageServiceBehavior for the given BucketInfo
101+
func createStorageService(
102+
authService: AWSAuthCredentialsProviderBehavior,
103+
bucketInfo: BucketInfo
104+
) throws -> AWSS3StorageServiceBehavior {
105+
let storageService = try AWSS3StorageService(
106+
authService: authService,
107+
region: bucketInfo.region,
108+
bucket: bucketInfo.bucketName,
109+
httpClientEngineProxy: httpClientEngineProxy
110+
)
111+
storageService.urlRequestDelegate = urlRequestDelegate
112+
return storageService
82113
}
83114

84115
// MARK: Private helper methods
@@ -127,6 +158,21 @@ extension AWSS3StoragePlugin {
127158
retrieveDefaultAccessLevel: defaultAccessLevelClosure)
128159
}
129160

161+
/// Retrieves the configured buckets from the configuration grouped by their names.
162+
/// If no buckets are provided in the configuration, an empty dictionary is returned instead.
163+
private func retrieveAdditionalBucketsByName(
164+
from configuration: AmplifyOutputsData.Storage?
165+
) -> [String: AmplifyOutputsData.Storage.Bucket] {
166+
guard let configuration,
167+
let buckets = configuration.buckets else {
168+
return [:]
169+
}
170+
171+
return buckets.reduce(into: [:]) { dictionary, bucket in
172+
dictionary[bucket.name] = bucket
173+
}
174+
}
175+
130176
/// Retrieves the region from configuration, validates, and returns it.
131177
private static func getRegion(_ configuration: [String: JSONValue]) throws -> String {
132178
guard let region = configuration[PluginConstants.region] else {

‎AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/AWSS3StoragePlugin+Reset.swift‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,19 @@ extension AWSS3StoragePlugin {
1818
///
1919
/// - Tag: AWSS3StoragePlugin.reset
2020
public func reset() async {
21-
if storageService != nil {
21+
if defaultBucket != nil {
22+
defaultBucket = nil
23+
}
24+
25+
for storageService in storageServicesByBucket.values {
2226
storageService.reset()
23-
storageService = nil
2427
}
28+
storageServicesByBucket.removeAll()
29+
30+
if additionalBucketsByName != nil {
31+
additionalBucketsByName = nil
32+
}
33+
2534
if authService != nil {
2635
authService = nil
2736
}

‎AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/AWSS3StoragePlugin+StorageBucket.swift‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
//
2+
// Copyright Amazon.com Inc. or its affiliates.
3+
// All Rights Reserved.
4+
//
5+
// SPDX-License-Identifier: Apache-2.0
6+
//
7+
8+
@_spi(InternalAmplifyConfiguration) import Amplify
9+
import Foundation
10+
11+
extension AWSS3StoragePlugin {
12+
/// Returns a AWSS3StorageServiceBehavior instance for the given StorageBucket
13+
func storageService(for bucket: (any StorageBucket)?) throws -> AWSS3StorageServiceBehavior {
14+
guard let bucket else {
15+
// When no bucket is provided, use the default one
16+
return defaultStorageService
17+
}
18+
19+
let bucketInfo = try bucketInfo(from: bucket)
20+
guard let storageService = storageServicesByBucket[bucketInfo.bucketName] else {
21+
// If no service was found for the bucket, create one
22+
let storageService = try createStorageService(
23+
authService: authService,
24+
bucketInfo: bucketInfo
25+
)
26+
storageServicesByBucket[bucketInfo.bucketName] = storageService
27+
return storageService
28+
}
29+
30+
return storageService
31+
}
32+
33+
/// Returns a AWSS3StorageServiceProvider callback for the given StorageBucket
34+
func storageServiceProvider(for bucket: (any StorageBucket)?) -> AWSS3StorageServiceProvider {
35+
let storageServiceResolver: () throws -> AWSS3StorageServiceBehavior = { [weak self] in
36+
guard let self = self else {
37+
throw StorageError.unknown("AWSS3StoragePlugin was deallocated", nil)
38+
}
39+
return try self.storageService(for: bucket)
40+
}
41+
return storageServiceResolver
42+
}
43+
44+
/// Returns a valid `BucketInfo` instance from the given StorageBucket
45+
private func bucketInfo(from bucket: any StorageBucket) throws -> BucketInfo {
46+
switch bucket {
47+
case let outputsBucket as OutputsStorageBucket:
48+
guard let additionalBucketsByName else {
49+
let errorDescription = "Amplify was not configured using an Amplify Outputs file"
50+
let recoverySuggestion = "Make sure that `Amplify.configure(with:)` is invoked"
51+
throw StorageError.validation("bucket", errorDescription, recoverySuggestion, nil)
52+
}
53+
54+
guard let awsBucket = additionalBucketsByName[outputsBucket.name] else {
55+
let errorDescription = "Unable to lookup bucket from provided name in Amplify Outputs"
56+
let recoverySuggestion = "Make sure the bucket name exists in the Amplify Outputs file"
57+
throw StorageError.validation("bucket", errorDescription, recoverySuggestion, nil)
58+
}
59+
60+
return .init(
61+
bucketName: awsBucket.bucketName,
62+
region: awsBucket.awsRegion
63+
)
64+
65+
case let resolvedBucket as ResolvedStorageBucket:
66+
return resolvedBucket.bucketInfo
67+
68+
default:
69+
let errorDescription = "The specified StorageBucket is not supported"
70+
let recoverySuggestion = "Please specify a StorageBucket from the Outputs file or from BucketInfo"
71+
throw StorageError.validation("bucket", errorDescription, recoverySuggestion, nil)
72+
}
73+
}
74+
}

0 commit comments

Comments
 (0)