feat(storage): Add pagination support.

Author: jcjimenez

Amplify/Categories/Storage/Operation/Request/StorageListRequest.swift

@@ -5,39 +5,84 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+/// - Tag: StorageListRequest
 public struct StorageListRequest: AmplifyOperationRequest {
+
     /// Options to adjust the behavior of this request, including plugin-options
+    /// - Tag: StorageListRequest
     public let options: Options
 
+    /// - Tag: StorageListRequest.init
     public init(options: Options) {
         self.options = options
     }
 }
 
 public extension StorageListRequest {
-    /// Options to adjust the behavior of this request, including plugin-options
+
+    /// Options available to callers of
+    /// [StorageCategoryBehavior.list](x-source-tag://StorageCategoryBehavior.list).
+    ///
+    /// Tag: StorageListRequestOptions
     struct Options {
+
         /// Access level of the storage system. Defaults to `public`
+        ///
+        /// - Tag: StorageListRequestOptions.accessLevel
         public let accessLevel: StorageAccessLevel
 
         /// Target user to apply the action on
+        ///
+        /// - Tag: StorageListRequestOptions.targetIdentityId
         public let targetIdentityId: String?
 
         /// Path to the keys
+        ///
+        /// - Tag: StorageListRequestOptions.path
         public let path: String?
 
+        /// Number between 1 and 1,000 that indicates the limit of how many entries to fetch when
+        /// retreiving file lists from the server.
+        ///
+        /// NOTE: Plugins may decide to throw or perform normalization when encoutering vaues outside
+        ///       the specified range.
+        ///
+        /// - SeeAlso:
+        /// [StorageListRequestOptions.nextToken](x-source-tag://StorageListRequestOptions.nextToken)
+        /// [StorageListResult.nextToken](x-source-tag://StorageListResult.nextToken)
+        ///
+        /// - Tag: StorageListRequestOptions.pageSize
+        public let pageSize: UInt
+
+        /// Opaque string indicating the page offset at which to resume a listing. This is usually a copy of
+        /// the value from [StorageListResult.nextToken](x-source-tag://StorageListResult.nextToken).
+        ///
+        /// - SeeAlso:
+        /// [StorageListRequestOptions.pageSize](x-source-tag://StorageListRequestOptions.pageSize)
+        /// [StorageListResult.nextToken](x-source-tag://StorageListResult.nextToken)
+        ///
+        /// - Tag: StorageListRequestOptions.nextToken
+        public let nextToken: String?
+
         /// Extra plugin specific options, only used in special circumstances when the existing options do not provide
         /// a way to utilize the underlying storage system's functionality. See plugin documentation for expected
         /// key/values
+        ///
+        /// - Tag: StorageListRequestOptions.pluginOptions
         public let pluginOptions: Any?
 
+        /// - Tag: StorageListRequestOptions.init
         public init(accessLevel: StorageAccessLevel = .guest,
                     targetIdentityId: String? = nil,
                     path: String? = nil,
+                    pageSize: UInt = 1000,
+                    nextToken: String? = nil,
                     pluginOptions: Any? = nil) {
             self.accessLevel = accessLevel
             self.targetIdentityId = targetIdentityId
             self.path = path
+            self.pageSize = pageSize
+            self.nextToken = nextToken
             self.pluginOptions = pluginOptions
         }
     }

Amplify/Categories/Storage/Result/StorageListResult.swift

@@ -7,35 +7,71 @@
 
 import Foundation
 
+/// Represents the output of a call to
+/// [StorageCategoryBehavior.list](x-source-tag://StorageCategoryBehavior.list)
+///
+/// - Tag: StorageListResult
 public struct StorageListResult {
-    public init(items: [Item]) {
+
+    /// This is meant to be called by plugins implementing
+    /// [StorageCategoryBehavior.list](x-source-tag://StorageCategoryBehavior.list).
+    ///
+    /// - Tag: StorageListResult.init
+    public init(items: [Item], nextToken: String? = nil) {
         self.items = items
+        self.nextToken = nextToken
     }
 
-    // Array of Items in the Result
+    /// Array of Items in the Result
+    ///
+    /// - Tag: StorageListResult.items
     public var items: [Item]
+
+    /// Opaque string indicating the page offset at which to resume a listing. This value is usually copied to
+    /// [StorageListRequestOptions.nextToken](x-source-tag://StorageListRequestOptions.nextToken).
+    ///
+    /// - SeeAlso:
+    /// [StorageListRequestOptions.nextToken](x-source-tag://StorageListRequestOptions.nextToken)
+    ///
+    /// - Tag: StorageListResult.nextToken
+    public let nextToken: String?
 }
 
 extension StorageListResult {
 
+    /// - Tag: StorageListResultItem
     public struct Item {
 
         /// The unique identifier of the object in storage.
+        ///
+        /// - Tag: StorageListResultItem.key
         public let key: String
 
         /// Size in bytes of the object
+        ///
+        /// - Tag: StorageListResultItem.size
         public let size: Int?
 
         /// The date the Object was Last Modified
+        ///
+        /// - Tag: StorageListResultItem.lastModified
         public let lastModified: Date?
 
         /// The entity tag is an MD5 hash of the object.
         /// ETag reflects only changes to the contents of an object, not its metadata.
+        ///
+        /// - Tag: StorageListResultItem.eTag
         public let eTag: String?
 
         /// Additional results specific to the plugin.
+        ///
+        /// - Tag: StorageListResultItem.pluginResults
         public let pluginResults: Any?
 
+        /// This is meant to be called by plugins implementing
+        /// [StorageCategoryBehavior.list](x-source-tag://StorageCategoryBehavior.list).
+        ///
+        /// - Tag: StorageListResultItem.init
         public init(key: String,
                     size: Int? = nil,
                     eTag: String? = nil,

Amplify/Categories/Storage/StorageCategoryBehavior.swift

@@ -7,7 +7,11 @@
 
 import Foundation
 
-/// Behavior of the Storage category that clients will use
+/// Behavior of the Storage category used though `Amplify.Storage.*`. Plugin implementations
+/// conform to this protocol indirectly though the
+/// [StorageCategoryPlugin](x-source-tag://StorageCategoryPlugin) protocol.
+///
+/// - Tag: StorageCategoryBehavior
 public protocol StorageCategoryBehavior {
 
     /// Retrieve the remote URL for the object from storage.
@@ -16,6 +20,8 @@ public protocol StorageCategoryBehavior {
     ///   - key: The unique identifier for the object in storage.
     ///   - options: Parameters to specific plugin behavior
     /// - Returns: requested Get URL
+    ///
+    /// - Tag: StorageCategoryBehavior.getURL
     @discardableResult
     func getURL(key: String,
                 options: StorageGetURLOperation.Request.Options?) async throws -> URL
@@ -26,6 +32,8 @@ public protocol StorageCategoryBehavior {
     ///   - key: The unique identifier for the object in storage
     ///   - options: Options to adjust the behavior of this request, including plugin-options
     /// - Returns: A task that provides progress updates and the key which was used to download
+    ///
+    /// - Tag: StorageCategoryBehavior.downloadData
     @discardableResult
     func downloadData(key: String,
                       options: StorageDownloadDataOperation.Request.Options?) -> StorageDownloadDataTask
@@ -37,6 +45,8 @@ public protocol StorageCategoryBehavior {
     ///   - local: The local file to download destination
     ///   - options: Parameters to specific plugin behavior
     /// - Returns: A task that provides progress updates and the key which was used to download
+    ///
+    /// - Tag: StorageCategoryBehavior.downloadFile
     @discardableResult
     func downloadFile(key: String,
                       local: URL,
@@ -49,6 +59,8 @@ public protocol StorageCategoryBehavior {
     ///   - data: The data in memory to be uploaded
     ///   - options: Parameters to specific plugin behavior
     /// - Returns: A task that provides progress updates and the key which was used to upload
+    ///
+    /// - Tag: StorageCategoryBehavior.uploadData
     @discardableResult
     func uploadData(key: String,
                     data: Data,
@@ -61,6 +73,8 @@ public protocol StorageCategoryBehavior {
     ///   - local: The path to a local file.
     ///   - options: Parameters to specific plugin behavior
     /// - Returns: A task that provides progress updates and the key which was used to upload
+    ///
+    /// - Tag: StorageCategoryBehavior.uploadFile
     @discardableResult
     func uploadFile(key: String,
                     local: URL,
@@ -72,6 +86,8 @@ public protocol StorageCategoryBehavior {
     ///   - key: The unique identifier of the object in storage.
     ///   - options: Parameters to specific plugin behavior
     /// - Returns: An operation object that provides notifications and actions related to the execution of the work
+    ///
+    /// - Tag: StorageCategoryBehavior.remove
     @discardableResult
     func remove(key: String,
                 options: StorageRemoveOperation.Request.Options?) async throws -> String
@@ -82,12 +98,16 @@ public protocol StorageCategoryBehavior {
     ///   - options: Parameters to specific plugin behavior
     ///   - resultListener: Triggered when the list is complete
     /// - Returns: An operation object that provides notifications and actions related to the execution of the work
+    ///
+    /// - Tag: StorageCategoryBehavior.list
     @discardableResult
     func list(options: StorageListOperation.Request.Options?) async throws -> StorageListResult
 
     /// Handles background events which are related to URLSession
     /// - Parameter identifier: identifier
     /// - Returns: returns true if the identifier is handled by Amplify
+    ///
+    /// - Tag: StorageCategoryBehavior.handleBackgroundEvents
     func handleBackgroundEvents(identifier: String) async -> Bool
 
 }

Amplify/Categories/Storage/StorageCategoryPlugin.swift

@@ -5,6 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+/// - Tag: StorageCategoryPlugin
 public protocol StorageCategoryPlugin: Plugin, StorageCategoryBehavior { }
 
 public extension StorageCategoryPlugin {

AmplifyPlugins/Storage/Sources/AWSS3StoragePlugin/Service/Storage/AWSS3StorageService+ListBehavior.swift

@@ -30,9 +30,9 @@ extension AWSS3StorageService {
             finalPrefix = prefix
         }
         let input = ListObjectsV2Input(bucket: bucket,
-                                       continuationToken: nil,
+                                       continuationToken: options.nextToken,
                                        delimiter: nil,
-                                       maxKeys: 1_000,
+                                       maxKeys: Int(options.pageSize),
                                        prefix: finalPrefix,
                                        startAfter: nil)
         do {
@@ -41,7 +41,7 @@ extension AWSS3StorageService {
             let items = try contents.map {
                 try StorageListResult.Item(s3Object: $0, prefix: prefix)
             }
-            return StorageListResult(items: items)
+            return StorageListResult(items: items, nextToken: response.nextContinuationToken)
         } catch let error as SdkError<ListObjectsV2OutputError> {
             throw error.storageError
         } catch {

AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Request/AWSS3StorageListRequestTests.swift

@@ -27,6 +27,23 @@ class StorageListRequestTests: XCTestCase {
         XCTAssertNil(storageErrorOptional)
     }
 
+    /// - Given: A an options parameter containing pagination options
+    /// - When: The containing request is validated
+    /// - Then: No errors are raised
+    func testValidateWithPaginationOptions() {
+        let pageSizeOnly = StorageListRequest(options: StorageListRequest.Options(pageSize: UInt.random(in: 1..<1_000)))
+        XCTAssertNil(pageSizeOnly.validate())
+
+        let nextTokenOnly = StorageListRequest(options: StorageListRequest.Options(nextToken: UUID().uuidString))
+        XCTAssertNil(nextTokenOnly.validate())
+
+        let pageSizeAndNextToken = StorageListRequest(options: StorageListRequest.Options(
+            pageSize: UInt.random(in: 1..<1_000),
+            nextToken: UUID().uuidString
+        ))
+        XCTAssertNil(pageSizeAndNextToken.validate())
+    }
+
     func testValidateEmptyTargetIdentityIdError() {
         let options = StorageListRequest.Options(accessLevel: .protected,
                                                  targetIdentityId: "",

AmplifyPlugins/Storage/Tests/AWSS3StoragePluginTests/Service/Storage/AWSS3StorageServiceListTests.swift

@@ -50,6 +50,25 @@ final class AWSS3StorageServiceListTests: XCTestCase {
         systemUnderTest = nil
     }
 
+    /// Given: Any S3 bucket (client)
+    /// When: A listing of it is requested using pagination options
+    /// Then: The service propagates the pagination options to its underlying S3 client.
+    func testPaginationOptionsPropagation() async throws {
+        var inputs: [ListObjectsV2Input] = []
+        client.listObjectsV2Handler = { input in
+            inputs.append(input)
+            return .init(contents: [])
+        }
+        let pageSize: UInt = UInt.random(in: 1..<1_000)
+        let nextToken = UUID().uuidString
+        let options = StorageListRequest.Options(pageSize: pageSize,
+                                                 nextToken: nextToken)
+        let listing = try await systemUnderTest.list(prefix: prefix, options: options)
+        XCTAssertEqual(listing.items.map { $0.key }, [])
+        XCTAssertEqual(inputs.map { $0.continuationToken }, [nextToken])
+        XCTAssertEqual(inputs.map { $0.maxKeys }, [Int(pageSize)])
+    }
+
     /// Given: A empty S3 bucket (client)
     /// When: A listing of it is requested using typical parameters
     /// Then: The service returns an empty list of StorageListResult.Item

AmplifyPlugins/Storage/Tests/StorageHostApp/AWSS3StoragePluginIntegrationTests/AWSS3StoragePluginBasicIntegrationTests.swift

@@ -12,6 +12,21 @@ import CryptoKit
 
 class AWSS3StoragePluginBasicIntegrationTests: AWSS3StoragePluginTestBase {
 
+    var uploadedKeys: [String]!
+
+    override func setUp() async throws {
+        try await super.setUp()
+        uploadedKeys = []
+    }
+
+    override func tearDown() async throws {
+        for key in uploadedKeys {
+            await self.remove(key: key)
+        }
+        uploadedKeys = nil
+        try await super.tearDown()
+    }
+
     /// Given: An data object
     /// When: Upload the data
     /// Then: The operation completes successfully
@@ -203,6 +218,49 @@ class AWSS3StoragePluginBasicIntegrationTests: AWSS3StoragePluginTestBase {
         await remove(key: key)
     }
 
+    /// Given: A collection of objects in storage numbering `objectCount`.
+    /// When: The list API is invoked twice using a pageSize of `((objectCount/2) - 1)` and its
+    ///      corresponding token options.
+    /// Then: All objects are listed.
+    func testListTwoPages() async throws {
+        let objectCount = UInt.random(in: 16..<32)
+        // One more than half in order to ensure there are only two pages
+        let pageSize = UInt(objectCount/2) + 1
+        let path = "pagination-\(UUID().uuidString)"
+        for i in 0..<objectCount {
+            let key = "\(path)/\(i).txt"
+            let data = Data("\(i)".utf8)
+            await uploadData(key: key, data: data)
+            uploadedKeys.append(key)
+        }
+
+        // First half of listing
+        let firstResult = try await Amplify.Storage.list(options: .init(
+            accessLevel: .guest,
+            path: path,
+            pageSize: pageSize
+        ))
+        let firstPage = try XCTUnwrap(firstResult.items)
+        XCTAssertEqual(firstPage.count, Int(pageSize))
+        let firstNextToken = try XCTUnwrap(firstResult.nextToken)
+
+        // Second half of listing
+        let secondResult = try await Amplify.Storage.list(options: .init(
+            accessLevel: .guest,
+            path: path,
+            pageSize: pageSize,
+            nextToken: firstNextToken
+        ))
+        let secondPage = try XCTUnwrap(secondResult.items)
+        XCTAssertEqual(secondPage.count, Int(objectCount - pageSize))
+        XCTAssertNil(secondResult.nextToken)
+
+        XCTAssertEqual(
+            uploadedKeys.sorted(),
+            Array((firstPage + secondPage).map { $0.key }).sorted()
+        )
+    }
+
     /// Given: No object in storage for the key
     /// When: Call the list API
     /// Then: The operation completes successfully with empty list of keys returned