feat: Support lazy load and pagination for API (#1009)

* chore: Separate DataStore List logic to ListProvider

* address PR comments

* Update CoreError - remove pluginError case

* enable some datastore integ tests

* fix Collection count to add implicit load

* update shouldDecode with ModelType

* - fixed List.load check for not loaded state
- fixed DataStoreListProvider association logic
- improved unit tests

* update doc comment

* - removed .unknown case from CoreError
- added more unit tests for DataStoreListProvider

* feat: address pr comments

* feat: support API fetch and pagination

* feat: address PR comments
- added AWSAPIPlugin readme file
- added back in the datastore list provider implementation of hasNextPage and getNextPage

* feat: address AWSAPIPlugin.md comments, update list decoder method name

* feat: update APIPlugin.md

Author: lawmicha

Amplify.xcodeproj/project.pbxproj

@@ -142,6 +142,8 @@
 		21A4ED77259CDF6800E1047D /* List+Combine.swift in Sources */ = {isa = PBXBuildFile; fileRef = 21A4ED76259CDF6800E1047D /* List+Combine.swift */; };
 		21A7C8AA25ACB6C8004355D6 /* ArrayLiteralListProviderTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 21A7C8A925ACB6C8004355D6 /* ArrayLiteralListProviderTests.swift */; };
 		21A7C90225ACC4D1004355D6 /* MockDataStoreResponders.swift in Sources */ = {isa = PBXBuildFile; fileRef = 21A7C90125ACC4D1004355D6 /* MockDataStoreResponders.swift */; };
+		21A4F26325A3CD3D00E1047D /* List+Pagination.swift in Sources */ = {isa = PBXBuildFile; fileRef = 21A4F26225A3CD3D00E1047D /* List+Pagination.swift */; };
+		21A4F8F325A77D9100E1047D /* ListPaginationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 21A4F8F225A77D9100E1047D /* ListPaginationTests.swift */; };
 		21AD424B249BF0DA0016FE95 /* AnyModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = FACBAD522386160100E29E56 /* AnyModel.swift */; };
 		21AD424C249BF0DE0016FE95 /* AnyModel+Codable.swift in Sources */ = {isa = PBXBuildFile; fileRef = FA8EE776238626D60097E4F1 /* AnyModel+Codable.swift */; };
 		21AD424D249BF0E50016FE95 /* AnyModel+Schema.swift in Sources */ = {isa = PBXBuildFile; fileRef = FA8EE78223862DDB0097E4F1 /* AnyModel+Schema.swift */; };
@@ -924,6 +926,8 @@
 		217D5EAF2577F9DF009F0639 /* Project2.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Project2.swift; sourceTree = "<group>"; };
 		217D5ECD2577FA1B009F0639 /* connection-schema.graphql */ = {isa = PBXFileReference; lastKnownFileType = text; path = "connection-schema.graphql"; sourceTree = "<group>"; };
 		217D5F63257830C2009F0639 /* ModelField+GraphQL.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "ModelField+GraphQL.swift"; sourceTree = "<group>"; };
+		218DC969258A73AF00D59FC6 /* CoreError.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CoreError.swift; sourceTree = "<group>"; };
+		218DC979258A73CA00D59FC6 /* Paginatable.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Paginatable.swift; sourceTree = "<group>"; };
 		219A887E23EB627100BBC5F2 /* ModelBasedGraphQLDocumentBuilder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ModelBasedGraphQLDocumentBuilder.swift; sourceTree = "<group>"; };
 		219A888023EB629800BBC5F2 /* ModelBasedGraphQLDocumentDecorator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ModelBasedGraphQLDocumentDecorator.swift; sourceTree = "<group>"; };
 		219A888423EB897700BBC5F2 /* GraphQLRequest+AnyModelWithSync.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "GraphQLRequest+AnyModelWithSync.swift"; sourceTree = "<group>"; };
@@ -944,6 +948,8 @@
 		21A4ED76259CDF6800E1047D /* List+Combine.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "List+Combine.swift"; sourceTree = "<group>"; };
 		21A7C8A925ACB6C8004355D6 /* ArrayLiteralListProviderTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ArrayLiteralListProviderTests.swift; sourceTree = "<group>"; };
 		21A7C90125ACC4D1004355D6 /* MockDataStoreResponders.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MockDataStoreResponders.swift; sourceTree = "<group>"; };
+		21A4F26225A3CD3D00E1047D /* List+Pagination.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "List+Pagination.swift"; sourceTree = "<group>"; };
+		21A4F8F225A77D9100E1047D /* ListPaginationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ListPaginationTests.swift; sourceTree = "<group>"; };
 		21AD4255249BFFDF0016FE95 /* DeprecatedTodo.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = DeprecatedTodo.swift; path = Deprecated/DeprecatedTodo.swift; sourceTree = "<group>"; };
 		21C395B2245729EC00597EA2 /* AppSyncErrorType.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppSyncErrorType.swift; sourceTree = "<group>"; };
 		21D79FD9237617C60057D00D /* SubscriptionEvent.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SubscriptionEvent.swift; sourceTree = "<group>"; };
@@ -1945,6 +1951,7 @@
 			children = (
 				21A7C8A825ACB68C004355D6 /* Collection */,
 				21665EF4259A947100841696 /* ListTests.swift */,
+				21A4F8F225A77D9100E1047D /* ListPaginationTests.swift */,
 			);
 			path = Model;
 			sourceTree = "<group>";
@@ -2892,6 +2899,8 @@
 				217856BD2383322700A30D19 /* GraphQLMutationType.swift */,
 				21409C4C23847E41000A53C9 /* LabelType.swift */,
 				217856BA2383320900A30D19 /* GraphQLQueryType.swift */,
+				218DC969258A73AF00D59FC6 /* CoreError.swift */,
+				218DC979258A73CA00D59FC6 /* Paginatable.swift */,
 			);
 			name = "Recovered References";
 			sourceTree = "<group>";
@@ -2969,6 +2978,7 @@
 				21A4ED76259CDF6800E1047D /* List+Combine.swift */,
 				21A4ED6D259CDA4600E1047D /* List+LazyLoad.swift */,
 				B9FAA1242388BE48009414B4 /* List+Model.swift */,
+				21A4F26225A3CD3D00E1047D /* List+Pagination.swift */,
 			);
 			path = Collection;
 			sourceTree = "<group>";
@@ -5065,6 +5075,7 @@
 				B9A329CC243559BF00C5B80C /* TimeInterval+Helper.swift in Sources */,
 				950A26DC23D15D7E00D92B19 /* PredictionsTextToSpeechOperation.swift in Sources */,
 				21A4ED6E259CDA4600E1047D /* List+LazyLoad.swift in Sources */,
+				21A4F26325A3CD3D00E1047D /* List+Pagination.swift in Sources */,
 				FAC235A3227A5ED000424678 /* HubChannel.swift in Sources */,
 				FAA2E8CE23A02A8100E420EA /* PredictionsCategory+Resettable.swift in Sources */,
 				FAC23516227A053D00424678 /* StorageCategoryBehavior.swift in Sources */,
@@ -5166,6 +5177,7 @@
 				FA5D76AF23947E9C00489864 /* Model+CodableTests.swift in Sources */,
 				FA58456B24DA31370028D65A /* AmplifyInProcessReportingOperationChainedTests.swift in Sources */,
 				FA1B964E24BF5FA70002B90A /* AmplifyOperationCombineTests.swift in Sources */,
+				21A4F8F325A77D9100E1047D /* ListPaginationTests.swift in Sources */,
 				FAAFAF3323904BA4002CF932 /* AtomicValue+NumericTests.swift in Sources */,
 				FA176EDD2385943000C5C5F9 /* NotificationListeningAnalyticsPlugin.swift in Sources */,
 				FA1C80BF258686CC006160E9 /* AWSHubPluginAmplifyVersionableTests.swift in Sources */,

Amplify/Categories/DataStore/Model/Collection/List+LazyLoad.swift

@@ -14,6 +14,35 @@ extension List {
 
     // MARK: - Asynchronous API
 
+    /// Call this to initialize the collection if you have retrieved the list by traversing from your model objects
+    /// to its associated children objects. For example, a Post model may contain a list of Comments. By retrieving the
+    /// post object and traversing to the comments, the comments are not retrieved from the data source until this
+    /// method is called. Data will be retrieved based on the plugin's data source and may have different failure
+    /// conditions--for example, a data source that requires network connectivity may fail if the network is
+    /// unavailable. Alternately, you can trigger an implicit `fetch` by invoking the Collection methods (such as using
+    /// `map`, or iterating in a `for/in` loop) on the List, which will retrieve data if it hasn't already been
+    /// retrieved. In such cases, the time to perform that operation will include the time required to request data
+    /// from the underlying data source.
+    ///
+    /// If you have directly created this list object (for example, by calling `List(elements:)`) then the collection
+    /// has already been initialized and calling this method will have no effect.
+    public func fetch(_ completion: @escaping (Result<Void, CoreError>) -> Void) {
+        guard case .notLoaded = loadedState else {
+            completion(.successfulVoid)
+            return
+        }
+
+        listProvider.load { result in
+            switch result {
+            case .success(let elements):
+                self.elements = elements
+                completion(.success(()))
+            case .failure(let coreError):
+                completion(.failure(coreError))
+            }
+        }
+    }
+
     /// Call this to initialize the collection if you have retrieved the list by traversing from your model objects
     /// to its associated children objects. For example, a Post model may contain a list of Comments. By retrieving the
     /// post object and traversing to the comments, the comments are not retrieved from the data source until this
@@ -26,6 +55,7 @@ extension List {
     ///
     /// If you have directly created this list object (for example, by calling `List(elements:)`) then the collection
     /// has already been initialized and calling this method will have no effect.
+    @available(*, deprecated, message: "Use fetch(completion:) instead.")
     public func load(_ completion: DataStoreCallback<[Element]>) {
         if case .loaded(let elements) = loadedState {
             completion(.success(elements))
@@ -56,7 +86,7 @@ extension List {
     ///
     /// - Returns: the current instance after data was loaded.
     /// - seealso: `load(completion:)`
-    @available(*, deprecated, message: "Use load(completion:) instead.")
+    @available(*, deprecated, message: "Use fetch(completion:) instead.")
     public func load() -> Self {
         guard case .notLoaded = loadedState else {
             return self

Amplify/Categories/DataStore/Model/Collection/List+Model.swift

@@ -14,7 +14,7 @@ import Combine
 /// before returning the data to you. Consumers must be aware that multiple calls to the data source and then stored
 /// into this object will happen simultaneously if the object is used from different threads, thus not thread safe.
 /// Lazy loading is idempotent and will return the stored results on subsequent access.
-public class List<ModelType: Model>: Collection, Codable, ExpressibleByArrayLiteral {
+public class List<ModelType: Model>: Collection, Codable, ExpressibleByArrayLiteral, ModelListMarker {
     public typealias Index = Int
     public typealias Element = ModelType
 
@@ -36,7 +36,7 @@ public class List<ModelType: Model>: Collection, Codable, ExpressibleByArrayLite
     /// provider's data source. This is not thread safe as it can be performed from multiple threads, however the
     /// provider's call to `load` should be idempotent and should result in the final loaded state. An attempt to set
     /// this again will result in no-op and will not overwrite the existing loaded data.
-    var elements: [Element] {
+    public internal(set) var elements: [Element] {
         get {
             switch loadedState {
             case .loaded(let elements):
@@ -144,7 +144,7 @@ public class List<ModelType: Model>: Collection, Codable, ExpressibleByArrayLite
     required convenience public init(from decoder: Decoder) throws {
         for listDecoder in ModelListDecoderRegistry.listDecoders.get() {
             if listDecoder.shouldDecode(modelType: ModelType.self, decoder: decoder) {
-                let listProvider = try listDecoder.getListProvider(modelType: ModelType.self, decoder: decoder)
+                let listProvider = try listDecoder.makeListProvider(modelType: ModelType.self, decoder: decoder)
                 self.init(listProvider: listProvider)
                 return
             }

Amplify/Categories/DataStore/Model/Collection/List+Pagination.swift

@@ -0,0 +1,50 @@
+//
+// Copyright Amazon.com Inc. or its affiliates.
+// All Rights Reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+import Foundation
+
+extension List {
+
+    /// Check if there is subsequent data to retrieve. If true, the next page can be retrieved using
+    /// `getNextPage(completion:)`. Calling `hasNextPage()` will load the underlying elements from the data source if not yet
+    /// loaded before.
+    public func hasNextPage() -> Bool {
+        switch loadedState {
+        case .loaded:
+            return listProvider.hasNextPage()
+        case .notLoaded:
+            let result = listProvider.load()
+            switch result {
+            case .success(let elements):
+                self.elements = elements
+                return listProvider.hasNextPage()
+            case .failure(let coreError):
+                Amplify.log.error(error: coreError)
+                return false
+            }
+        }
+    }
+
+    /// Retrieve the next page as a new in-memory List object. Calling `getNextPage(completion:)` will load the
+    /// underlying elements of the receiver from the data source if not yet loaded before
+    public func getNextPage(completion: @escaping (Result<List<Element>, CoreError>) -> Void) {
+        switch loadedState {
+        case .loaded:
+            listProvider.getNextPage(completion: completion)
+        case .notLoaded:
+            let result = listProvider.load()
+            switch result {
+            case .success(let elements):
+                self.elements = elements
+                listProvider.getNextPage(completion: completion)
+            case .failure(let coreError):
+                Amplify.log.error(error: coreError)
+                completion(.failure(coreError))
+            }
+        }
+    }
+}

Amplify/Categories/DataStore/Model/Internal/ModelListDecoder.swift

@@ -37,6 +37,6 @@ extension ModelListDecoderRegistry {
 /// change.
 public protocol ModelListDecoder {
     static func shouldDecode<ModelType: Model>(modelType: ModelType.Type, decoder: Decoder) -> Bool
-    static func getListProvider<ModelType: Model>(
+    static func makeListProvider<ModelType: Model>(
         modelType: ModelType.Type, decoder: Decoder) throws -> AnyModelListProvider<ModelType>
 }

Amplify/Categories/DataStore/Model/Internal/ModelListProvider.swift

@@ -8,6 +8,14 @@
 import Foundation
 import Combine
 
+/// Empty protocol used as a marker to detect when the type is a `List`
+///
+/// - Warning: Although this has `public` access, it is intended for internal & codegen use and should not be used
+/// directly by host applications. The behavior of this may change without warning. Though it is not used by host
+/// application making any change to these `public` types should be backward compatible, otherwise it will be a breaking
+/// change.
+public protocol ModelListMarker { }
+
 /// - Warning: Although this has `public` access, it is intended for internal & codegen use and should not be used
 /// directly by host applications. The behavior of this may change without warning. Though it is not used by host
 /// application making any change to these `public` types should be backward compatible, otherwise it will be a breaking
@@ -21,11 +29,13 @@ public protocol ModelListProvider {
     ///  Retrieve the array of `Element` from the data source asychronously.
     func load(completion: @escaping (Result<[Element], CoreError>) -> Void)
 
-    /// Checks if there is subsequent data to retrieve. If true, the next page can be retrieved using
-    /// `getNextPage(completion:)`
+    /// Check if there is subsequent data to retrieve. This method always returns false if the underlying provider is
+    /// not loaded. Make sure the underlying data is loaded by calling `load(completion)` before calling this method.
+    /// If true, the next page can be retrieved using `getNextPage(completion:)`.
     func hasNextPage() -> Bool
 
-    /// Retrieves the next page as a new in-memory List object asynchronously.
+    /// Asynchronously retrieve the next page as a new in-memory List object. Returns a failure if there
+    /// is no next page of results. You can validate whether the list has another page with `hasNextPage()`.
     func getNextPage(completion: @escaping (Result<List<Element>, CoreError>) -> Void)
 }