fix(datastore): observe API mutation event decode to model successfully (#2684)

* chore: enable lazy loading integration tests

* fix(datastore): observe API mutation event decode to model successfully

* update branches for GH workflows

* fix typo

Author: lawmicha

.github/workflows/integ_test_datastore_lazy_load.yml

@@ -1,6 +1,8 @@
 name: DataStore Lazy Load Tests
 on:
   workflow_dispatch:
+  push:
+    branches: [data-dev-preview]
 
 permissions:
     id-token: write
@@ -27,7 +29,7 @@ jobs:
           aws_region: ${{ secrets.AWS_REGION }}
           aws_s3_bucket: ${{ secrets.AWS_S3_BUCKET_INTEG }}
 
-  datastore-integration-v2-test:
+  datastore-integration-lazy-load-test:
     timeout-minutes: 30
     needs: prepare-for-test
     runs-on: macos-12
@@ -53,5 +55,3 @@ jobs:
         with:
           project_path: ./AmplifyPlugins/DataStore/Tests/DataStoreHostApp
           scheme: AWSDataStorePluginLazyLoadTests
-
- 

Amplify/Categories/DataStore/Model/Internal/ModelListProvider.swift

@@ -52,6 +52,9 @@ public protocol ModelListProvider {
     /// Asynchronously retrieve the next page as a new in-memory List object. Returns a failure if there
     /// is no next page of results. You can validate whether the list has another page with `hasNextPage()`.
     func getNextPage() async throws -> List<Element>
+    
+    /// Custom encoder
+    func encode(to encoder: Encoder) throws
 }
 
 /// - Warning: Although this has `public` access, it is intended for internal & codegen use and should not be used
@@ -63,14 +66,16 @@ public struct AnyModelListProvider<Element: Model>: ModelListProvider {
     private let loadAsync: () async throws -> [Element]
     private let hasNextPageClosure: () -> Bool
     private let getNextPageAsync: () async throws -> List<Element>
-
+    private let encodeClosure: (Encoder) throws -> Void
+    
     public init<Provider: ModelListProvider>(
         provider: Provider
     ) where Provider.Element == Self.Element {
         self.getStateClosure = provider.getState
         self.loadAsync = provider.load
         self.hasNextPageClosure = provider.hasNextPage
         self.getNextPageAsync = provider.getNextPage
+        self.encodeClosure = provider.encode
     }
 
     public func getState() -> ModelListProviderState<Element> {
@@ -88,6 +93,10 @@ public struct AnyModelListProvider<Element: Model>: ModelListProvider {
     public func getNextPage() async throws -> List<Element> {
         try await getNextPageAsync()
     }
+    
+    public func encode(to encoder: Encoder) throws {
+        try encodeClosure(encoder)
+    }
 }
 
 /// - Warning: Although this has `public` access, it is intended for internal & codegen use and should not be used

Amplify/Categories/DataStore/Model/Internal/ModelProvider.swift

@@ -47,6 +47,8 @@ public protocol ModelProvider {
     func load() async throws -> Element?
 
     func getState() -> ModelProviderState<Element>
+    
+    func encode(to encoder: Encoder) throws
 }
 
 /// - Warning: Although this has `public` access, it is intended for internal & codegen use and should not be used
@@ -57,18 +59,25 @@ public struct AnyModelProvider<Element: Model>: ModelProvider {
 
     private let loadAsync: () async throws -> Element?
     private let getStateClosure: () -> ModelProviderState<Element>
+    private let encodeClosure: (Encoder) throws -> Void
 
     public init<Provider: ModelProvider>(provider: Provider) where Provider.Element == Self.Element {
         self.loadAsync = provider.load
         self.getStateClosure = provider.getState
+        self.encodeClosure = provider.encode
     }
+    
     public func load() async throws -> Element? {
         try await loadAsync()
     }
 
     public func getState() -> ModelProviderState<Element> {
         getStateClosure()
     }
+    
+    public func encode(to encoder: Encoder) throws {
+        try encodeClosure(encoder)
+    }
 }
 
 /// - Warning: Although this has `public` access, it is intended for internal & codegen use and should not be used

Amplify/Categories/DataStore/Model/Lazy/ArrayLiteralListProvider.swift

@@ -45,4 +45,8 @@ public struct ArrayLiteralListProvider<Element: Model>: ModelListProvider {
                                                        "Don't call this method",
                                                        nil)
     }
+    
+    public func encode(to encoder: Encoder) throws {
+        try elements.encode(to: encoder)
+    }
 }

Amplify/Categories/DataStore/Model/Lazy/DefaultModelProvider.swift

@@ -33,4 +33,14 @@ public struct DefaultModelProvider<Element: Model>: ModelProvider {
     public func getState() -> ModelProviderState<Element> {
         loadedState
     }
+    
+    public func encode(to encoder: Encoder) throws {
+        switch loadedState {
+        case .notLoaded(let identifiers):
+            var container = encoder.singleValueContainer()
+            try container.encode(identifiers)
+        case .loaded(let element):
+            try element.encode(to: encoder)
+        }
+    }
 }

Amplify/Categories/DataStore/Model/Lazy/LazyReference.swift

@@ -86,7 +86,7 @@ public class LazyReference<ModelType: Model>: Codable, _LazyReferenceValue {
 
     // MARK: - Codable implementation
 
-    /// Decodable implementation is delegated to the underlying `self.reference`.
+    /// Decodable implementation is delegated to the ModelProviders.
     required convenience public init(from decoder: Decoder) throws {
         for modelDecoder in ModelProviderRegistry.decoders.get() {
             if let modelProvider = modelDecoder.decode(modelType: ModelType.self, decoder: decoder) {
@@ -107,15 +107,9 @@ public class LazyReference<ModelType: Model>: Codable, _LazyReferenceValue {
         self.init(identifiers: nil)
     }
 
-    /// Encodable implementation is delegated to the underlying `self.reference`.
+    /// Encodable implementation is delegated to the underlying ModelProviders.
     public func encode(to encoder: Encoder) throws {
-        switch loadedState {
-        case .notLoaded(let identifiers):
-            var container = encoder.singleValueContainer()
-            try container.encode(identifiers)
-        case .loaded(let element):
-            try element.encode(to: encoder)
-        }
+        try modelProvider.encode(to: encoder)
     }
 
     // MARK: - APIs

Amplify/Categories/DataStore/Model/Lazy/List+Model.swift

@@ -158,11 +158,6 @@ public class List<ModelType: Model>: Collection, Codable, ExpressibleByArrayLite
     }
 
     public func encode(to encoder: Encoder) throws {
-        switch loadedState {
-        case .notLoaded:
-            try [Element]().encode(to: encoder)
-        case .loaded(let elements):
-            try elements.encode(to: encoder)
-        }
+        try listProvider.encode(to: encoder)
     }
 }

AmplifyPlugins/API/Sources/AWSAPIPlugin/Core/AppSyncListProvider.swift

@@ -68,7 +68,9 @@ public class AppSyncListProvider<Element: Model>: ModelListProvider {
          apiName: String? = nil,
          limit: Int? = nil,
          filter: [String: Any]? = nil) {
-        self.loadedState = .loaded(elements: elements, nextToken: nextToken, filter: filter)
+        self.loadedState = .loaded(elements: elements,
+                                   nextToken: nextToken,
+                                   filter: filter)
         self.apiName = apiName
         self.limit = limit
     }
@@ -100,8 +102,6 @@ public class AppSyncListProvider<Element: Model>: ModelListProvider {
         }
     }
 
-    
-    
     //// Internal `load` to perform the retrieval of the first page and storing it in memory
     func load(associatedIdentifiers: [String],
               associatedField: String) async throws -> [Element] {
@@ -146,14 +146,14 @@ public class AppSyncListProvider<Element: Model>: ModelListProvider {
                                            filter: filter)
                 return listResponse.items
             case .failure(let graphQLError):
-                Amplify.API.log.error(error: graphQLError)
+                log.error(error: graphQLError)
                 throw CoreError.listOperation(
                     "The AppSync response returned successfully with GraphQL errors.",
                     "Check the underlying error for the failed GraphQL response.",
                     graphQLError)
             }
         } catch let apiError as APIError {
-            Amplify.API.log.error(error: apiError)
+            log.error(error: apiError)
             throw CoreError.listOperation("The AppSync request failed",
                                           "See underlying `APIError` for more details.",
                                           apiError)
@@ -203,13 +203,13 @@ public class AppSyncListProvider<Element: Model>: ModelListProvider {
                 _ = try await nextPageList.fetch()
                 return nextPageList
             case .failure(let graphQLError):
-                Amplify.API.log.error(error: graphQLError)
+                log.error(error: graphQLError)
                 throw CoreError.listOperation("""
                     The AppSync request was processed by the service, but the response contained GraphQL errors.
                     """, "Check the underlying error for the failed GraphQL response.", graphQLError)
             }
         } catch let apiError as APIError {
-            Amplify.API.log.error(error: apiError)
+            log.error(error: apiError)
             throw CoreError.listOperation("The AppSync request failed",
                                           "See underlying `APIError` for more details.",
                                           apiError)
@@ -218,6 +218,26 @@ public class AppSyncListProvider<Element: Model>: ModelListProvider {
         }
     }
 
+    public func encode(to encoder: Encoder) throws {
+        switch loadedState {
+        case .notLoaded(let associatedIdentifiers, let associatedField):
+            let metadata = AppSyncListDecoder.Metadata.init(
+                appSyncAssociatedIdentifiers: associatedIdentifiers,
+                appSyncAssociatedField: associatedField,
+                apiName: apiName)
+            var container = encoder.singleValueContainer()
+            try container.encode(metadata)
+        case .loaded(let elements, _, _):
+            // This does not encode the `nextToken` or `filter`, which means the data is essentially dropped.
+            // If the encoded list is later decoded, the existing `elements` will be available but will be missing
+            // the metadata to reflect whether there's a next page or not. Encoding the metadata is rather difficult
+            // with the filter being `Any` type and is not a call pattern that is recommended/supported. The extent
+            // of this supported flow is to allow encoding and decoding of the `elements`. To get the
+            // latest data, make the call to your data source directly through **Amplify.API**.
+            try elements.encode(to: encoder)
+        }
+    }
+    
     // MARK: - Helpers
 
     /// Retrieve the column names for the specified field `field` for this schema.
@@ -239,3 +259,5 @@ public class AppSyncListProvider<Element: Model>: ModelListProvider {
     }
 
 }
+
+extension AppSyncListProvider: DefaultLogger { }

AmplifyPlugins/API/Sources/AWSAPIPlugin/Core/AppSyncModelProvider.swift

@@ -58,6 +58,16 @@ public class AppSyncModelProvider<ModelType: Model>: ModelProvider {
     public func getState() -> ModelProviderState<ModelType> {
         loadedState
     }
+    
+    public func encode(to encoder: Encoder) throws {
+        switch loadedState {
+        case .notLoaded(let identifiers):
+            var container = encoder.singleValueContainer()
+            try container.encode(identifiers)
+        case .loaded(let element):
+            try element.encode(to: encoder)
+        }
+    }
 }
 
 extension AppSyncModelProvider: DefaultLogger { }

AmplifyPlugins/API/Tests/APIHostApp/AWSAPIPluginLazyLoadTests/README.md

@@ -8,9 +8,11 @@
 
 1. `amplify init`
 
-These tests were provisioned with V2 Transform:, and updates to `cli.json` 
-- "respectprimarykeyattributesonconnectionfield": true
-- "TODOlazyLoadiOS": true 
+These tests were provisioned with V2 Transform, CPK enabled, and with the lazy loading feature flag. Review or make updates to cli.json
+
+"transformerversion":2
+"respectprimarykeyattributesonconnectionfield": true
+"generateModelsForLazyLoadAndCustomSelectionSet": true
 
 2. `amplify add api`