test: fix failing datastore integration tests (#3317)

atierian · web-flow · commit 99f66d390af9 · 2023-10-20T17:19:29.000-04:00
diff --git a/.github/workflows/integ_test_datastore_auth_cognito.yml b/.github/workflows/integ_test_datastore_auth_cognito.yml
@@ -35,6 +35,7 @@ jobs:
           scheme: AWSDataStorePluginAuthCognitoTests
           destination: 'platform=iOS Simulator,name=iPhone 14,OS=16.4'
           xcode_path: '/Applications/Xcode_14.3.app'
+          other_flags: '-test-iterations 3 -retry-tests-on-failure'
 
   datastore-integration-auth-cognito-test-tvOS:
     timeout-minutes: 60
@@ -64,6 +65,7 @@ jobs:
           destination: platform=tvOS Simulator,name=Apple TV 4K (3rd generation),OS=latest
           sdk: appletvsimulator
           xcode_path: '/Applications/Xcode_14.3.app'
+          other_flags: '-test-iterations 3 -retry-tests-on-failure'
 
   datastore-integration-auth-cognito-test-watchOS:
     timeout-minutes: 60
@@ -93,3 +95,4 @@ jobs:
           destination: platform=watchOS Simulator,name=Apple Watch Series 8 (45mm),OS=latest
           sdk: watchsimulator
           xcode_path: '/Applications/Xcode_14.3.app'
+          other_flags: '-test-iterations 3 -retry-tests-on-failure'
diff --git a/.github/workflows/integ_test_datastore_auth_iam.yml b/.github/workflows/integ_test_datastore_auth_iam.yml
@@ -35,6 +35,7 @@ jobs:
           scheme: AWSDataStorePluginAuthIAMTests
           destination: 'platform=iOS Simulator,name=iPhone 14,OS=16.4'
           xcode_path: '/Applications/Xcode_14.3.app'
+          other_flags: '-test-iterations 3 -retry-tests-on-failure'
 
   datastore-integration-auth-iam-test-tvOS:
     timeout-minutes: 60
@@ -64,6 +65,7 @@ jobs:
           destination: platform=tvOS Simulator,name=Apple TV 4K (3rd generation),OS=latest
           sdk: appletvsimulator
           xcode_path: '/Applications/Xcode_14.3.app'
+          other_flags: '-test-iterations 3 -retry-tests-on-failure'
 
   datastore-integration-auth-iam-test-watchOS:
     timeout-minutes: 60
@@ -93,3 +95,4 @@ jobs:
           destination: platform=watchOS Simulator,name=Apple Watch Series 8 (45mm),OS=latest
           sdk: watchsimulator
           xcode_path: '/Applications/Xcode_14.3.app'
+          other_flags: '-test-iterations 3 -retry-tests-on-failure'
diff --git a/.github/workflows/integ_test_datastore_base.yml b/.github/workflows/integ_test_datastore_base.yml
@@ -35,6 +35,7 @@ jobs:
           scheme: AWSDataStorePluginIntegrationTests
           destination: 'platform=iOS Simulator,name=iPhone 14,OS=16.4'
           xcode_path: '/Applications/Xcode_14.3.app'
+          other_flags: '-test-iterations 3 -retry-tests-on-failure'
 
   datastore-integration-test-base-tvOS:
     timeout-minutes: 60
@@ -64,6 +65,7 @@ jobs:
           destination: platform=tvOS Simulator,name=Apple TV 4K (3rd generation),OS=latest
           sdk: appletvsimulator
           xcode_path: '/Applications/Xcode_14.3.app'
+          other_flags: '-test-iterations 3 -retry-tests-on-failure'
 
   datastore-integration-test-base-watchOS:
     timeout-minutes: 60
@@ -93,3 +95,4 @@ jobs:
           destination: platform=watchOS Simulator,name=Apple Watch Series 8 (45mm),OS=latest
           sdk: watchsimulator
           xcode_path: '/Applications/Xcode_14.3.app'
+          other_flags: '-test-iterations 3 -retry-tests-on-failure'
diff --git a/.github/workflows/integ_test_datastore_cpk.yml b/.github/workflows/integ_test_datastore_cpk.yml
@@ -35,6 +35,7 @@ jobs:
           scheme: AWSDataStorePluginCPKTests
           destination: 'platform=iOS Simulator,name=iPhone 14,OS=16.4'
           xcode_path: '/Applications/Xcode_14.3.app'
+          other_flags: '-test-iterations 3 -retry-tests-on-failure'
 
   datastore-integration-cpk-test-tvOS:
     timeout-minutes: 60
@@ -64,6 +65,7 @@ jobs:
           destination: platform=tvOS Simulator,name=Apple TV 4K (3rd generation),OS=latest
           sdk: appletvsimulator
           xcode_path: '/Applications/Xcode_14.3.app'
+          other_flags: '-test-iterations 3 -retry-tests-on-failure'
 
   datastore-integration-cpk-test-watchOS:
     timeout-minutes: 60
@@ -93,3 +95,4 @@ jobs:
           destination: platform=watchOS Simulator,name=Apple Watch Series 8 (45mm),OS=latest
           sdk: watchsimulator
           xcode_path: '/Applications/Xcode_14.3.app'
+          other_flags: '-test-iterations 3 -retry-tests-on-failure'
diff --git a/.github/workflows/integ_test_datastore_lazy_load.yml b/.github/workflows/integ_test_datastore_lazy_load.yml
@@ -35,6 +35,7 @@ jobs:
           scheme: AWSDataStorePluginLazyLoadTests
           destination: 'platform=iOS Simulator,name=iPhone 14,OS=16.4'
           xcode_path: '/Applications/Xcode_14.3.app'
+          other_flags: '-test-iterations 3 -retry-tests-on-failure'
 
   datastore-integration-lazy-load-test-tvOS:
     timeout-minutes: 60
@@ -64,6 +65,7 @@ jobs:
           destination: platform=tvOS Simulator,name=Apple TV 4K (3rd generation),OS=latest
           sdk: appletvsimulator
           xcode_path: '/Applications/Xcode_14.3.app'
+          other_flags: '-test-iterations 3 -retry-tests-on-failure'
 
   datastore-integration-lazy-load-test-watchOS:
     timeout-minutes: 60
@@ -93,3 +95,4 @@ jobs:
           destination: platform=watchOS Simulator,name=Apple Watch Series 8 (45mm),OS=latest
           sdk: watchsimulator
           xcode_path: '/Applications/Xcode_14.3.app'
+          other_flags: '-test-iterations 3 -retry-tests-on-failure'
diff --git a/.github/workflows/integ_test_datastore_multi_auth.yml b/.github/workflows/integ_test_datastore_multi_auth.yml
@@ -35,6 +35,7 @@ jobs:
           scheme: AWSDataStorePluginMultiAuthTests
           destination: 'platform=iOS Simulator,name=iPhone 14,OS=16.4'
           xcode_path: '/Applications/Xcode_14.3.app'
+          other_flags: '-test-iterations 3 -retry-tests-on-failure'
 
   datastore-integration-multi-auth-test-tvOS:
     timeout-minutes: 60
@@ -64,6 +65,7 @@ jobs:
           destination: platform=tvOS Simulator,name=Apple TV 4K (3rd generation),OS=latest
           sdk: appletvsimulator
           xcode_path: '/Applications/Xcode_14.3.app'
+          other_flags: '-test-iterations 3 -retry-tests-on-failure'
 
   datastore-integration-multi-auth-test-watchOS:
     timeout-minutes: 60
@@ -93,3 +95,4 @@ jobs:
           destination: platform=watchOS Simulator,name=Apple Watch Series 8 (45mm),OS=latest
           sdk: watchsimulator
           xcode_path: '/Applications/Xcode_14.3.app'
+          other_flags: '-test-iterations 3 -retry-tests-on-failure'
diff --git a/.github/workflows/integ_test_datastore_v2.yml b/.github/workflows/integ_test_datastore_v2.yml
@@ -35,6 +35,7 @@ jobs:
           scheme: AWSDataStorePluginV2Tests
           destination: 'platform=iOS Simulator,name=iPhone 14,OS=16.4'
           xcode_path: '/Applications/Xcode_14.3.app'
+          other_flags: '-test-iterations 3 -retry-tests-on-failure'
 
   datastore-integration-v2-test-tvOS:
     timeout-minutes: 60
@@ -64,6 +65,7 @@ jobs:
           destination: platform=tvOS Simulator,name=Apple TV 4K (3rd generation),OS=latest
           sdk: appletvsimulator
           xcode_path: '/Applications/Xcode_14.3.app'
+          other_flags: '-test-iterations 3 -retry-tests-on-failure'
 
   datastore-integration-v2-test-watchOS:
     timeout-minutes: 60
@@ -93,3 +95,4 @@ jobs:
           destination: platform=watchOS Simulator,name=Apple Watch Series 8 (45mm),OS=latest
           sdk: watchsimulator
           xcode_path: '/Applications/Xcode_14.3.app'
+          other_flags: '-test-iterations 3 -retry-tests-on-failure'
diff --git a/AmplifyPlugins/DataStore/Tests/DataStoreHostApp/AWSDataStorePluginAuthCognitoTests/AWSDataStoreCategoryPluginAuthIntegrationTests.swift b/AmplifyPlugins/DataStore/Tests/DataStoreHostApp/AWSDataStorePluginAuthCognitoTests/AWSDataStoreCategoryPluginAuthIntegrationTests.swift
@@ -60,7 +60,7 @@ class AWSDataStoreCategoryPluginAuthIntegrationTests: AWSDataStoreAuthBaseTest {
 
         try await signIn(user: user1)
 
-        await fulfillment(of: [syncReceivedInvoked], timeout: TestCommonConstants.networkTimeout)
+        await fulfillment(of: [syncReceivedInvoked], timeout: 60)
         Amplify.Hub.removeListener(syncReceivedListener)
         guard let remoteTodo = remoteTodoOptional else {
             XCTFail("Should have received a SyncReceived event with the remote note reconciled to local store")
diff --git a/AmplifyPlugins/DataStore/Tests/DataStoreHostApp/AWSDataStorePluginIntegrationTests/DataStoreObserveQueryTests.swift b/AmplifyPlugins/DataStore/Tests/DataStoreHostApp/AWSDataStorePluginIntegrationTests/DataStoreObserveQueryTests.swift
@@ -97,9 +97,16 @@ class DataStoreObserveQueryTests: SyncEngineIntegrationTestBase {
             }
         }.store(in: &cancellables)
         let receivedPost = expectation(description: "received Post")
-        try await savePostAndWaitForSync(Post(title: "title", content: "content", createdAt: .now()),
-                                         postSyncedExpctation: receivedPost)
-        await fulfillment(of: [snapshotWithIsSynced, receivedPost], timeout: 100)
+        receivedPost.assertForOverFulfill = false
+        try await savePostAndWaitForSync(
+            Post(
+                title: "title",
+                content: "content",
+                createdAt: .now()
+            ),
+            postSyncedExpctation: receivedPost
+        )
+        await fulfillment(of: [snapshotWithIsSynced], timeout: 100)
         
         XCTAssertTrue(snapshots.count >= 2)
         XCTAssertFalse(snapshots[0].isSynced)
@@ -296,7 +303,7 @@ class DataStoreObserveQueryTests: SyncEngineIntegrationTestBase {
         let receivedPost = expectation(description: "received Post")
         try await savePostAndWaitForSync(Post(title: "title", content: "content", createdAt: .now()),
                                          postSyncedExpctation: receivedPost)
-        await fulfillment(of: [snapshotWithIsSynced, receivedPost], timeout: 30)
+        await fulfillment(of: [snapshotWithIsSynced], timeout: 30)
         XCTAssertTrue(snapshots.count >= 2)
         XCTAssertFalse(snapshots[0].isSynced)
         XCTAssertTrue(snapshots.last!.isSynced)
