Implement retry logic for integration tests to avoid flakiness

VladislavFitz · VladislavFitz · commit 14bfdccf0318 · 2020-07-28T00:56:33.000+02:00
diff --git a/Tests/AlgoliaSearchClientTests/Integration/APIKeysIntegrationTests.swift b/Tests/AlgoliaSearchClientTests/Integration/APIKeysIntegrationTests.swift
@@ -17,7 +17,12 @@ class APIKeysIntegrationTests: OnlineTestCase {
   
   var keyToDelete: APIKey?
   
-  func testAPIKeys() throws {
+  override var retryableTests: [() throws -> Void] {
+    [apiKeys]
+  }
+
+  
+  func apiKeys() throws {
     
     let parameters = APIKeyParameters(ACLs: [.search])
       .set(\.description, to: "A description")
diff --git a/Tests/AlgoliaSearchClientTests/Integration/AnalyticsIntegrationsTests.swift b/Tests/AlgoliaSearchClientTests/Integration/AnalyticsIntegrationsTests.swift
@@ -11,12 +11,20 @@ import XCTest
 
 class AnalyticsIntegrationTests: OnlineTestCase {
   
-  func testBrowsing() throws {
+  override var retryableTests: [() throws -> Void] {
+    [
+      browsing,
+      abTesting,
+      aaTesting
+    ]
+  }
+  
+  func browsing() throws {
     let analyticsClient = AnalyticsClient(appID: client.applicationID, apiKey: client.apiKey)
     let _ = try analyticsClient.browseAllABTests(hitsPerPage: 3)
   }
 
-  func testABTesting() throws {
+  func abTesting() throws {
     
     let analyticsClient = AnalyticsClient(appID: client.applicationID, apiKey: client.apiKey)
 
@@ -60,7 +68,7 @@ class AnalyticsIntegrationTests: OnlineTestCase {
     try AssertThrowsHTTPError(try analyticsClient.getABTest(withID: creation.wrapped.abTestID), statusCode: 404)
   }
     
-  func testAATesting() throws {
+  func aaTesting() throws {
     
     let analyticsClient = AnalyticsClient(appID: client.applicationID, apiKey: client.apiKey)
     let index = client.index(withName: "aa_testing")
diff --git a/Tests/AlgoliaSearchClientTests/Integration/BatchingIntegrationsTests.swift b/Tests/AlgoliaSearchClientTests/Integration/BatchingIntegrationsTests.swift
@@ -15,7 +15,11 @@ class BatchingIntegrationTests: OnlineTestCase {
     return "index_batching"
   }
   
-  func testBatching() throws {
+  override var retryableTests: [() throws -> Void] {
+    [batching]
+  }
+
+  func batching() throws {
     
     let records: [JSON] = [
       ["objectID":"one", "key": "value"],
diff --git a/Tests/AlgoliaSearchClientTests/Integration/BrowseIntegrationTests.swift b/Tests/AlgoliaSearchClientTests/Integration/BrowseIntegrationTests.swift
@@ -21,7 +21,15 @@ class BrowseIntegrationTests: OnlineTestCase {
     return "index_browse"
   }
   
-  func testBrowseObjects() throws {
+  override var retryableTests: [() throws -> Void] {
+    [
+      browseObjects,
+      browseRules,
+      browseSynonyms
+    ]
+  }
+  
+  func browseObjects() throws {
     
     struct Record: Codable {
       let objectID: String
@@ -46,7 +54,7 @@ class BrowseIntegrationTests: OnlineTestCase {
     
   }
   
-  func testBrowseRules() throws {
+  func browseRules() throws {
     
     let rules: [Rule] = (0...50).map { _ in
       return Rule(objectID: .random)
@@ -64,7 +72,7 @@ class BrowseIntegrationTests: OnlineTestCase {
     
   }
   
-  func testBrowseSynonyms() throws {
+  func browseSynonyms() throws {
     
     let synonyms: [Synonym] = (0...50).map { _ in
       switch Int.random(in: 0...3) {
diff --git a/Tests/AlgoliaSearchClientTests/Integration/CopyIndexIntegrationTests.swift b/Tests/AlgoliaSearchClientTests/Integration/CopyIndexIntegrationTests.swift
@@ -15,6 +15,15 @@ class CopyIndexIntergrationTests: OnlineTestCase {
     return index!
   }
   
+  override var retryableTests: [() throws -> Void] {
+    [
+      copySettings,
+      copyRules,
+      copySynonyms,
+      fullCopy
+    ]
+  }
+  
   let records: [JSON] = [
     ["objectID": "one", "company": "apple"],
     ["objectID": "two", "company": "algolia"]
@@ -26,7 +35,7 @@ class CopyIndexIntergrationTests: OnlineTestCase {
     .set(\.consequence, to: Rule.Consequence().set(\.automaticFacetFilters, to: [Rule.AutomaticFacetFilters(attribute: "company")]))
   let synonym = Synonym.placeholder(objectID: "google_placeholder", placeholder: "<GOOG>", replacements: ["Google", "GOOG"])
       
-  func testCopySettings() throws {
+  func copySettings() throws {
     
     let targetIndex = client.index(withName: "copy_index_settings")
     
@@ -43,7 +52,7 @@ class CopyIndexIntergrationTests: OnlineTestCase {
     
   }
   
-  func testCopyRules() throws {
+  func copyRules() throws {
     
     let targetIndex = client.index(withName: "copy_index_rules")
     
@@ -61,7 +70,7 @@ class CopyIndexIntergrationTests: OnlineTestCase {
 
   }
   
-  func testCopySynonyms() throws {
+  func copySynonyms() throws {
     
     let targetIndex = client.index(withName: "copy_index_synonyms")
     
@@ -78,7 +87,7 @@ class CopyIndexIntergrationTests: OnlineTestCase {
     
   }
   
-  func testFullCopy() throws {
+  func fullCopy() throws {
     
     let targetIndex = client.index(withName: "copy_index_full_copy")
     
diff --git a/Tests/AlgoliaSearchClientTests/Integration/ExistsIntegrationTests.swift b/Tests/AlgoliaSearchClientTests/Integration/ExistsIntegrationTests.swift
@@ -15,7 +15,11 @@ class ExistsIntegrationTests: OnlineTestCase {
     return "exists"
   }
   
-  func testExists() throws {
+  override var retryableTests: [() throws -> Void] {
+    [exists]
+  }
+  
+  func exists() throws {
     XCTAssertFalse(try index.exists())
     try index.saveObject(TestRecord(), autoGeneratingObjectID: true).wait()
     XCTAssertTrue(try index.exists())
diff --git a/Tests/AlgoliaSearchClientTests/Integration/IndexingIntegrationTests.swift b/Tests/AlgoliaSearchClientTests/Integration/IndexingIntegrationTests.swift
@@ -15,7 +15,15 @@ class IndexingIntegrationTests: OnlineTestCase {
     return "indexing"
   }
   
-  func testIndexing() throws {
+  override var retryableTests: [() throws -> Void] {
+    [
+      indexing,
+      saveGetObject,
+      saveGetObjectCallback
+    ]
+  }
+  
+  func indexing() throws {
     
     let objectID = ObjectID(rawValue: .init(randomWithLength: 10))
     let object = TestRecord(objectID: objectID)
@@ -107,7 +115,7 @@ class IndexingIntegrationTests: OnlineTestCase {
     
   }
 
-  func testSaveGetObject() throws {
+  func saveGetObject() throws {
 
     let object: JSON = [
       "testField1": "testValue1",
@@ -122,7 +130,7 @@ class IndexingIntegrationTests: OnlineTestCase {
     XCTAssertEqual(fetchedObject["testField3"], true)
   }
 
-  func testSaveGetObjectCallback() throws {
+  func saveGetObjectCallback() throws {
 
     let object: JSON = [
       "testField1": "testValue1",
diff --git a/Tests/AlgoliaSearchClientTests/Integration/InsightsIntegrationTests.swift b/Tests/AlgoliaSearchClientTests/Integration/InsightsIntegrationTests.swift
@@ -15,7 +15,11 @@ class InsightsIntegrationTests: OnlineTestCase {
     return "sending_events"
   }
   
-  func testInsights() throws {
+  override var retryableTests: [() throws -> Void] {
+    [insights]
+  }
+
+  func insights() throws {
     
     let insightsClient = InsightsClient(appID: client.applicationID, apiKey: client.apiKey)
     
diff --git a/Tests/AlgoliaSearchClientTests/Integration/LogsIntergrationTests.swift b/Tests/AlgoliaSearchClientTests/Integration/LogsIntergrationTests.swift
@@ -14,8 +14,12 @@ class LogsIntergrationTests: OnlineTestCase {
   override var indexNameSuffix: String? {
     "logs"
   }
+  
+  override var retryableTests: [() throws -> Void] {
+    [getLogs]
+  }
 
-  func testGetLogs() throws {
+  func getLogs() throws {
     try client.listIndices()
     try client.listIndices()
     let logs = try client.getLogs(offset: 0, length: 2, type: .all).logs
diff --git a/Tests/AlgoliaSearchClientTests/Integration/MultiClusterIntegrationTests.swift b/Tests/AlgoliaSearchClientTests/Integration/MultiClusterIntegrationTests.swift
@@ -56,11 +56,15 @@ class MultipleClusterIntegrationTests: OnlineTestCase {
   
   let date: Date = .init()
   
+  override var retryableTests: [() throws -> Void] {
+    [multiCluster]
+  }
+  
   func userID(_ id: String) -> UserID {
     return UserID(rawValue: TestIdentifier(date: date, suffix: id).rawValue)
   }
     
-  func testMultiCluster() throws {
+  func multiCluster() throws {
   
     let userID0 = userID("0")
     let userID1 = userID("1")
diff --git a/Tests/AlgoliaSearchClientTests/Integration/MultipleOperationsIntegrationTests.swift b/Tests/AlgoliaSearchClientTests/Integration/MultipleOperationsIntegrationTests.swift
@@ -23,6 +23,10 @@ class MultipleOperationsIntegrationTests: OnlineTestCase {
     return client.index(withName: "multiple_operations_dev")
   }()
   
+  override var retryableTests: [() throws -> Void] {
+    [multipleOperations]
+  }
+  
   override func setUpWithError() throws {
     try super.setUpWithError()
     try firstIndex.delete()
@@ -35,7 +39,7 @@ class MultipleOperationsIntegrationTests: OnlineTestCase {
     try secondIndex.delete()
   }
   
-  func testMultipleOperationsTests() throws {
+  func multipleOperations() throws {
     
     let object: JSON = ["firstName": "Jimmie"]
     
diff --git a/Tests/AlgoliaSearchClientTests/Integration/OnlineTestCase.swift b/Tests/AlgoliaSearchClientTests/Integration/OnlineTestCase.swift
@@ -14,6 +14,9 @@ class OnlineTestCase: XCTestCase {
   var client: SearchClient!
   var index: Index!
   let expectationTimeout: TimeInterval = 100
+  let retryCount: Int = 3
+  
+  var retryableTests: [() throws -> Void] { return [] }
 
   /// Abstract base class for online test cases.
   ///
@@ -43,6 +46,19 @@ class OnlineTestCase: XCTestCase {
 
     try index.delete()
   }
+  
+  func testRetryable() throws {
+    for test in retryableTests {
+      for attemptCount in 1...retryCount {
+        do {
+          try test()
+          break
+        } catch let error where attemptCount == retryCount {
+          throw Error.retryFailed(error)
+        }
+      }
+    }
+  }
 
   override func tearDownWithError() throws {
     try super.tearDownWithError()
@@ -54,5 +70,6 @@ class OnlineTestCase: XCTestCase {
 extension OnlineTestCase {
   enum Error: Swift.Error {
     case missingCredentials
+    case retryFailed(Swift.Error)
   }
 }
diff --git a/Tests/AlgoliaSearchClientTests/Integration/PersonalizationIntegrationTests.swift b/Tests/AlgoliaSearchClientTests/Integration/PersonalizationIntegrationTests.swift
@@ -11,7 +11,11 @@ import XCTest
 
 class PersonalizationIntegrationTests: OnlineTestCase {
   
-  func testPersonalization() throws {
+  override var retryableTests: [() throws -> Void] {
+    [personalization]
+  }
+
+  func personalization() throws {
     let recommendationClient = RecommendationClient(appID: client.applicationID, apiKey: client.apiKey, region: .custom("eu"))
     let _ = try recommendationClient.getPersonalizationStrategy()
   }
diff --git a/Tests/AlgoliaSearchClientTests/Integration/QueryRulesIntegrationTests.swift b/Tests/AlgoliaSearchClientTests/Integration/QueryRulesIntegrationTests.swift
@@ -13,7 +13,11 @@ class QueryRulesIntegrationTests: OnlineTestCase {
   
   override var indexNameSuffix: String? { return "rules" }
   
-  func testQueryRules() throws {
+  override var retryableTests: [() throws -> Void] {
+    [queryRules]
+  }
+  
+  func queryRules() throws {
     
     Logger.minSeverityLevel = .warning
     
diff --git a/Tests/AlgoliaSearchClientTests/Integration/SearchIntegrationTests.swift b/Tests/AlgoliaSearchClientTests/Integration/SearchIntegrationTests.swift
@@ -10,8 +10,12 @@ import XCTest
 @testable import AlgoliaSearchClient
 
 class SearchIntegrationTests: OnlineTestCase {
+  
+  override var retryableTests: [() throws -> Void] {
+    [search]
+  }
 
-  func testSearch() throws {
+  func search() throws {
     
     let employeesData = try Data(filename: "Employees.json")
     let employees = try JSONDecoder().decode([JSON].self, from: employeesData)
diff --git a/Tests/AlgoliaSearchClientTests/Integration/SecuredAPIKeysIntegrationTests.swift b/Tests/AlgoliaSearchClientTests/Integration/SecuredAPIKeysIntegrationTests.swift
@@ -10,8 +10,16 @@ import XCTest
 @testable import AlgoliaSearchClient
 
 class SecuredAPIKeysIntegrationTests: OnlineTestCase {
+  
+  override var retryableTests: [() throws -> Void] {
+    [
+      securedAPIKeys,
+      notExpiredKey,
+      expiredKey
+    ]
+  }
 
-  func testSecuredAPIKeys() throws {
+  func securedAPIKeys() throws {
     
     let index1 = client!.index(withName: "secured_api_keys")
     let index2 = client!.index(withName: "secured_api_keys_dev")
@@ -35,15 +43,15 @@ class SecuredAPIKeysIntegrationTests: OnlineTestCase {
 
   }
   
-  func testNotExpiredKey() throws {
+  func notExpiredKey() throws {
     let restriction = SecuredAPIKeyRestriction().set(\.validUntil, to: Date().addingTimeInterval(.minutes(10)).timeIntervalSince1970)
     let parentAPIKey = APIKey(rawValue: .init(randomWithLength: 64))
     let generatedAPIKey = client.generateSecuredApiKey(parentApiKey: parentAPIKey, with: restriction)
     let remainingValidity = try XCTUnwrap(client.getSecuredApiKeyRemainingValidity(generatedAPIKey))
     XCTAssertGreaterThan(remainingValidity, 0)
   }
   
-  func testExpiredKey() throws {
+  func expiredKey() throws {
     let restriction = SecuredAPIKeyRestriction().set(\.validUntil, to: Date().addingTimeInterval(-.minutes(10)).timeIntervalSince1970)
     let parentAPIKey = APIKey(rawValue: .init(randomWithLength: 64))
     let generatedAPIKey = client.generateSecuredApiKey(parentApiKey: parentAPIKey, with: restriction)
diff --git a/Tests/AlgoliaSearchClientTests/Integration/SettingsIntegrationTests.swift b/Tests/AlgoliaSearchClientTests/Integration/SettingsIntegrationTests.swift
@@ -16,7 +16,11 @@ class SettingsIntegrationTests: OnlineTestCase {
     return "settings"
   }
   
-  func testSettings() throws {
+  override var retryableTests: [() throws -> Void] {
+    [settings]
+  }
+  
+  func settings() throws {
     
     let indexInitialization = try index.saveObject(TestRecord(), autoGeneratingObjectID: true)
     try indexInitialization.wait()
diff --git a/Tests/AlgoliaSearchClientTests/Integration/SynonymsIntegrationTests.swift b/Tests/AlgoliaSearchClientTests/Integration/SynonymsIntegrationTests.swift
@@ -12,8 +12,12 @@ import XCTest
 class SynonymsIntegrationTests: OnlineTestCase {
 
   override var indexNameSuffix: String? { return "synonyms" }
+  
+  override var retryableTests: [() throws -> Void] {
+    [synonyms]
+  }
 
-  func testSynonyms() throws {
+  func synonyms() throws {
     
     let records: [JSON] = [
       ["console": "Sony PlayStation <PLAYSTATIONVERSION>"],
