fix(clients): WithTransformation methods retry logic (#5183)

Author: shortcuts

clients/algoliasearch-client-java/algoliasearch/src/main/java/com/algolia/utils/TaskUtils.java

@@ -16,6 +16,12 @@ private TaskUtils() {
 
   public static <T> T retryUntil(Supplier<T> func, Predicate<T> validate, int maxRetries, IntUnaryOperator timeout)
     throws AlgoliaRuntimeException {
+    if (timeout == null) {
+      timeout = DEFAULT_TIMEOUT;
+    }
+    if (maxRetries == 0) {
+      maxRetries = DEFAULT_MAX_RETRIES;
+    }
     int retryCount = 0;
     while (retryCount < maxRetries) {
       T resp = func.get();

clients/algoliasearch-client-php/lib/Support/Helpers.php

@@ -193,7 +193,7 @@ public static function retryForApiKeyUntil(
      *
      * @return float|int
      */
-    private static function linearTimeout($defaultTimeout, $retries)
+    public static function linearTimeout($defaultTimeout, $retries)
     {
         // minimum between timeout and 200 milliseconds * number of retries
         // Convert into microseconds for usleep (* 1000)

scripts/cts/testServer/index.ts

@@ -14,7 +14,7 @@ import { benchmarkServer } from './benchmark.ts';
 import { chunkWrapperServer } from './chunkWrapper.ts';
 import { errorServer, errorServerRetriedOnce, errorServerRetriedTwice } from './error.ts';
 import { gzipServer } from './gzip.ts';
-import { pushMockServer } from './pushMock.ts';
+import { pushMockServer, pushMockServerRetriedOnce } from './pushMock.ts';
 import { replaceAllObjectsServer } from './replaceAllObjects.ts';
 import { replaceAllObjectsServerFailed } from './replaceAllObjectsFailed.ts';
 import { replaceAllObjectsScopesServer } from './replaceAllObjectsScopes.ts';
@@ -42,6 +42,7 @@ export async function startTestServer(suites: Record<CTSType, boolean>): Promise
       apiKeyServer(),
       algoliaMockServer(),
       pushMockServer(),
+      pushMockServerRetriedOnce(),
       replaceAllObjectsWithTransformationServer(),
     );
   }

scripts/cts/testServer/pushMock.ts

@@ -6,7 +6,15 @@ import express from 'express';
 
 import { setupServer } from './index.ts';
 
-const pushMockState: Record<string, any> = {};
+const pushMockState: Record<
+  string,
+  {
+    retriedEvents: boolean;
+    retriedHosts: boolean;
+    saveObjectsWithTransformation: number;
+    partialUpdateObjectsWithTransformation: number;
+  }
+> = {};
 
 export function assertPushMockValid(expectedCount: number): void {
   if (Object.values(pushMockState).length !== expectedCount) {
@@ -20,6 +28,8 @@ export function assertPushMockValid(expectedCount: number): void {
     }
 
     expect(state).to.deep.equal({
+      retriedHosts: true,
+      retriedEvents: true,
       saveObjectsWithTransformation: Number(numberOfTestSuites),
       partialUpdateObjectsWithTransformation: Number(numberOfTestSuites),
     });
@@ -40,7 +50,16 @@ function addRoutes(app: Express): void {
     const lang = match?.[2] as string;
 
     if (!pushMockState[lang]) {
-      pushMockState[lang] = {};
+      // simulate a retry at the hosts level
+      pushMockState[lang] = {
+        retriedHosts: true,
+        retriedEvents: false,
+        saveObjectsWithTransformation: 0,
+        partialUpdateObjectsWithTransformation: 0,
+      };
+
+      res.status(500).json({ message: 'error test server response' });
+      return;
     }
 
     pushMockState[lang][helper] = (pushMockState[lang][helper] ?? 0) + 1;
@@ -55,7 +74,7 @@ function addRoutes(app: Express): void {
         });
 
         res.json({
-          runID: 'b1b7a982-524c-40d2-bb7f-48aab075abda',
+          runID: `b1b7a982-524c-40d2-bb7f-48aab075abda_${lang}`,
           eventID: '113b2068-6337-4c85-b5c2-e7b213d82925',
           message: 'OK',
           createdAt: '2022-05-12T06:24:30.049Z',
@@ -72,7 +91,7 @@ function addRoutes(app: Express): void {
         });
 
         res.json({
-          runID: 'b1b7a982-524c-40d2-bb7f-48aab075abda',
+          runID: `b1b7a982-524c-40d2-bb7f-48aab075abda_${lang}`,
           eventID: '113b2068-6337-4c85-b5c2-e7b213d82925',
           message: 'OK',
           createdAt: '2022-05-12T06:24:30.049Z',
@@ -82,8 +101,32 @@ function addRoutes(app: Express): void {
         throw new Error('unknown helper');
     }
   });
+
+  app.get('/1/runs/:runID/events/:eventID', (req, res) => {
+    const lang = req.params.runID.match(/^b1b7a982-524c-40d2-bb7f-48aab075abda_(.*)$/)?.[1] as string;
+
+    if (pushMockState[lang] && !pushMockState[lang]?.retriedEvents) {
+      pushMockState[lang].retriedEvents = true;
+      res.status(404).json({ message: 'error test server response' });
+
+      return;
+    }
+
+    res.json({
+      status: 'succeeded',
+      eventID: req.params.eventID,
+      runID: req.params.runID,
+      type: 'fetch',
+      batchSize: 1,
+      publishedAt: '2022-05-12T06:24:30.049Z',
+    });
+  });
+}
+
+export function pushMockServerRetriedOnce(): Promise<Server> {
+  return setupServer('pushMockRetriedOnce', 6689, addRoutes);
 }
 
 export function pushMockServer(): Promise<Server> {
-  return setupServer('pushMock', 6689, addRoutes);
+  return setupServer('pushMock', 6688, addRoutes);
 }

templates/java/api_helpers.mustache

@@ -776,6 +776,25 @@ public <T> List<WatchResponse> saveObjectsWithTransformation(String indexName, I
   return saveObjectsWithTransformation(indexName, objects, null);
 }
 
+/**
+ * Helper: Similar to the `saveObjects` method but requires a Push connector
+ * (https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/connectors/push/)
+ * to be created first, in order to transform records before indexing them to Algolia. The
+ * `region` must have been passed to the client instantiation method.
+ *
+ * @param indexName The `indexName` to replace `objects` in.
+ * @param objects The array of `objects` to store in the given Algolia `indexName`.
+ * @param waitForTasks - Whether or not we should wait until every `batch` tasks has been
+ *     processed, this operation may slow the total execution time of this method but is more
+ *     reliable.
+ * @throws AlgoliaRetryException When the retry has failed on all hosts
+ * @throws AlgoliaApiException When the API sends an http error code
+ * @throws AlgoliaRuntimeException When an error occurred during the serialization
+ */
+public <T> List<WatchResponse> saveObjectsWithTransformation(String indexName, Iterable<T> objects, boolean waitForTasks) {
+  return saveObjectsWithTransformation(indexName, objects, waitForTasks, null);
+}
+
 /**
  * Helper: Similar to the `saveObjects` method but requires a Push connector
  * (https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/connectors/push/)

templates/php/client_config.mustache

@@ -30,7 +30,17 @@ class {{configClassname}} extends {{#hasRegionalHost}}ConfigWithRegion{{/hasRegi
     }
     {{/hasRegionalHost}}
 
-    {{#isSearchClient}}private $defaultWaitTaskTimeBeforeRetry = 5000; // 5 sec in milliseconds
+    {{#isIngestionClient}}
+    private $defaultWaitTaskTimeBeforeRetry = 5000; // 5 sec in milliseconds
+
+    public function getWaitTaskTimeBeforeRetry()
+    {
+        return $this->config['waitTaskTimeBeforeRetry'];
+    }
+
+    {{/isIngestionClient}}
+    {{#isSearchClient}}
+    private $defaultWaitTaskTimeBeforeRetry = 5000; // 5 sec in milliseconds
     private $defaultMaxRetries = 50;
 
     public static function create($appId, $apiKey)
@@ -98,6 +108,9 @@ class {{configClassname}} extends {{#hasRegionalHost}}ConfigWithRegion{{/hasRegi
             'writeTimeout' => {{#lambda.toSeconds}}{{x-timeouts.server.write}}{{/lambda.toSeconds}},
             'connectTimeout' => {{#lambda.toSeconds}}{{x-timeouts.server.connect}}{{/lambda.toSeconds}},
             'defaultHeaders' => [],
+            {{#isIngestionClient}}
+            'waitTaskTimeBeforeRetry' => $this->defaultWaitTaskTimeBeforeRetry,
+            {{/isIngestionClient}}
             {{#isSearchClient}}
             'region' => null,
             'waitTaskTimeBeforeRetry' => $this->defaultWaitTaskTimeBeforeRetry,

tests/CTS/client/search/partialUpdateObjectsWithTransformation.json

@@ -9,6 +9,9 @@
           "appId": "test-app-id",
           "apiKey": "test-api-key",
           "customHosts": [
+            {
+              "port": 6688
+            },
             {
               "port": 6689
             }
@@ -31,12 +34,13 @@
               "name": "Benoit"
             }
           ],
-          "createIfNotExists": true
+          "createIfNotExists": true,
+          "waitForTasks": true
         },
         "expected": {
           "type": "response",
           "match": [{
-              "runID": "b1b7a982-524c-40d2-bb7f-48aab075abda",
+              "runID": "b1b7a982-524c-40d2-bb7f-48aab075abda_${{language}}",
               "eventID": "113b2068-6337-4c85-b5c2-e7b213d82925",
               "message": "OK",
               "createdAt": "2022-05-12T06:24:30.049Z"

tests/CTS/client/search/saveObjectsWithTransformation.json

@@ -9,6 +9,9 @@
           "appId": "test-app-id",
           "apiKey": "test-api-key",
           "customHosts": [
+            {
+              "port": 6688
+            },
             {
               "port": 6689
             }
@@ -30,12 +33,13 @@
               "objectID": "2",
               "name": "Benoit"
             }
-          ]
+          ],
+          "waitForTasks": true
         },
         "expected": {
           "type": "response",
           "match": [{
-              "runID": "b1b7a982-524c-40d2-bb7f-48aab075abda",
+              "runID": "b1b7a982-524c-40d2-bb7f-48aab075abda_${{language}}",
               "eventID": "113b2068-6337-4c85-b5c2-e7b213d82925",
               "message": "OK",
               "createdAt": "2022-05-12T06:24:30.049Z"