fix(clients): reduce chances of Push rate limiting (#5153) (generated) [skip ci]

Co-authored-by: Clément Vannicatte <[email protected]>

Author: algolia-bot

clients/algoliasearch-client-go/algolia/ingestion/api_ingestion.go

@@ -14,6 +14,7 @@
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.EnumSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
@@ -4997,9 +4998,19 @@ public <T> List<WatchResponse> chunkedPush(
   ) {
     List<WatchResponse> responses = new ArrayList<>();
     List<T> records = new ArrayList<>();
+    int offset = 0;
+    int waitBatchSize = batchSize / 10;
+    if (waitBatchSize < 1) {
+      waitBatchSize = batchSize;
+    }
+
+    Iterator<T> it = objects.iterator();
+    T current = it.next();
+
+    while (true) {
+      records.add(current);
 
-    for (T item : objects) {
-      if (records.size() == batchSize) {
+      if (records.size() == batchSize || !it.hasNext()) {
         WatchResponse watch = this.push(
           indexName,
           new PushTaskPayload().setAction(action).setRecords(this.objectsToPushTaskRecords(records)),
@@ -5011,41 +5022,38 @@ public <T> List<WatchResponse> chunkedPush(
         records.clear();
       }
 
-      records.add(item);
-    }
+      if (waitForTasks && responses.size() > 0 && (responses.size() % waitBatchSize == 0 || !it.hasNext())) {
+        responses
+          .subList(offset, Math.min(offset + waitBatchSize, responses.size()))
+          .forEach(response -> {
+            TaskUtils.retryUntil(
+              () -> {
+                try {
+                  return this.getEvent(response.getRunID(), response.getEventID());
+                } catch (AlgoliaApiException e) {
+                  if (e.getStatusCode() == 404) {
+                    return null;
+                  }
+
+                  throw e;
+                }
+              },
+              (Event resp) -> {
+                return resp != null;
+              },
+              50,
+              null
+            );
+          });
+
+        offset += waitBatchSize;
+      }
 
-    if (records.size() > 0) {
-      WatchResponse watch = this.push(
-        indexName,
-        new PushTaskPayload().setAction(action).setRecords(this.objectsToPushTaskRecords(records)),
-        waitForTasks,
-        referenceIndexName,
-        requestOptions
-      );
-      responses.add(watch);
-    }
+      if (!it.hasNext()) {
+        break;
+      }
 
-    if (waitForTasks) {
-      responses.forEach(response -> {
-        TaskUtils.retryUntil(
-          () -> {
-            try {
-              return this.getEvent(response.getRunID(), response.getEventID());
-            } catch (AlgoliaApiException e) {
-              if (e.getStatusCode() == 404) {
-                return null;
-              }
-
-              throw e;
-            }
-          },
-          (Event resp) -> {
-            return resp != null;
-          },
-          50,
-          null
-        );
-      });
+      current = it.next();
     }
 
     return responses;

clients/algoliasearch-client-java/algoliasearch/src/main/java/com/algolia/api/IngestionClient.java

@@ -268,7 +268,9 @@ export function createIngestionClient({
       requestOptions?: RequestOptions,
     ): Promise<Array<WatchResponse>> {
       let records: Array<PushTaskRecords> = [];
+      let offset = 0;
       const responses: Array<WatchResponse> = [];
+      const waitBatchSize = Math.floor(batchSize / 10) || batchSize;
 
       const objectEntries = objects.entries();
       for (const [i, obj] of objectEntries) {
@@ -279,44 +281,48 @@ export function createIngestionClient({
           );
           records = [];
         }
-      }
-
-      let retryCount = 0;
-
-      if (waitForTasks) {
-        for (const resp of responses) {
-          if (!resp.eventID) {
-            throw new Error('received unexpected response from the push endpoint, eventID must not be undefined');
-          }
 
-          await createIterablePromise({
-            func: async () => {
-              if (resp.eventID === undefined || !resp.eventID) {
-                throw new Error('received unexpected response from the push endpoint, eventID must not be undefined');
-              }
-
-              return this.getEvent({ runID: resp.runID, eventID: resp.eventID }).catch((error: ApiError) => {
-                if (error.status === 404) {
-                  return undefined;
+        if (
+          waitForTasks &&
+          responses.length > 0 &&
+          (responses.length % waitBatchSize === 0 || i === objects.length - 1)
+        ) {
+          for (const resp of responses.slice(offset, offset + waitBatchSize)) {
+            if (!resp.eventID) {
+              throw new Error('received unexpected response from the push endpoint, eventID must not be undefined');
+            }
+
+            let retryCount = 0;
+
+            await createIterablePromise({
+              func: async () => {
+                if (resp.eventID === undefined || !resp.eventID) {
+                  throw new Error('received unexpected response from the push endpoint, eventID must not be undefined');
                 }
 
-                throw error;
-              });
-            },
-            validate: (response) => response !== undefined,
-            aggregator: () => (retryCount += 1),
-            error: {
-              validate: () => retryCount >= 50,
-              message: () => `The maximum number of retries exceeded. (${retryCount}/${50})`,
-            },
-            timeout: (): number => Math.min(retryCount * 500, 5000),
-          });
+                return this.getEvent({ runID: resp.runID, eventID: resp.eventID }).catch((error: ApiError) => {
+                  if (error.status === 404) {
+                    return undefined;
+                  }
+
+                  throw error;
+                });
+              },
+              validate: (response) => response !== undefined,
+              aggregator: () => (retryCount += 1),
+              error: {
+                validate: () => retryCount >= 50,
+                message: () => `The maximum number of retries exceeded. (${retryCount}/${50})`,
+              },
+              timeout: (): number => Math.min(retryCount * 500, 5000),
+            });
+          }
+          offset += waitBatchSize;
         }
       }
 
       return responses;
     },
-
     /**
      * Creates a new authentication resource.
      *

clients/algoliasearch-client-javascript/packages/ingestion/src/ingestionClient.ts

@@ -2861,49 +2861,50 @@ public function chunkedPush(
         $responses = [];
         $records = [];
         $count = 0;
+        $offset = 0;
+        $waitBatchSize = (int) ($batchSize / 10);
+        if ($waitBatchSize < 1) {
+            $waitBatchSize = $batchSize;
+        }
 
         foreach ($objects as $object) {
             $records[] = $object;
             $ok = false;
+            ++$count;
 
-            if (sizeof($records) === $batchSize || $count === sizeof($objects) - 1) {
+            if (sizeof($records) === $batchSize || $count === sizeof($objects)) {
                 $responses[] = $this->push($indexName, ['action' => $action, 'records' => $records], false, $referenceIndexName, $requestOptions);
                 $records = [];
             }
 
-            ++$count;
-        }
+            if ($waitForTasks && !empty($responses) && (0 === sizeof($responses) % $waitBatchSize || $count === sizeof($objects))) {
+                $timeoutCalculation = 'Algolia\AlgoliaSearch\Support\Helpers::linearTimeout';
 
-        if (!empty($records)) {
-            $responses[] = $this->push($indexName, ['action' => $action, 'records' => $records], false, $referenceIndexName, $requestOptions);
-        }
+                foreach (array_slice($responses, $offset, $waitBatchSize) as $response) {
+                    $retry = 0;
 
-        if ($waitForTasks && !empty($responses)) {
-            $timeoutCalculation = 'Algolia\AlgoliaSearch\Support\Helpers::linearTimeout';
+                    while ($retry < 50) {
+                        try {
+                            $this->getEvent($response['runID'], $response['eventID']);
 
-            foreach ($responses as $response) {
-                $retry = 0;
+                            $ok = true;
 
-                while ($retry < 50) {
-                    try {
-                        $this->getEvent($response['runID'], $response['eventID']);
+                            break;
+                        } catch (NotFoundException $e) {
+                            // just retry
+                        }
 
-                        $ok = true;
-
-                        break;
-                    } catch (NotFoundException $e) {
-                        // just retry
+                        ++$retry;
+                        usleep(
+                            call_user_func_array($timeoutCalculation, [$this->config->getWaitTaskTimeBeforeRetry(), $retry])
+                        );
                     }
 
-                    ++$retry;
-                    usleep(
-                        call_user_func_array($timeoutCalculation, [$this->config->getWaitTaskTimeBeforeRetry(), $retry])
-                    );
-                }
-
-                if (false === $ok) {
-                    throw new ExceededRetriesException('Maximum number of retries (50) exceeded.');
+                    if (false === $ok) {
+                        throw new ExceededRetriesException('Maximum number of retries (50) exceeded.');
+                    }
                 }
+                $offset = $offset + $waitBatchSize;
             }
         }
 

clients/algoliasearch-client-php/lib/Api/IngestionClient.php

@@ -12,9 +12,9 @@
  */
 
 /**
- * Ingestion API.
+ * Search API.
  *
- * The Ingestion API lets you connect third-party services and platforms with Algolia and schedule tasks to ingest your data. The Ingestion API powers the no-code [data connectors](https://dashboard.algolia.com/connectors).  ## Base URLs  The base URLs for requests to the Ingestion API are:  - `https://data.us.algolia.com` - `https://data.eu.algolia.com`  Use the URL that matches your [analytics region](https://dashboard.algolia.com/account/infrastructure/analytics).  **All requests must use HTTPS.**  ## Authentication  To authenticate your API requests, add these headers:  - `x-algolia-application-id`. Your Algolia application ID. - `x-algolia-api-key`. An API key with the necessary permissions to make the request.   The required access control list (ACL) to make a request is listed in each endpoint's reference.  You can find your application ID and API key in the [Algolia dashboard](https://dashboard.algolia.com/account).  ## Request format  Request bodies must be JSON objects.  ## Response status and errors  Response bodies are JSON objects.  Successful responses return a `2xx` status. Client errors return a `4xx` status. Server errors are indicated by a `5xx` status. Error responses have a `message` property with more information.  ## Version  The current version of the Ingestion API is version 1, as indicated by the `/1/` in each endpoint's URL.
+ * The Algolia Search API lets you search, configure, and manage your indices and records.  ## Client libraries  Use Algolia's API clients and libraries to reliably integrate Algolia's APIs with your apps. The official API clients are covered by Algolia's [Service Level Agreement](https://www.algolia.com/policies/sla/).  See: [Algolia's ecosystem](https://www.algolia.com/doc/guides/getting-started/how-algolia-works/in-depth/ecosystem/)  ## Base URLs  The base URLs for requests to the Search API are:  - `https://{APPLICATION_ID}.algolia.net` - `https://{APPLICATION_ID}-dsn.algolia.net`.   If your subscription includes a [Distributed Search Network](https://dashboard.algolia.com/infra),   this ensures that requests are sent to servers closest to users.  Both URLs provide high availability by distributing requests with load balancing.  **All requests must use HTTPS.**  ## Retry strategy  To guarantee high availability, implement a retry strategy for all API requests using the URLs of your servers as fallbacks:  - `https://{APPLICATION_ID}-1.algolianet.com` - `https://{APPLICATION_ID}-2.algolianet.com` - `https://{APPLICATION_ID}-3.algolianet.com`  These URLs use a different DNS provider than the primary URLs. You should randomize this list to ensure an even load across the three servers.  All Algolia API clients implement this retry strategy.  ## Authentication  To authenticate your API requests, add these headers:  - `x-algolia-application-id`. Your Algolia application ID. - `x-algolia-api-key`. An API key with the necessary permissions to make the request.   The required access control list (ACL) to make a request is listed in each endpoint's reference.  You can find your application ID and API key in the [Algolia dashboard](https://dashboard.algolia.com/account).  ## Request format  Depending on the endpoint, request bodies are either JSON objects or arrays of JSON objects,  ## Parameters  Parameters are passed as query parameters for GET and DELETE requests, and in the request body for POST and PUT requests.  Query parameters must be [URL-encoded](https://developer.mozilla.org/en-US/docs/Glossary/Percent-encoding). Non-ASCII characters must be UTF-8 encoded. Plus characters (`+`) are interpreted as spaces. Arrays as query parameters must be one of:  - A comma-separated string: `attributesToRetrieve=title,description` - A URL-encoded JSON array: `attributesToRetrieve=%5B%22title%22,%22description%22%D`  ## Response status and errors  The Search API returns JSON responses. Since JSON doesn't guarantee any specific ordering, don't rely on the order of attributes in the API response.  Successful responses return a `2xx` status. Client errors return a `4xx` status. Server errors are indicated by a `5xx` status. Error responses have a `message` property with more information.  ## Version  The current version of the Search API is version 1, as indicated by the `/1/` in each endpoint's URL.
  *
  * The version of the OpenAPI document: 1.0.0
  * Generated by: https://openapi-generator.tech
@@ -29,7 +29,7 @@
 
 namespace Algolia\AlgoliaSearch;
 
-use Algolia\AlgoliaSearch\Model\Ingestion\ModelInterface;
+use Algolia\AlgoliaSearch\Model\Search\ModelInterface;
 use DateTime;
 use GuzzleHttp\Psr7\Utils;
 use Psr\Http\Message\StreamInterface;