fix(clients): allow chunked requests on WithTransformation methods

Author: shortcuts

templates/javascript/clients/algoliasearch/builds/definition.mustache

@@ -44,7 +44,10 @@ export type Algoliasearch = SearchClient & {
    * @param saveObjects.waitForTasks - Whether or not we should wait until every `batch` tasks has been processed, this operation may slow the total execution time of this method but is more reliable.
    * @param requestOptions - The requestOptions to send along with the query, they will be forwarded to the `batch` method and merged with the transporter requestOptions.
    */
-  saveObjectsWithTransformation: (options: SaveObjectsOptions, requestOptions?: RequestOptions | undefined) => Promise<WatchResponse>;
+  saveObjectsWithTransformation: (
+    options: SaveObjectsOptions,
+    requestOptions?: RequestOptions | undefined,
+  ) => Promise<WatchResponse[]>;
 
   /**
    * Helper: Similar to the `partialUpdateObjects` method but requires a Push connector (https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/connectors/push/) to be created first, in order to transform records before indexing them to Algolia. The `region` must have been passed to the client instantiation method.
@@ -58,7 +61,10 @@ export type Algoliasearch = SearchClient & {
    * @param partialUpdateObjects.waitForTasks - Whether or not we should wait until every `batch` tasks has been processed, this operation may slow the total execution time of this method but is more reliable.
    * @param requestOptions - The requestOptions to send along with the query, they will be forwarded to the `getTask` method and merged with the transporter requestOptions.
    */
-  partialUpdateObjectsWithTransformation: (options: PartialUpdateObjectsOptions, requestOptions?: RequestOptions | undefined) => Promise<WatchResponse>;
+  partialUpdateObjectsWithTransformation: (
+    options: PartialUpdateObjectsOptions,
+    requestOptions?: RequestOptions | undefined,
+  ) => Promise<WatchResponse[]>;
 
   /**
    * Helper: Similar to the `replaceAllObjects` method but requires a Push connector (https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/connectors/push/) to be created first, in order to transform records before indexing them to Algolia. The `region` must have been passed to the client instantiation method.
@@ -86,9 +92,13 @@ export type Algoliasearch = SearchClient & {
    * @param chunkedPush.action - The `batch` `action` to perform on the given array of `objects`, defaults to `addObject`.
    * @param chunkedPush.waitForTasks - Whether or not we should wait until every `batch` tasks has been processed, this operation may slow the total execution time of this method but is more reliable.
    * @param chunkedPush.batchSize - The size of the chunk of `objects`. The number of `batch` calls will be equal to `length(objects) / batchSize`. Defaults to 1000.
+   * @param chunkedPush.referenceIndexName - This is required when targeting an index that does not have a push connector setup (e.g. a tmp index), but you wish to attach another index's transformation to it (e.g. the source index name).
    * @param requestOptions - The requestOptions to send along with the query, they will be forwarded to the `getTask` method and merged with the transporter requestOptions.
    */
-  chunkedPush: (options: ChunkedBatchOptions, requestOptions?: RequestOptions) => Promise<Array<WatchResponse>>;
+  chunkedPush: (
+    options: ChunkedBatchOptions & { referenceIndexName?: string },
+    requestOptions?: RequestOptions,
+  ) => Promise<Array<WatchResponse>>;
 };
 
 export type TransformationOptions = {
@@ -127,55 +137,37 @@ export function algoliasearch(
   return {
     ...client,
 
-    async saveObjectsWithTransformation({ indexName, objects, waitForTasks }, requestOptions): Promise<WatchResponse> {
-      if (!ingestionTransporter) {
-        throw new Error('`transformation.region` must be provided at client instantiation before calling this method.');
-      }
-
-      if (!options?.transformation?.region) {
-        throw new Error('`region` must be provided when leveraging the transformation pipeline');
-      }
-
-      return ingestionTransporter?.push(
-        {
-          indexName,
-          watch: waitForTasks,
-          pushTaskPayload: {
-            action: 'addObject',
-            records: objects as PushTaskRecords[],
-          },
-        },
-        requestOptions,
-      );
+    async saveObjectsWithTransformation(
+      { indexName, objects, waitForTasks },
+      requestOptions,
+    ): Promise<WatchResponse[]> {
+      return this.chunkedPush({ indexName, objects, action: 'addObject', waitForTasks }, requestOptions);
     },
 
     async partialUpdateObjectsWithTransformation(
       { indexName, objects, createIfNotExists, waitForTasks },
       requestOptions,
-    ): Promise<WatchResponse> {
-      if (!ingestionTransporter) {
-        throw new Error('`transformation.region` must be provided at client instantiation before calling this method.');
-      }
-
-      if (!options?.transformation?.region) {
-        throw new Error('`region` must be provided when leveraging the transformation pipeline');
-      }
-
-      return ingestionTransporter?.push(
+    ): Promise<WatchResponse[]> {
+      return this.chunkedPush(
         {
           indexName,
-          watch: waitForTasks,
-          pushTaskPayload: {
-            action: createIfNotExists ? 'partialUpdateObject' : 'partialUpdateObjectNoCreate',
-            records: objects as PushTaskRecords[],
-          },
+          objects,
+          action: createIfNotExists ? 'partialUpdateObject' : 'partialUpdateObjectNoCreate',
+          waitForTasks,
         },
         requestOptions,
       );
     },
 
     async chunkedPush(
-      { indexName, objects, action = 'addObject', waitForTasks, batchSize = 1000 }: ChunkedBatchOptions,
+      {
+        referenceIndexName,
+        indexName,
+        objects,
+        action = 'addObject',
+        waitForTasks,
+        batchSize = 1000,
+      }: ChunkedBatchOptions & { referenceIndexName?: string },
       requestOptions?: RequestOptions,
     ): Promise<Array<WatchResponse>> {
       if (!ingestionTransporter) {
@@ -195,7 +187,7 @@ export function algoliasearch(
         if (records.length === batchSize || i === objects.length - 1) {
           responses.push(
             await ingestionTransporter.push(
-              { indexName, pushTaskPayload: { action, records }, watch: waitForTasks },
+              { indexName, pushTaskPayload: { action, records }, watch: waitForTasks, referenceIndexName },
               requestOptions,
             ),
           );
@@ -217,13 +209,15 @@ export function algoliasearch(
                 throw new Error('received unexpected response from the push endpoint, eventID must not be undefined');
               }
 
-              return ingestionTransporter.getEvent({ runID: resp.runID, eventID: resp.eventID }).catch((error: ApiError) => {
-                if (error.status === 404) {
-                  return undefined;
-                }
+              return ingestionTransporter
+                .getEvent({ runID: resp.runID, eventID: resp.eventID })
+                .catch((error: ApiError) => {
+                  if (error.status === 404) {
+                    return undefined;
+                  }
 
-                throw error;
-              })
+                  throw error;
+                });
             },
             validate: (response) => response !== undefined,
             aggregator: () => (retryCount += 1),
@@ -272,7 +266,7 @@ export function algoliasearch(
         );
 
         const watchResponses = await this.chunkedPush(
-          { indexName: tmpIndexName, objects, waitForTasks: true, batchSize },
+          { indexName: tmpIndexName, objects, waitForTasks: true, batchSize, referenceIndexName: indexName },
           requestOptions,
         );
 

tests/CTS/client/search/partialUpdateObjectsWithTransformation.json

@@ -35,12 +35,12 @@
         },
         "expected": {
           "type": "response",
-          "match": {
+          "match": [{
               "runID": "b1b7a982-524c-40d2-bb7f-48aab075abda",
               "eventID": "113b2068-6337-4c85-b5c2-e7b213d82925",
               "message": "OK",
               "createdAt": "2022-05-12T06:24:30.049Z"
-            }
+            }]
         }
       }
     ]

tests/CTS/client/search/saveObjectsWithTransformation.json

@@ -34,12 +34,12 @@
         },
         "expected": {
           "type": "response",
-          "match": {
+          "match": [{
               "runID": "b1b7a982-524c-40d2-bb7f-48aab075abda",
               "eventID": "113b2068-6337-4c85-b5c2-e7b213d82925",
               "message": "OK",
               "createdAt": "2022-05-12T06:24:30.049Z"
-            }
+            }]
         }
       }
     ]