Merge #1039

1039: Add methods to automatically add/update documents in batches r=bidoubiwa a=vishnugt

Fixes #1036 


Co-authored-by: vishnugt <[email protected]>
Co-authored-by: Vishnu Gt <[email protected]>

Author: bors[bot]

README.md

@@ -397,10 +397,18 @@ Or using the index object:
 
 `index.addDocuments(documents: Document<T>[]): Promise<EnqueuedUpdate>`
 
+- [Add or replace multiple documents in batches](https://docs.meilisearch.com/reference/api/documents.html#add-or-replace-documents):
+
+`index.addDocumentsInBatch(documents: Document<T>[], batchSize = 1000): Promise<EnqueuedUpdate[]>`
+
 - [Add or update multiple documents](https://docs.meilisearch.com/reference/api/documents.html#add-or-update-documents):
 
 `index.updateDocuments(documents: Document<T>[]): Promise<EnqueuedUpdate>`
 
+- [Add or update multiple documents in batches](https://docs.meilisearch.com/reference/api/documents.html#add-or-update-documents):
+
+`index.updateDocumentsInBatch(documents: Document<T>[], batchSize = 1000): Promise<EnqueuedUpdate[]>`
+
 - [Get Documents](https://docs.meilisearch.com/reference/api/documents.html#get-documents):
 
 `index.getDocuments(params: getDocumentsParams): Promise<Document<T>[]>`

src/lib/indexes.ts

@@ -316,6 +316,25 @@ class Index<T = Record<string, any>> {
     return await this.httpRequest.post(url, documents, options)
   }
 
+  /**
+   * Add or replace multiples documents to an index in batches
+   * @memberof Index
+   * @method addDocumentsInBatch
+   */
+  async addDocumentsInBatch(
+    documents: Array<Document<T>>,
+    batchSize = 1000,
+    options?: AddDocumentParams
+  ): Promise<EnqueuedUpdate[]> {
+    const updates = []
+    for (let i = 0; i < documents.length; i += batchSize) {
+      updates.push(
+        await this.addDocuments(documents.slice(i, i + batchSize), options)
+      )
+    }
+    return updates
+  }
+
   /**
    * Add or update multiples documents to an index
    * @memberof Index
@@ -329,6 +348,25 @@ class Index<T = Record<string, any>> {
     return await this.httpRequest.put(url, documents, options)
   }
 
+  /**
+   * Add or update multiples documents to an index in batches
+   * @memberof Index
+   * @method updateDocuments
+   */
+  async updateDocumentsInBatch(
+    documents: Array<Document<T>>,
+    batchSize = 1000,
+    options?: AddDocumentParams
+  ): Promise<EnqueuedUpdate[]> {
+    const updates = []
+    for (let i = 0; i < documents.length; i += batchSize) {
+      updates.push(
+        await this.updateDocuments(documents.slice(i, i + batchSize), options)
+      )
+    }
+    return updates
+  }
+
   /**
    * Delete one document
    * @memberof Index

tests/documents_tests.ts

@@ -71,6 +71,22 @@ describe.each([
     await client.index(indexPk.uid).waitForPendingUpdate(updateId)
   })
 
+  test(`${permission} key: Add documents to uid with primary key in batch`, async () => {
+    const response: EnqueuedUpdate[] = await client
+      .index(indexPk.uid)
+      .addDocumentsInBatch(dataset, 4)
+    expect(response).toBeInstanceOf(Array)
+    expect(response).toHaveLength(2)
+    expect(response[0]).toHaveProperty('updateId', expect.any(Number))
+    for (const enqueuedUpdate of response) {
+      const addResponse = await client
+        .index(indexPk.uid)
+        .waitForPendingUpdate(enqueuedUpdate.updateId)
+      expect(addResponse.status).toBe('processed')
+      expect(addResponse.type.name).toBe('DocumentsAddition')
+    }
+  })
+
   test(`${permission} key: Get documents with string attributesToRetrieve`, async () => {
     await client
       .index(indexNoPk.uid)
@@ -208,6 +224,22 @@ describe.each([
       })
   })
 
+  test(`${permission} key: Update document from index that has a primary key in batch`, async () => {
+    const response: EnqueuedUpdate[] = await client
+      .index(indexPk.uid)
+      .updateDocumentsInBatch(dataset, 2)
+    expect(response).toBeInstanceOf(Array)
+    expect(response).toHaveLength(4)
+    expect(response[0]).toHaveProperty('updateId', expect.any(Number))
+    for (const enqueuedUpdate of response) {
+      const addResponse = await client
+        .index(indexPk.uid)
+        .waitForPendingUpdate(enqueuedUpdate.updateId)
+      expect(addResponse.status).toBe('processed')
+      expect(addResponse.type.name).toBe('DocumentsPartial')
+    }
+  })
+
   test(`${permission} key: Add document with update documents function from index that has NO primary key`, async () => {
     const { updateId: addDocUpdate } = await client
       .index(indexNoPk.uid)