Merge #1474

1474: Add method to add/update documents from a string r=bidoubiwa a=bidoubiwa

Partial: #1061 

Introduces two methods on Index(): 
- `addDocumentsFromString`
- `updateDocumentsFromString` 

These allow the user to add documents using any of the following format: `csv`, `json` or `jsonld`.

They require 2 parameters and 1 optional parameter:
- the documents in a string
- the content type, either one of the following options:
   -  `'text/csv'` or `ContentType.CSV` in typescript
   - `'application/x-ndjson'` or `ContentType.NDJSON` in typescript
   - 'application/json' or `ContentType.JSON` in typescript
- The document addition options. for example the primary key

The content of the string must match its content-type. Only when using the `CSV` format it is possible to use `csvDelimiter`

Usage:

```ts
client.index('myIndex').addDocumentsFromString(documents: string, contentType: ContentType, queryParams: RawDocumentAdditionOptions): Promise<EnqueuedTask>
```

```ts
client.index('myIndex').updateDocumentsFromString(documents: string, contentType: ContentType, queryParams: RawDocumentAdditionOptions): Promise<EnqueuedTask>
```

### Example: 

https://github.com/meilisearch/meilisearch-js/blob/d948a7551108b18def7e1e69c05997c47b9ac2ae/tests/raw_document.test.ts#L164-L174



Co-authored-by: Charlotte Vermandel <[email protected]>

Author: meili-bors[bot]

README.md

@@ -420,6 +420,12 @@ client.multiSearch(queries?: MultiSearchParams, config?: Partial<Request>): Prom
 client.index('myIndex').addDocuments(documents: Document<T>[]): Promise<EnqueuedTask>
 ```
 
+#### [Add or replace multiple documents in string format](https://docs.meilisearch.com/reference/api/documents.html#add-or-update-documents)
+
+```ts
+client.index('myIndex').addDocumentsFromString(documents: string, contentType: ContentType, queryParams: RawDocumentAdditionOptions): Promise<EnqueuedTask>
+```
+
 #### [Add or replace multiple documents in batches](https://docs.meilisearch.com/reference/api/documents.html#add-or-replace-documents)
 
 ```ts
@@ -432,6 +438,12 @@ client.index('myIndex').addDocumentsInBatches(documents: Document<T>[], batchSiz
 client.index('myIndex').updateDocuments(documents: Array<Document<Partial<T>>>): Promise<EnqueuedTask>
 ```
 
+#### [Add or update multiple documents in string format](https://docs.meilisearch.com/reference/api/documents.html#add-or-update-documents)
+
+```ts
+client.index('myIndex').updateDocumentsFromString(documents: string, contentType: ContentType, queryParams: RawDocumentAdditionOptions): Promise<EnqueuedTask>
+```
+
 #### [Add or update multiple documents in batches](https://docs.meilisearch.com/reference/api/documents.html#add-or-update-documents)
 
 ```ts

src/http-requests.ts

@@ -112,7 +112,7 @@ class HttpRequests {
     url,
     params,
     body,
-    config,
+    config = {},
   }: {
     method: string
     url: string
@@ -129,14 +129,22 @@ class HttpRequests {
       constructURL.search = queryParams.toString()
     }
 
+    // in case a custom content-type is provided
+    // do not stringify body
+    if (!config.headers?.['Content-Type']) {
+      body = JSON.stringify(body)
+    }
+
+    const headers = { ...this.headers, ...config.headers }
+
     try {
       const fetchFn = this.httpClient ? this.httpClient : fetch
       const result = fetchFn(constructURL.toString(), {
         ...config,
         ...this.requestConfig,
         method,
-        body: JSON.stringify(body),
-        headers: this.headers,
+        body,
+        headers,
       })
 
       // When using a custom HTTP client, the response is returned to allow the user to parse/handle it as they see fit

src/indexes.ts

@@ -37,6 +37,8 @@ import {
   PaginationSettings,
   Faceting,
   ResourceResults,
+  RawDocumentAdditionOptions,
+  ContentType,
 } from './types'
 import { removeUndefinedFromObject } from './utils'
 import { HttpRequests } from './http-requests'
@@ -372,6 +374,32 @@ class Index<T extends Record<string, any> = Record<string, any>> {
     return new EnqueuedTask(task)
   }
 
+  /**
+   * Add or replace multiples documents in a string format to an index. It only
+   * supports csv, ndjson and json formats.
+   *
+   * @param documents - Documents provided in a string to add/replace
+   * @param contentType - Content type of your document:
+   *   'text/csv'|'application/x-ndjson'|'application/json'
+   * @param options - Options on document addition
+   * @returns Promise containing an EnqueuedTask
+   */
+  async addDocumentsFromString(
+    documents: string,
+    contentType: ContentType,
+    queryParams?: RawDocumentAdditionOptions
+  ): Promise<EnqueuedTask> {
+    const url = `indexes/${this.uid}/documents`
+
+    const task = await this.httpRequest.post(url, documents, queryParams, {
+      headers: {
+        'Content-Type': contentType,
+      },
+    })
+
+    return new EnqueuedTask(task)
+  }
+
   /**
    * Add or replace multiples documents to an index in batches
    *
@@ -433,6 +461,32 @@ class Index<T extends Record<string, any> = Record<string, any>> {
     return updates
   }
 
+  /**
+   * Add or update multiples documents in a string format to an index. It only
+   * supports csv, ndjson and json formats.
+   *
+   * @param documents - Documents provided in a string to add/update
+   * @param contentType - Content type of your document:
+   *   'text/csv'|'application/x-ndjson'|'application/json'
+   * @param queryParams - Options on raw document addition
+   * @returns Promise containing an EnqueuedTask
+   */
+  async updateDocumentsFromString(
+    documents: string,
+    contentType: ContentType,
+    queryParams?: RawDocumentAdditionOptions
+  ): Promise<EnqueuedTask> {
+    const url = `indexes/${this.uid}/documents`
+
+    const task = await this.httpRequest.put(url, documents, queryParams, {
+      headers: {
+        'Content-Type': contentType,
+      },
+    })
+
+    return new EnqueuedTask(task)
+  }
+
   /**
    * Delete one document
    *

src/types/types.ts

@@ -201,6 +201,22 @@ export type DocumentOptions = {
   primaryKey?: string
 }
 
+export const ContentTypeEnum: Readonly<Record<string, ContentType>> = {
+  JSON: 'application/json',
+  CSV: 'text/csv',
+  NDJSON: 'application/x-ndjson',
+}
+
+export type ContentType =
+  | 'text/csv'
+  | 'application/x-ndjson'
+  | 'application/json'
+  | 'PUT'
+
+export type RawDocumentAdditionOptions = DocumentOptions & {
+  csvDelimiter?: string
+}
+
 export type DocumentsQuery<T = Record<string, any>> = ResourceQuery & {
   fields?: Fields<T>
 }

tests/raw_document.test.ts

@@ -0,0 +1,240 @@
+import {
+  clearAllIndexes,
+  config,
+  getClient,
+} from './utils/meilisearch-test-utils'
+import { TaskStatus, ContentTypeEnum } from '../src/types'
+
+beforeEach(async () => {
+  await clearAllIndexes(config)
+})
+
+describe.each([{ permission: 'Master' }, { permission: 'Admin' }])(
+  'Test on raw documents addition using `addDocumentsFromString`',
+  ({ permission }) => {
+    test(`${permission} key: Add documents in CSV format`, async () => {
+      const client = await getClient(permission)
+      const data = `id,title,comment
+123,Pride and Prejudice, A great book
+546,Le Petit Prince,a french book`
+
+      const { taskUid } = await client
+        .index('csv_index')
+        .addDocumentsFromString(data, ContentTypeEnum.CSV)
+      const task = await client.waitForTask(taskUid)
+
+      expect(task.details.receivedDocuments).toBe(2)
+      expect(task.status).toBe(TaskStatus.TASK_SUCCEEDED)
+    })
+
+    test(`${permission} key: Add documents in CSV format with custom primary key`, async () => {
+      const client = await getClient(permission)
+      const data = `name,title,comment
+123,Pride and Prejudice, A great book
+546,Le Petit Prince,a french book`
+
+      const { taskUid } = await client
+        .index('csv_index')
+        .addDocumentsFromString(data, ContentTypeEnum.CSV, {
+          primaryKey: 'name',
+        })
+      const task = await client.waitForTask(taskUid)
+
+      expect(task.details.receivedDocuments).toBe(2)
+      expect(task.status).toBe(TaskStatus.TASK_SUCCEEDED)
+    })
+
+    test(`${permission} key: Add documents in CSV format with custom delimiter`, async () => {
+      const client = await getClient(permission)
+      const data = `name;title;comment
+123;Pride and Prejudice; A great book
+546;Le Petit Prince;a french book`
+
+      const { taskUid } = await client
+        .index('csv_index')
+        .addDocumentsFromString(data, ContentTypeEnum.CSV, {
+          primaryKey: 'name',
+          csvDelimiter: ';',
+        })
+      const task = await client.waitForTask(taskUid)
+
+      expect(task.details.receivedDocuments).toBe(2)
+      expect(task.status).toBe(TaskStatus.TASK_SUCCEEDED)
+    })
+
+    test(`${permission} key: Add documents in JSON lines format`, async () => {
+      const client = await getClient(permission)
+      const data = `{ "id": 123, "title": "Pride and Prejudice", "comment": "A great book" }
+{ "id": 456, "title": "Le Petit Prince", "comment": "A french book" }`
+
+      const { taskUid } = await client
+        .index('jsonl_index')
+        .addDocumentsFromString(data, ContentTypeEnum.NDJSON, {})
+      const task = await client.waitForTask(taskUid)
+
+      expect(task.details.receivedDocuments).toBe(2)
+      expect(task.status).toBe(TaskStatus.TASK_SUCCEEDED)
+    })
+
+    test(`${permission} key: Add documents in JSON lines with custom primary key`, async () => {
+      const client = await getClient(permission)
+      const data = `{ "name": 123, "title": "Pride and Prejudice", "comment": "A great book" }
+{ "name": 456, "title": "Le Petit Prince", "comment": "A french book" }`
+
+      const { taskUid } = await client
+        .index('jsonl_index')
+        .addDocumentsFromString(data, ContentTypeEnum.NDJSON, {
+          primaryKey: 'name',
+        })
+      const task = await client.waitForTask(taskUid)
+
+      expect(task.details.receivedDocuments).toBe(2)
+      expect(task.status).toBe(TaskStatus.TASK_SUCCEEDED)
+    })
+
+    test(`${permission} key: Add documents in JSON format`, async () => {
+      const client = await getClient(permission)
+      const data = `[{ "id": 123, "title": "Pride and Prejudice", "comment": "A great book" },
+{ "id": 456, "title": "Le Petit Prince", "comment": "A french book" }]`
+
+      const { taskUid } = await client
+        .index('json_index')
+        .addDocumentsFromString(data, ContentTypeEnum.JSON)
+      const task = await client.waitForTask(taskUid)
+
+      expect(task.details.receivedDocuments).toBe(2)
+      expect(task.status).toBe(TaskStatus.TASK_SUCCEEDED)
+    })
+
+    test(`${permission} key: Add documents in JSON format with custom primary key`, async () => {
+      const client = await getClient(permission)
+      const data = `[{ "name": 123, "title": "Pride and Prejudice", "comment": "A great book" },
+{ "name": 456, "title": "Le Petit Prince", "comment": "A french book" }]`
+
+      const { taskUid } = await client
+        .index('json_index')
+        .addDocumentsFromString(data, ContentTypeEnum.JSON, {
+          primaryKey: 'name',
+        })
+      const task = await client.waitForTask(taskUid)
+
+      expect(task.details.receivedDocuments).toBe(2)
+      expect(task.status).toBe(TaskStatus.TASK_SUCCEEDED)
+    })
+  }
+)
+
+describe.each([{ permission: 'Master' }, { permission: 'Admin' }])(
+  'Test on raw documents update using `updateDocumentsFromString`',
+  ({ permission }) => {
+    test(`${permission} key: Update documents in CSV format`, async () => {
+      const client = await getClient(permission)
+      const data = `id,title,comment
+123,Pride and Prejudice, A great book
+546,Le Petit Prince,a french book`
+
+      const { taskUid } = await client
+        .index('csv_index')
+        .updateDocumentsFromString(data, ContentTypeEnum.CSV)
+      const task = await client.waitForTask(taskUid)
+
+      expect(task.details.receivedDocuments).toBe(2)
+      expect(task.status).toBe(TaskStatus.TASK_SUCCEEDED)
+    })
+
+    test(`${permission} key: Update documents in CSV format with custom primary key`, async () => {
+      const client = await getClient(permission)
+      const data = `name,title,comment
+123,Pride and Prejudice, A great book
+546,Le Petit Prince,a french book`
+
+      const { taskUid } = await client
+        .index('csv_index')
+        .updateDocumentsFromString(data, ContentTypeEnum.CSV, {
+          primaryKey: 'name',
+        })
+      const task = await client.waitForTask(taskUid)
+
+      expect(task.details.receivedDocuments).toBe(2)
+      expect(task.status).toBe(TaskStatus.TASK_SUCCEEDED)
+    })
+
+    test(`${permission} key: Update documents in CSV format with custom delimiter`, async () => {
+      const client = await getClient(permission)
+      const data = `name;title;comment
+123;Pride and Prejudice; A great book
+546;Le Petit Prince;a french book`
+
+      const { taskUid } = await client
+        .index('csv_index')
+        .updateDocumentsFromString(data, ContentTypeEnum.CSV, {
+          primaryKey: 'name',
+          csvDelimiter: ';',
+        })
+      const task = await client.waitForTask(taskUid)
+
+      expect(task.details.receivedDocuments).toBe(2)
+      expect(task.status).toBe(TaskStatus.TASK_SUCCEEDED)
+    })
+
+    test(`${permission} key: Update documents in JSON lines format`, async () => {
+      const client = await getClient(permission)
+      const data = `{ "id": 123, "title": "Pride and Prejudice", "comment": "A great book" }
+{ "id": 456, "title": "Le Petit Prince", "comment": "A french book" }`
+
+      const { taskUid } = await client
+        .index('jsonl_index')
+        .updateDocumentsFromString(data, ContentTypeEnum.NDJSON)
+      const task = await client.waitForTask(taskUid)
+
+      expect(task.details.receivedDocuments).toBe(2)
+      expect(task.status).toBe(TaskStatus.TASK_SUCCEEDED)
+    })
+
+    test(`${permission} key: Update documents in JSON lines with custom primary key`, async () => {
+      const client = await getClient(permission)
+      const data = `{ "name": 123, "title": "Pride and Prejudice", "comment": "A great book" }
+{ "name": 456, "title": "Le Petit Prince", "comment": "A french book" }`
+
+      const { taskUid } = await client
+        .index('jsonl_index')
+        .updateDocumentsFromString(data, ContentTypeEnum.NDJSON, {
+          primaryKey: 'name',
+        })
+      const task = await client.waitForTask(taskUid)
+
+      expect(task.details.receivedDocuments).toBe(2)
+      expect(task.status).toBe(TaskStatus.TASK_SUCCEEDED)
+    })
+
+    test(`${permission} key: Update documents in JSON format`, async () => {
+      const client = await getClient(permission)
+      const data = `[{ "id": 123, "title": "Pride and Prejudice", "comment": "A great book" },
+{ "id": 456, "title": "Le Petit Prince", "comment": "A french book" }]`
+
+      const { taskUid } = await client
+        .index('json_index')
+        .updateDocumentsFromString(data, ContentTypeEnum.JSON, {})
+      const task = await client.waitForTask(taskUid)
+
+      expect(task.details.receivedDocuments).toBe(2)
+      expect(task.status).toBe(TaskStatus.TASK_SUCCEEDED)
+    })
+
+    test(`${permission} key: Update documents in JSON format with custom primary key`, async () => {
+      const client = await getClient(permission)
+      const data = `[{ "name": 123, "title": "Pride and Prejudice", "comment": "A great book" },
+{ "name": 456, "title": "Le Petit Prince", "comment": "A french book" }]`
+
+      const { taskUid } = await client
+        .index('json_index')
+        .updateDocumentsFromString(data, ContentTypeEnum.JSON, {
+          primaryKey: 'name',
+        })
+      const task = await client.waitForTask(taskUid)
+
+      expect(task.details.receivedDocuments).toBe(2)
+      expect(task.status).toBe(TaskStatus.TASK_SUCCEEDED)
+    })
+  }
+)