proto-beta: Tokenizer customization

Author: atoulmet

.prettierignore

@@ -1,2 +1,3 @@
 node_modules
 dist
+*.md

README.md

@@ -897,6 +897,67 @@ client.index('myIndex').updateTypoTolerance(typoTolerance: TypoTolerance | null)
 client.index('myIndex').resetTypoTolerance(): Promise<EnqueuedTask>
 ```
 
+
+### Separator tokens <!-- omit in toc -->
+
+#### Get separator tokens
+
+```ts
+client.index('myIndex').getSeparatorTokens(): Promise<SeparatorTokens>
+```
+
+#### Update separator tokens
+
+```ts
+client.index('myIndex').updateSeparatorTokens(separatorTokens: SeparatorTokens | null): Promise<EnqueuedTask>
+```
+
+#### Reset separator tokens
+
+```ts
+client.index('myIndex').resetSeparatorTokens(): Promise<EnqueuedTask>
+```
+
+### Non Separator tokens <!-- omit in toc -->
+
+#### Get non separator tokens
+
+```ts
+client.index('myIndex').getNonSeparatorTokens(): Promise<NonSeparatorTokens>
+```
+
+#### Update non separator tokens
+
+```ts
+client.index('myIndex').updateNonSeparatorTokens(nonSeparatorTokens: NonSeparatorTokens | null): Promise<EnqueuedTask>
+```
+
+#### Reset non separator tokens
+
+```ts
+client.index('myIndex').resetNonSeparatorTokens(): Promise<EnqueuedTask>
+```
+
+### Dictionary <!-- omit in toc -->
+
+#### Get dictionary
+
+```ts
+client.index('myIndex').getDictionary(): Promise<Dictionary>
+```
+
+#### Update dictionary
+
+```ts
+client.index('myIndex').updateDictionary(dictionary: Dictionary | null): Promise<EnqueuedTask>
+```
+
+#### Reset dictionary
+
+```ts
+client.index('myIndex').resetDictionary(): Promise<EnqueuedTask>
+```
+
 ### Keys <!-- omit in toc -->
 
 #### [Get keys](https://www.meilisearch.com/docs/reference/api/keys#get-all-keys)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "meilisearch",
-  "version": "0.34.1",
+  "version": "0.34.2-tokenizer-customization.0",
   "description": "The Meilisearch JS client for Node.js and the browser.",
   "keywords": [
     "meilisearch",

src/indexes.ts

@@ -47,6 +47,9 @@ import {
   DocumentsDeletionQuery,
   SearchForFacetValuesParams,
   SearchForFacetValuesResponse,
+  SeparatorTokens,
+  NonSeparatorTokens,
+  Dictionary,
 } from './types'
 import { removeUndefinedFromObject } from './utils'
 import { HttpRequests } from './http-requests'
@@ -1117,6 +1120,133 @@ class Index<T extends Record<string, any> = Record<string, any>> {
 
     return new EnqueuedTask(task)
   }
+
+  ///
+  /// SEPARATOR TOKENS
+  ///
+
+  /**
+   * Get the list of all separator tokens.
+   *
+   * @returns Promise containing array of separator tokens
+   */
+  async getSeparatorTokens(): Promise<string[]> {
+    const url = `indexes/${this.uid}/settings/separator-tokens`
+    return await this.httpRequest.get<string[]>(url)
+  }
+
+  /**
+   * Update the list of separator tokens. Overwrite the old list.
+   *
+   * @param separatorTokens - Array that contains separator tokens.
+   * @returns Promise containing an EnqueuedTask or null
+   */
+  async updateSeparatorTokens(
+    separatorTokens: SeparatorTokens
+  ): Promise<EnqueuedTask> {
+    const url = `indexes/${this.uid}/settings/separator-tokens`
+    const task = await this.httpRequest.put(url, separatorTokens)
+
+    return new EnqueuedTask(task)
+  }
+
+  /**
+   * Reset the separator tokens list to its default value
+   *
+   * @returns Promise containing an EnqueuedTask
+   */
+  async resetSeparatorTokens(): Promise<EnqueuedTask> {
+    const url = `indexes/${this.uid}/settings/separator-tokens`
+    const task = await this.httpRequest.delete<EnqueuedTask>(url)
+
+    task.enqueuedAt = new Date(task.enqueuedAt)
+
+    return task
+  }
+
+  ///
+  /// NON-SEPARATOR TOKENS
+  ///
+
+  /**
+   * Get the list of all non-separator tokens.
+   *
+   * @returns Promise containing array of non-separator tokens
+   */
+  async getNonSeparatorTokens(): Promise<string[]> {
+    const url = `indexes/${this.uid}/settings/non-separator-tokens`
+    return await this.httpRequest.get<string[]>(url)
+  }
+
+  /**
+   * Update the list of non-separator tokens. Overwrite the old list.
+   *
+   * @param nonSeparatorTokens - Array that contains non-separator tokens.
+   * @returns Promise containing an EnqueuedTask or null
+   */
+  async updateNonSeparatorTokens(
+    nonSeparatorTokens: NonSeparatorTokens
+  ): Promise<EnqueuedTask> {
+    const url = `indexes/${this.uid}/settings/non-separator-tokens`
+    const task = await this.httpRequest.put(url, nonSeparatorTokens)
+
+    return new EnqueuedTask(task)
+  }
+
+  /**
+   * Reset the non-separator tokens list to its default value
+   *
+   * @returns Promise containing an EnqueuedTask
+   */
+  async resetNonSeparatorTokens(): Promise<EnqueuedTask> {
+    const url = `indexes/${this.uid}/settings/non-separator-tokens`
+    const task = await this.httpRequest.delete<EnqueuedTask>(url)
+
+    task.enqueuedAt = new Date(task.enqueuedAt)
+
+    return task
+  }
+
+  ///
+  /// DICTIONARY
+  ///
+
+  /**
+   * Get the dictionary settings of a Meilisearch index.
+   *
+   * @returns Promise containing the dictionary settings
+   */
+  async getDictionary(): Promise<string[]> {
+    const url = `indexes/${this.uid}/settings/dictionary`
+    return await this.httpRequest.get<string[]>(url)
+  }
+
+  /**
+   * Update the the dictionary settings. Overwrite the old settings.
+   *
+   * @param dictionary - Array that contains the new dictionary settings.
+   * @returns Promise containing an EnqueuedTask or null
+   */
+  async updateDictionary(dictionary: Dictionary): Promise<EnqueuedTask> {
+    const url = `indexes/${this.uid}/settings/dictionary`
+    const task = await this.httpRequest.put(url, dictionary)
+
+    return new EnqueuedTask(task)
+  }
+
+  /**
+   * Reset the dictionary settings to its default value
+   *
+   * @returns Promise containing an EnqueuedTask
+   */
+  async resetDictionary(): Promise<EnqueuedTask> {
+    const url = `indexes/${this.uid}/settings/dictionary`
+    const task = await this.httpRequest.delete<EnqueuedTask>(url)
+
+    task.enqueuedAt = new Date(task.enqueuedAt)
+
+    return task
+  }
 }
 
 export { Index }

src/package-version.ts

@@ -1 +1 @@
-export const PACKAGE_VERSION = '0.34.1'
+export const PACKAGE_VERSION = '0.34.2-tokenizer-customization.0'

src/types/types.ts

@@ -312,6 +312,9 @@ export type TypoTolerance = {
     twoTypos?: number | null
   }
 } | null
+export type SeparatorTokens = string[] | null
+export type NonSeparatorTokens = string[] | null
+export type Dictionary = string[] | null
 
 export type FacetOrder = 'alpha' | 'count'
 
@@ -336,6 +339,9 @@ export type Settings = {
   typoTolerance?: TypoTolerance
   faceting?: Faceting
   pagination?: PaginationSettings
+  separatorTokens?: SeparatorTokens
+  nonSeparatorTokens?: NonSeparatorTokens
+  dictionary?: Dictionary
 }
 
 /*