Merge pull request #827 from meilisearch/feat/add-ai-powered-search

Add AI-powered search features

Author: brunoocasali

.github/workflows/tests.yml

@@ -29,7 +29,7 @@ jobs:
       - name: Meilisearch (latest) setup with Docker
         run: docker run -d -p 7700:7700 getmeili/meilisearch:latest meilisearch --master-key=masterKey --no-analytics
       - name: Build and run unit and integration tests
-        run: ./gradlew build integrationTest
+        run: ./gradlew build integrationTest --info
       - name: Archive test report
         uses: actions/upload-artifact@v4
         if: failure()

src/main/java/com/meilisearch/sdk/Index.java

@@ -1242,11 +1242,57 @@ public TaskInfo resetSearchCutoffMsSettings() throws MeilisearchException {
         return this.settingsHandler.resetSearchCutoffMsSettings(this.uid);
     }
 
+    /**
+     * Retrieves documents that are semantically similar to a given document
+     *
+     * @param query SimilarDocumentRequest containing parameters for the similar documents search
+     * @return SimilarDocumentsResults containing the search results
+     * @throws MeilisearchException if an error occurs
+     * @see <a href="https://www.meilisearch.com/docs/reference/api/similar">API specification</a>
+     */
     public SimilarDocumentsResults searchSimilarDocuments(SimilarDocumentRequest query)
             throws MeilisearchException {
         return this.config.httpClient.post(
                 new URLBuilder("/indexes").addSubroute(this.uid).addSubroute("/similar").getURL(),
                 query,
                 SimilarDocumentsResults.class);
     }
+
+    /**
+     * Gets the embedders settings of the index
+     *
+     * @return a Map that contains all embedders settings
+     * @throws MeilisearchException if an error occurs
+     * @see <a href="https://www.meilisearch.com/docs/reference/api/settings#get-embedders">API
+     *     specification</a>
+     */
+    public Map<String, Embedder> getEmbeddersSettings() throws MeilisearchException {
+        return this.settingsHandler.getEmbedders(this.uid);
+    }
+
+    /**
+     * Updates the embedders settings of the index
+     *
+     * @param embedders a Map that contains the new embedders settings
+     * @return TaskInfo instance
+     * @throws MeilisearchException if an error occurs
+     * @see <a href="https://www.meilisearch.com/docs/reference/api/settings#update-embedders">API
+     *     specification</a>
+     */
+    public TaskInfo updateEmbeddersSettings(Map<String, Embedder> embedders)
+            throws MeilisearchException {
+        return this.settingsHandler.updateEmbedders(this.uid, embedders);
+    }
+
+    /**
+     * Resets the embedders settings of the index
+     *
+     * @return TaskInfo instance
+     * @throws MeilisearchException if an error occurs
+     * @see <a href="https://www.meilisearch.com/docs/reference/api/settings#reset-embedders">API
+     *     specification</a>
+     */
+    public TaskInfo resetEmbeddersSettings() throws MeilisearchException {
+        return this.settingsHandler.resetEmbedders(this.uid);
+    }
 }

src/main/java/com/meilisearch/sdk/IndexSearchRequest.java

@@ -38,13 +38,14 @@ public class IndexSearchRequest {
     private FederationOptions federationOptions;
     protected String[] locales;
     protected String distinct;
+    protected Boolean retrieveVectors;
 
     /**
      * Constructor for MultiSearchRequest for building search queries with the default values:
      * offset: 0, limit: 20, attributesToRetrieve: ["*"], attributesToCrop: null, cropLength: 200,
      * attributesToHighlight: null, filter: null, showMatchesPosition: false, facets: null, sort:
      * null, showRankingScore: false, showRankingScoreDetails: false, rankingScoreThreshold: null
-     * distinct: null
+     * distinct: null, retrieveVectors: false
      *
      * @param indexUid uid of the requested index String
      */
@@ -106,7 +107,8 @@ public String toString() {
                         .putOpt("rankingScoreThreshold", this.rankingScoreThreshold)
                         .putOpt("attributesToSearchOn", this.attributesToSearchOn)
                         .putOpt("locales", this.locales)
-                        .putOpt("distinct", this.distinct);
+                        .putOpt("distinct", this.distinct)
+                        .putOpt("retrieveVectors", this.retrieveVectors);
 
         return jsonObject.toString();
     }

src/main/java/com/meilisearch/sdk/SearchRequest.java

@@ -1,5 +1,6 @@
 package com.meilisearch.sdk;
 
+import com.meilisearch.sdk.model.Hybrid;
 import com.meilisearch.sdk.model.MatchingStrategy;
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -42,12 +43,15 @@ public class SearchRequest {
     protected Double rankingScoreThreshold;
     protected String[] locales;
     protected String distinct;
-
+    protected Hybrid hybrid;
+    protected Double[] vector;
+    protected Boolean retrieveVectors;
     /**
      * Constructor for SearchRequest for building search queries with the default values: offset: 0,
      * limit: 20, attributesToRetrieve: ["*"], attributesToCrop: null, cropLength: 200,
      * attributesToHighlight: null, filter: null, showMatchesPosition: false, facets: null, sort:
-     * null, showRankingScore: false, showRankingScoreDetails: false, rankingScoreThreshold: null
+     * null, showRankingScore: false, showRankingScoreDetails: false, rankingScoreThreshold: null,
+     * retrieveVectors: false
      *
      * @param q Query String
      */
@@ -104,7 +108,13 @@ public String toString() {
                         .putOpt("showRankingScoreDetails", this.showRankingScoreDetails)
                         .putOpt("rankingScoreThreshold", this.rankingScoreThreshold)
                         .putOpt("locales", this.locales)
-                        .putOpt("distinct", this.distinct);
+                        .putOpt("distinct", this.distinct)
+                        .putOpt("vector", this.vector)
+                        .putOpt("retrieveVectors", this.retrieveVectors);
+
+        if (this.hybrid != null) {
+            jsonObject.put("hybrid", this.hybrid.toJSONObject());
+        }
 
         return jsonObject.toString();
     }

src/main/java/com/meilisearch/sdk/SettingsHandler.java

@@ -2,6 +2,7 @@
 
 import com.meilisearch.sdk.exceptions.MeilisearchException;
 import com.meilisearch.sdk.http.URLBuilder;
+import com.meilisearch.sdk.model.Embedder;
 import com.meilisearch.sdk.model.Faceting;
 import com.meilisearch.sdk.model.LocalizedAttribute;
 import com.meilisearch.sdk.model.Pagination;
@@ -769,4 +770,47 @@ public TaskInfo resetNonSeparatorTokensSettings(String uid) {
         return httpClient.delete(
                 settingsPath(uid).addSubroute("non-separator-tokens").getURL(), TaskInfo.class);
     }
+
+    /**
+     * Gets the embedders settings of the index
+     *
+     * @param uid Index identifier
+     * @return a Map that contains all embedders settings
+     * @throws MeilisearchException if an error occurs
+     */
+    Map<String, Embedder> getEmbedders(String uid) throws MeilisearchException {
+        return httpClient.get(
+                settingsPath(uid).addSubroute("embedders").getURL(),
+                Map.class,
+                String.class,
+                Embedder.class);
+    }
+
+    /**
+     * Updates the embedders settings of the index
+     *
+     * @param uid Index identifier
+     * @param embedders a Map that contains the new embedders settings
+     * @return TaskInfo instance
+     * @throws MeilisearchException if an error occurs
+     */
+    TaskInfo updateEmbedders(String uid, Map<String, Embedder> embedders)
+            throws MeilisearchException {
+        return httpClient.patch(
+                settingsPath(uid).addSubroute("embedders").getURL(),
+                embedders == null ? null : httpClient.jsonHandler.encode(embedders),
+                TaskInfo.class);
+    }
+
+    /**
+     * Resets the embedders settings of the index
+     *
+     * @param uid Index identifier
+     * @return TaskInfo instance
+     * @throws MeilisearchException if an error occurs
+     */
+    TaskInfo resetEmbedders(String uid) throws MeilisearchException {
+        return httpClient.delete(
+                settingsPath(uid).addSubroute("embedders").getURL(), TaskInfo.class);
+    }
 }

src/main/java/com/meilisearch/sdk/SimilarDocumentRequest.java

@@ -21,28 +21,22 @@ public class SimilarDocumentRequest {
     private Double rankingScoreThreshold;
     private Boolean retrieveVectors;
 
-    /**
-     * Constructor for SimilarDocumentsRequest for building search request for similar documents
-     * with the default values: id null, embedder "default", attributesToRetrieve ["*"], offset 0,
-     * limit 20, filter null, showRankingScore false, showRankingScoreDetails false,
-     * rankingScoreThreshold null, retrieveVectors false
-     */
+    /** Constructor for SimilarDocumentsRequest for building search request for similar documents */
     public SimilarDocumentRequest() {}
 
     @Override
     public String toString() {
-        JSONObject jsonObject =
-                new JSONObject()
-                        .put("id", this.id)
-                        .put("embedder", this.embedder)
-                        .put("attributesToRetrieve", this.attributesToRetrieve)
-                        .put("offset", this.offset)
-                        .put("limit", this.limit)
-                        .put("filter", this.filter)
-                        .put("showRankingScore", this.showRankingScore)
-                        .put("showRankingScoreDetails", this.showRankingScoreDetails)
-                        .put("rankingScoreThreshold", this.rankingScoreThreshold)
-                        .put("retrieveVectors", this.retrieveVectors);
+        JSONObject jsonObject = new JSONObject();
+        jsonObject.put("id", this.id);
+        jsonObject.put("embedder", this.embedder);
+        jsonObject.putOpt("attributesToRetrieve", this.attributesToRetrieve);
+        jsonObject.putOpt("offset", this.offset);
+        jsonObject.putOpt("limit", this.limit);
+        jsonObject.putOpt("filter", this.filter);
+        jsonObject.putOpt("showRankingScore", this.showRankingScore);
+        jsonObject.putOpt("showRankingScoreDetails", this.showRankingScoreDetails);
+        jsonObject.putOpt("rankingScoreThreshold", this.rankingScoreThreshold);
+        jsonObject.putOpt("retrieveVectors", this.retrieveVectors);
 
         return jsonObject.toString();
     }

src/main/java/com/meilisearch/sdk/model/Embedder.java

@@ -0,0 +1,73 @@
+package com.meilisearch.sdk.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import java.util.Map;
+import lombok.*;
+import lombok.experimental.Accessors;
+
+@Builder
+@AllArgsConstructor(access = AccessLevel.PACKAGE)
+@Getter
+@Setter
+@Accessors(chain = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class Embedder {
+    /** Source of the embedder. Accepts: ollama, rest, openAI, huggingFace and userProvided */
+    protected EmbedderSource source;
+
+    /**
+     * API key for authentication with the embedder service. Optional: Only applicable for openAi,
+     * ollama, and rest sources.
+     */
+    protected String apiKey;
+
+    /**
+     * Model to use for generating embeddings. Optional: Only applicable for ollama, openAI, and
+     * huggingFace sources.
+     */
+    protected String model;
+
+    /** Template for document embedding. Optional. */
+    protected String documentTemplate;
+
+    /**
+     * Dimensions of the embedding vectors. Optional: Only applicable for openAi, huggingFace,
+     * ollama, and rest sources.
+     */
+    protected Integer dimensions;
+
+    /** Distribution configuration. Optional. */
+    protected EmbedderDistribution distribution;
+
+    /** Request configuration. Mandatory only when using rest embedder, optional otherwise. */
+    protected Map<String, Object> request;
+
+    /** Response configuration. Mandatory only when using rest embedder, optional otherwise. */
+    protected Map<String, Object> response;
+
+    /** Maximum bytes for document template. Optional. */
+    protected Integer documentTemplateMaxBytes;
+
+    /** Revision identifier. Optional: Only applicable for huggingFace. */
+    protected String revision;
+
+    /** HTTP headers. Optional: Only applicable for rest. */
+    protected Map<String, String> headers;
+
+    /** Whether to use binary quantization. Optional. */
+    protected Boolean binaryQuantized;
+
+    /** URL for the embedder service. Optional. */
+    protected String url;
+
+    /** Input fields for the embedder. Optional. */
+    protected String[] inputField;
+
+    /** Type of input for the embedder. Optional. */
+    protected EmbedderInputType inputType;
+
+    /** Query for the embedder. Optional. */
+    protected String query;
+
+    public Embedder() {}
+}

src/main/java/com/meilisearch/sdk/model/EmbedderDistribution.java

@@ -0,0 +1,56 @@
+package com.meilisearch.sdk.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+
+/**
+ * Describes the natural distribution of search results for embedders. Contains mean and sigma
+ * values, each between 0 and 1.
+ */
+@Builder
+@AllArgsConstructor(access = AccessLevel.PACKAGE)
+@NoArgsConstructor(access = AccessLevel.PUBLIC)
+@Getter
+@Setter
+@Accessors(chain = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class EmbedderDistribution {
+    /** Mean value of the distribution, between 0 and 1 */
+    private Double mean;
+
+    /** Sigma (standard deviation) value of the distribution, between 0 and 1 */
+    private Double sigma;
+
+    /**
+     * Creates a uniform distribution with default values
+     *
+     * @return An EmbedderDistribution instance with mean=0.5 and sigma=0.5
+     */
+    public static EmbedderDistribution uniform() {
+        return new EmbedderDistribution().setMean(0.5).setSigma(0.5);
+    }
+
+    /**
+     * Creates a custom distribution with specified mean and sigma values
+     *
+     * @param mean Mean value between 0 and 1
+     * @param sigma Sigma value between 0 and 1
+     * @return An EmbedderDistribution instance with the specified values
+     * @throws IllegalArgumentException if mean or sigma are outside the valid range
+     */
+    public static EmbedderDistribution custom(double mean, double sigma) {
+        if (mean < 0 || mean > 1) {
+            throw new IllegalArgumentException("Mean must be between 0 and 1");
+        }
+        if (sigma < 0 || sigma > 1) {
+            throw new IllegalArgumentException("Sigma must be between 0 and 1");
+        }
+        return new EmbedderDistribution().setMean(mean).setSigma(sigma);
+    }
+}