Merge pull request #1167 from awais786/enable-multi-modal

Add support for multi-modal search

Author: Strift

.code-samples.meilisearch.yaml

@@ -767,6 +767,25 @@ get_batch_1: |-
   client.get_batch(BATCH_UID)
 get_similar_post_1: |-
   client.index("INDEX_NAME").get_similar_documents({"id": "TARGET_DOCUMENT_ID", "embedder": "default"})
+search_parameter_reference_media_1: |-
+  client.index('movies_fragments').search(
+      "a futuristic space movie",
+      {
+          "media": {
+              "textAndPoster": {
+                  "text": "a futuristic space movie",
+                  "image": {
+                      "mime": "image/jpeg",
+                      "data": "BASE64_ENCODED_IMAGE_DATA"
+                  }
+              }
+          },
+          "hybrid": {
+              "embedder": "multimodal",
+              "semanticRatio": 1.0
+          },
+      }
+  )
 webhooks_get_1: |-
   client.get_webhooks()
 webhooks_get_single_1: |-

datasets/multimodal_movies.json

@@ -0,0 +1,9 @@
+[
+{"id":"287947","title":"Shazam!","poster":"https://image.tmdb.org/t/p/w1280/xnopI5Xtky18MPhK40cZAGAOVeV.jpg","overview":"A boy is given the ability to become an adult superhero in times of need with a single magic word.","release_date":1553299200},
+{"id":"299537","title":"Captain Marvel","poster":"https://image.tmdb.org/t/p/w1280/AtsgWhDnHTq68L0lLsUrCnM7TjG.jpg","overview":"The story follows Carol Danvers as she becomes one of the universe’s most powerful heroes when Earth is caught in the middle of a galactic war between two alien races. Set in the 1990s, Captain Marvel is an all-new adventure from a previously unseen period in the history of the Marvel Cinematic Universe.","release_date":1551830400},
+{"id":"522681","title":"Escape Room","poster":"https://image.tmdb.org/t/p/w1280/8yZAx7tlKRZIg7pJfaPhl00yHIQ.jpg","overview":"Six strangers find themselves in circumstances beyond their control, and must use their wits to survive.","release_date":1546473600},
+{"id":"166428","title":"How to Train Your Dragon: The Hidden World","poster":"https://image.tmdb.org/t/p/w1280/xvx4Yhf0DVH8G4LzNISpMfFBDy2.jpg","overview":"As Hiccup fulfills his dream of creating a peaceful dragon utopia, Toothless’ discovery of an untamed, elusive mate draws the Night Fury away. When danger mounts at home and Hiccup’s reign as village chief is tested, both dragon and rider must make impossible decisions to save their kind.","release_date":1546473600},
+{"id":"450465","title":"Glass","poster":"https://image.tmdb.org/t/p/w1280/svIDTNUoajS8dLEo7EosxvyAsgJ.jpg","overview":"In a series of escalating encounters, security guard David Dunn uses his supernatural abilities to track Kevin Wendell Crumb, a disturbed man who has twenty-four personalities. Meanwhile, the shadowy presence of Elijah Price emerges as an orchestrator who holds secrets critical to both men.","release_date":1547596800},
+{"id":"495925","title":"Doraemon the Movie: Nobita's Treasure Island","poster":"https://image.tmdb.org/t/p/w1280/cmJ71gdZxCqkMUvGwWgSg3MK7pC.jpg","overview":"The story is based on Robert Louis Stevenson's Treasure Island novel.","release_date":1520035200},
+{"id":"329996","title":"Dumbo","poster":"https://image.tmdb.org/t/p/w1280/279PwJAcelI4VuBtdzrZASqDPQr.jpg","overview":"A young elephant, whose oversized ears enable him to fly, helps save a struggling circus, but when the circus plans a new venture, Dumbo and his friends discover dark secrets beneath its shiny veneer.","release_date":1553644800},
+{"id":"299536","title":"Avengers: Infinity War","poster":"https://image.tmdb.org/t/p/w1280/7WsyChQLEftFiDOVTGkv3hFpyyt.jpg","overview":"As the Avengers and their allies have continued to protect the world from threats too large for any one hero to handle, a new danger has emerged from the cosmic shadows: Thanos. A despot of intergalactic infamy, his goal is to collect all six Infinity Stones, artifacts of unimaginable power, and use them to inflict his twisted will on all of reality. Everything the Avengers have fought for has led up to this moment - the fate of Earth and existence itself has never been more uncertain.","release_date":1524618000}]

meilisearch/client.py

@@ -1117,3 +1117,41 @@ def _valid_uuid(uuid: str) -> bool:
         )
         match = uuid4hex.match(uuid)
         return bool(match)
+
+    def get_experimental_features(self) -> Dict[str, Any]:
+        """Retrieve the current settings for all experimental features.
+
+        Returns
+        -------
+        features:
+            A dictionary mapping feature names to their enabled/disabled state.
+            For example: {"multimodal": True, "vectorStore": False}
+
+        Raises
+        ------
+        MeilisearchApiError
+            An error containing details about why Meilisearch can't process your request. Meilisearch error codes are described here: https://www.meilisearch.com/docs/reference/errors/error_codes#meilisearch-errors
+        """
+        return self.http.get(self.config.paths.experimental_features)
+
+    def update_experimental_features(self, features: Dict[str, bool]) -> Dict[str, Any]:
+        """Update one or more experimental features.
+
+        Parameters
+        ----------
+        features:
+            A dictionary mapping feature names to booleans.
+            For example, {"multimodal": True} to enable multimodal,
+            or {"multimodal": True, "vectorStore": False} to update multiple features.
+
+        Returns
+        -------
+        features:
+            The updated experimental features settings as a dictionary.
+
+        Raises
+        ------
+        MeilisearchApiError
+            An error containing details about why Meilisearch can't process your request. Meilisearch error codes are described here: https://www.meilisearch.com/docs/reference/errors/error_codes#meilisearch-errors
+        """
+        return self.http.patch(self.config.paths.experimental_features, body=features)

meilisearch/config.py

@@ -47,6 +47,7 @@ class Paths:
         localized_attributes = "localized-attributes"
         edit = "edit"
         network = "network"
+        experimental_features = "experimental-features"
         webhooks = "webhooks"
 
     def __init__(

meilisearch/models/embedders.py

@@ -167,6 +167,12 @@ class RestEmbedder(CamelBase):
         Template defining the data Meilisearch sends to the embedder
     document_template_max_bytes: Optional[int]
         Maximum allowed size of rendered document template (defaults to 400)
+    indexing_fragments: Optional[Dict[str, Dict[str, Any]]]
+        Defines how to fragment documents for indexing (multi-modal search).
+        Fragments can contain complex nested structures (e.g., lists of objects).
+    search_fragments: Optional[Dict[str, Dict[str, Any]]]
+        Defines how to fragment search queries (multi-modal search).
+        Fragments can contain complex nested structures (e.g., lists of objects).
     request: Dict[str, Any]
         A JSON value representing the request Meilisearch makes to the remote embedder
     response: Dict[str, Any]
@@ -185,6 +191,8 @@ class RestEmbedder(CamelBase):
     dimensions: Optional[int] = None
     document_template: Optional[str] = None
     document_template_max_bytes: Optional[int] = None
+    indexing_fragments: Optional[Dict[str, Dict[str, Any]]] = None
+    search_fragments: Optional[Dict[str, Dict[str, Any]]] = None
     request: Dict[str, Any]
     response: Dict[str, Any]
     headers: Optional[Dict[str, str]] = None

tests/client/test_experimental_features.py

@@ -0,0 +1,22 @@
+"""Tests for experimental features API."""
+
+
+def test_get_experimental_features(client):
+    """Test getting experimental features."""
+    response = client.get_experimental_features()
+    assert isinstance(response, dict)
+    # Check that at least one known experimental feature is present
+    assert "multimodal" in response or "vectorStoreSetting" in response
+
+
+def test_update_experimental_features(client):
+    """Test updating experimental features."""
+    # Enable multimodal
+    response = client.update_experimental_features({"multimodal": True})
+    assert isinstance(response, dict)
+    assert response.get("multimodal") is True
+
+    # Disable multimodal
+    response = client.update_experimental_features({"multimodal": False})
+    assert isinstance(response, dict)
+    assert response.get("multimodal") is False