Weaviate multitenancy support (#2199)

* feat: weaviate multitenancy support

* update `.tenant` to `.with_tenant`

* Update weaviate_rm.py

* Update WeaviateRM.md

---------

Co-authored-by: Connor Shorten <[email protected]>

Author: Timtech4u

docs/docs/deep-dive/retrieval_models_clients/WeaviateRM.md

@@ -19,6 +19,22 @@ WeaviateRM(
 )
 ```
 
+## Using Multitenancy
+Multi-tenancy allows a collection to efficiently serve isolated groups of data. Each "tenant" in a multi-tenant collection can only access its own data, while sharing the same data structure and settings.
+
+If your Weaviate instance is tenant-aware, you can provide a tenant_id in the WeaviateRM constructor or as a keyword argument:
+
+```python
+retriever_model = WeaviateRM(
+    weaviate_collection_name="<WEAVIATE_COLLECTION>",
+    weaviate_client=weaviate_client,
+    tenant_id="tenant123"
+)
+
+results = retriever_model("Your query here", tenant_id="tenantXYZ")
+```
+When tenant_id is specified, this will scope all retrieval requests to the tenant ID provided.
+
 ## Under the Hood
 
 `forward(self, query_or_queries: Union[str, List[str]], k: Optional[int] = None, **kwargs) -> dspy.Prediction`

dspy/retrieve/weaviate_rm.py

@@ -24,6 +24,7 @@ class WeaviateRM(dspy.Retrieve):
         weaviate_collection_name (str): The name of the Weaviate collection.
         weaviate_client (WeaviateClient): An instance of the Weaviate client.
         k (int, optional): The default number of top passages to retrieve. Default to 3.
+        tenant_id (str, optional): The tenant to retrieve objects from.
 
     Examples:
         Below is a code snippet that shows how to use Weaviate as the default retriever:
@@ -51,11 +52,13 @@ def __init__(
         weaviate_client: Union[weaviate.WeaviateClient, weaviate.Client],
         weaviate_collection_text_key: Optional[str] = "content",
         k: int = 3,
+        tenant_id: Optional[str] = None,
     ):
         self._weaviate_collection_name = weaviate_collection_name
         self._weaviate_client = weaviate_client
         self._weaviate_collection = self._weaviate_client.collections.get(self._weaviate_collection_name)
         self._weaviate_collection_text_key = weaviate_collection_text_key
+        self._tenant_id = tenant_id
 
         # Check the type of weaviate_client (this is added to support v3 and v4)
         if hasattr(weaviate_client, "collections"):
@@ -82,26 +85,23 @@ def forward(self, query_or_queries: Union[str, List[str]], k: Optional[int] = No
         queries = [query_or_queries] if isinstance(query_or_queries, str) else query_or_queries
         queries = [q for q in queries if q]
         passages, parsed_results = [], []
+        tenant = kwargs.pop("tenant_id", self._tenant_id)
         for query in queries:
             if self._client_type == "WeaviateClient":
-                results = self._weaviate_collection.query.hybrid(
-                    query=query,
-                    limit=k,
-                    **kwargs,
-                )
+                if tenant:
+                    results = self._weaviate_collection.query.with_tenant(tenant).hybrid(query=query, limit=k, **kwargs)
+                else:
+                    results = self._weaviate_collection.query.hybrid(query=query, limit=k, **kwargs)
 
                 parsed_results = [result.properties[self._weaviate_collection_text_key] for result in results.objects]
 
             elif self._client_type == "Client":
-                results = (
-                    self._weaviate_client.query.get(
-                        self._weaviate_collection_name,
-                        [self._weaviate_collection_text_key],
+                q = self._weaviate_client.query.get(
+                        self._weaviate_collection_name, [self._weaviate_collection_text_key]
                     )
-                    .with_hybrid(query=query)
-                    .with_limit(k)
-                    .do()
-                )
+                if tenant:
+                    q = q.with_tenant(tenant)
+                results = q.with_hybrid(query=query).with_limit(k).do()
 
                 results = results["data"]["Get"][self._weaviate_collection_name]
                 parsed_results = [result[self._weaviate_collection_text_key] for result in results]
@@ -115,7 +115,7 @@ def get_objects(self, num_samples: int, fields: List[str]) -> List[dict]:
         if self._client_type == "WeaviateClient":
             objects = []
             counter = 0
-            for item in self._weaviate_collection.iterator():
+            for item in self._weaviate_collection.iterator(): # TODO: add tenancy scoping
                 if counter >= num_samples:
                     break
                 new_object = {}
@@ -133,6 +133,6 @@ def insert(self, new_object_properties: dict):
             self._weaviate_collection.data.insert(
                 properties=new_object_properties,
                 uuid=get_valid_uuid(uuid4())
-            )
+            ) # TODO: add tenancy scoping
         else:
             raise AttributeError("`insert` is not supported for the v3 Weaviate Python client, please upgrade to v4.")