wip

Author: miguelgrinberg

elasticsearch/dsl/utils.py

@@ -609,11 +609,11 @@ def to_dict(self, skip_empty: bool = True) -> Dict[str, Any]:
             if isinstance(v, AttrList):
                 v = v._l_
 
-            if skip_empty:
-                # don't serialize empty values
-                # careful not to include numeric zeros
-                if v in ([], {}, None):
-                    continue
+            # if skip_empty:
+            #     # don't serialize empty values
+            #     # careful not to include numeric zeros
+            #     if v in ([], {}, None):
+            #         continue
 
             out[k] = v
         return out

examples/quotes/backend/quotes.py

@@ -1,4 +1,5 @@
 import asyncio
+import base64
 import csv
 import os
 from time import time
@@ -8,19 +9,23 @@
 from pydantic import BaseModel, Field, ValidationError
 from sentence_transformers import SentenceTransformer
 
-from elasticsearch import NotFoundError
+from elasticsearch import NotFoundError, OrjsonSerializer
 from elasticsearch.dsl.pydantic import AsyncBaseESModel
 from elasticsearch import dsl
+from elasticsearch.dsl.types import DenseVectorIndexOptions
 
 model = SentenceTransformer("all-MiniLM-L6-v2")
-dsl.async_connections.create_connection(hosts=[os.environ['ELASTICSEARCH_URL']])
+dsl.async_connections.create_connection(hosts=[os.environ['ELASTICSEARCH_URL']], serializer=OrjsonSerializer()
+)
 
 
 class Quote(AsyncBaseESModel):
     quote: str
     author: Annotated[str, dsl.Keyword()]
     tags: Annotated[list[str], dsl.Keyword()]
-    embedding: Annotated[list[float], dsl.DenseVector()] = Field(init=False, default=[])
+    embedding: Annotated[list[float], dsl.DenseVector(
+        index_options=DenseVectorIndexOptions(type="flat"),
+    )] = Field(init=False, default=[])
 
     class Index:
         name = 'quotes'
@@ -135,7 +140,10 @@ async def search_quotes(req: SearchRequest) -> SearchResponse:
 def embed_quotes(quotes):
     embeddings = model.encode([q.quote for q in quotes])
     for q, e in zip(quotes, embeddings):
-        q.embedding = e.tolist()
+        q.embedding = e
+        # q.embedding = e.tolist()
+        ##byte_array = e.byteswap().tobytes()
+        ##q.embedding = base64.b64encode(byte_array).decode()
 
 
 async def ingest_quotes():

utils/templates/field.py.tpl

@@ -159,7 +159,11 @@ class Field(DslBase):
     def clean(self, data: Any) -> Any:
         if data is not None:
             data = self.deserialize(data)
-        if data in (None, [], {}) and self._required:
+        # the "data is ..." comparisons below work well when data is a numpy
+        # array (only for dense vector fields)
+        # unfortunately numpy overrides the == operator in a way that causes
+        # errors when used instead of "is"
+        if (data is None or data is [] or data is {}) and self._required:
             raise ValidationException("Value required for this field.")
         return data
 
@@ -417,13 +421,6 @@ class {{ k.name }}({{ k.parent }}):
         if self._element_type in ["float", "byte"]:
             kwargs["multi"] = True
         super().__init__(*args, **kwargs)
-
-    def _deserialize(self, data: Any) -> Any:
-        if self._element_type == "float":
-            return float(data)
-        elif self._element_type == "byte":
-            return int(data)
-        return data
         {% elif k.field == "scaled_float" %}
         if 'scaling_factor' not in kwargs:
             if len(args) > 0: