Reinstate binary-encoding of vectors in Collections write path (#338)

* reinstate binary encoding of vectors in collection write path (Data API fixed 1710)

* adjust tests on collection bin-enc + docstring for APIOptions

* Adapt tests to binenc in collections and docker lagging behind

Author: hemidactylus

CHANGES

@@ -16,6 +16,7 @@ Spawner methods for databases/admins standardized; they don't issue DevOps API c
 Support for Astra DB "custom domain" endpoints for database
     - in which case: `.id`, `.region`, `.get_database_admin()`, `.info()` and `.name()` aren't available.
 Support for the `indexType` field to describe table indexes (for compatibility, said field is not mandatory).
+Collections write path now obeys the binary-encoding API Option (which in turn defaults to True. Formerly bin-encoding was always turned off.)
 DataAPITime: support for "hh:mm" no-seconds format.
 DataAPIDuration: improved parse performance by caching regexpes.
 DataAPIDuration: support for "P4W"-type strings and for zeroes such as "P", "-PR".

astrapy/data/utils/collection_converters.py

@@ -59,10 +59,7 @@ def preprocess_collection_payload_value(
             _value = convert_vector_to_floats(_value)
         # now _value is either a list or a DataAPIVector.
         # can/should it be binary-encoded?
-        can_bin_encode = False
-        # TODO: reinstate the following condition once the Data API
-        # correctly excludes $binary from indexing for collections:
-        # can_bin_encode = path[0] in {"insertOne", "insertMany"}
+        can_bin_encode = path[0] in {"insertOne", "insertMany"}
         # will it be bin-encoded?
         if isinstance(_value, DataAPIVector):
             # if I can, I will

astrapy/utils/api_options.py

@@ -281,7 +281,6 @@ class SerdesOptions:
             as instances of `DataAPIVector`, while for collections this affects
             the encoding of the quantity found in the "$vector" field, if present,
             regardless of its representation in the method argument. Defaults to True.
-            *Note: For release `2.0.0-preview`, binary encoding in collections is OFF.*
         custom_datatypes_in_reading: Read-Path. This setting determines whether return
             values from read methods should use astrapy custom classes (default setting
             of True), or try to use only standard-library data types instead (False).

tests/base/integration/collections/test_collection_dml_async.py

@@ -212,21 +212,21 @@ async def test_collection_vector_insertion_options_async(
         )
 
         # check how the documents are stored
-        # TODO: reinstate the expectation once collection regains conditional binenc.
         expect_binaries = {
-            "Yb_Yc_()": False,  # True,
-            "Yb_Yc_[]": False,  # True,
-            "Yb_Yc_DV": False,  # True,
-            "Nb_Yc_()": False,  # False,
-            "Nb_Yc_[]": False,  # False,
-            "Nb_Yc_DV": False,  # False,
+            "Yb_Yc_()": True,
+            "Yb_Yc_[]": True,
+            "Yb_Yc_DV": True,
+            "Nb_Yc_()": False,
+            "Nb_Yc_[]": False,
+            "Nb_Yc_DV": False,
             #
-            "Yb_Nc_[]": False,  # True,
-            "Yb_Nc_DV": False,  # True,
+            "Yb_Nc_[]": True,
+            "Yb_Nc_DV": True,
             #
-            "Nb_Nc_[]": False,  # False,
-            "Nb_Nc_DV": False,  # False,
+            "Nb_Nc_[]": False,
+            "Nb_Nc_DV": False,
         }
+
         raw_find_response = await async_empty_collection.command(
             body={"find": {"projection": {"_id": True, "$vector": True}}},
         )

tests/base/integration/collections/test_collection_dml_sync.py

@@ -183,21 +183,21 @@ def test_collection_vector_insertion_options_sync(
         )
 
         # check how the documents are stored
-        # TODO: reinstate the expectation once collection regains conditional binenc.
         expect_binaries = {
-            "Yb_Yc_()": False,  # True,
-            "Yb_Yc_[]": False,  # True,
-            "Yb_Yc_DV": False,  # True,
-            "Nb_Yc_()": False,  # False,
-            "Nb_Yc_[]": False,  # False,
-            "Nb_Yc_DV": False,  # False,
+            "Yb_Yc_()": True,
+            "Yb_Yc_[]": True,
+            "Yb_Yc_DV": True,
+            "Nb_Yc_()": False,
+            "Nb_Yc_[]": False,
+            "Nb_Yc_DV": False,
             #
-            "Yb_Nc_[]": False,  # True,
-            "Yb_Nc_DV": False,  # True,
+            "Yb_Nc_[]": True,
+            "Yb_Nc_DV": True,
             #
-            "Nb_Nc_[]": False,  # False,
-            "Nb_Nc_DV": False,  # False,
+            "Nb_Nc_[]": False,
+            "Nb_Nc_DV": False,
         }
+
         raw_find_response = sync_empty_collection.command(
             body={"find": {"projection": {"_id": True, "$vector": True}}},
         )

tests/base/integration/collections/test_collection_vectorize_methods_async.py

@@ -19,13 +19,18 @@
 import pytest
 
 from astrapy import AsyncDatabase
+from astrapy.api_options import APIOptions, SerdesOptions
 from astrapy.constants import DefaultDocumentType
 from astrapy.cursors import AsyncCollectionFindCursor
 from astrapy.data_types import DataAPIVector
 from astrapy.exceptions import CollectionInsertManyException, DataAPIResponseException
 from astrapy.info import CollectionDefinition
 
-from ..conftest import HEADER_EMBEDDING_API_KEY_OPENAI, DefaultAsyncCollection
+from ..conftest import (
+    HEADER_EMBEDDING_API_KEY_OPENAI,
+    IS_ASTRA_DB,
+    DefaultAsyncCollection,
+)
 
 
 @pytest.mark.skipif(
@@ -42,13 +47,19 @@ async def test_collection_methods_vectorize_async(
         acol = async_empty_service_collection
         service_vector_dimension = service_collection_parameters["dimension"]
 
+        # TODO we lift storage of binencoded vectors on nonAstra because docker image
+        # 1.0.20-ct1 does not have fix 1738 yet (long nonindexed binenc strings)
+        binencoptions = APIOptions(
+            serdes_options=SerdesOptions(binary_encode_vectors=IS_ASTRA_DB)
+        )
+
         await acol.insert_one({"t": "tower", "$vectorize": "How high is this tower?"})
         await acol.insert_one({"t": "vectorless"})
-        await acol.insert_one(
+        await acol.with_options(api_options=binencoptions).insert_one(
             {"t": "vectorful", "$vector": [0.01] * service_vector_dimension},
         )
 
-        await acol.insert_many(
+        await acol.with_options(api_options=binencoptions).insert_many(
             [
                 {
                     "t": "guide",

tests/base/integration/collections/test_collection_vectorize_methods_sync.py

@@ -19,11 +19,16 @@
 import pytest
 
 from astrapy import Database
+from astrapy.api_options import APIOptions, SerdesOptions
 from astrapy.data_types import DataAPIVector
 from astrapy.exceptions import CollectionInsertManyException, DataAPIResponseException
 from astrapy.info import CollectionDefinition
 
-from ..conftest import HEADER_EMBEDDING_API_KEY_OPENAI, DefaultCollection
+from ..conftest import (
+    HEADER_EMBEDDING_API_KEY_OPENAI,
+    IS_ASTRA_DB,
+    DefaultCollection,
+)
 
 
 @pytest.mark.skipif(
@@ -40,11 +45,19 @@ def test_collection_methods_vectorize_sync(
         col = sync_empty_service_collection
         service_vector_dimension = service_collection_parameters["dimension"]
 
+        # TODO we lift storage of binencoded vectors on nonAstra because docker image
+        # 1.0.20-ct1 does not have fix 1738 yet (long nonindexed binenc strings)
+        binencoptions = APIOptions(
+            serdes_options=SerdesOptions(binary_encode_vectors=IS_ASTRA_DB)
+        )
+
         col.insert_one({"t": "tower", "$vectorize": "How high is this tower?"})
         col.insert_one({"t": "vectorless"})
-        col.insert_one({"t": "vectorful", "$vector": [0.01] * service_vector_dimension})
+        col.with_options(api_options=binencoptions).insert_one(
+            {"t": "vectorful", "$vector": [0.01] * service_vector_dimension}
+        )
 
-        col.insert_many(
+        col.with_options(api_options=binencoptions).insert_many(
             [
                 {
                     "t": "guide",

tests/base/integration/misc/test_vectorize_ops_async.py

@@ -24,7 +24,7 @@
 
 class TestVectorizeOpsAsync:
     @pytest.mark.describe("test of find_embedding_providers, async")
-    async def test_collection_methods_vectorize_async(
+    async def test_findembeddingproviders_async(
         self,
         async_database: AsyncDatabase,
     ) -> None:

tests/base/integration/misc/test_vectorize_ops_sync.py

@@ -24,7 +24,7 @@
 
 class TestVectorizeOpsSync:
     @pytest.mark.describe("test of find_embedding_providers, sync")
-    def test_collection_methods_vectorize_sync(
+    def test_findembeddingproviders_sync(
         self,
         sync_database: Database,
     ) -> None: