Merge branch 'main' into npapa/timetravel

Author: Nikos Papailiou

.github/workflows/ci_python.yml

@@ -35,4 +35,8 @@ jobs:
           #pip uninstall -y tiledb.vector_search
           #pip install -e .
           #pytest
+          pip install -r test/ipynb/requirements.txt
+          pytest --nbmake test/ipynb
+        env:
+          TILEDB_REST_TOKEN: ${{ secrets.TILEDB_CLOUD_HELPER_VAR }}
         shell: bash -el {0}

README.md

@@ -42,3 +42,15 @@ development-build instructions. For large new
 features, please open an issue to discuss goals and approach in order
 to ensure a smooth PR integration and review process. All contributions
 must be licensed under the repository's [MIT License](../LICENSE).
+
+# Testing
+
+* Unit tests: `pytest`
+* Demo notebooks:
+  * ```
+    pip install -r test/ipynb/requirements.txt
+    pytest --nbmake test/ipynb
+    ```
+* Credentials:
+  * Some tests run on TileDB Cloud using your current environment variable `TILEDB_REST_TOKEN` -- you will need a valid API token for the tests to pass 
+  * For continuous integration, the token is configured for the `unittest` user and all tests should pass

apis/python/pyproject.toml

@@ -26,7 +26,7 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-test = ["pytest"]
+test = ["nbmake", "pytest"]
 
 
 [project.urls]

apis/python/src/tiledb/vector_search/flat_index.py

@@ -38,10 +38,12 @@ def __init__(
             config=config,
             timestamp=self.base_array_timestamp,
         )
-        self.ids_uri = self.group[
-            storage_formats[self.storage_version]["IDS_ARRAY_NAME"] + self.index_version
-        ].uri
-        if tiledb.array_exists(self.ids_uri, self.ctx):
+        # Check for existence of ids array. Previous versions were not using external_ids in the ingestion assuming
+        # that the external_ids were the position of the vector in the array.
+        if storage_formats[self.storage_version]["IDS_ARRAY_NAME"] + self.index_version in self.group:
+            self.ids_uri = self.group[
+                storage_formats[self.storage_version]["IDS_ARRAY_NAME"] + self.index_version
+            ].uri
             self._ids = read_vector_u64(self.ctx, self.ids_uri, 0, 0, self.base_array_timestamp)
         else:
             self._ids = StdVector_u64(np.arange(self.size).astype(np.uint64))

apis/python/src/tiledb/vector_search/index.py

@@ -279,3 +279,15 @@ def consolidate_updates(self):
             config=self.config,
         )
         return new_index
+
+    @staticmethod
+    def delete_index(uri, config):
+        try:
+            group = tiledb.Group(uri, "m", config=config)
+        except tiledb.TileDBError as err:
+            message = str(err)
+            if "group does not exist" in message:
+                return
+            else:
+                raise err
+        group.delete()

apis/python/src/tiledb/vector_search/ingestion.py

@@ -1751,11 +1751,12 @@ def consolidate_and_vacuum(
             conf = tiledb.Config(config)
             conf["sm.consolidation.mode"] = mode
             conf["sm.vacuum.mode"] = mode
-            tiledb.consolidate(group[PARTS_ARRAY_NAME].uri, config=conf)
-            tiledb.vacuum(group[PARTS_ARRAY_NAME].uri, config=conf)
-            if index_type == "IVF_FLAT":
-                tiledb.consolidate(group[IDS_ARRAY_NAME].uri, config=conf)
-                tiledb.vacuum(group[IDS_ARRAY_NAME].uri, config=conf)
+            ids_uri = group[IDS_ARRAY_NAME].uri
+            parts_uri = group[PARTS_ARRAY_NAME].uri
+            tiledb.consolidate(parts_uri, config=conf)
+            tiledb.vacuum(parts_uri, config=conf)
+            tiledb.consolidate(ids_uri, config=conf)
+            tiledb.vacuum(ids_uri, config=conf)
 
         # TODO remove temp data for tiledb URIs
         if not index_group_uri.startswith("tiledb://"):

apis/python/test/common.py

@@ -2,6 +2,8 @@
 
 import numpy as np
 
+import random
+import string
 import tiledb
 
 
@@ -192,3 +194,8 @@ def accuracy(result, gt, external_ids_offset=0, updated_ids=None, only_updated_i
         total += len(temp_result)
         found += len(np.intersect1d(temp_result, gt[i]))
     return found / total
+
+# Generate random names for test array uris
+def random_name(name: str) -> str:
+    suffix = "".join(random.choices(string.ascii_letters, k=10))
+    return f"zzz_unittest_{name}_{suffix}"

apis/python/test/ipynb/README.md

@@ -0,0 +1,17 @@
+# Cloud Notebook tests
+
+Run all tests:
+```
+pip install -r test/ipynb/requirements.txt
+pytest --nbmake test/ipynb
+```
+
+This is using [nbmake](https://github.com/treebeardtech/nbmake) to test notebooks.
+Credentials:
+* Local tests: use `TILEDB_REST_TOKEN` -- you will need a valid API token for the tests to pass
+* For continuous integration, the token is configured for the `unittest` user and all tests should pass
+
+When changes get merged in these files make sure that you also propagate the changes to the Cloud registered notebooks:
+* [tiledb_101_vector_search.ipynb](https://cloud.tiledb.com/notebooks/details/TileDB-Inc/b05dd4b4-ba1c-41c6-a3c9-a1de70abb039/preview)
+* [staging-vector-search-checks-py.ipynb](https://console.dev.tiledb.io/notebooks/details/TileDB-Inc/299dd052-6b45-4943-88ae-37639b7b4b48/preview)
+* [image-search-dashboard](https://cloud.tiledb.com/notebooks/details/TileDB-Inc/9289229a-1742-4f99-9b86-0bb1339b31a0/preview)

apis/python/test/ipynb/image-search-dashboard.ipynb

@@ -0,0 +1,4 @@
+efficientnet
+ipywidgets
+panel
+tensorflow