Use groups instead of os.path.join and register arrays with relative paths

Author: Nikos Papailiou

apis/python/src/tiledb/vector_search/index.py

@@ -3,6 +3,10 @@
 import numpy as np
 from tiledb.vector_search.module import *
 
+CENTROIDS_ARRAY_NAME = "centroids.tdb"
+INDEX_ARRAY_NAME = "index.tdb"
+IDS_ARRAY_NAME = "ids.tdb"
+PARTS_ARRAY_NAME = "parts.tdb"
 
 class Index:
     def query(self, targets: np.ndarray, k=10, nqueries=10, nthreads=8, nprobe=1):
@@ -83,10 +87,11 @@ class IVFFlatIndex(Index):
     def __init__(
         self, uri, dtype: np.dtype, memory_budget: int = -1, ctx: "Ctx" = None
     ):
-        self.parts_db_uri = os.path.join(uri, "parts.tdb")
-        self.centroids_uri = os.path.join(uri, "centroids.tdb")
-        self.index_uri = os.path.join(uri, "index.tdb")
-        self.ids_uri = os.path.join(uri, "ids.tdb")
+        group = tiledb.Group(uri)
+        self.parts_db_uri = group[PARTS_ARRAY_NAME].uri
+        self.centroids_uri = group[CENTROIDS_ARRAY_NAME].uri
+        self.index_uri = group[INDEX_ARRAY_NAME].uri
+        self.ids_uri = group[IDS_ARRAY_NAME].uri
         self.dtype = dtype
         self.memory_budget = memory_budget
         self.ctx = ctx

apis/python/src/tiledb/vector_search/ingestion.py

@@ -224,7 +224,7 @@ def create_arrays(
                 )
                 logger.info(parts_schema)
                 tiledb.Array.create(parts_uri, parts_schema)
-                group.add(parts_uri, name=PARTS_ARRAY_NAME)
+                group.add(PARTS_ARRAY_NAME, name=PARTS_ARRAY_NAME, relative=True)
 
         elif index_type == "IVF_FLAT":
             centroids_uri = f"{group.uri}/{CENTROIDS_ARRAY_NAME}"
@@ -272,7 +272,7 @@ def create_arrays(
                 )
                 logger.info(centroids_schema)
                 tiledb.Array.create(centroids_uri, centroids_schema)
-                group.add(centroids_uri, name=CENTROIDS_ARRAY_NAME)
+                group.add(CENTROIDS_ARRAY_NAME, name=CENTROIDS_ARRAY_NAME, relative=True)
 
             if not tiledb.array_exists(index_uri):
                 logger.info("Creating index array")
@@ -294,7 +294,7 @@ def create_arrays(
                 )
                 logger.info(index_schema)
                 tiledb.Array.create(index_uri, index_schema)
-                group.add(index_uri, name=INDEX_ARRAY_NAME)
+                group.add(INDEX_ARRAY_NAME, name=INDEX_ARRAY_NAME, relative=True)
 
             if not tiledb.array_exists(ids_uri):
                 logger.info("Creating ids array")
@@ -316,7 +316,7 @@ def create_arrays(
                 )
                 logger.info(ids_schema)
                 tiledb.Array.create(ids_uri, ids_schema)
-                group.add(ids_uri, name=IDS_ARRAY_NAME)
+                group.add(IDS_ARRAY_NAME, name=IDS_ARRAY_NAME, relative=True)
 
             if not tiledb.array_exists(parts_uri):
                 logger.info("Creating parts array")
@@ -346,7 +346,7 @@ def create_arrays(
                 )
                 logger.info(parts_schema)
                 tiledb.Array.create(parts_uri, parts_schema)
-                group.add(parts_uri, name=PARTS_ARRAY_NAME)
+                group.add(PARTS_ARRAY_NAME, name=PARTS_ARRAY_NAME, relative=True)
 
             vfs = tiledb.VFS()
             if vfs.is_dir(partial_write_array_dir_uri):