Bump storage version

Author: Nikos Papailiou

apis/python/src/tiledb/vector_search/index.py

@@ -40,17 +40,21 @@ def __init__(
         self.ctx = Ctx(config)
         self.group = tiledb.Group(self.uri, "r", ctx=tiledb.Ctx(config))
         self.storage_version = self.group.meta.get("storage_version", "0.1")
+        if not storage_formats[self.storage_version]["SUPPORT_TIMETRAVEL"] and timestamp is not None:
+            raise ValueError(f"Time traveling is not supported for index storage_version={self.storage_version}")
+
         updates_array_name = storage_formats[self.storage_version][
             "UPDATES_ARRAY_NAME"
         ]
         self.updates_array_uri = f"{self.group.uri}/{updates_array_name}"
         self.index_version = self.group.meta.get("index_version", "")
-        self.ingestion_timestamps = list(json.loads(self.group.meta.get("ingestion_timestamps", "[]")))
+        self.ingestion_timestamps = [int(x) for x in
+                                     list(json.loads(self.group.meta.get("ingestion_timestamps", "[]")))]
         if len(self.ingestion_timestamps) > 0:
             self.latest_ingestion_timestamp = self.ingestion_timestamps[len(self.ingestion_timestamps)-1]
         else:
             self.latest_ingestion_timestamp = MAX_UINT64
-        self.base_sizes = list(json.loads(self.group.meta.get("base_sizes", "[]")))
+        self.base_sizes = [int(x) for x in list(json.loads(self.group.meta.get("base_sizes", "[]")))]
         if len(self.base_sizes) > 0:
             self.base_size = self.base_sizes[len(self.ingestion_timestamps)-1]
         else:
@@ -245,9 +249,12 @@ def get_updates_uri(self):
         return self.updates_array_uri
 
     def open_updates_array(self, timestamp: int = None):
-        if timestamp is not None and timestamp <= self.latest_ingestion_timestamp:
-            raise ValueError(f"Updates at a timestamp before the latest_ingestion_timestamp are not supported. "
-                             f"timestamp: {timestamp}, latest_ingestion_timestamp: {self.latest_ingestion_timestamp}")
+        if timestamp is not None:
+            if not storage_formats[self.storage_version]["SUPPORT_TIMETRAVEL"]:
+                raise ValueError(f"Time traveling is not supported for index storage_version={self.storage_version}")
+            if timestamp <= self.latest_ingestion_timestamp:
+                raise ValueError(f"Updates at a timestamp before the latest_ingestion_timestamp are not supported. "
+                                 f"timestamp: {timestamp}, latest_ingestion_timestamp: {self.latest_ingestion_timestamp}")
         if not tiledb.array_exists(self.updates_array_uri):
             updates_array_name = storage_formats[self.storage_version][
                 "UPDATES_ARRAY_NAME"

apis/python/src/tiledb/vector_search/ingestion.py

@@ -1886,6 +1886,7 @@ def consolidate_and_vacuum(
         group.meta["dtype"] = np.dtype(vector_type).name
         group.meta["partitions"] = partitions
         group.meta["storage_version"] = STORAGE_VERSION
+        group.meta["index_type"] = index_type
         group.meta["base_sizes"] = json.dumps(base_sizes)
 
         if external_ids is not None:

apis/python/src/tiledb/vector_search/storage_formats.py

@@ -11,6 +11,7 @@
         "PARTIAL_WRITE_ARRAY_DIR": "write_temp",
         "DEFAULT_ATTR_FILTERS": None,
         "UPDATES_ARRAY_NAME": "updates",
+        "SUPPORT_TIMETRAVEL": False,
     },
     "0.2": {
         "CENTROIDS_ARRAY_NAME": "partition_centroids",
@@ -22,7 +23,20 @@
         "PARTIAL_WRITE_ARRAY_DIR": "temp_data",
         "DEFAULT_ATTR_FILTERS": tiledb.FilterList([tiledb.ZstdFilter()]),
         "UPDATES_ARRAY_NAME": "updates",
+        "SUPPORT_TIMETRAVEL": False,
+    },
+    "0.3": {
+        "CENTROIDS_ARRAY_NAME": "partition_centroids",
+        "INDEX_ARRAY_NAME": "partition_indexes",
+        "IDS_ARRAY_NAME": "shuffled_vector_ids",
+        "PARTS_ARRAY_NAME": "shuffled_vectors",
+        "INPUT_VECTORS_ARRAY_NAME": "input_vectors",
+        "EXTERNAL_IDS_ARRAY_NAME": "external_ids",
+        "PARTIAL_WRITE_ARRAY_DIR": "temp_data",
+        "DEFAULT_ATTR_FILTERS": tiledb.FilterList([tiledb.ZstdFilter()]),
+        "UPDATES_ARRAY_NAME": "updates",
+        "SUPPORT_TIMETRAVEL": True,
     },
 }
 
-STORAGE_VERSION = "0.2"
+STORAGE_VERSION = "0.3"

apis/python/test/test_ingestion.py

@@ -4,6 +4,7 @@
 
 from tiledb.vector_search.utils import load_fvecs
 from tiledb.vector_search.ingestion import ingest
+from tiledb.vector_search.index import Index
 from tiledb.vector_search.flat_index import FlatIndex
 from tiledb.vector_search.ivf_flat_index import IVFFlatIndex
 from tiledb.cloud.dag import Mode