Fix consolidate_updates error for tiledb:// URIs. (#329)

Fix `consolidate_updates` error for `tiledb://` URIs.

`consolidate_updates` performs `timestamp` consolidation of all updates between consequent ingestions.
This caused failures for file indexing when appending to an existing `tiledb://` index.

This is an optional performance enhancement. We skip it for remote arrays as consolidation of remote arrays currently only supports modes `fragment_meta, commits, metadata`.

Author: NikolaosPapailiou

apis/python/src/tiledb/vector_search/index.py

@@ -403,11 +403,15 @@ def consolidate_updates(self, retrain_index: bool = False, **kwargs):
             if fragment_info.timestamp_range[1] > max_timestamp:
                 max_timestamp = fragment_info.timestamp_range[1]
         max_timestamp += 1
-        conf = tiledb.Config(self.config)
-        conf["sm.consolidation.timestamp_start"] = self.latest_ingestion_timestamp
-        conf["sm.consolidation.timestamp_end"] = max_timestamp
-        tiledb.consolidate(self.updates_array_uri, config=conf)
-        tiledb.vacuum(self.updates_array_uri, config=conf)
+        # Consolidate all updates since the previous ingestion_timestamp.
+        # This is a performance optimization. We skip this for remote arrays as consolidation
+        # of remote arrays currently only supports modes `fragment_meta, commits, metadata`.
+        if not self.updates_array_uri.startswith("tiledb://"):
+            conf = tiledb.Config(self.config)
+            conf["sm.consolidation.timestamp_start"] = self.latest_ingestion_timestamp
+            conf["sm.consolidation.timestamp_end"] = max_timestamp
+            tiledb.consolidate(self.updates_array_uri, config=conf)
+            tiledb.vacuum(self.updates_array_uri, config=conf)
 
         # We don't copy the centroids if self.partitions=0 because this means our index was previously empty.
         should_pass_copy_centroids_uri = (

apis/python/test/test_cloud.py

@@ -187,10 +187,18 @@ def test_cloud_ivf_flat(self):
                 resources=resources,
             )
 
+        index = vs.ivf_flat_index.IVFFlatIndex(
+            uri=index_uri,
+            config=tiledb.cloud.Config().dict(),
+        )
         index.delete(external_id=42)
         _, result_i = index.query(queries, k=k, nprobe=nprobe)
         assert accuracy(result_i, gt_i) > MINIMUM_ACCURACY
 
+        index = index.consolidate_updates()
+        _, result_i = index.query(queries, k=k, nprobe=nprobe)
+        assert accuracy(result_i, gt_i) > MINIMUM_ACCURACY
+
     def test_cloud_ivf_flat_random_sampling(self):
         # NOTE(paris): This was also tested with the following (and also with mode=Mode.BATCH):
         # source_uri = "tiledb://TileDB-Inc/ann_sift1b_raw_vectors_col_major"