Skip to content

Commit 8ce2d7f

Browse files
jparismorganjparismorgan
authored
Remove overwrite from write_index() (#325)
1 parent 21eda51 commit 8ce2d7f

17 files changed

+41
-90
lines changed

‎apis/python/src/tiledb/vector_search/type_erased_module.cc‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -283,11 +283,11 @@ void init_type_erased_module(py::module_& m) {
283283
"write_index",
284284
[](IndexVamana& index,
285285
const tiledb::Context& ctx,
286-
const std::string& group_uri,
287-
bool overwrite) { index.write_index(ctx, group_uri, overwrite); },
286+
const std::string& group_uri) {
287+
index.write_index(ctx, group_uri);
288+
},
288289
py::arg("ctx"),
289-
py::arg("group_uri"),
290-
py::arg_v("overwrite", true))
290+
py::arg("group_uri"))
291291
.def("feature_type_string", &IndexVamana::feature_type_string)
292292
.def("id_type_string", &IndexVamana::id_type_string)
293293
.def(

‎src/include/api/ivf_flat_index.h‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -427,14 +427,12 @@ class IndexIVFFlat {
427427
}
428428

429429
void write_index(
430-
const tiledb::Context& ctx,
431-
const std::string& group_uri,
432-
bool overwrite = false) const {
430+
const tiledb::Context& ctx, const std::string& group_uri) const {
433431
if (!index_) {
434432
throw std::runtime_error(
435433
"Cannot write_index() because there is no index.");
436434
}
437-
index_->write_index(ctx, group_uri, overwrite);
435+
index_->write_index(ctx, group_uri);
438436
}
439437

440438
constexpr auto dimension() const {
@@ -505,9 +503,7 @@ class IndexIVFFlat {
505503
virtual void remove(const IdVector& ids) const = 0;
506504

507505
virtual void write_index(
508-
const tiledb::Context& ctx,
509-
const std::string& group_uri,
510-
bool overwrite) const = 0;
506+
const tiledb::Context& ctx, const std::string& group_uri) const = 0;
511507

512508
[[nodiscard]] virtual size_t dimension() const = 0;
513509

@@ -691,11 +687,9 @@ class IndexIVFFlat {
691687
// index_.update(vectors_uri, ids, options);
692688
}
693689

694-
void write_index(
695-
const tiledb::Context& ctx,
696-
const std::string& group_uri,
697-
bool overwrite) const override {
698-
impl_index_.write_index(ctx, group_uri, overwrite);
690+
void write_index(const tiledb::Context& ctx, const std::string& group_uri)
691+
const override {
692+
impl_index_.write_index(ctx, group_uri);
699693
}
700694

701695
// WIP

‎src/include/api/vamana_index.h‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -233,14 +233,12 @@ class IndexVamana {
233233
}
234234

235235
void write_index(
236-
const tiledb::Context& ctx,
237-
const std::string& group_uri,
238-
bool overwrite = false) const {
236+
const tiledb::Context& ctx, const std::string& group_uri) const {
239237
if (!index_) {
240238
throw std::runtime_error(
241239
"Cannot write_index() because there is no index.");
242240
}
243-
index_->write_index(ctx, group_uri, overwrite);
241+
index_->write_index(ctx, group_uri);
244242
}
245243

246244
constexpr auto dimension() const {
@@ -289,9 +287,7 @@ class IndexVamana {
289287
std::optional<size_t> opt_L) = 0;
290288

291289
virtual void write_index(
292-
const tiledb::Context& ctx,
293-
const std::string& group_uri,
294-
bool overwrite) const = 0;
290+
const tiledb::Context& ctx, const std::string& group_uri) const = 0;
295291

296292
[[nodiscard]] virtual size_t dimension() const = 0;
297293
};
@@ -394,11 +390,9 @@ class IndexVamana {
394390
}
395391
}
396392

397-
void write_index(
398-
const tiledb::Context& ctx,
399-
const std::string& group_uri,
400-
bool overwrite) const override {
401-
impl_index_.write_index(ctx, group_uri, overwrite);
393+
void write_index(const tiledb::Context& ctx, const std::string& group_uri)
394+
const override {
395+
impl_index_.write_index(ctx, group_uri);
402396
}
403397

404398
size_t dimension() const override {

‎src/include/index/flatpq_index.h‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -640,15 +640,8 @@ class flatpq_index {
640640
return un_pq;
641641
}
642642

643-
auto write_index(const std::string& group_uri, bool overwrite) {
643+
auto write_index(const std::string& group_uri) {
644644
tiledb::Context ctx;
645-
tiledb::VFS vfs(ctx);
646-
if (vfs.is_dir(group_uri)) {
647-
if (overwrite == false) {
648-
return false;
649-
}
650-
vfs.remove_dir(group_uri);
651-
}
652645

653646
tiledb::Config cfg;
654647
tiledb::Group::create(ctx, group_uri);

‎src/include/index/ivf_flat_index.h‎

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -738,22 +738,10 @@ class ivf_flat_index {
738738
* we write from the PartitionedMatrix base class.
739739
*
740740
* @param group_uri
741-
* @param overwrite
742741
* @return bool indicating success or failure
743742
*/
744743
auto write_index(
745-
const tiledb::Context& ctx,
746-
const std::string& group_uri,
747-
bool overwrite) const {
748-
tiledb::VFS vfs(ctx);
749-
750-
// @todo Deal with this in right way vis a vis timestamping
751-
if (vfs.is_dir(group_uri)) {
752-
if (overwrite == false) {
753-
return false;
754-
}
755-
}
756-
744+
const tiledb::Context& ctx, const std::string& group_uri) const {
757745
// Write the group
758746
auto write_group =
759747
ivf_flat_index_group(*this, ctx, group_uri, TILEDB_WRITE, timestamp_);
@@ -799,8 +787,7 @@ class ivf_flat_index {
799787
const std::string& centroids_uri,
800788
const std::string& parts_uri,
801789
const std::string& ids_uri,
802-
const std::string& indices_uri,
803-
bool overwrite) const {
790+
const std::string& indices_uri) const {
804791
tiledb::VFS vfs(ctx);
805792

806793
write_matrix(ctx, centroids_, centroids_uri, 0, true);

‎src/include/index/vamana_index.h‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -824,7 +824,6 @@ class vamana_index {
824824
/**
825825
* @brief Write the index to a TileDB group
826826
* @param group_uri The URI of the TileDB group where the index will be saved
827-
* @param overwrite Whether to overwrite an existing group
828827
* @return Whether the write was successful
829828
*
830829
* The group consists of the original feature vectors, and the graph index,
@@ -837,19 +836,10 @@ class vamana_index {
837836
* the group?
838837
*/
839838
auto write_index(
840-
const tiledb::Context& ctx,
841-
const std::string& group_uri,
842-
bool overwrite = false) const {
839+
const tiledb::Context& ctx, const std::string& group_uri) const {
843840
// metadata: dimension, ntotal, L, R, B, alpha_min, alpha_max, medoid
844841
// Save as a group: metadata, feature_vectors, graph edges, offsets
845842

846-
tiledb::VFS vfs(ctx);
847-
if (vfs.is_dir(group_uri)) {
848-
if (overwrite == false) {
849-
return false;
850-
}
851-
}
852-
853843
auto write_group =
854844
vamana_index_group(*this, ctx, group_uri, TILEDB_WRITE, timestamp_);
855845

‎src/include/test/query_common.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@ struct siftsmall_test_init : public siftsmall_test_init_defaults {
181181
if (vfs.is_dir(tmp_ivf_index_uri)) {
182182
vfs.remove_dir(tmp_ivf_index_uri);
183183
}
184-
idx.write_index(ctx_, tmp_ivf_index_uri, true);
184+
idx.write_index(ctx_, tmp_ivf_index_uri);
185185
auto idx0 =
186186
// ivf_flat_l2_index<feature_type, id_type, px_type>(ctx_,
187187
// tmp_ivf_index_uri);

‎src/include/test/unit_api_ivf_flat_index.cc‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ TEST_CASE(
172172
auto training_set = FeatureVectorArray(ctx, siftsmall_inputs_uri);
173173
a.train(training_set, kmeans_init::random);
174174
a.add(training_set);
175-
a.write_index(ctx, api_ivf_flat_index_uri, true);
175+
a.write_index(ctx, api_ivf_flat_index_uri);
176176

177177
auto b = IndexIVFFlat(ctx, api_ivf_flat_index_uri);
178178

@@ -244,7 +244,7 @@ TEST_CASE(
244244
auto training_set = FeatureVectorArray(ctx, siftsmall_inputs_uri);
245245
a.train(training_set, kmeans_init::random);
246246
a.add(training_set);
247-
a.write_index(ctx, api_ivf_flat_index_uri, true);
247+
a.write_index(ctx, api_ivf_flat_index_uri);
248248
auto b = IndexIVFFlat(ctx, api_ivf_flat_index_uri);
249249

250250
auto query_set = FeatureVectorArray(ctx, siftsmall_query_uri);

‎src/include/test/unit_api_vamana_index.cc‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@ TEST_CASE(
181181
auto training_vector_array = FeatureVectorArray(training);
182182
index.train(training_vector_array);
183183
index.add(training_vector_array);
184-
index.write_index(ctx, index_uri, true);
184+
index.write_index(ctx, index_uri);
185185

186186
CHECK(index.feature_type_string() == feature_type);
187187
CHECK(index.id_type_string() == id_type);
@@ -255,7 +255,7 @@ TEST_CASE(
255255
auto training_vector_array = FeatureVectorArray(training);
256256
index.train(training_vector_array);
257257
index.add(training_vector_array);
258-
index.write_index(ctx, index_uri, true);
258+
index.write_index(ctx, index_uri);
259259

260260
CHECK(index.feature_type_string() == feature_type);
261261
CHECK(index.id_type_string() == id_type);
@@ -306,7 +306,7 @@ TEST_CASE(
306306
siftsmall_dimension, num_vectors, feature_type, id_type);
307307
index.train(empty_training_vector_array);
308308
index.add(empty_training_vector_array);
309-
index.write_index(ctx, index_uri, true);
309+
index.write_index(ctx, index_uri);
310310

311311
CHECK(index.feature_type_string() == feature_type);
312312
CHECK(index.id_type_string() == id_type);
@@ -323,7 +323,7 @@ TEST_CASE(
323323
auto training_set = FeatureVectorArray(ctx, siftsmall_inputs_uri);
324324
index.train(training_set);
325325
index.add(training_set);
326-
index.write_index(ctx, index_uri, true);
326+
index.write_index(ctx, index_uri);
327327

328328
CHECK(index.feature_type_string() == feature_type);
329329
CHECK(index.id_type_string() == id_type);
@@ -376,7 +376,7 @@ TEST_CASE(
376376
auto training_set = FeatureVectorArray(ctx, siftsmall_inputs_uri);
377377
a.train(training_set);
378378
a.add(training_set);
379-
a.write_index(ctx, api_vamana_index_uri, true);
379+
a.write_index(ctx, api_vamana_index_uri);
380380

381381
auto b = IndexVamana(ctx, api_vamana_index_uri);
382382

@@ -423,7 +423,7 @@ TEST_CASE("api_vamana_index: read index and query", "[api_vamana_index]") {
423423
auto training_set = FeatureVectorArray(ctx, siftsmall_inputs_uri);
424424
a.train(training_set);
425425
a.add(training_set);
426-
a.write_index(ctx, api_vamana_index_uri, true);
426+
a.write_index(ctx, api_vamana_index_uri);
427427
auto b = IndexVamana(ctx, api_vamana_index_uri);
428428

429429
auto query_set = FeatureVectorArray(ctx, siftsmall_query_uri);

‎src/include/test/unit_flatpq_index.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -841,7 +841,7 @@ TEST_CASE("flatpq_index: flatpq_index write and read", "[flatpq_index]") {
841841
idx.train(training_set);
842842
idx.add(training_set);
843843

844-
idx.write_index(flatpq_index_uri, true);
844+
idx.write_index(flatpq_index_uri);
845845
auto idx2 = flatpq_index<
846846
siftsmall_feature_type,
847847
siftsmall_ids_type,

0 commit comments

Comments
 (0)