Skip to content

Commit c11cb8f

Browse files
LiuRuoyu01yuhaijun999
LiuRuoyu01
authored and
yuhaijun999
committed
[fix][store]Fixup checking if Max exceeds the limit
1 parent 264b76a commit c11cb8f

17 files changed

+33
-25
lines changed

‎src/server/index_service.cc‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -415,7 +415,7 @@ static butil::Status ValidateVectorAddRequest(StoragePtr storage, const pb::inde
415415
fmt::format("Vector index {} not ready, please retry.", region->Id()));
416416
}
417417

418-
if (vector_index_wrapper->IsExceedsMaxElements()) {
418+
if (vector_index_wrapper->IsExceedsMaxElements(request->vectors_size())) {
419419
return butil::Status(pb::error::EVECTOR_INDEX_EXCEED_MAX_ELEMENTS,
420420
fmt::format("Vector index {} exceeds max elements.", region->Id()));
421421
}
@@ -1217,7 +1217,7 @@ static butil::Status ValidateVectorImportRequestForAdd(StoragePtr storage,
12171217
fmt::format("Vector index {} not ready, please retry.", region->Id()));
12181218
}
12191219

1220-
if (vector_index_wrapper->IsExceedsMaxElements()) {
1220+
if (vector_index_wrapper->IsExceedsMaxElements(request->vectors_size())) {
12211221
return butil::Status(pb::error::EVECTOR_INDEX_EXCEED_MAX_ELEMENTS,
12221222
fmt::format("Vector index {} exceeds max elements.", region->Id()));
12231223
}
@@ -2684,7 +2684,7 @@ static butil::Status ValidateIndexTxnPrewriteRequest(StoragePtr storage, const p
26842684
// check if vector_id is legal
26852685
const auto& vector = mutation.vector();
26862686
if (mutation.op() == pb::store::Op::Put || mutation.op() == pb::store::PutIfAbsent) {
2687-
if (vector_index_wrapper->IsExceedsMaxElements()) {
2687+
if (vector_index_wrapper->IsExceedsMaxElements(request->mutations_size())) {
26882688
return butil::Status(pb::error::EVECTOR_INDEX_EXCEED_MAX_ELEMENTS,
26892689
fmt::format("Vector index {} exceeds max elements.", region->Id()));
26902690
}
@@ -2926,7 +2926,7 @@ static butil::Status ValidateTxnCommitRequest(const pb::store::TxnCommitRequest*
29262926
fmt::format("Vector index {} not ready, please retry.", region->Id()));
29272927
}
29282928

2929-
if (region->VectorIndexWrapper()->IsExceedsMaxElements()) {
2929+
if (region->VectorIndexWrapper()->IsExceedsMaxElements(request->keys_size())) {
29302930
return butil::Status(pb::error::EVECTOR_INDEX_EXCEED_MAX_ELEMENTS,
29312931
fmt::format("Vector index {} exceeds max elements.", region->Id()));
29322932
}

‎src/vector/vector_index.cc‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -787,20 +787,20 @@ butil::Status VectorIndexWrapper::GetMemorySize(int64_t& memory_size) {
787787
return status;
788788
}
789789

790-
bool VectorIndexWrapper::IsExceedsMaxElements() {
790+
bool VectorIndexWrapper::IsExceedsMaxElements(int64_t vector_size) {
791791
auto vector_index = GetVectorIndex();
792792
if (vector_index == nullptr) {
793793
return true;
794794
}
795795

796796
auto sibling_vector_index = SiblingVectorIndex();
797797
if (sibling_vector_index != nullptr) {
798-
if (!sibling_vector_index->IsExceedsMaxElements()) {
798+
if (!sibling_vector_index->IsExceedsMaxElements(vector_size)) {
799799
return false;
800800
}
801801
}
802802

803-
return vector_index->IsExceedsMaxElements();
803+
return vector_index->IsExceedsMaxElements(vector_size);
804804
}
805805

806806
bool VectorIndexWrapper::NeedToRebuild() {

‎src/vector/vector_index.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ class VectorIndex {
150150
virtual butil::Status GetCount(int64_t& count);
151151
virtual butil::Status GetDeletedCount(int64_t& deleted_count);
152152
virtual butil::Status GetMemorySize(int64_t& memory_size);
153-
virtual bool IsExceedsMaxElements() = 0;
153+
virtual bool IsExceedsMaxElements(int64_t vector_size) = 0;
154154

155155
virtual butil::Status Add(const std::vector<pb::common::VectorWithId>& vector_with_ids) = 0;
156156
virtual butil::Status Add(const std::vector<pb::common::VectorWithId>& vector_with_ids, bool is_priority);
@@ -192,7 +192,7 @@ class VectorIndex {
192192
virtual void LockWrite() = 0;
193193
virtual void UnlockWrite() = 0;
194194
virtual butil::Status Train(std::vector<float>& train_datas) = 0;
195-
virtual butil::Status Train(std::vector<uint8_t>& ) { return butil::Status::OK(); }
195+
virtual butil::Status Train(std::vector<uint8_t>&) { return butil::Status::OK(); }
196196
virtual butil::Status TrainByParallel(std::vector<float>& train_datas);
197197
virtual butil::Status Train(const std::vector<pb::common::VectorWithId>& vectors) = 0;
198198
virtual bool NeedToRebuild() = 0;
@@ -407,7 +407,7 @@ class VectorIndexWrapper : public std::enable_shared_from_this<VectorIndexWrappe
407407
butil::Status GetCount(int64_t& count);
408408
butil::Status GetDeletedCount(int64_t& deleted_count);
409409
butil::Status GetMemorySize(int64_t& memory_size);
410-
bool IsExceedsMaxElements();
410+
bool IsExceedsMaxElements(int64_t vector_size);
411411

412412
bool NeedToRebuild();
413413
bool NeedToSave(std::string& reason);

‎src/vector/vector_index_bruteforce.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ butil::Status VectorIndexBruteforce::GetMemorySize(int64_t& memory_size) {
102102
return butil::Status::OK();
103103
}
104104

105-
bool VectorIndexBruteforce::IsExceedsMaxElements() { return false; }
105+
bool VectorIndexBruteforce::IsExceedsMaxElements(int64_t /*vector_size*/) { return false; }
106106

107107
bool VectorIndexBruteforce::NeedToSave(int64_t last_save_log_behind) {
108108
return last_save_log_behind > FLAGS_bruteforce_need_save_count;

‎src/vector/vector_index_bruteforce.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ class VectorIndexBruteforce : public VectorIndex {
7171
butil::Status GetCount([[maybe_unused]] int64_t& count) override;
7272
butil::Status GetDeletedCount([[maybe_unused]] int64_t& deleted_count) override;
7373
butil::Status GetMemorySize([[maybe_unused]] int64_t& memory_size) override;
74-
bool IsExceedsMaxElements() override;
74+
bool IsExceedsMaxElements(int64_t vector_size) override;
7575
butil::Status Train([[maybe_unused]] std::vector<float>& train_datas) override { return butil::Status::OK(); }
7676
butil::Status Train([[maybe_unused]] const std::vector<pb::common::VectorWithId>& vectors) override {
7777
return butil::Status::OK();

‎src/vector/vector_index_diskann.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -732,7 +732,7 @@ butil::Status VectorIndexDiskANN::GetMemorySize(int64_t& memory_size) {
732732
return butil::Status::OK();
733733
}
734734

735-
bool VectorIndexDiskANN::IsExceedsMaxElements() { return false; }
735+
bool VectorIndexDiskANN::IsExceedsMaxElements(int64_t /*vector_size*/) { return false; }
736736

737737
bool VectorIndexDiskANN::NeedToSave(int64_t /*last_save_log_behind*/) { return false; }
738738

‎src/vector/vector_index_diskann.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ class VectorIndexDiskANN : public VectorIndex, public std::enable_shared_from_th
103103
butil::Status GetCount(int64_t& count) override;
104104
butil::Status GetDeletedCount(int64_t& deleted_count) override;
105105
butil::Status GetMemorySize(int64_t& memory_size) override;
106-
bool IsExceedsMaxElements() override;
106+
bool IsExceedsMaxElements(int64_t vector_size) override;
107107
butil::Status Train([[maybe_unused]] std::vector<float>& train_datas) override { return butil::Status::OK(); }
108108
butil::Status Train([[maybe_unused]] const std::vector<pb::common::VectorWithId>& vectors) override {
109109
return butil::Status::OK();

‎src/vector/vector_index_flat.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -507,7 +507,7 @@ butil::Status VectorIndexFlat<T, U>::GetMemorySize(int64_t& memory_size) {
507507
}
508508

509509
template <typename T, typename U>
510-
bool VectorIndexFlat<T, U>::IsExceedsMaxElements() {
510+
bool VectorIndexFlat<T, U>::IsExceedsMaxElements(int64_t /*vector_size*/) {
511511
return false;
512512
}
513513

‎src/vector/vector_index_flat.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ class VectorIndexFlat : public VectorIndex {
102102
butil::Status GetCount(int64_t& count) override;
103103
butil::Status GetDeletedCount(int64_t& deleted_count) override;
104104
butil::Status GetMemorySize(int64_t& memory_size) override;
105-
bool IsExceedsMaxElements() override;
105+
bool IsExceedsMaxElements(int64_t vector_size) override;
106106
butil::Status Train([[maybe_unused]] std::vector<float>& train_datas) override { return butil::Status::OK(); }
107107
butil::Status Train([[maybe_unused]] const std::vector<pb::common::VectorWithId>& vectors) override {
108108
return butil::Status::OK();

‎src/vector/vector_index_hnsw.cc‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -528,12 +528,20 @@ butil::Status VectorIndexHnsw::GetMaxElements(int64_t& max_elements) {
528528
return butil::Status::OK();
529529
}
530530

531-
bool VectorIndexHnsw::IsExceedsMaxElements() {
531+
bool VectorIndexHnsw::IsExceedsMaxElements(int64_t vector_size) {
532532
if (hnsw_index_ == nullptr) {
533533
return true;
534534
}
535+
bool is_exceeds = hnsw_index_->getCurrentElementCount() + vector_size > max_element_limit_;
536+
if (is_exceeds) {
537+
DINGO_LOG(ERROR) << fmt::format(
538+
"[vector_index.hnsw][id({})] exceeds max elements, current_element_count({}) , delete_element_count({}) , "
539+
"vector_size({}) , "
540+
"max_element_limit({}).",
541+
Id(), hnsw_index_->getCurrentElementCount(), hnsw_index_->getDeletedCount(), vector_size, max_element_limit_);
542+
}
535543

536-
return hnsw_index_->getCurrentElementCount() >= max_element_limit_;
544+
return is_exceeds;
537545
}
538546

539547
hnswlib::HierarchicalNSW<float>* VectorIndexHnsw::GetHnswIndex() { return this->hnsw_index_; }

0 commit comments

Comments
 (0)