[0.6] [MOD-10559] Decouple the shrinking and growing logic of large containers in Flat and HNSW

src/VecSim/algorithms/brute_force/brute_force.h

@@ -54,6 +54,17 @@ class BruteForceIndex : public VecSimIndexAbstract<DistType> {
     inline vecsim_stl::vector<VectorBlock *> getVectorBlocks() const { return vectorBlocks; }
     virtual ~BruteForceIndex();
 
+#ifdef BUILD_TESTS
+    size_t getStoredVectorsCount() const {
+        size_t actual_stored_vec = 0;
+        for (auto &block : vectorBlocks) {
+            actual_stored_vec += block->getLength();
+        }
+
+        return actual_stored_vec;
+    }
+#endif
+
 protected:
     // Private internal function that implements generic single vector insertion.
     virtual int appendVector(const void *vector_data, labelType label);
@@ -142,10 +153,14 @@ int BruteForceIndex<DataType, DistType>::appendVector(const void *vector_data, l
     size_t idToLabelMapping_size = this->idToLabelMapping.size();
 
     if (id >= idToLabelMapping_size) {
+        assert(indexCapacity() == idToLabelMapping.capacity());
+        assert(idToLabelMapping.size() == idToLabelMapping.capacity());
         size_t last_block_vectors_count = id % this->blockSize;
-        this->idToLabelMapping.resize(
-            idToLabelMapping_size + this->blockSize - last_block_vectors_count, 0);
+        size_t new_size = idToLabelMapping_size + this->blockSize - last_block_vectors_count;
+        assert(new_size % this->blockSize == 0);
+        this->idToLabelMapping.resize(new_size, 0);
         this->idToLabelMapping.shrink_to_fit();
+        assert(idToLabelMapping.size() == idToLabelMapping.capacity());
     }
 
     // add label to idToLabelMapping
@@ -196,12 +211,20 @@ int BruteForceIndex<DataType, DistType>::removeVector(idType id_to_delete) {
 
         // Resize and align the idToLabelMapping.
         size_t idToLabel_size = idToLabelMapping.size();
-        // If the new size is smaller by at least one block comparing to the idToLabelMapping
+        // If the new size is smaller by at least two blocks comparing to the idToLabelMapping,
+        // or if the new size is 0 and the capacity is at least one block,
         // align to be a multiplication of blocksize  and resize by one block.
-        if (this->count + this->blockSize <= idToLabel_size) {
+        if ((this->count + 2 * this->blockSize <= idToLabel_size) ||
+            // Handle last block
+            (this->count == 0 && idToLabel_size >= this->blockSize)) {
             size_t vector_to_align_count = idToLabel_size % this->blockSize;
-            this->idToLabelMapping.resize(idToLabel_size - this->blockSize - vector_to_align_count);
+            size_t new_size = idToLabel_size - this->blockSize - vector_to_align_count;
+            assert(new_size >= this->count);
+            assert(new_size % this->blockSize == 0);
+            assert(idToLabelMapping.size() == idToLabelMapping.capacity());
+            this->idToLabelMapping.resize(new_size);
             this->idToLabelMapping.shrink_to_fit();
+            assert(idToLabelMapping.size() == idToLabelMapping.capacity());
         }
     }
 

src/VecSim/algorithms/brute_force/brute_force_friend_tests.h

@@ -10,11 +10,13 @@
 INDEX_TEST_FRIEND_CLASS(BruteForceTest_brute_force_vector_update_test_Test)
 INDEX_TEST_FRIEND_CLASS(BruteForceTest_resize_and_align_index_Test)
 INDEX_TEST_FRIEND_CLASS(BruteForceTest_resize_and_align_index_largeInitialCapacity_Test)
+INDEX_TEST_FRIEND_CLASS(BruteForceTest_resize_and_align_index_smallInitialCapacity_Test)
 INDEX_TEST_FRIEND_CLASS(BruteForceTest_brute_force_empty_index_Test)
 INDEX_TEST_FRIEND_CLASS(BruteForceTest_brute_force_reindexing_same_vector_Test)
 INDEX_TEST_FRIEND_CLASS(BruteForceTest_brute_force_reindexing_same_vector_different_id_Test)
 INDEX_TEST_FRIEND_CLASS(BruteForceTest_test_delete_swap_block_Test)
 INDEX_TEST_FRIEND_CLASS(BruteForceTest_test_dynamic_bf_info_iterator_Test)
 INDEX_TEST_FRIEND_CLASS(BruteForceTest_brute_force_zero_minimal_capacity_Test)
 INDEX_TEST_FRIEND_CLASS(BruteForceTest_preferAdHocOptimization_Test)
+INDEX_TEST_FRIEND_CLASS(IndexAllocatorTest_test_bf_index_block_size_1_Test)
 INDEX_TEST_FRIEND_CLASS(BM_VecSimBasics)

src/VecSim/algorithms/hnsw/hnsw.h

@@ -1151,13 +1151,16 @@ int HNSWIndex<DataType, DistType>::removeVector(const idType element_internal_id
     --cur_element_count;
     --max_id;
 
-    // If we need to free a complete block & there is a least one block between the
-    // capacity and the size.
+    // If the new size is smaller by at least two blocks comparing to the idToLabelMapping,
+    // or if the new size is 0 and the capacity is at least one block,
+    // align to be a multiplication of blocksize  and resize by one block.
     if (cur_element_count % this->blockSize == 0 &&
-        cur_element_count + this->blockSize <= max_elements_) {
+        ((cur_element_count + 2 * this->blockSize <= max_elements_) ||
+         (cur_element_count == 0 && max_elements_ >= this->blockSize))) {
 
         // Check if the capacity is aligned to block size.
         size_t extra_space_to_free = max_elements_ % this->blockSize;
+        assert(max_elements_ - this->blockSize - extra_space_to_free >= cur_element_count);
 
         // Remove one block from the capacity.
         this->resizeIndex(max_elements_ - this->blockSize - extra_space_to_free);