Avoid checking deleted_ids when there are no updates

Author: Nikos Papailiou

apis/python/test/test_ingestion.py

@@ -283,7 +283,6 @@ def test_ivf_flat_ingestion_external_ids_numpy(tmp_path):
 def test_ivf_flat_ingestion_with_updates(tmp_path):
     dataset_dir = os.path.join(tmp_path, "dataset")
     index_uri = os.path.join(tmp_path, "array")
-    index_uri_2 = os.path.join(tmp_path, "array_2")
     k = 10
     size = 100000
     partitions = 100

src/include/detail/ivf/index.h

@@ -77,11 +77,18 @@ int ivf_index(
     std::unordered_set<ids_type> deleted_ids_set(deleted_ids.begin(), deleted_ids.end());
     std::vector<size_t> degrees(centroids.num_cols());
     std::vector<ids_type> indices(centroids.num_cols() + 1);
-    for (size_t i = 0; i < db.num_cols(); ++i) {
-      if (auto iter = deleted_ids_set.find(external_ids[i]); iter == deleted_ids_set.end()) {
+    if (deleted_ids.empty()) {
+      for (size_t i = 0; i < db.num_cols(); ++i) {
         auto j = parts[i];
         ++degrees[j];
       }
+    } else {
+      for (size_t i = 0; i < db.num_cols(); ++i) {
+        if (auto iter = deleted_ids_set.find(external_ids[i]); iter == deleted_ids_set.end()) {
+          auto j = parts[i];
+          ++degrees[j];
+        }
+      }
     }
     indices[0] = 0;
     std::inclusive_scan(begin(degrees), end(degrees), begin(indices) + 1);
@@ -112,8 +119,8 @@ int ivf_index(
     // which will group them nicely -- but a distributed parallel sort may
     // be difficult to implement.  Even this algorithm is not trivial to
     // parallelize, because of the random access to the indices array.
-    for (size_t i = 0; i < db.num_cols(); ++i) {
-      if (auto iter = deleted_ids_set.find(external_ids[i]); iter == deleted_ids_set.end()) {
+    if (deleted_ids.empty()) {
+      for (size_t i = 0; i < db.num_cols(); ++i) {
         size_t bin = parts[i];
         size_t ibin = indices[bin];
 
@@ -125,6 +132,21 @@ int ivf_index(
         }
         ++indices[bin];
       }
+    } else {
+      for (size_t i = 0; i < db.num_cols(); ++i) {
+        if (auto iter = deleted_ids_set.find(external_ids[i]); iter == deleted_ids_set.end()) {
+          size_t bin = parts[i];
+          size_t ibin = indices[bin];
+
+          shuffled_ids[ibin] = external_ids[i];
+
+          assert(ibin < shuffled_db.num_cols());
+          for (size_t j = 0; j < db.num_rows(); ++j) {
+            shuffled_db(j, ibin) = db(j, i);
+          }
+          ++indices[bin];
+        }
+      }
     }
 
     std::shift_right(begin(indices), end(indices), 1);