Cleanup C++ group code (#331)

Author: jparismorgan

src/include/index/index_defs.h

@@ -98,25 +98,15 @@ using StorageFormat =
      }},
     {"0.3",
      {
-         // From external/data/pytest-170/test_ivf_flat_ingestion_f320/array
-         // (generated by test/test_ingestion.py on 2023-12-01)
-         // Arrays comprising the index (note that temp_data has these as well)
          {"centroids_array_name", "partition_centroids"},
          {"index_array_name", "partition_indexes"},
          {"ids_array_name", "shuffled_vector_ids"},
          {"parts_array_name", "shuffled_vectors"},
-
-         // temp_data is a group, not an array
-         {"partial_write_array_name", "temp_data"},
-
-         // In storage_formats.py but not part of group?
          {"input_vectors_array_name", "input_vectors"},
          {"external_ids_array_name", "external_ids"},
-         {"updates_array_name", "updates"},
-
-         // Not arrays
-         // @todo Where are these stored?
+         {"partial_write_array_dir", "temp_data"},
          {"default_attr_filters", "zstd"},
+         {"updates_array_name", "updates"},
          {"support_timetravel", "true"},
      }},
 };

src/include/index/index_group.h

@@ -457,29 +457,11 @@ class base_index_group {
    * Getters for names and uris
    **************************************************************************/
 
-  [[nodiscard]] auto partial_write_array_dir() const {
-    return array_key_to_uri("partial_write_array_dir");
+  [[nodiscard]] auto ids_uri() const {
+    return array_key_to_uri("ids_array_name");
   }
-  [[nodiscard]] auto input_vectors_uri() const {
-    return array_key_to_uri("input_vectors_array_name");
-  }
-  [[nodiscard]] auto external_ids_uri() const {
-    return array_key_to_uri("external_ids_array_name");
-  }
-  [[nodiscard]] auto updates_array_uri() const {
-    return array_key_to_uri("updates_array_name");
-  }
-  [[nodiscard]] auto partial_write_array_name() const {
-    return array_key_to_array_name("partial_write_array_dir");
-  }
-  [[nodiscard]] auto input_vectors_array_name() const {
-    return array_key_to_array_name("input_vectors_array_name");
-  }
-  [[nodiscard]] auto external_ids_array_name() const {
-    return array_key_to_array_name("external_ids_array_name");
-  }
-  [[nodiscard]] auto updates_array_name() const {
-    return array_key_to_array_name("updates_array_name");
+  [[nodiscard]] auto ids_array_name() const {
+    return array_key_to_array_name("ids_array_name");
   }
   [[nodiscard]] const std::reference_wrapper<const tiledb::Context> cached_ctx()
       const {

src/include/index/ivf_flat_group.h

@@ -112,7 +112,7 @@ class ivf_flat_index_group
   auto append_num_partitions(size_t size) {
     metadata_.partition_history_.push_back(size);
   }
-  auto get_all_num_partitions() {
+  auto get_all_num_partitions() const {
     return metadata_.partition_history_;
   }
 
@@ -129,9 +129,6 @@ class ivf_flat_index_group
   [[nodiscard]] auto parts_uri() const {
     return this->array_key_to_uri("parts_array_name");
   }
-  [[nodiscard]] auto ids_uri() const {
-    return this->array_key_to_uri("ids_array_name");
-  }
   [[nodiscard]] auto indices_uri() const {
     return this->array_key_to_uri("index_array_name");
   }
@@ -141,9 +138,6 @@ class ivf_flat_index_group
   [[nodiscard]] auto parts_array_name() const {
     return this->array_key_to_array_name("parts_array_name");
   }
-  [[nodiscard]] auto ids_array_name() const {
-    return this->array_key_to_array_name("ids_array_name");
-  }
   [[nodiscard]] auto indices_array_name() const {
     return this->array_key_to_array_name("index_array_name");
   }
@@ -213,8 +207,13 @@ class ivf_flat_index_group
     tiledb_helpers::add_to_group(write_group, parts_uri(), parts_array_name());
 
     create_empty_for_vector<typename index_type::id_type>(
-        cached_ctx_, ids_uri(), default_domain, tile_size, default_compression);
-    tiledb_helpers::add_to_group(write_group, ids_uri(), ids_array_name());
+        cached_ctx_,
+        this->ids_uri(),
+        default_domain,
+        tile_size,
+        default_compression);
+    tiledb_helpers::add_to_group(
+        write_group, this->ids_uri(), this->ids_array_name());
 
     create_empty_for_vector<typename index_type::indices_type>(
         cached_ctx_,

src/include/index/vamana_group.h

@@ -135,7 +135,7 @@ class vamana_index_group : public base_index_group<vamana_index_group<Index>> {
   auto append_num_edges(size_t size) {
     metadata_.num_edges_history_.push_back(size);
   }
-  auto get_all_num_edges() {
+  auto get_all_num_edges() const {
     return metadata_.num_edges_history_;
   }
   auto set_num_edges(size_t size) {
@@ -184,9 +184,6 @@ class vamana_index_group : public base_index_group<vamana_index_group<Index>> {
   [[nodiscard]] auto feature_vectors_uri() const {
     return this->array_key_to_uri("parts_array_name");
   }
-  [[nodiscard]] auto feature_vector_ids_uri() const {
-    return this->array_key_to_uri("ids_array_name");
-  }
   [[nodiscard]] auto adjacency_scores_uri() const {
     return this->array_key_to_uri("adjacency_scores_array_name");
   }
@@ -199,9 +196,6 @@ class vamana_index_group : public base_index_group<vamana_index_group<Index>> {
   [[nodiscard]] auto feature_vectors_array_name() const {
     return this->array_key_to_array_name("parts_array_name");
   }
-  [[nodiscard]] auto feature_vector_ids_name() const {
-    return this->array_key_to_array_name("ids_array_name");
-  }
   [[nodiscard]] auto adjacency_scores_array_name() const {
     return this->array_key_to_array_name("adjacency_scores_array_name");
   }
@@ -286,12 +280,12 @@ class vamana_index_group : public base_index_group<vamana_index_group<Index>> {
 
     create_empty_for_vector<typename index_type::id_type>(
         cached_ctx_,
-        feature_vector_ids_uri(),
+        this->ids_uri(),
         default_domain,
         tile_size,
         default_compression);
     tiledb_helpers::add_to_group(
-        write_group, feature_vector_ids_uri(), feature_vector_ids_name());
+        write_group, this->ids_uri(), this->ids_array_name());
 
     create_empty_for_vector<typename index_type::score_type>(
         cached_ctx_,

src/include/index/vamana_index.h

@@ -521,7 +521,7 @@ class vamana_index {
         std::move(tdbColMajorPreLoadMatrixWithIds<feature_type, id_type>(
             group_->cached_ctx(),
             group_->feature_vectors_uri(),
-            group_->feature_vector_ids_uri(),
+            group_->ids_uri(),
             dimension_,
             num_vectors_,
             0,
@@ -871,7 +871,7 @@ class vamana_index {
     write_vector(
         ctx,
         feature_vectors_.ids(),
-        write_group.feature_vector_ids_uri(),
+        write_group.ids_uri(),
         0,
         false,
         timestamp_);

src/include/test/unit_vamana_index.cc

@@ -1238,6 +1238,10 @@ TEST_CASE("vamana: vamana_index write and read", "[vamana]") {
   tiledb::Context ctx;
   std::string vamana_index_uri =
       (std::filesystem::temp_directory_path() / "tmp_vamana_index").string();
+  tiledb::VFS vfs(ctx);
+  if (vfs.is_dir(vamana_index_uri)) {
+    vfs.remove_dir(vamana_index_uri);
+  }
   auto training_set = tdbColMajorMatrix<float>(ctx, siftsmall_inputs_uri, 0);
   load(training_set);
   std::vector<siftsmall_ids_type> ids(num_vectors(training_set));