[#28911] DocDB: Remove DummyANN from DocDB

Summary:
DummyANN was created to provide ability to create postgres side part of vector index stuff, while DocDB lack vector index support.
Now DummyANN is not used, and just consumes time in case of refactoring or API changes.

This diff removes DummyANN from DocDB.

**Upgrade/rollback safety:** Marked test only enum entry as deprecated.
Jira: DB-18634

Test Plan: Jenkins

Reviewers: arybochkin

Reviewed By: arybochkin

Subscribers: ybase, yql

Tags: #jenkins-ready

Differential Revision: https://phorge.dev.yugabyte.com/D47396

Author: spolitov

src/postgres/third-party-extensions/pgvector/test/expected/yb.orig.index.out

@@ -9,8 +9,7 @@ CREATE TABLE items (id bigserial PRIMARY KEY, embedding vector(3));
 ERROR:  all nodes in the cluster need to upgrade before creating a vector table
 SET yb_enable_docdb_vector_type = true;
 CREATE TABLE items (id bigserial PRIMARY KEY, embedding vector(3)) SPLIT INTO 1 TABLETS;
-CREATE INDEX ON items USING ybdummyann (embedding vector_l2_ops);
-WARNING:  ybdummyann is meant for internal-testing only and does not yield ordered results
+CREATE INDEX ON items USING ybhnsw (embedding vector_l2_ops);
 INSERT INTO items VALUES (1, '[1.0, 0.4, 0.3]');
 INSERT INTO items VALUES (2, '[0.001, 0.432, 0.32]');
 \d items
@@ -21,7 +20,7 @@ INSERT INTO items VALUES (2, '[0.001, 0.432, 0.32]');
  embedding | vector(3) |           |          | 
 Indexes:
     "items_pkey" PRIMARY KEY, lsm (id HASH)
-    "items_embedding_idx" ybdummyann (embedding)
+    "items_embedding_idx" ybhnsw (embedding)
 
 EXPLAIN (COSTS OFF) SELECT * FROM items ORDER BY embedding <-> '[1.0, 0.4, 0.3]' LIMIT 5;
                        QUERY PLAN                        
@@ -39,10 +38,10 @@ SELECT * FROM items ORDER BY embedding <-> '[1.0, 0.4, 0.3]' LIMIT 5;
 (2 rows)
 
 EXPLAIN (COSTS OFF) SELECT embedding FROM items ORDER BY embedding <-> '[1.0, 0.4, 0.3]' LIMIT 5;
-                        QUERY PLAN                        
-----------------------------------------------------------
+                       QUERY PLAN                        
+---------------------------------------------------------
  Limit
-   ->  Index Only Scan using items_embedding_idx on items
+   ->  Index Scan using items_embedding_idx on items
          Order By: (embedding <-> '[1,0.4,0.3]'::vector)
 (3 rows)
 
@@ -289,8 +288,7 @@ ERROR:  different vector dimensions 10 and 3
 
 DROP INDEX items_embedding_idx;
 -- Dummy implementation, should only provide Exact ANN within a tablet.
-CREATE INDEX ON items USING ybdummyann (embedding vector_l2_ops);
-WARNING:  ybdummyann is meant for internal-testing only and does not yield ordered results
+CREATE INDEX ON items USING ybhnsw (embedding vector_l2_ops);
 EXPLAIN (COSTS OFF) SELECT * FROM items ORDER BY embedding <-> '[1,1,1,1,1,1,1,1,1,1]';
                          QUERY PLAN                          
 -------------------------------------------------------------

src/postgres/third-party-extensions/pgvector/test/sql/yb.orig.index.sql

@@ -3,7 +3,7 @@ SET yb_enable_docdb_vector_type = false;
 CREATE TABLE items (id bigserial PRIMARY KEY, embedding vector(3));
 SET yb_enable_docdb_vector_type = true;
 CREATE TABLE items (id bigserial PRIMARY KEY, embedding vector(3)) SPLIT INTO 1 TABLETS;
-CREATE INDEX ON items USING ybdummyann (embedding vector_l2_ops);
+CREATE INDEX ON items USING ybhnsw (embedding vector_l2_ops);
 INSERT INTO items VALUES (1, '[1.0, 0.4, 0.3]');
 INSERT INTO items VALUES (2, '[0.001, 0.432, 0.32]');
 \d items
@@ -92,7 +92,7 @@ SELECT * FROM items ORDER BY embedding <-> '[1.0, 0.4, 0.3]' LIMIT 5;
 DROP INDEX items_embedding_idx;
 
 -- Dummy implementation, should only provide Exact ANN within a tablet.
-CREATE INDEX ON items USING ybdummyann (embedding vector_l2_ops);
+CREATE INDEX ON items USING ybhnsw (embedding vector_l2_ops);
 EXPLAIN (COSTS OFF) SELECT * FROM items ORDER BY embedding <-> '[1,1,1,1,1,1,1,1,1,1]';
 SELECT * FROM items ORDER BY embedding <-> '[1,1,1,1,1,1,1,1,1,1]';
 

src/yb/common/common.proto

@@ -129,7 +129,7 @@ enum PgVectorDistanceType {
 
 enum PgVectorIndexType {
   UNKNOWN_IDX = 0;
-  DUMMY = 1;
+  DEPRECATED_DUMMY = 1;
   IVFFLAT = 2;
   HNSW = 3;
 }

src/yb/docdb/doc_vector_index.cc

@@ -109,7 +109,7 @@ auto GetVectorLSMFactory(
   switch (options.idx_type()) {
     case PgVectorIndexType::HNSW:
       return VectorLSMFactory<LSM>(block_cache, options, mem_tracker);
-    case PgVectorIndexType::DUMMY: [[fallthrough]];
+    case PgVectorIndexType::DEPRECATED_DUMMY: [[fallthrough]];
     case PgVectorIndexType::IVFFLAT: [[fallthrough]];
     case PgVectorIndexType::UNKNOWN_IDX:
       break;
@@ -123,7 +123,7 @@ std::string GetFileExtension(const PgVectorIdxOptionsPB& options) {
   switch (options.idx_type()) {
     case PgVectorIndexType::HNSW:
       return "." + boost::to_lower_copy(HnswBackend_Name(options.hnsw().backend()));
-    case PgVectorIndexType::DUMMY: [[fallthrough]];
+    case PgVectorIndexType::DEPRECATED_DUMMY: [[fallthrough]];
     case PgVectorIndexType::IVFFLAT: [[fallthrough]];
     case PgVectorIndexType::UNKNOWN_IDX:
       break;

src/yb/docdb/pgsql_operation.cc

@@ -78,19 +78,13 @@
 #include "yb/util/trace.h"
 #include "yb/util/yb_pg_errcodes.h"
 
-#include "yb/vector_index/vectorann.h"
+#include "yb/vector_index/vector_index_if.h"
 
 #include "yb/yql/pggate/util/pg_doc_data.h"
 #include "yb/yql/pgwrapper/pg_wrapper.h"
 
 using namespace std::literals;
 
-using yb::vector_index::ANNPagingState;
-using yb::vector_index::DocKeyWithDistance;
-using yb::vector_index::IndexableVectorType;
-using yb::vector_index::DummyANNFactory;
-using yb::vector_index::VectorANN;
-
 DECLARE_bool(ysql_disable_index_backfill);
 
 DEPRECATE_FLAG(double, ysql_scan_timeout_multiplier, "10_2022");
@@ -2115,59 +2109,6 @@ class PgsqlReadRequestYbctidProvider {
   std::optional<int64> current_order_;
 };
 
-template<IndexableVectorType Vector>
-class ANNKeyProvider {
- public:
-  explicit ANNKeyProvider(
-      const DocReadContext& doc_read_context, PgsqlResponsePB& response, size_t prefetch_size,
-      const Vector& query_vec, VectorANN<Vector>* ann, const ANNPagingState& paging_state)
-      : response_(response), prefetch_size_(prefetch_size), query_vec_(query_vec), ann_(ann) {
-    next_batch_paging_state_ = paging_state;
-    CHECK_GT(prefetch_size_, 0);
-  }
-
-  bool RefillBatch() {
-    CHECK(key_batch_.empty());
-    auto batch = ann_->GetTopKVectors(
-        query_vec_, prefetch_size_, next_batch_paging_state_.distance(),
-        next_batch_paging_state_.main_key(), false);
-
-    std::sort(batch.begin(), batch.end(), std::less<DocKeyWithDistance>());
-    key_batch_.insert(key_batch_.end(), batch.begin(), batch.end());
-    if (key_batch_.empty()) {
-      return false;
-    }
-    next_batch_paging_state_ =
-        ANNPagingState(key_batch_.back().distance_, key_batch_.back().dockey_);
-    return true;
-  }
-
-  Slice FetchKey() {
-    if (key_batch_.empty() && !RefillBatch()) {
-      return Slice();
-    }
-
-    auto ret = key_batch_.front();
-    current_entry_ = ret;
-    key_batch_.pop_front();
-    return ret.dockey_;
-  }
-
-  ANNPagingState GetNextBatchPagingState() const { return next_batch_paging_state_; }
-
-  void AddedKeyToResultSet() { response_.add_vector_index_distances(current_entry_->distance_); }
-
- private:
-  PgsqlResponsePB& response_;
-  size_t prefetch_size_;
-  const FloatVector& query_vec_;
-  VectorANN<Vector>* ann_;
-  std::deque<DocKeyWithDistance> key_batch_;
-  ANNPagingState next_batch_paging_state_;
-
-  std::optional<DocKeyWithDistance> current_entry_;
-};
-
 Result<size_t> PgsqlReadOperation::Execute() {
   // Verify that this request references no columns marked for deletion.
   RETURN_NOT_OK(VerifyNoRefColsMarkedForDeletion(data_.doc_read_context.schema(), request_));
@@ -2205,9 +2146,6 @@ Result<size_t> PgsqlReadOperation::Execute() {
     } else {
       std::tie(fetched_rows, has_paging_state) = VERIFY_RESULT(ExecuteSample());
     }
-  } else if (request_.has_vector_idx_options()) {
-    std::tie(fetched_rows, has_paging_state) = VERIFY_RESULT(ExecuteVectorSearch(
-        data_.doc_read_context, request_.vector_idx_options()));
   } else if (request_.index_request().has_vector_idx_options()) {
     fetched_rows = VERIFY_RESULT(ExecuteVectorLSMSearch(
         request_.index_request().vector_idx_options()));
@@ -2680,91 +2618,6 @@ Result<std::tuple<size_t, bool>> PgsqlReadOperation::ExecuteSampleBlockBased() {
   return std::tuple<size_t, bool>{fetched_items, false};
 }
 
-Result<std::tuple<size_t, bool>> PgsqlReadOperation::ExecuteVectorSearch(
-     const DocReadContext& doc_read_context, const PgVectorReadOptionsPB& options) {
-  // Build the vectorann and then make an index_doc_read_context on the vectorann
-  // to get the index iterator. Then do ExecuteBatchKeys on the index iterator.
-  RSTATUS_DCHECK(options.has_vector(), InvalidArgument, "Query vector not provided");
-
-  auto query_vec = options.vector().binary_value();
-
-  auto ysql_query_vec = pointer_cast<const vector_index::YSQLVector*>(query_vec.data());
-
-  auto query_vec_ref = VERIFY_RESULT(
-      VectorANN<FloatVector>::GetVectorFromYSQLWire(*ysql_query_vec, query_vec.size()));
-
-  DummyANNFactory<FloatVector> ann_factory;
-  auto ann_store = ann_factory.Create(ysql_query_vec->dim);
-
-  dockv::ReaderProjection index_doc_projection;
-
-  auto key_col_id = doc_read_context.schema().column_id(0);
-
-  // Building the schema to extract the vector and key from the main DocDB store.
-  // Vector should be the first value after the key.
-  auto vector_col_id =
-      doc_read_context.schema().column_id(doc_read_context.schema().num_key_columns());
-  index_doc_projection.Init(doc_read_context.schema(), {key_col_id, vector_col_id});
-
-  FilteringIterator table_iter(&table_iter_);
-  RETURN_NOT_OK(table_iter.Init(data_, request_, index_doc_projection, doc_read_context));
-  dockv::PgTableRow row(index_doc_projection);
-  const auto& table_id = request_.table_id();
-
-  // Build the VectorANN.
-  for (;;) {
-    const auto fetch_result =
-        VERIFY_RESULT(FetchTableRow(table_id, &table_iter, nullptr /* index */, &row));
-    // If changing this code, see also PgsqlReadOperation::ExecuteBatchKeys.
-    if (fetch_result == FetchResult::NotFound) {
-      break;
-    }
-    ++scanned_table_rows_;
-    if (fetch_result == FetchResult::Found) {
-      auto vec_value = row.GetValueByColumnId(vector_col_id);
-      if (!vec_value.has_value()) {
-        continue;
-      }
-
-      // Add the vector to the ANN store
-      auto encoded = dockv::EncodedDocVectorValue::FromSlice(vec_value->binary_value());
-      auto vec = VERIFY_RESULT(VectorANN<FloatVector>::GetVectorFromYSQLWire(encoded.data));
-      auto doc_iter = down_cast<DocRowwiseIterator*>(table_iter_.get());
-      ann_store->Add(VERIFY_RESULT(encoded.DecodeId()), std::move(vec), doc_iter->GetTupleId());
-    }
-  }
-
-  // Check for paging state.
-  ANNPagingState ann_paging_state;
-  if (request_.has_paging_state()) {
-    ann_paging_state =
-        ANNPagingState{request_.paging_state().distance(), request_.paging_state().main_key()};
-  }
-
-  // All rows have been added to the ANN store, now we can create the iterator.
-  auto initial_prefetch_size = request_.vector_idx_options().prefetch_size();
-  initial_prefetch_size = std::max(initial_prefetch_size, 25);
-
-  ANNKeyProvider key_provider(
-      doc_read_context, response_, initial_prefetch_size, query_vec_ref, ann_store.get(),
-      ann_paging_state);
-  auto fetched_rows = VERIFY_RESULT(ExecuteBatchKeys(key_provider));
-
-  auto next_paging_state = key_provider.GetNextBatchPagingState();
-
-  // Set paging state.
-  bool has_paging_state = !next_paging_state.valid();
-  if (has_paging_state) {
-    auto* paging_state = response_.mutable_paging_state();
-    paging_state->set_distance(next_paging_state.distance());
-    paging_state->set_main_key(next_paging_state.main_key().ToBuffer());
-
-    BindReadTimeToPagingState(data_.read_operation_data.read_time);
-  }
-
-  return std::tuple<size_t, bool>{fetched_rows, has_paging_state};
-}
-
 Result<size_t> PgsqlReadOperation::ExecuteVectorLSMSearch(const PgVectorReadOptionsPB& options) {
   RSTATUS_DCHECK(
       data_.vector_index, IllegalState, "Search vector when vector index is null: $0", request_);

src/yb/docdb/pgsql_operation.h

@@ -212,10 +212,6 @@ class PgsqlReadOperation : public DocExprExecutor {
   // Execute a READ operator for a given scalar argument.
   Result<std::tuple<size_t, bool>> ExecuteScalar();
 
-  // Execute a READ operator for a given vector search.
-  Result<std::tuple<size_t, bool>> ExecuteVectorSearch(
-      const DocReadContext& doc_read_context, const PgVectorReadOptionsPB& options);
-
   // Execute a READ operator for a given batch of keys.
   template <class KeyProvider>
   Result<size_t> ExecuteBatchKeys(KeyProvider& key_provider);

src/yb/tablet/tablet_metadata.cc

@@ -463,7 +463,7 @@ TableInfoPtr TableInfo::TEST_CreateWithLogPrefix(
 
 bool TableInfo::NeedVectorIndex() const {
   return index_info && index_info->is_vector_index() &&
-         index_info->vector_idx_options().idx_type() != PgVectorIndexType::DUMMY;
+         index_info->vector_idx_options().idx_type() != PgVectorIndexType::DEPRECATED_DUMMY;
 }
 
 Status KvStoreInfo::LoadTablesFromPB(

src/yb/vector_index/CMakeLists.txt

@@ -34,7 +34,6 @@ set(VECTOR_INDEX_SRCS
   vector_id.cc
   vector_lsm.cc
   vector_lsm_metadata.cc
-  vectorann.cc
   )
 
 set(VECTOR_INDEX_LIBS