[#27095] docdb: Vector Index: Filter tombstoned reverse mapping during compaction

Summary:
The diff updates the compaction logic to correctly handle tombstoned reverse-mapping vector index keys. The same approach as for regular compactions is used for cleaning such entries.
Additionally, the following changes are included:
- `SubDocKey::DebugSliceToStringAsResult()` is updated to correctly render vector index reverse mappings.
- A new term `MetaKey` is introduced to describe different types of internal/metadata records from DocDB perspective such as `kVectorIndexMapping` and `kTransactionApplyState` (TBD for intents meta keys). Probably, it'd be good to introduced a common term for MetaKey and SubDocKey/DocKey in future.
Jira: DB-16582

Test Plan:
./yb_build.sh --cxx-test pgwrapper_pg_vector_index-test --gtest_filter PgVectorIndexSingleServerTest.ReverseMappingCleanup/Colocated
./yb_build.sh --cxx-test pgwrapper_pg_vector_index-test --gtest_filter PgVectorIndexSingleServerTest.ReverseMappingCleanup/ColocatedHnswlib
./yb_build.sh --cxx-test pgwrapper_pg_vector_index-test --gtest_filter PgVectorIndexSingleServerTest.ReverseMappingCleanup/ColocatedYbHnsw
./yb_build.sh --cxx-test pgwrapper_pg_vector_index-test --gtest_filter PgVectorIndexSingleServerTest.ReverseMappingCleanup/Distributed
./yb_build.sh --cxx-test pgwrapper_pg_vector_index-test --gtest_filter PgVectorIndexSingleServerTest.ReverseMappingCleanup/DistributedHnswlib
./yb_build.sh --cxx-test pgwrapper_pg_vector_index-test --gtest_filter PgVectorIndexSingleServerTest.ReverseMappingCleanup/DistributedYbHnsw

Reviewers: sergei, rthallam, timur

Reviewed By: sergei, rthallam

Subscribers: ybase

Differential Revision: https://phorge.dev.yugabyte.com/D47021

Author: arybochkin

src/yb/common/common_flags.cc

@@ -87,7 +87,7 @@ TAG_FLAG(wait_for_ysql_backends_catalog_version_client_master_rpc_margin_ms, adv
 // TODO(#13369): use this flag in tserver.
 DEFINE_NON_RUNTIME_uint32(master_ts_ysql_catalog_lease_ms, 10000, // 10s
     "Lease period between master and tserver that guarantees YSQL system catalog is not stale."
-    " Must be higher than --heartbeat_interval_ms, preferrably many times higher.");
+    " Must be higher than --heartbeat_interval_ms, preferably many times higher.");
 TAG_FLAG(master_ts_ysql_catalog_lease_ms, advanced);
 TAG_FLAG(master_ts_ysql_catalog_lease_ms, hidden);
 
@@ -130,8 +130,8 @@ DEFINE_RUNTIME_AUTO_PG_FLAG(
     "Requires yb_enable_replication_commands to be true.");
 
 DEFINE_RUNTIME_PG_PREVIEW_FLAG(bool, yb_enable_consistent_replication_from_hash_range, false,
-                       "Enable consumption of consistent changes via replication slots from "
-                       "a hash range of a table.");
+    "Enable consumption of consistent changes via replication slots from "
+    "a hash range of a table.");
 
 DEFINE_NON_RUNTIME_PREVIEW_bool(ysql_yb_enable_implicit_dynamic_tables_logical_replication, false,
     "When set to true, modifications to publication will be reflected implicitly. "
@@ -163,7 +163,7 @@ DEFINE_NON_RUNTIME_string(placement_zone, "rack1",
     "The cloud availability zone in which this instance is started.");
 
 DEFINE_test_flag(bool, check_catalog_version_overflow, false,
-                 "Check whether received catalog version is unreasonably too big");
+    "Check whether received catalog version is unreasonably too big");
 
 DEFINE_RUNTIME_PG_FLAG(bool, yb_enable_invalidation_messages, true,
     "True to enable invalidation messages");
@@ -197,13 +197,13 @@ DEFINE_validator(TEST_ysql_yb_enable_ddl_savepoint_support,
 // is relatively new, it is advisable that we have the flag enabled for now. The value can be
 // increased once the feature hardens and the above referred issue is resolved.
 DEFINE_RUNTIME_uint64(refresh_waiter_timeout_ms, 30000,
-                      "The maximum amount of time a waiter transaction waits in the wait-queue "
-                      "before its callback is invoked. On invocation, the waiter transaction "
-                      "re-runs conflicts resolution and might enter the wait-queue again with "
-                      "updated blocker(s) information. Setting the value to 0 disables "
-                      "automatically re-running conflict resolution due to timeout. It follows "
-                      "that the waiter callback would only be invoked when a blocker txn commits/ "
-                      "aborts/gets promoted.");
+    "The maximum amount of time a waiter transaction waits in the wait-queue "
+    "before its callback is invoked. On invocation, the waiter transaction "
+    "re-runs conflicts resolution and might enter the wait-queue again with "
+    "updated blocker(s) information. Setting the value to 0 disables "
+    "automatically re-running conflict resolution due to timeout. It follows "
+    "that the waiter callback would only be invoked when a blocker txn commits/ "
+    "aborts/gets promoted.");
 TAG_FLAG(refresh_waiter_timeout_ms, advanced);
 TAG_FLAG(refresh_waiter_timeout_ms, hidden);
 
@@ -282,41 +282,40 @@ DEFINE_NON_RUNTIME_bool(use_fast_backward_scan, true,
     "Use backward scan optimization to build a row in the reverse order for YSQL.");
 
 DEFINE_RUNTIME_bool(ysql_enable_auto_analyze_service, false,
-                    "Enable the Auto Analyze service which automatically triggers ANALYZE to "
-                    "update table statistics for tables which have changed more than a "
-                    "configurable threshold.");
+    "Enable the Auto Analyze service which automatically triggers ANALYZE to "
+    "update table statistics for tables which have changed more than a "
+    "configurable threshold.");
 TAG_FLAG(ysql_enable_auto_analyze_service, experimental);
 
 DEFINE_RUNTIME_AUTO_bool(cdcsdk_enable_dynamic_table_addition_with_table_cleanup,
-                        kLocalPersisted,
-                        false,
-                        true,
-                        "This flag needs to be true in order to support addition of dynamic tables "
-                        "along with removal of not of interest/expired tables from a CDCSDK "
-                        "stream.");
+    kLocalPersisted,
+    false,
+    true,
+    "This flag needs to be true in order to support addition of dynamic tables "
+    "along with removal of not of interest/expired tables from a CDCSDK "
+    "stream.");
 TAG_FLAG(cdcsdk_enable_dynamic_table_addition_with_table_cleanup, advanced);
 
 DEFINE_RUNTIME_AUTO_PG_FLAG(bool, yb_update_optimization_infra, kLocalPersisted, false, true,
-                            "Enables optimizations of YSQL UPDATE queries. This includes "
-                            "(but not limited to) skipping redundant secondary index updates "
-                            "and redundant constraint checks.");
+    "Enables optimizations of YSQL UPDATE queries. This includes "
+    "(but not limited to) skipping redundant secondary index updates "
+    "and redundant constraint checks.");
 
 DEFINE_RUNTIME_PG_FLAG(bool, yb_skip_redundant_update_ops, true,
-                       "Enables the comparison of old and new values of columns specified in the "
-                       "SET clause of YSQL UPDATE queries to skip redundant secondary index "
-                       "updates and redundant constraint checks.");
+    "Enables the comparison of old and new values of columns specified in the "
+    "SET clause of YSQL UPDATE queries to skip redundant secondary index "
+    "updates and redundant constraint checks.");
 TAG_FLAG(ysql_yb_skip_redundant_update_ops, advanced);
 
-DEFINE_RUNTIME_bool(cdc_disable_sending_composite_values,
-                    true,
-                    "When this flag is set to true, cdc service will send null values for columns "
-                    "of composite types");
+DEFINE_RUNTIME_bool(cdc_disable_sending_composite_values, true,
+    "When this flag is set to true, cdc service will send null values for columns "
+    "of composite types");
 
-DEFINE_UNKNOWN_int32(timestamp_history_retention_interval_sec, 900,
-                     "The time interval in seconds to retain DocDB history for. Point-in-time "
-                     "reads at a hybrid time further than this in the past might not be allowed "
-                     "after a compaction. Set this to be higher than the expected maximum duration "
-                     "of any single transaction in your application.");
+DEFINE_RUNTIME_int32(timestamp_history_retention_interval_sec, 900,
+    "The time interval in seconds to retain DocDB history for. Point-in-time "
+    "reads at a hybrid time further than this in the past might not be allowed "
+    "after a compaction. Set this to be higher than the expected maximum duration "
+    "of any single transaction in your application.");
 
 namespace yb {
 

src/yb/docdb/doc_boundary_values_extractor.cc

@@ -39,7 +39,7 @@ class DocBoundaryValuesExtractor : public rocksdb::BoundaryValuesExtractor {
   virtual ~DocBoundaryValuesExtractor() {}
 
   Status Extract(Slice user_key, rocksdb::UserBoundaryValueRefs* values) override {
-    if (dockv::IsInternalRecordKeyType(dockv::DecodeKeyEntryType(user_key))) {
+    if (dockv::IsMetaKeyType(dockv::DecodeKeyEntryType(user_key))) {
       // Skipping internal DocDB records.
       return Status::OK();
     }

src/yb/docdb/doc_operation-test.cc

@@ -135,10 +135,7 @@ class DiscardUntilFileFilterFactory : public rocksdb::CompactionFileFilterFactor
 // rocksdb_max_file_size_for_compaction flag if it is set to a positive number, and returns
 // the max uint64 otherwise. It does NOT take the schema's table TTL into consideration.
 auto MakeExcludeFromCompactionFunction() {
-  using ExcludeFromCompaction = decltype(std::declval<rocksdb::Options>().exclude_from_compaction);
-  using ExcludeFromCompactionFunction = typename ExcludeFromCompaction::element_type;
-
-  return std::make_shared<ExcludeFromCompactionFunction>([](const rocksdb::FileMetaData& file) {
+  return std::make_shared<rocksdb::CompactionFileExcluder>([](const rocksdb::FileMetaData& file) {
     if (FLAGS_rocksdb_max_file_size_for_compaction > 0) {
       return file.fd.GetTotalFileSize() > FLAGS_rocksdb_max_file_size_for_compaction;
     }