Search deleted terms tracker (#25254)

This PR adds the logic for the search deleted terms tracker. We'll write three tracker files - id tracker, deleted bitset, and deleted terms since not all of them are needed everywhere. The deleted terms file format is pretty similar to id tracker but instead of using perfect hashing, we store the term ordinals and number of deleted documents corresponding to that term in separate structures: `EliasFano` for the `TermOrdinal`s because they are monotonic increasing integers, and `DacsOpt` for the counts of deleted documents because they will be small positive integers.

GitOrigin-RevId: 8e42c994af566b27d0a16219898e36459ec19af1

Author: emmaling27

Cargo.lock

@@ -109,8 +109,8 @@ sourcemap = "7"
 strum = { version = "0.26", features = [ "derive" ] }
 sucds = { version = "0.8.1", features = [ "intrinsics" ] }
 syn = { version = "2.0", features = [ "full" ] }
-tantivy = { git = "https://github.com/get-convex/tantivy", branch = "rakeeb/reexport-tantivy-fst" }
-tantivy-common = { git = "https://github.com/get-convex/tantivy", branch = "rakeeb/reexport-tantivy-fst" }
+tantivy = { git = "https://github.com/get-convex/tantivy", rev = "a0f4d3c3843dd255ac6a7210830b3f91cf16db8f" }
+tantivy-common = { git = "https://github.com/get-convex/tantivy", rev = "a0f4d3c3843dd255ac6a7210830b3f91cf16db8f" }
 tempfile = "3"
 thiserror = "1"
 thousands = "0.2.0"

Cargo.toml

@@ -7,7 +7,10 @@ use std::{
         Write,
     },
     mem,
-    path::PathBuf,
+    path::{
+        Path,
+        PathBuf,
+    },
 };
 
 use bitvec::{
@@ -30,7 +33,7 @@ use byteorder::{
 /// - bitset blocks (dense array of little-endian u64s): bitset contents
 pub const DELETED_BITSET_VERSION: u8 = 1;
 
-#[derive(Clone)]
+#[derive(Clone, Default)]
 pub struct DeletedBitset {
     deleted: BitVec,
     num_deleted: usize,
@@ -121,7 +124,10 @@ impl DeletedBitset {
         Ok(())
     }
 
-    pub fn load_from_path(path: PathBuf) -> anyhow::Result<Self> {
+    pub fn load_from_path<P>(path: P) -> anyhow::Result<Self>
+    where
+        P: AsRef<Path>,
+    {
         let deleted_file = File::open(path)?;
         Self::load(
             deleted_file.metadata()?.len() as usize,

crates/common/src/deleted_bitset.rs

@@ -40,6 +40,7 @@ sucds = { workspace = true }
 tantivy = { workspace = true }
 tantivy-common = { workspace = true }
 tempfile = { workspace = true }
+text_search = { path = "../text_search" }
 tokio = { workspace = true }
 tracing = { workspace = true }
 uuid = { workspace = true }
@@ -60,6 +61,7 @@ proptest-derive = { workspace = true }
 rand = { workspace = true }
 runtime = { path = "../runtime", features = ["testing"] }
 storage = { path = "../storage", features = ["testing"] }
+text_search = { path = "../text_search", features = ["testing"] }
 value = { path = "../value", features = ["testing"] }
 vector = { path = "../vector", features = ["testing"] }
 
@@ -72,6 +74,7 @@ testing = [
     "proptest",
     "proptest-derive",
     "storage/testing",
+    "text_search/testing",
     "value/testing",
     "vector/testing",
 ]

crates/search/Cargo.toml

@@ -377,6 +377,9 @@ fn is_immutable(search_file_type: SearchFileType) -> bool {
         SearchFileType::VectorIdTracker => true,
         // Text indexes do not appear to be read in readonly mode.
         SearchFileType::Text => false,
+        SearchFileType::TextIdTracker => true,
+        SearchFileType::TextDeletedBitset => true,
+        SearchFileType::TextDeletedTerms => true,
     }
 }
 

crates/search/src/archive/cache.rs

@@ -3,6 +3,7 @@ use std::{
     fmt,
 };
 
+use common::deleted_bitset::DeletedBitset;
 use tantivy::{
     query::{
         intersect_scorers,
@@ -25,7 +26,6 @@ use tantivy::{
     Term,
     TERMINATED,
 };
-use tantivy_common::ReadOnlyBitSet;
 
 /// A query for documents that:
 /// 1. Contain at least one of the OR terms.
@@ -151,29 +151,26 @@ impl Weight for ConvexSearchWeight {
 #[derive(Clone)]
 pub struct DeletedDocuments {
     pub memory_deleted: BTreeSet<DocId>,
-    pub segment_deleted: ReadOnlyBitSet,
-    // NB: `ReadOnlyBitSet::len` is linear time, so use our precomputed count
-    // of the number of documents deleted.
-    pub num_segment_deleted: usize,
+    pub segment_deleted: DeletedBitset,
 }
 
 impl fmt::Debug for DeletedDocuments {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_struct("DeletedDocuments")
             .field("memory_deleted", &self.memory_deleted)
             .field("segment_deleted", &"<bitset>")
-            .field("num_segment_deleted", &self.num_segment_deleted)
+            .field("num_segment_deleted", &self.segment_deleted.num_deleted())
             .finish()
     }
 }
 
 impl DeletedDocuments {
     pub fn contains(&self, doc: DocId) -> bool {
-        self.memory_deleted.contains(&doc) || self.segment_deleted.contains(doc)
+        self.memory_deleted.contains(&doc) || self.segment_deleted.is_deleted(doc)
     }
 
     pub fn len(&self) -> usize {
-        self.memory_deleted.len() + self.num_segment_deleted
+        self.memory_deleted.len() + self.segment_deleted.num_deleted()
     }
 }
 

crates/search/src/convex_query.rs

@@ -875,6 +875,9 @@ pub enum SearchFileType {
     VectorDeletedBitset,
     VectorIdTracker,
     Text,
+    TextIdTracker,
+    TextDeletedBitset,
+    TextDeletedTerms,
 }
 
 #[cfg(test)]

crates/search/src/lib.rs

@@ -425,6 +425,9 @@ impl SearchFileType {
             SearchFileType::VectorDeletedBitset => "vector_deleted_bitset",
             SearchFileType::VectorIdTracker => "vector_id_tracker",
             SearchFileType::Text => "text",
+            SearchFileType::TextIdTracker => "text_id_tracker",
+            SearchFileType::TextDeletedBitset => "text_deleted_bitset",
+            SearchFileType::TextDeletedTerms => "text_deleted_terms",
             SearchFileType::FragmentedVectorSegment => "fragmented_vector_segment",
         };
         StaticMetricLabel::new(SEARCH_FILE_TYPE, search_type_str)