Add timers and size metrics for loading search index files (#25648)

Adds timers and size metrics for loading id tracker, deleted terms table, and alive bitset.

GitOrigin-RevId: 6e2df08a36f00a4103f17ddfd304fe70a58b9803

Author: emmaling27

Cargo.lock

@@ -16,6 +16,11 @@ use byteorder::{
 };
 use csf::ls::Map as CsfMap;
 
+use crate::metrics::{
+    load_id_tracker_timer,
+    log_id_tracker_size,
+};
+
 /// Version 1 of the id table has the following format:
 /// ```
 /// [ version ] [ count ] [ index_len ] [ ID ]* [ index ]
@@ -44,15 +49,15 @@ pub struct StaticIdTracker {
 
 impl StaticIdTracker {
     pub fn load_from_path(id_table_path: PathBuf) -> anyhow::Result<Self> {
+        let _timer = load_id_tracker_timer();
         let uuid_file = File::open(id_table_path)?;
-        StaticIdTracker::load((
-            uuid_file.metadata()?.len() as usize,
-            BufReader::new(uuid_file),
-        ))
+        let size = uuid_file.metadata()?.len() as usize;
+        log_id_tracker_size(size);
+        StaticIdTracker::load(size, BufReader::new(uuid_file))
     }
 
-    pub fn load(uuid_file: (usize, impl Read)) -> anyhow::Result<Self> {
-        let (count, uuid_buf, csf_map) = Self::load_ids(uuid_file.0, uuid_file.1)?;
+    pub fn load(file_len: usize, reader: impl Read) -> anyhow::Result<Self> {
+        let (count, uuid_buf, csf_map) = Self::load_ids(file_len, reader)?;
         Ok(Self {
             count,
             id_buf: uuid_buf,

crates/common/src/id_tracker.rs

@@ -21,7 +21,10 @@ use metrics::{
     Timer,
     CONVEX_METRICS_REGISTRY,
 };
-use prometheus::VMHistogramVec;
+use prometheus::{
+    VMHistogram,
+    VMHistogramVec,
+};
 use sync_types::backoff::Backoff;
 
 use crate::runtime::Runtime;
@@ -152,3 +155,13 @@ pub fn register_prometheus_exporter<RT: Runtime>(
     };
     (handle, flush)
 }
+
+register_convex_histogram!(LOAD_ID_TRACKER_SECONDS, "Time to load IdTracker in seconds");
+pub fn load_id_tracker_timer() -> Timer<VMHistogram> {
+    Timer::new(&LOAD_ID_TRACKER_SECONDS)
+}
+
+register_convex_histogram!(ID_TRACKER_SIZE_BYTES, "IdTracker file size");
+pub fn log_id_tracker_size(size: usize) {
+    log_distribution(&ID_TRACKER_SIZE_BYTES, size as f64);
+}

crates/common/src/metrics.rs

@@ -13,6 +13,7 @@ anyhow = { workspace = true }
 byteorder = { workspace = true }
 common = { path = "../common" }
 csf = { workspace = true }
+metrics = { path = "../metrics" }
 sucds = { workspace = true }
 tantivy = { workspace = true }
 tantivy-common = { workspace = true }
@@ -21,10 +22,8 @@ value = { path = "../value" }
 
 [dev-dependencies]
 common = { path = "../common", features = ["testing"] }
+metrics = { path = "../metrics", features = ["testing"] }
 value = { path = "../value", features = ["testing"] }
 
 [features]
-testing = [
-    "common/testing",
-    "value/testing",
-]
+testing = ["common/testing", "metrics/testing", "value/testing"]

crates/text_search/Cargo.toml

@@ -1,3 +1,5 @@
+#![feature(lazy_cell)]
+mod metrics;
 #[cfg(test)]
 mod tests;
 pub mod tracker;

crates/text_search/src/lib.rs

@@ -0,0 +1,35 @@
+use metrics::{
+    log_distribution,
+    prometheus::VMHistogram,
+    register_convex_histogram,
+    Timer,
+};
+
+register_convex_histogram!(
+    LOAD_ALIVE_BITSET_SECONDS,
+    "Time to load AliveBitSet in seconds"
+);
+pub fn load_alive_bitset_timer() -> Timer<VMHistogram> {
+    Timer::new(&LOAD_ALIVE_BITSET_SECONDS)
+}
+
+register_convex_histogram!(ALIVE_BITSET_SIZE_BYTES, "Size of AliveBitSet in bytes");
+pub fn log_alive_bitset_size(size: usize) {
+    log_distribution(&ALIVE_BITSET_SIZE_BYTES, size as f64)
+}
+
+register_convex_histogram!(
+    LOAD_DELETED_TERMS_TABLE_SECONDS,
+    "Time to load deleted terms table in seconds"
+);
+pub fn load_deleted_terms_table_timer() -> Timer<VMHistogram> {
+    Timer::new(&LOAD_DELETED_TERMS_TABLE_SECONDS)
+}
+
+register_convex_histogram!(
+    DELETED_TERMS_TABLE_SIZE_BYTES,
+    "Size of deleted terms table in bytes"
+);
+pub fn log_deleted_terms_table_size(size: usize) {
+    log_distribution(&DELETED_TERMS_TABLE_SIZE_BYTES, size as f64)
+}

crates/text_search/src/metrics.rs

@@ -46,6 +46,13 @@ use tantivy_common::{
 };
 use value::InternalId;
 
+use crate::metrics::{
+    load_alive_bitset_timer,
+    load_deleted_terms_table_timer,
+    log_alive_bitset_size,
+    log_deleted_terms_table_size,
+};
+
 /// Version 1 of the deletion tracker has the following format:
 /// ```
 /// [ version ] [ num_terms_deleted ] [ deleted_term_ordinals_size ] [ counts_size ] [ deleted_term_ordinals ] [ counts ]
@@ -93,7 +100,10 @@ impl DeletedTermsTable {
 }
 
 pub fn load_alive_bitset(path: &Path) -> anyhow::Result<AliveBitSet> {
+    let _timer = load_alive_bitset_timer();
     let mut file = File::open(path)?;
+    let size = file.metadata()?.len();
+    log_alive_bitset_size(size as usize);
     let mut buf = vec![];
     file.read_to_end(&mut buf)?;
     let owned = OwnedBytes::new(buf);
@@ -156,6 +166,8 @@ impl StaticDeletionTracker {
         file_len: usize,
         mut reader: impl Read,
     ) -> anyhow::Result<(u32, Option<DeletedTermsTable>)> {
+        log_deleted_terms_table_size(file_len);
+        let _timer = load_deleted_terms_table_timer();
         let mut expected_len = 0;
         let version = reader.read_u8()?;
         expected_len += 1;

crates/text_search/src/tracker.rs

@@ -444,7 +444,7 @@ mod tests {
         let mut uuid_buf = vec![];
         tracker.write_uuids(&mut uuid_buf).unwrap();
 
-        let id_tracker = StaticIdTracker::load((uuid_buf.len(), &uuid_buf[..])).unwrap();
+        let id_tracker = StaticIdTracker::load(uuid_buf.len(), &uuid_buf[..]).unwrap();
         VectorStaticIdTracker {
             id_tracker,
             deleted_bitset,
@@ -560,7 +560,7 @@ mod tests {
             tracker.write_uuids(&mut uuid_buf).unwrap();
 
             let id_tracker = StaticIdTracker::load(
-                (uuid_buf.len(), &uuid_buf[..]),
+                uuid_buf.len(), &uuid_buf[..],
             ).unwrap();
             let tracker = VectorStaticIdTracker {
                 id_tracker,
@@ -662,7 +662,7 @@ mod tests {
 
             let deleted_bitset = DeletedBitset::load(deleted_buf.len(), &deleted_buf[..]).unwrap();
             let id_tracker = StaticIdTracker::load(
-                (uuid_buf.len(), &uuid_buf[..]),
+                uuid_buf.len(), &uuid_buf[..],
             ).unwrap();
             let static_ids = VectorStaticIdTracker {
                 id_tracker,
@@ -703,7 +703,7 @@ mod tests {
 
             let deleted_bitset = DeletedBitset::load(deleted_buf.len(), &deleted_buf[..]).unwrap();
             let id_tracker = StaticIdTracker::load(
-                (uuid_buf.len(), &uuid_buf[..]),
+                uuid_buf.len(), &uuid_buf[..],
             ).unwrap();
             let static_ids = VectorStaticIdTracker {
                 id_tracker,