Record how long it takes to wait for a writer connection to become available.

Author: sandhose

crates/syn2mas/src/mas_writer/mod.rs

@@ -31,7 +31,10 @@ use self::{
     constraint_pausing::{ConstraintDescription, IndexDescription},
     locking::LockedMasDatabase,
 };
-use crate::{Progress, telemetry::WRITER_FLUSH_TIME};
+use crate::{
+    Progress,
+    telemetry::{WRITER_FLUSH_TIME, WRITER_WAIT_TIME},
+};
 
 pub mod checks;
 pub mod locking;
@@ -119,8 +122,12 @@ impl WriterConnectionPool {
             + Sync
             + 'static,
     {
+        let start = Instant::now();
         match self.connection_rx.recv().await {
             Some(Ok(mut connection)) => {
+                let elapsed = start.elapsed();
+                WRITER_WAIT_TIME.record(elapsed.as_millis().try_into().unwrap_or(u64::MAX), &[]);
+
                 let connection_tx = self.connection_tx.clone();
                 tokio::task::spawn(
                     async move {

crates/syn2mas/src/telemetry.rs

@@ -45,6 +45,14 @@ pub static WRITER_FLUSH_TIME: LazyLock<Histogram<u64>> = LazyLock::new(|| {
         .build()
 });
 
+pub static WRITER_WAIT_TIME: LazyLock<Histogram<u64>> = LazyLock::new(|| {
+    METER
+        .u64_histogram("syn2mas.writer.wait_time")
+        .with_description("Time spent waiting for a writer connection to become available")
+        .with_unit("ms")
+        .build()
+});
+
 /// Attribute key for syn2mas.entity metrics representing what entity.
 pub const K_ENTITY: &str = "entity";