chore: add persist and RootStore metrics

Author: RodrigoVillar

firewood/src/persist_worker.rs

@@ -64,6 +64,7 @@ use parking_lot::{Condvar, Mutex, MutexGuard};
 use crate::{manager::CommittedRevision, root_store::RootStore};
 use crossbeam::channel::{self, Receiver, Sender};
 
+use firewood_metrics::firewood_increment;
 use firewood_storage::logger::error;
 
 /// Error type for persistence operations.
@@ -266,8 +267,11 @@ impl PersistSemaphore {
     #[inline]
     fn acquire(&self) {
         let mut permits = self.state.lock();
-        while *permits == 0 {
-            self.condvar.wait(&mut permits);
+        if *permits == 0 {
+            firewood_increment!(crate::registry::PERSIST_THROTTLE_BLOCKS, 1);
+            while *permits == 0 {
+                self.condvar.wait(&mut permits);
+            }
         }
         // The background loop guarantees permits > 0
         *permits = permits.saturating_sub(1);
@@ -385,6 +389,7 @@ impl PersistLoop {
     }
 
     /// Persists the given revision and releases semaphore permits.
+    #[crate::metrics("persist.total", "persist revision to storage")]
     fn persist(
         &mut self,
         revision: &CommittedRevision,
@@ -444,9 +449,17 @@ impl PersistLoop {
         if let Some(ref store) = self.shared.root_store
             && let (Some(hash), Some(addr)) = (revision.root_hash(), revision.root_address())
         {
-            store
+            let start = coarsetime::Instant::now();
+
+            let result = store
                 .add_root(&hash, &addr)
-                .map_err(|e| PersistError::RootStore(e.into()))?;
+                .map_err(|e| PersistError::RootStore(e.into()));
+
+            let success = if result.is_err() { "false" } else { "true" };
+            firewood_increment!(crate::registry::PERSIST_ROOT_STORE, 1, "success" => success);
+            firewood_increment!(crate::registry::PERSIST_ROOT_STORE_MS, start.elapsed().as_millis(), "success" => success);
+
+            result?;
         }
 
         Ok(())

firewood/src/registry.rs

@@ -35,6 +35,21 @@ pub const ACTIVE_REVISIONS: &str = "active_revisions";
 /// Maximum number of revisions configured.
 pub const MAX_REVISIONS: &str = "max_revisions";
 
+/// Number of times a commit blocked waiting for persistence to catch up.
+pub const PERSIST_THROTTLE_BLOCKS: &str = "persist.throttle_blocks";
+
+/// Number of persist operations.
+pub const PERSIST_TOTAL: &str = "persist.total";
+
+/// Persist latency in milliseconds.
+pub const PERSIST_TOTAL_MS: &str = "persist.total_ms";
+
+/// Number of root store save operations.
+pub const PERSIST_ROOT_STORE: &str = "persist.root_store";
+
+/// Root store save latency in milliseconds.
+pub const PERSIST_ROOT_STORE_MS: &str = "persist.root_store_ms";
+
 /// Registers all firewood metric descriptions.
 pub fn register() {
     describe_counter!(PROPOSALS, "Number of proposals created");
@@ -59,4 +74,12 @@ pub fn register() {
     describe_counter!(COMMIT_LATENCY_MS, "Commit latency (ms)");
     describe_gauge!(ACTIVE_REVISIONS, "Current number of active revisions");
     describe_gauge!(MAX_REVISIONS, "Maximum number of revisions configured");
+    describe_counter!(
+        PERSIST_THROTTLE_BLOCKS,
+        "Number of times a commit blocked waiting for persistence"
+    );
+    describe_counter!(PERSIST_TOTAL, "Number of persist operations");
+    describe_counter!(PERSIST_TOTAL_MS, "Persist latency (ms)");
+    describe_counter!(PERSIST_ROOT_STORE, "Number of root store save operations");
+    describe_counter!(PERSIST_ROOT_STORE_MS, "Root store save latency (ms)");
 }