better tracking

Author: jayantk

apps/fortuna/src/keeper/block.rs

@@ -119,6 +119,10 @@ pub async fn process_single_block_batch(
     metrics: Arc<KeeperMetrics>,
     fulfilled_requests_cache: Arc<RwLock<HashSet<u64>>>,
 ) {
+    let label = ChainIdLabel {
+        chain_id: chain_state.id.clone(),
+    };
+
     loop {
         let events_res = chain_state
             .contract
@@ -129,6 +133,31 @@ pub async fn process_single_block_batch(
             )
             .await;
 
+        // Only update metrics if we successfully retrieved events.
+        if events_res.is_ok() {
+            // Track the last time blocks were processed. If anything happens to the processing thread, the
+            // timestamp will lag, which will trigger an alert.
+            let server_timestamp = SystemTime::now()
+                .duration_since(UNIX_EPOCH)
+                .map(|duration| duration.as_secs() as i64)
+                .unwrap_or(0);
+            metrics
+                .process_event_timestamp
+                .get_or_create(&label)
+                .set(server_timestamp);
+
+            let current_block = metrics
+                .process_event_block_number
+                .get_or_create(&label)
+                .get();
+            if block_range.to > current_block as u64 {
+                metrics
+                    .process_event_block_number
+                    .get_or_create(&label)
+                    .set(block_range.to as i64);
+            }
+        }
+
         match events_res {
             Ok(events) => {
                 tracing::info!(num_of_events = &events.len(), "Processing",);
@@ -262,22 +291,8 @@ pub async fn process_new_blocks(
     block_delays: Vec<u64>,
 ) {
     tracing::info!("Waiting for new block ranges to process");
-    let label = ChainIdLabel {
-        chain_id: chain_state.id.clone(),
-    };
     loop {
         if let Some(block_range) = rx.recv().await {
-            // Track the last time blocks were processed. If anything happens to the processing thread, the
-            // timestamp will lag, which will trigger an alert.
-            let server_timestamp = SystemTime::now()
-                .duration_since(UNIX_EPOCH)
-                .map(|duration| duration.as_secs() as i64)
-                .unwrap_or(0);
-            metrics
-                .process_event_timestamp
-                .get_or_create(&label)
-                .set(server_timestamp);
-
             // Process blocks immediately first
             process_block_range(
                 block_range.clone(),

apps/fortuna/src/keeper/keeper_metrics.rs

@@ -46,6 +46,8 @@ pub struct KeeperMetrics {
     pub block_timestamp_lag: Family<ChainIdLabel, Gauge>,
     pub block_timestamp: Family<ChainIdLabel, Gauge>,
     pub process_event_timestamp: Family<ChainIdLabel, Gauge>,
+    pub block_number: Family<ChainIdLabel, Gauge>,
+    pub process_event_block_number: Family<ChainIdLabel, Gauge>,
 }
 
 impl Default for KeeperMetrics {
@@ -91,6 +93,8 @@ impl Default for KeeperMetrics {
             block_timestamp_lag: Family::default(),
             block_timestamp: Family::default(),
             process_event_timestamp: Family::default(),
+            block_number: Family::default(),
+            process_event_block_number: Family::default(),
         }
     }
 }
@@ -244,6 +248,18 @@ impl KeeperMetrics {
             keeper_metrics.process_event_timestamp.clone(),
         );
 
+        writable_registry.register(
+            "block_number",
+            "The current block number",
+            keeper_metrics.block_number.clone(),
+        );
+
+        writable_registry.register(
+            "process_event_block_number",
+            "The highest block number for which events have been successfully retrieved and processed",
+            keeper_metrics.process_event_block_number.clone(),
+        );
+
         // *Important*: When adding a new metric:
         // 1. Register it above using `writable_registry.register(...)`
         // 2. Add a get_or_create call in the add_chain function below to initialize it for each chain/provider pair
@@ -259,6 +275,10 @@ impl KeeperMetrics {
         let _ = self.block_timestamp_lag.get_or_create(&chain_id_label);
         let _ = self.block_timestamp.get_or_create(&chain_id_label);
         let _ = self.process_event_timestamp.get_or_create(&chain_id_label);
+        let _ = self.block_number.get_or_create(&chain_id_label);
+        let _ = self
+            .process_event_block_number
+            .get_or_create(&chain_id_label);
 
         let account_label = AccountLabel {
             chain_id,

apps/fortuna/src/keeper/track.rs

@@ -53,14 +53,24 @@ pub async fn track_block_timestamp_lag(
     };
 
     let block = provider.get_block(BlockNumber::Latest).await?;
-    let block_timestamp = block.ok_or(anyhow!("block was none"))?.timestamp.as_u64();
+    let block = block.ok_or(anyhow!("block was none"))?;
+    let block_timestamp = block.timestamp.as_u64();
     let block_timestamp = i64::try_from(block_timestamp)?;
+    let block_number = block
+        .number
+        .ok_or(anyhow!("block number was none"))?
+        .as_u64();
 
     metrics
         .block_timestamp
         .get_or_create(&label)
         .set(block_timestamp);
 
+    metrics
+        .block_number
+        .get_or_create(&label)
+        .set(block_number as i64);
+
     let server_timestamp = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs())?;
 
     let lag = server_timestamp - block_timestamp;