feat: re-add wait_for in metrics checks

Author: hstove

testnet/stacks-node/src/tests/nakamoto_integrations.rs

@@ -1500,16 +1500,19 @@ fn simple_neon_integration() {
     // query for prometheus metrics
     #[cfg(feature = "monitoring_prom")]
     {
-        let prom_http_origin = format!("http://{}", prom_bind);
-        let client = reqwest::blocking::Client::new();
-        let res = client
-            .get(&prom_http_origin)
-            .send()
-            .unwrap()
-            .text()
-            .unwrap();
-        let expected_result = format!("stacks_node_stacks_tip_height {block_height_pre_3_0}");
-        assert!(res.contains(&expected_result));
+        wait_for(10, || {
+            let prom_http_origin = format!("http://{}", prom_bind);
+            let client = reqwest::blocking::Client::new();
+            let res = client
+                .get(&prom_http_origin)
+                .send()
+                .unwrap()
+                .text()
+                .unwrap();
+            let expected_result = format!("stacks_node_stacks_tip_height {block_height_pre_3_0}");
+            Ok(res.contains(&expected_result))
+        })
+        .expect("Prometheus metrics did not update");
     }
 
     info!("Nakamoto miner started...");
@@ -1621,16 +1624,27 @@ fn simple_neon_integration() {
     // make sure prometheus returns an updated number of processed blocks
     #[cfg(feature = "monitoring_prom")]
     {
-        let prom_http_origin = format!("http://{}", prom_bind);
-        let client = reqwest::blocking::Client::new();
-        let res = client
-            .get(&prom_http_origin)
-            .send()
-            .unwrap()
-            .text()
-            .unwrap();
-        let expected_result = format!("stacks_node_stacks_tip_height {}", tip.stacks_block_height);
-        assert!(res.contains(&expected_result));
+        wait_for(10, || {
+            let prom_http_origin = format!("http://{}", prom_bind);
+            let client = reqwest::blocking::Client::new();
+            let res = client
+                .get(&prom_http_origin)
+                .send()
+                .unwrap()
+                .text()
+                .unwrap();
+            let expected_result_1 = format!(
+                "stacks_node_stx_blocks_processed_total {}",
+                tip.stacks_block_height
+            );
+
+            let expected_result_2 = format!(
+                "stacks_node_stacks_tip_height {}",
+                tip.stacks_block_height - 1
+            );
+            Ok(res.contains(&expected_result_1) && res.contains(&expected_result_2))
+        })
+        .expect("Prometheus metrics did not update");
     }
 
     check_nakamoto_empty_block_heuristics();
@@ -6360,15 +6374,18 @@ fn signer_chainstate() {
                 .unwrap()
                 .stacks_block_height;
         let prom_http_origin = format!("http://{}", prom_bind);
-        let client = reqwest::blocking::Client::new();
-        let res = client
-            .get(&prom_http_origin)
-            .send()
-            .unwrap()
-            .text()
-            .unwrap();
-        let expected_result = format!("stacks_node_stacks_tip_height {block_height_pre_3_0}");
-        assert!(res.contains(&expected_result));
+        wait_for(10, || {
+            let client = reqwest::blocking::Client::new();
+            let res = client
+                .get(&prom_http_origin)
+                .send()
+                .unwrap()
+                .text()
+                .unwrap();
+            let expected_result = format!("stacks_node_stacks_tip_height {block_height_pre_3_0}");
+            Ok(res.contains(&expected_result))
+        })
+        .expect("Failed waiting for prometheus metrics to update")
     }
 
     info!("Nakamoto miner started...");
@@ -6972,15 +6989,18 @@ fn continue_tenure_extend() {
     #[cfg(feature = "monitoring_prom")]
     {
         let prom_http_origin = format!("http://{}", prom_bind);
-        let client = reqwest::blocking::Client::new();
-        let res = client
-            .get(&prom_http_origin)
-            .send()
-            .unwrap()
-            .text()
-            .unwrap();
-        let expected_result = format!("stacks_node_stacks_tip_height {block_height_pre_3_0}");
-        assert!(res.contains(&expected_result));
+        wait_for(10, || {
+            let client = reqwest::blocking::Client::new();
+            let res = client
+                .get(&prom_http_origin)
+                .send()
+                .unwrap()
+                .text()
+                .unwrap();
+            let expected_result = format!("stacks_node_stacks_tip_height {block_height_pre_3_0}");
+            Ok(res.contains(&expected_result))
+        })
+        .expect("Prometheus metrics did not update");
     }
 
     info!("Nakamoto miner started...");