Merge pull request #5283 from stacks-network/fix/metrics-in-tests

Fix metric breakage

Author: kantai

testnet/stacks-node/src/tests/nakamoto_integrations.rs

@@ -1488,16 +1488,19 @@ fn simple_neon_integration() {
     // query for prometheus metrics
     #[cfg(feature = "monitoring_prom")]
     {
-        let prom_http_origin = format!("http://{}", prom_bind);
-        let client = reqwest::blocking::Client::new();
-        let res = client
-            .get(&prom_http_origin)
-            .send()
-            .unwrap()
-            .text()
-            .unwrap();
-        let expected_result = format!("stacks_node_stacks_tip_height {block_height_pre_3_0}");
-        assert!(res.contains(&expected_result));
+        wait_for(10, || {
+            let prom_http_origin = format!("http://{}", prom_bind);
+            let client = reqwest::blocking::Client::new();
+            let res = client
+                .get(&prom_http_origin)
+                .send()
+                .unwrap()
+                .text()
+                .unwrap();
+            let expected_result = format!("stacks_node_stacks_tip_height {block_height_pre_3_0}");
+            Ok(res.contains(&expected_result))
+        })
+        .expect("Prometheus metrics did not update");
     }
 
     info!("Nakamoto miner started...");
@@ -1599,19 +1602,30 @@ fn simple_neon_integration() {
     let bhh = u64::from(tip.burn_header_height);
     test_observer::contains_burn_block_range(220..=bhh).unwrap();
 
-    // make sure prometheus returns an updated height
+    // make sure prometheus returns an updated number of processed blocks
     #[cfg(feature = "monitoring_prom")]
     {
-        let prom_http_origin = format!("http://{}", prom_bind);
-        let client = reqwest::blocking::Client::new();
-        let res = client
-            .get(&prom_http_origin)
-            .send()
-            .unwrap()
-            .text()
-            .unwrap();
-        let expected_result = format!("stacks_node_stacks_tip_height {}", tip.stacks_block_height);
-        assert!(res.contains(&expected_result));
+        wait_for(10, || {
+            let prom_http_origin = format!("http://{}", prom_bind);
+            let client = reqwest::blocking::Client::new();
+            let res = client
+                .get(&prom_http_origin)
+                .send()
+                .unwrap()
+                .text()
+                .unwrap();
+            let expected_result_1 = format!(
+                "stacks_node_stx_blocks_processed_total {}",
+                tip.stacks_block_height
+            );
+
+            let expected_result_2 = format!(
+                "stacks_node_stacks_tip_height {}",
+                tip.stacks_block_height - 1
+            );
+            Ok(res.contains(&expected_result_1) && res.contains(&expected_result_2))
+        })
+        .expect("Prometheus metrics did not update");
     }
 
     check_nakamoto_empty_block_heuristics();

testnet/stacks-node/src/tests/signer/v0.rs

@@ -558,18 +558,22 @@ fn miner_gather_signatures() {
     // Test prometheus metrics response
     #[cfg(feature = "monitoring_prom")]
     {
-        let metrics_response = signer_test.get_signer_metrics();
-
-        // Because 5 signers are running in the same process, the prometheus metrics
-        // are incremented once for every signer. This is why we expect the metric to be
-        // `5`, even though there is only one block proposed.
-        let expected_result = format!("stacks_signer_block_proposals_received {}", num_signers);
-        assert!(metrics_response.contains(&expected_result));
-        let expected_result = format!(
-            "stacks_signer_block_responses_sent{{response_type=\"accepted\"}} {}",
-            num_signers
-        );
-        assert!(metrics_response.contains(&expected_result));
+        wait_for(30, || {
+            let metrics_response = signer_test.get_signer_metrics();
+
+            // Because 5 signers are running in the same process, the prometheus metrics
+            // are incremented once for every signer. This is why we expect the metric to be
+            // `10`, even though there are only two blocks proposed.
+            let expected_result_1 =
+                format!("stacks_signer_block_proposals_received {}", num_signers * 2);
+            let expected_result_2 = format!(
+                "stacks_signer_block_responses_sent{{response_type=\"accepted\"}} {}",
+                num_signers * 2
+            );
+            Ok(metrics_response.contains(&expected_result_1)
+                && metrics_response.contains(&expected_result_2))
+        })
+        .expect("Failed to advance prometheus metrics");
     }
 }