Merge pull request #5327 from stacks-network/chore/event-dispatcher-logs

Improvements to event dispatcher retry logic

Author: kantai

testnet/stacks-node/src/event_dispatcher.rs

@@ -26,6 +26,7 @@ use clarity::vm::analysis::contract_interface_builder::build_contract_interface;
 use clarity::vm::costs::ExecutionCost;
 use clarity::vm::events::{FTEventType, NFTEventType, STXEventType};
 use clarity::vm::types::{AssetIdentifier, QualifiedContractIdentifier, Value};
+use rand::Rng;
 use rusqlite::{params, Connection};
 use serde_json::json;
 use stacks::burnchains::{PoxConstants, Txid};
@@ -429,6 +430,10 @@ impl EventObserver {
             .unwrap_or(PeerHost::DNS(host.to_string(), port));
 
         let mut backoff = Duration::from_millis(100);
+        let mut attempts: i32 = 0;
+        // Cap the backoff at 3x the timeout
+        let max_backoff = timeout.saturating_mul(3);
+
         loop {
             let mut request = StacksHttpRequest::new_for_peer(
                 peerhost.clone(),
@@ -438,7 +443,6 @@ impl EventObserver {
             )
             .unwrap_or_else(|_| panic!("FATAL: failed to encode infallible data as HTTP request"));
             request.add_header("Connection".into(), "close".into());
-
             match send_http_request(host, port, request, timeout) {
                 Ok(response) => {
                     if response.preamble().status_code == 200 {
@@ -455,7 +459,9 @@ impl EventObserver {
                 Err(err) => {
                     warn!(
                         "Event dispatcher: connection or request failed to {}:{} - {:?}",
-                        &host, &port, err
+                        &host, &port, err;
+                        "backoff" => ?backoff,
+                        "attempts" => attempts
                     );
                 }
             }
@@ -471,7 +477,12 @@ impl EventObserver {
             }
 
             sleep(backoff);
-            backoff *= 2;
+            let jitter: u64 = rand::thread_rng().gen_range(0..100);
+            backoff = std::cmp::min(
+                backoff.saturating_mul(2) + Duration::from_millis(jitter),
+                max_backoff,
+            );
+            attempts = attempts.saturating_add(1);
         }
     }