CRC: retry post block up to limit if response status is not a success

Signed-off-by: Jacinta Ferrant <[email protected]>

Author: jferrant

stacks-signer/src/client/mod.rs

@@ -28,7 +28,7 @@ use libstackerdb::Error as StackerDBError;
 pub use stackerdb::*;
 pub use stacks_client::*;
 use stacks_common::codec::Error as CodecError;
-use stacks_common::debug;
+use stacks_common::{debug, warn};
 
 /// Backoff timer initial interval in milliseconds
 const BACKOFF_INITIAL_INTERVAL: u64 = 128;
@@ -103,7 +103,7 @@ pub enum ClientError {
 pub fn retry_with_exponential_backoff<F, E, T>(request_fn: F) -> Result<T, ClientError>
 where
     F: FnMut() -> Result<T, backoff::Error<E>>,
-    E: std::fmt::Debug,
+    E: std::fmt::Debug + Into<ClientError>,
 {
     let notify = |err, dur| {
         debug!(
@@ -117,7 +117,16 @@ where
         .with_max_elapsed_time(Some(Duration::from_secs(BACKOFF_MAX_ELAPSED)))
         .build();
 
-    backoff::retry_notify(backoff_timer, request_fn, notify).map_err(|_| ClientError::RetryTimeout)
+    backoff::retry_notify(backoff_timer, request_fn, notify).map_err(|e| match e {
+        backoff::Error::Permanent(err) => {
+            warn!("Non-retry error during request: {err:?}");
+            err.into()
+        }
+        backoff::Error::Transient { err, .. } => {
+            warn!("Exceeded max retries during request: {err:?}");
+            err.into()
+        }
+    })
 }
 
 #[cfg(test)]

stacks-signer/src/client/stacks_client.rs

@@ -612,22 +612,30 @@ impl StacksClient {
         let path = format!("{}{}?broadcast=1", self.http_origin, postblock_v3::PATH);
         let timer = crate::monitoring::actions::new_rpc_call_timer(&path, &self.http_origin);
         let send_request = || {
-            self.stacks_node_client
+            let response = self
+                .stacks_node_client
                 .post(&path)
                 .header("Content-Type", "application/octet-stream")
                 .header(AUTHORIZATION, self.auth_password.clone())
                 .body(block.serialize_to_vec())
                 .send()
                 .map_err(|e| {
                     debug!("Failed to submit block to the Stacks node: {e:?}");
-                    backoff::Error::transient(e)
-                })
+                    backoff::Error::transient(ClientError::from(e))
+                })?;
+            if !response.status().is_success() {
+                warn!(
+                    "Failed to post block to stacks-node, will retry until limit reached";
+                    "http_status" => %response.status(),
+                );
+                return Err(backoff::Error::transient(ClientError::RequestFailure(
+                    response.status(),
+                )));
+            }
+            Ok(response)
         };
         let response = retry_with_exponential_backoff(send_request)?;
         timer.stop_and_record();
-        if !response.status().is_success() {
-            return Err(ClientError::RequestFailure(response.status()));
-        }
         let post_block_resp = response.json::<StacksBlockAcceptedData>()?;
         Ok(post_block_resp.accepted)
     }