fix: a handful of issues causing timing-related test failures in CI

* make `not_available_try_again` error in `GetStackers`, and make it transient in the signer binary
* make signer binary timeout on retries in client
* update signer outer runloop to differentiate between 'not in signer set' and 'have not loaded info yet'
* update signer outer runloop to handle errors and non-presence differently in the signer config refresh
* update signer outer runloop to perform signer config refresh on the current cycle (if not loaded yet) and on the next cycle (if in prepare phase for the next cycle). This was causing an issue on exactly the first cycle of Nakamoto, because the signer set cannot be loaded for the first cycle until after the prepare phase
* update the signer outer runloop to check the node’s block height on event receipt as well
* update the testing harnesses to wait and check more appropriately for status checks from signers, etc.

Author: kantai

libsigner/src/runloop.rs

@@ -246,13 +246,14 @@ impl<
         let (event_send, event_recv) = channel();
         event_receiver.add_consumer(event_send);
 
+        let bind_port = bind_addr.port();
         event_receiver.bind(bind_addr)?;
         let stop_signaler = event_receiver.get_stop_signaler()?;
         let mut ret_stop_signaler = event_receiver.get_stop_signaler()?;
 
         // start a thread for the event receiver
         let event_thread = thread::Builder::new()
-            .name("event_receiver".to_string())
+            .name(format!("event_receiver:{bind_port}"))
             .stack_size(THREAD_STACK_SIZE)
             .spawn(move || event_receiver.main_loop())
             .map_err(|e| {
@@ -262,7 +263,7 @@ impl<
 
         // start receiving events and doing stuff with them
         let runloop_thread = thread::Builder::new()
-            .name(format!("signer_runloop:{}", bind_addr.port()))
+            .name(format!("signer_runloop:{bind_port}"))
             .stack_size(THREAD_STACK_SIZE)
             .spawn(move || {
                 signer_loop.main_loop(event_recv, command_receiver, result_sender, stop_signaler)

stacks-signer/src/client/mod.rs

@@ -34,6 +34,8 @@ use stacks_common::debug;
 const BACKOFF_INITIAL_INTERVAL: u64 = 128;
 /// Backoff timer max interval in milliseconds
 const BACKOFF_MAX_INTERVAL: u64 = 16384;
+/// Backoff timer max elapsed seconds
+const BACKOFF_MAX_ELAPSED: u64 = 5;
 
 #[derive(thiserror::Error, Debug)]
 /// Client error type
@@ -109,6 +111,7 @@ where
     let backoff_timer = backoff::ExponentialBackoffBuilder::new()
         .with_initial_interval(Duration::from_millis(BACKOFF_INITIAL_INTERVAL))
         .with_max_interval(Duration::from_millis(BACKOFF_MAX_INTERVAL))
+        .with_max_elapsed_time(Some(Duration::from_secs(BACKOFF_MAX_ELAPSED)))
         .build();
 
     backoff::retry_notify(backoff_timer, request_fn, notify).map_err(|_| ClientError::RetryTimeout)

stacks-signer/src/client/stacks_client.rs

@@ -44,6 +44,7 @@ use clarity::vm::types::{PrincipalData, QualifiedContractIdentifier};
 use clarity::vm::{ClarityName, ContractName, Value as ClarityValue};
 use libsigner::v0::messages::PeerInfo;
 use reqwest::header::AUTHORIZATION;
+use serde::Deserialize;
 use serde_json::json;
 use slog::{slog_debug, slog_warn};
 use stacks_common::codec::StacksMessageCodec;
@@ -80,6 +81,12 @@ pub struct StacksClient {
     auth_password: String,
 }
 
+#[derive(Deserialize)]
+struct GetStackersErrorResp {
+    err_type: String,
+    err_msg: String,
+}
+
 impl From<&GlobalConfig> for StacksClient {
     fn from(config: &GlobalConfig) -> Self {
         Self {
@@ -514,23 +521,38 @@ impl StacksClient {
         &self,
         reward_cycle: u64,
     ) -> Result<Option<Vec<NakamotoSignerEntry>>, ClientError> {
-        debug!("Getting reward set for reward cycle {reward_cycle}...");
         let timer = crate::monitoring::new_rpc_call_timer(
             &self.reward_set_path(reward_cycle),
             &self.http_origin,
         );
         let send_request = || {
-            self.stacks_node_client
+            let response = self
+                .stacks_node_client
                 .get(self.reward_set_path(reward_cycle))
                 .send()
-                .map_err(backoff::Error::transient)
+                .map_err(|e| backoff::Error::transient(e.into()))?;
+            let status = response.status();
+            if status.is_success() {
+                return response
+                    .json()
+                    .map_err(|e| backoff::Error::permanent(e.into()));
+            }
+            let error_data = response.json::<GetStackersErrorResp>().map_err(|e| {
+                warn!("Failed to parse the GetStackers error response: {e}");
+                backoff::Error::permanent(e.into())
+            })?;
+            if error_data.err_type == "not_available_try_again" {
+                return Err(backoff::Error::transient(ClientError::NoSortitionOnChain));
+            } else {
+                warn!("Got error response ({status}): {}", error_data.err_msg);
+                return Err(backoff::Error::permanent(ClientError::RequestFailure(
+                    status,
+                )));
+            }
         };
-        let response = retry_with_exponential_backoff(send_request)?;
+        let stackers_response =
+            retry_with_exponential_backoff::<_, ClientError, GetStackersResponse>(send_request)?;
         timer.stop_and_record();
-        if !response.status().is_success() {
-            return Err(ClientError::RequestFailure(response.status()));
-        }
-        let stackers_response = response.json::<GetStackersResponse>()?;
         Ok(stackers_response.stacker_set.signers)
     }
 

stacks-signer/src/lib.rs

@@ -61,8 +61,6 @@ use crate::runloop::{RunLoop, RunLoopCommand};
 pub trait Signer<T: SignerEventTrait>: Debug + Display {
     /// Create a new `Signer` instance
     fn new(config: SignerConfig) -> Self;
-    /// Update the `Signer` instance's with the next reward cycle data `SignerConfig`
-    fn update_signer(&mut self, next_signer_config: &SignerConfig);
     /// Get the reward cycle of the signer
     fn reward_cycle(&self) -> u64;
     /// Process an event