Skip to content

Commit 0528da2

Browse files
Zhang Yanpodrmingdrmer
authored andcommitted
fix: increase raft_client retry window and return error instead of panic
The previous retry loop attempted only 3 times with 50ms intervals (150ms total), which is too short for slow CI environments. On timeout it called `panic!()`, losing structured error context. Increase to 20 attempts at 200ms intervals (4s total) and return `anyhow::Error` with attempt count, elapsed time, and last connection error instead of panicking.
1 parent 413c391 commit 0528da2

File tree

1 file changed

+18
-5
lines changed

1 file changed

+18
-5
lines changed

crates/common/test-harness/src/service.rs

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -246,19 +246,32 @@ impl<R: RuntimeApi> MetaSrvTestContext<R> {
246246
) -> anyhow::Result<RaftServiceClient<tonic::transport::Channel>> {
247247
let addr = self.config.raft_config.raft_api_addr::<R>().await?;
248248

249-
// retry 3 times until server starts listening.
250-
for _ in 0..3 {
249+
let max_retries = 20;
250+
let interval = tokio::time::Duration::from_millis(200);
251+
let mut last_err = None;
252+
253+
for attempt in 1..=max_retries {
251254
let client = RaftServiceClient::connect(format!("http://{}", addr)).await;
252255
match client {
253256
Ok(x) => return Ok(x),
254257
Err(err) => {
255-
info!("can not yet connect to {}, {}, sleep a while", addr, err);
256-
tokio::time::sleep(tokio::time::Duration::from_millis(50)).await;
258+
info!(
259+
"can not yet connect to {} (attempt {}/{}): {}",
260+
addr, attempt, max_retries, err
261+
);
262+
last_err = Some(err);
263+
tokio::time::sleep(interval).await;
257264
}
258265
}
259266
}
260267

261-
panic!("can not connect to raft server: {:?}", self.config);
268+
Err(anyhow::anyhow!(
269+
"can not connect to raft server at {} after {} attempts over {:.1}s: {}",
270+
addr,
271+
max_retries,
272+
max_retries as f64 * interval.as_secs_f64(),
273+
last_err.unwrap()
274+
))
262275
}
263276

264277
pub async fn assert_raft_server_connection(&self) -> anyhow::Result<()> {

0 commit comments

Comments
 (0)