feat: retry timed out broker requests

Author: Threated

broker/src/crypto.rs

@@ -40,11 +40,11 @@ impl GetCertsFromPki {
         health: Arc<RwLock<Health>>,
         config: &Config,
     ) -> Result<Self, SamplyBeamError> {
-        let hyper_client = http_client::build(
+        let hyper_client = http_client::builder(
             &config.tls_ca_certificates,
             Some(Duration::from_secs(30)),
             Some(Duration::from_secs(20)),
-        )?;
+        ).build()?;
 
         Ok(Self {
             pki_realm: config.pki_realm.clone(),

proxy/src/main.rs

@@ -36,11 +36,26 @@ pub async fn main() -> anyhow::Result<()> {
     banner::print_banner();
 
     let config = Config::load()?;
-    let client = http_client::build(
+    let retry_policy = reqwest::retry::for_host(config.broker_uri.host_str().unwrap().to_string())
+        .classify_fn(|res|  {
+            if res.method() != reqwest::Method::GET {
+                return res.success();
+            }
+            if let Some(StatusCode::BAD_GATEWAY | StatusCode::SERVICE_UNAVAILABLE | StatusCode::GATEWAY_TIMEOUT) = res.status() {
+                res.retryable()
+            } else if res.error().and_then(|e| e.downcast_ref::<reqwest::Error>()).is_some_and(reqwest::Error::is_timeout) {
+                res.retryable()
+            } else {
+                res.success()
+            }
+        })
+        .max_retries_per_request(3);
+
+    let client = http_client::builder(
         &config.tls_ca_certificates,
         Some(Duration::from_secs(PROXY_TIMEOUT)),
         Some(Duration::from_secs(20)),
-    )?;
+    ).retry(retry_policy).build()?;
 
     if let Err(err) = retry_notify(|| get_broker_health(&config, &client), |err, dur| {
         warn!("Still trying to reach Broker: {err}. Retrying in {}s", dur.as_secs());

shared/src/http_client.rs

@@ -11,11 +11,11 @@ use crate::{errors::SamplyBeamError};
 
 pub type SamplyHttpClient = reqwest::Client;
 
-pub fn build(
+pub fn builder(
     ca_certificates: &Vec<Certificate>,
     timeout: Option<Duration>,
     keepalive: Option<Duration>,
-) -> Result<SamplyHttpClient, SamplyBeamError> {
+) -> ClientBuilder {
     let mut builder = Client::builder().tcp_keepalive(keepalive);
     if let Some(to) = timeout {
         builder = builder.connect_timeout(to);
@@ -43,7 +43,7 @@ pub fn build(
     };
     info!("Using {proxies} and {certs} for TLS termination.");
 
-    builder.build().map_err(|e| SamplyBeamError::ConfigurationFailed(e.to_string()))
+    builder
 }
 
 #[cfg(test)]
@@ -60,13 +60,13 @@ mod test {
 
     #[tokio::test]
     async fn https() {
-        let client = http_client::build(&vec![], None, None).unwrap();
+        let client = http_client::builder(&vec![], None, None).build().unwrap();
         run(HTTPS.parse().unwrap(), client).await;
     }
 
     #[tokio::test]
     async fn http() {
-        let client = http_client::build(&vec![], None, None).unwrap();
+        let client = http_client::builder(&vec![], None, None).build().unwrap();
         run(HTTP.parse().unwrap(), client).await;
     }