silence cometbft rpc unavailable block err logs

Author: sug0

chain/src/main.rs

@@ -16,7 +16,7 @@ use clap::Parser;
 use namada_sdk::masp_primitives::transaction::Transaction as MaspTransaction;
 use shared::block::Block;
 use shared::client::Client;
-use shared::error::{IntoMainError, MainError};
+use shared::error::{CometWrapperError, IntoMainError, MainError};
 use shared::height::{BlockHeight, FollowingHeights, UnprocessedBlocks};
 use shared::indexed_tx::MaspIndexedTx;
 use shared::transaction::Transaction;
@@ -194,7 +194,8 @@ fn fetch_blocks_and_get_handle(
                                 &client,
                                 block_height,
                             )
-                            .await?;
+                            .await
+                            .map_err(CometWrapperError)?;
 
                         with_time_taken(&mut checkpoint, |time_taken| {
                             tracing::info!(
@@ -204,7 +205,7 @@ fn fetch_blocks_and_get_handle(
                             );
                         });
 
-                        anyhow::Ok(block_data)
+                        Ok::<_, CometWrapperError>(block_data)
                     })
                     .await
                 else {

shared/src/error.rs

@@ -1,5 +1,72 @@
 use std::fmt;
 
+use tendermint_rpc::error::{
+    Error as TendermintRpcError, ErrorDetail as TendermintRpcErrorDetail,
+};
+use tendermint_rpc::response_error::Code as TendermintRpcCode;
+
+/// Override the log level of errors.
+///
+/// Changes logs from [`tracing::error`] to [`tracing::debug`].
+pub trait SilenceLog {
+    fn silence_log(&self) -> bool;
+}
+
+impl SilenceLog for CometWrapperError {
+    fn silence_log(&self) -> bool {
+        // Check if the error originates from a block that hasn't been committed
+        // yet
+        if let Some(err) = self
+            .0
+            .chain()
+            .find_map(|err| err.downcast_ref::<TendermintRpcError>())
+        {
+            let TendermintRpcErrorDetail::Response(resp) = err.detail() else {
+                return false;
+            };
+
+            if resp.source.code() != TendermintRpcCode::InternalError {
+                return false;
+            }
+
+            return resp.source.data().is_some_and(|msg| {
+                msg.contains("could not find results for height")
+            });
+        }
+
+        // By default, never silence error logs
+        false
+    }
+}
+
+impl SilenceLog for MainError {
+    fn silence_log(&self) -> bool {
+        false
+    }
+}
+
+impl SilenceLog for anyhow::Error {
+    fn silence_log(&self) -> bool {
+        false
+    }
+}
+
+pub struct CometWrapperError(pub anyhow::Error);
+
+impl fmt::Debug for CometWrapperError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        <anyhow::Error as fmt::Debug>::fmt(&self.0, f)
+    }
+}
+
+impl fmt::Display for CometWrapperError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        <anyhow::Error as fmt::Display>::fmt(&self.0, f)
+    }
+}
+
+impl std::error::Error for CometWrapperError {}
+
 #[derive(PartialEq, Eq)]
 pub struct MainError;
 

shared/src/retry.rs

@@ -7,6 +7,7 @@ use std::task::Poll;
 
 use tokio::time::Duration;
 
+use crate::error::SilenceLog;
 use crate::exit_handle;
 
 /// Retry a future generated by `future_generator`, if it fails.
@@ -16,7 +17,7 @@ pub async fn every<F, T, E>(
 ) -> ControlFlow<(), T>
 where
     F: AsyncFnMut() -> Result<T, E>,
-    E: Display + Debug,
+    E: Display + Debug + SilenceLog,
 {
     loop {
         let fut = future_generator();
@@ -42,19 +43,31 @@ where
     }
 }
 
-async fn retry_sleep_with_jitter<E: Display + Debug>(duration: Duration, e: E) {
+async fn retry_sleep_with_jitter<E>(duration: Duration, e: E)
+where
+    E: Display + Debug + SilenceLog,
+{
     if exit_handle::must_exit() {
         return;
     }
 
     let jitter = duration.mul_f64(rand::random_range(0.75f64..=1.25));
 
-    tracing::error!(
-        summary = %e,
-        full = ?e,
-        after = ?jitter,
-        "Retrying execution"
-    );
+    if !e.silence_log() {
+        tracing::error!(
+            summary = %e,
+            full = ?e,
+            after = ?jitter,
+            "Retrying execution"
+        );
+    } else {
+        tracing::debug!(
+            summary = %e,
+            full = ?e,
+            after = ?jitter,
+            "Retrying execution"
+        );
+    }
 
     let fut = tokio::time::sleep(jitter);
     futures::pin_mut!(fut);