feat: report reorg if checkpoint index decreased but block height stayed the same or increased

rust/main/agents/relayer/src/msg/db_loader.rs

@@ -422,7 +422,7 @@ pub mod test {
         },
     };
     use hyperlane_core::{
-        identifiers::UniqueIdentifier, test_utils::dummy_domain, GasPaymentKey,
+        identifiers::UniqueIdentifier, test_utils::dummy_domain, CheckpointInfo, GasPaymentKey,
         InterchainGasPayment, InterchainGasPaymentMeta, MerkleTreeInsertion,
         PendingOperationStatus, H256,
     };
@@ -736,6 +736,9 @@ pub mod test {
             fn store_payload_uuids_by_message_id(&self, message_id: &H256, payload_uuids: Vec<UniqueIdentifier>) -> DbResult<()>;
 
             fn retrieve_payload_uuids_by_message_id(&self, message_id: &H256) -> DbResult<Option<Vec<UniqueIdentifier>>>;
+
+            fn store_latest_checkpoint_info(&self, checkpoint_info: &CheckpointInfo) -> DbResult<()>;
+            fn retrieve_latest_checkpoint_info(&self) -> DbResult<Option<CheckpointInfo>>;
         }
     }
 

rust/main/agents/relayer/src/msg/pending_message.rs

@@ -1202,6 +1202,9 @@ mod test {
             fn retrieve_highest_seen_message_nonce_number(&self) -> DbResult<Option<u32>>;
             fn store_payload_uuids_by_message_id(&self, message_id: &H256, payload_uuids: Vec<UniqueIdentifier>) -> DbResult<()>;
             fn retrieve_payload_uuids_by_message_id(&self, message_id: &H256) -> DbResult<Option<Vec<UniqueIdentifier>>>;
+
+            fn store_latest_checkpoint_info(&self, checkpoint_info: &CheckpointInfo) -> DbResult<()>;
+            fn retrieve_latest_checkpoint_info(&self) -> DbResult<Option<CheckpointInfo>>;
         }
     }
 

rust/main/agents/validator/src/submit.rs

@@ -16,7 +16,7 @@ use hyperlane_core::{
     HyperlaneSignerExt, IncrementalMerkleAtBlock,
 };
 use hyperlane_core::{
-    ChainResult, HyperlaneSigner, MerkleTreeHook, ReorgEvent, ReorgPeriod, SignedType,
+    ChainResult, HyperlaneSigner, MerkleTreeHook, ReorgEvent, ReorgPeriod, SignedType, H256,
 };
 use hyperlane_ethereum::{Signers, SingletonSignerHandle};
 
@@ -118,6 +118,12 @@ impl ValidatorSubmitter {
             true
         };
 
+        let mut latest_seen_checkpoint = self
+            .db
+            .retrieve_latest_checkpoint_info()
+            .unwrap_or_default()
+            .unwrap_or_default();
+
         loop {
             // Lag by reorg period because this is our correctness checkpoint.
             let latest_checkpoint = call_and_retry_indefinitely(|| {
@@ -127,6 +133,42 @@ impl ValidatorSubmitter {
             })
             .await;
 
+            if let Some(block_height) = latest_checkpoint.block_height {
+                if latest_checkpoint.index < latest_seen_checkpoint.checkpoint_index
+                    && block_height >= latest_seen_checkpoint.block_height
+                {
+                    tracing::error!(
+                        ?latest_checkpoint,
+                        ?latest_seen_checkpoint,
+                        "Latest checkpoint index is lower than previously seen, but has a block height equal or greater.");
+
+                    let checkpoint = self.checkpoint(&tree);
+                    Self::panic_with_reorg(
+                        &self.reorg_reporter,
+                        &self.reorg_period,
+                        &self.checkpoint_syncer,
+                        tree.root(),
+                        &latest_checkpoint,
+                        &checkpoint,
+                    )
+                    .await;
+                }
+                tracing::debug!(
+                    ?latest_checkpoint,
+                    ?latest_seen_checkpoint,
+                    "Updating latest seen checkpoint index"
+                );
+                if block_height < latest_seen_checkpoint.block_height {
+                    tracing::warn!(
+                        ?latest_checkpoint,
+                        ?latest_seen_checkpoint,
+                        "Receive a checkpoint with a higher index, but lower block height"
+                    );
+                }
+                latest_seen_checkpoint.block_height = block_height;
+                latest_seen_checkpoint.checkpoint_index = latest_checkpoint.index;
+            }
+
             self.metrics
                 .set_latest_checkpoint_observed(&latest_checkpoint);
 
@@ -238,41 +280,15 @@ impl ValidatorSubmitter {
         // If the tree's checkpoint doesn't match the correctness checkpoint, something went wrong
         // and we bail loudly.
         if checkpoint != correctness_checkpoint.checkpoint {
-            let reorg_event = ReorgEvent::new(
+            Self::panic_with_reorg(
+                &self.reorg_reporter,
+                &self.reorg_period,
+                &self.checkpoint_syncer,
                 tree.root(),
-                correctness_checkpoint.root,
-                checkpoint.index,
-                chrono::Utc::now().timestamp() as u64,
-                self.reorg_period.clone(),
-            );
-            error!(
-                ?checkpoint,
-                ?correctness_checkpoint,
-                ?reorg_event,
-                "Incorrect tree root. Most likely a reorg has occurred. Please reach out for help, this is a potentially serious error impacting signed messages. Do NOT forcefully resume operation of this validator. Keep it crashlooping or shut down until you receive support."
-            );
-
-            if let Some(height) = correctness_checkpoint.block_height {
-                self.reorg_reporter.report_at_block(height).await;
-            } else {
-                info!("Blockchain does not support block height, reporting with reorg period");
-                self.reorg_reporter
-                    .report_with_reorg_period(&self.reorg_period)
-                    .await;
-            }
-
-            let mut panic_message = "Incorrect tree root. Most likely a reorg has occurred. Please reach out for help, this is a potentially serious error impacting signed messages. Do NOT forcefully resume operation of this validator. Keep it crashlooping or shut down until you receive support.".to_owned();
-            if let Err(e) = self
-                .checkpoint_syncer
-                .write_reorg_status(&reorg_event)
-                .await
-            {
-                panic_message.push_str(&format!(
-                    " Reorg troubleshooting details couldn't be written to checkpoint storage: {}",
-                    e
-                ));
-            }
-            panic!("{panic_message}");
+                correctness_checkpoint,
+                &checkpoint,
+            )
+            .await;
         }
 
         tracing::info!(
@@ -296,6 +312,55 @@ impl ValidatorSubmitter {
         }
     }
 
+    async fn panic_with_reorg(
+        reorg_reporter: &Arc<dyn ReorgReporter>,
+        reorg_period: &ReorgPeriod,
+        checkpoint_syncer: &Arc<dyn CheckpointSyncer>,
+        tree_root: H256,
+        correctness_checkpoint: &CheckpointAtBlock,
+        incorrect_checkpoint: &Checkpoint,
+    ) {
+        let reorg_event = ReorgEvent {
+            local_merkle_root: tree_root,
+            local_checkpoint_index: incorrect_checkpoint.index,
+            canonical_merkle_root: correctness_checkpoint.root,
+            canonical_checkpoint_index: correctness_checkpoint.index,
+            unix_timestamp: chrono::Utc::now().timestamp() as u64,
+            reorg_period: reorg_period.clone(),
+        };
+        error!(
+            ?incorrect_checkpoint,
+            ?correctness_checkpoint,
+            ?reorg_event,
+            "Incorrect tree root. Most likely a reorg has occurred. Please reach out for help, this is a potentially serious error impacting signed messages. Do NOT forcefully resume operation of this validator. Keep it crashlooping or shut down until you receive support."
+        );
+
+        Self::report_reorg_with_checkpoint(reorg_reporter, reorg_period, correctness_checkpoint)
+            .await;
+
+        let mut panic_message = "Incorrect tree root. Most likely a reorg has occurred. Please reach out for help, this is a potentially serious error impacting signed messages. Do NOT forcefully resume operation of this validator. Keep it crashlooping or shut down until you receive support.".to_owned();
+        if let Err(e) = checkpoint_syncer.write_reorg_status(&reorg_event).await {
+            panic_message.push_str(&format!(
+                " Reorg troubleshooting details couldn't be written to checkpoint storage: {}",
+                e
+            ));
+        }
+        panic!("{panic_message}");
+    }
+
+    async fn report_reorg_with_checkpoint(
+        reorg_reporter: &Arc<dyn ReorgReporter>,
+        reorg_period: &ReorgPeriod,
+        correctness_checkpoint: &CheckpointAtBlock,
+    ) {
+        if let Some(height) = correctness_checkpoint.block_height {
+            reorg_reporter.report_at_block(height).await;
+        } else {
+            info!("Blockchain does not support block height, reporting with reorg period");
+            reorg_reporter.report_with_reorg_period(reorg_period).await;
+        }
+    }
+
     async fn sign_checkpoint(
         &self,
         checkpoint: CheckpointWithMessageId,

rust/main/agents/validator/src/submit/tests.rs

@@ -9,10 +9,11 @@ use hyperlane_base::db::{
     DbResult, HyperlaneDb, InterchainGasExpenditureData, InterchainGasPaymentData,
 };
 use hyperlane_core::{
-    identifiers::UniqueIdentifier, test_utils::dummy_domain, GasPaymentKey, HyperlaneChain,
-    HyperlaneContract, HyperlaneDomain, HyperlaneMessage, HyperlaneProvider, InterchainGasPayment,
-    InterchainGasPaymentMeta, MerkleTreeHook, MerkleTreeInsertion, PendingOperationStatus,
-    ReorgEvent, SignedAnnouncement, SignedCheckpointWithMessageId, H160, H256,
+    identifiers::UniqueIdentifier, test_utils::dummy_domain, CheckpointInfo, GasPaymentKey,
+    HyperlaneChain, HyperlaneContract, HyperlaneDomain, HyperlaneMessage, HyperlaneProvider,
+    InterchainGasPayment, InterchainGasPaymentMeta, MerkleTreeHook, MerkleTreeInsertion,
+    PendingOperationStatus, ReorgEvent, SignedAnnouncement, SignedCheckpointWithMessageId, H160,
+    H256,
 };
 
 use super::*;
@@ -130,6 +131,9 @@ mockall::mock! {
         fn retrieve_highest_seen_message_nonce_number(&self) -> DbResult<Option<u32>>;
         fn store_payload_uuids_by_message_id(&self, message_id: &H256, payload_uuids: Vec<UniqueIdentifier>) -> DbResult<()>;
         fn retrieve_payload_uuids_by_message_id(&self, message_id: &H256) -> DbResult<Option<Vec<UniqueIdentifier>>>;
+
+        fn store_latest_checkpoint_info(&self, checkpoint_info: &CheckpointInfo) -> DbResult<()>;
+        fn retrieve_latest_checkpoint_info(&self) -> DbResult<Option<CheckpointInfo>>;
     }
 }
 
@@ -218,12 +222,16 @@ fn reorg_event_is_correct(
         reorg_event.canonical_merkle_root,
         mock_onchain_merkle_tree.root()
     );
+    assert_eq!(
+        reorg_event.canonical_checkpoint_index,
+        mock_onchain_merkle_tree.index()
+    );
     assert_eq!(
         reorg_event.local_merkle_root,
         expected_local_merkle_tree.root()
     );
     assert_eq!(
-        reorg_event.checkpoint_index,
+        reorg_event.local_checkpoint_index,
         expected_local_merkle_tree.index()
     );
     // timestamp diff should be less than 1 second

rust/main/hyperlane-base/src/db/mod.rs

@@ -3,9 +3,9 @@ pub use error::*;
 pub use rocks::*;
 
 use hyperlane_core::{
-    identifiers::UniqueIdentifier, GasPaymentKey, HyperlaneDomain, HyperlaneMessage,
-    InterchainGasPayment, InterchainGasPaymentMeta, MerkleTreeInsertion, PendingOperationStatus,
-    H256,
+    identifiers::UniqueIdentifier, CheckpointInfo, GasPaymentKey, HyperlaneDomain,
+    HyperlaneMessage, InterchainGasPayment, InterchainGasPaymentMeta, MerkleTreeInsertion,
+    PendingOperationStatus, H256,
 };
 
 mod error;
@@ -173,4 +173,9 @@ pub trait HyperlaneDb: Send + Sync {
         &self,
         message_id: &H256,
     ) -> DbResult<Option<Vec<UniqueIdentifier>>>;
+
+    /// Store latest seen checkpoint info
+    fn store_latest_checkpoint_info(&self, checkpoint_info: &CheckpointInfo) -> DbResult<()>;
+    /// Retrieve latest seen checkpoint info
+    fn retrieve_latest_checkpoint_info(&self) -> DbResult<Option<CheckpointInfo>>;
 }

rust/main/hyperlane-base/src/db/rocks/hyperlane_db.rs

@@ -5,7 +5,7 @@ use eyre::{bail, Result};
 use tracing::{debug, instrument, trace};
 
 use hyperlane_core::{
-    identifiers::UniqueIdentifier, Decode, Encode, GasPaymentKey, HyperlaneDomain,
+    identifiers::UniqueIdentifier, CheckpointInfo, Decode, Encode, GasPaymentKey, HyperlaneDomain,
     HyperlaneLogStore, HyperlaneMessage, HyperlaneSequenceAwareIndexerStoreReader,
     HyperlaneWatermarkedLogStore, Indexed, InterchainGasExpenditure, InterchainGasPayment,
     InterchainGasPaymentMeta, LogMeta, MerkleTreeInsertion, PendingOperationStatus, H256,
@@ -40,6 +40,7 @@ const MERKLE_TREE_INSERTION_BLOCK_NUMBER_BY_LEAF_INDEX: &str =
     "merkle_tree_insertion_block_number_by_leaf_index_";
 const LATEST_INDEXED_GAS_PAYMENT_BLOCK: &str = "latest_indexed_gas_payment_block";
 const PAYLOAD_UUIDS_BY_MESSAGE_ID: &str = "payload_uuids_by_message_id_";
+const LATEST_CHECKPOINT_INFO: &str = "latest_checkpoint_info";
 
 /// Rocks DB result type
 pub type DbResult<T> = std::result::Result<T, DbError>;
@@ -698,6 +699,13 @@ impl HyperlaneDb for HyperlaneRocksDB {
     ) -> DbResult<Option<Vec<UniqueIdentifier>>> {
         self.retrieve_value_by_key(PAYLOAD_UUIDS_BY_MESSAGE_ID, message_id)
     }
+
+    fn store_latest_checkpoint_info(&self, checkpoint_info: &CheckpointInfo) -> DbResult<()> {
+        self.store_value_by_key(LATEST_CHECKPOINT_INFO, &bool::default(), checkpoint_info)
+    }
+    fn retrieve_latest_checkpoint_info(&self) -> DbResult<Option<CheckpointInfo>> {
+        self.retrieve_value_by_key(LATEST_CHECKPOINT_INFO, &bool::default())
+    }
 }
 
 impl HyperlaneRocksDB {

rust/main/hyperlane-base/src/settings/checkpoint_syncer.rs

@@ -207,13 +207,15 @@ mod test {
         let dummy_canonical_merkle_root =
             H256::from_str("0xb437b888332ef12f7260c7f679aad3c96b91ab81c2dc7242f8b290f0b6bba92b")
                 .unwrap();
-        let dummy_checkpoint_index = 56;
+        let dummy_local_checkpoint_index = 56;
+        let dummy_canonical_checkpoint_index = 56;
         let unix_timestamp = 1620000000;
         let reorg_period = ReorgPeriod::from_blocks(5);
         let dummy_reorg_event = ReorgEvent {
             local_merkle_root: dummy_local_merkle_root,
+            local_checkpoint_index: dummy_local_checkpoint_index,
             canonical_merkle_root: dummy_canonical_merkle_root,
-            checkpoint_index: dummy_checkpoint_index,
+            canonical_checkpoint_index: dummy_canonical_checkpoint_index,
             unix_timestamp,
             reorg_period,
         };

rust/main/hyperlane-core/src/types/checkpoint.rs

@@ -4,7 +4,7 @@ use derive_more::Deref;
 use serde::{Deserialize, Serialize};
 use sha3::{digest::Update, Digest, Keccak256};
 
-use crate::{utils::domain_hash, Signable, Signature, SignedType, H256};
+use crate::{utils::domain_hash, Decode, Encode, Signable, Signature, SignedType, H256};
 
 /// An Hyperlane checkpoint
 #[derive(Copy, Clone, Eq, PartialEq, Serialize, Deserialize, Debug)]
@@ -98,3 +98,38 @@ impl TryFrom<&mut Vec<SignedCheckpointWithMessageId>> for MultisigSignedCheckpoi
         })
     }
 }
+
+/// Information about the checkpoint
+#[derive(Copy, Clone, Eq, PartialEq, Serialize, Deserialize, Debug, Default)]
+pub struct CheckpointInfo {
+    /// block height of checkpoint
+    pub block_height: u64,
+    /// index of checkpoint
+    pub checkpoint_index: u32,
+}
+
+impl Encode for CheckpointInfo {
+    fn write_to<W>(&self, writer: &mut W) -> std::io::Result<usize>
+    where
+        W: std::io::Write,
+    {
+        let mut written: usize = self.checkpoint_index.write_to(writer)?;
+        written = written.saturating_add(self.block_height.write_to(writer)?);
+        Ok(written)
+    }
+}
+
+impl Decode for CheckpointInfo {
+    fn read_from<R>(reader: &mut R) -> Result<Self, crate::HyperlaneProtocolError>
+    where
+        R: std::io::Read,
+        Self: Sized,
+    {
+        let checkpoint_index = u32::read_from(reader)?;
+        let block_height = u64::read_from(reader)?;
+        Ok(Self {
+            checkpoint_index,
+            block_height,
+        })
+    }
+}

rust/main/hyperlane-core/src/types/reorg.rs

@@ -10,9 +10,10 @@ pub struct ReorgEvent {
     pub local_merkle_root: H256,
     /// the onchain merkle root
     pub canonical_merkle_root: H256,
-    /// the index of the checkpoint when the reorg was detected
-    /// (due to a mismatch between local and canonical merkle roots)
-    pub checkpoint_index: u32,
+    /// the latest local checkpoint index
+    pub local_checkpoint_index: u32,
+    /// the latest onchain checkpoint index
+    pub canonical_checkpoint_index: u32,
     /// the timestamp when the reorg was detected, in seconds since the Unix epoch
     pub unix_timestamp: u64,
     /// the reorg period configured for the agent