From 43ddb1f55c0c966f49597818e991bc68c70ece84 Mon Sep 17 00:00:00 2001 From: kamiyaa Date: Wed, 15 Oct 2025 14:38:10 -0400 Subject: [PATCH 1/8] feat: report reorg if checkpoint index decreased but block height stayed the same or increased --- rust/main/agents/relayer/src/msg/db_loader.rs | 11 ++ .../agents/relayer/src/msg/pending_message.rs | 9 ++ rust/main/agents/validator/src/submit.rs | 146 +++++++++++++----- .../main/agents/validator/src/submit/tests.rs | 9 ++ rust/main/hyperlane-base/src/db/mod.rs | 10 ++ .../src/db/rocks/hyperlane_db.rs | 20 +++ 6 files changed, 170 insertions(+), 35 deletions(-) diff --git a/rust/main/agents/relayer/src/msg/db_loader.rs b/rust/main/agents/relayer/src/msg/db_loader.rs index 9c9712bfb3..f34feceffc 100644 --- a/rust/main/agents/relayer/src/msg/db_loader.rs +++ b/rust/main/agents/relayer/src/msg/db_loader.rs @@ -736,6 +736,17 @@ pub mod test { fn store_payload_uuids_by_message_id(&self, message_id: &H256, payload_uuids: Vec) -> DbResult<()>; fn retrieve_payload_uuids_by_message_id(&self, message_id: &H256) -> DbResult>>; + + fn store_latest_checkpoint_block_height( + &self, + checkpoint_index: u64, + ) -> DbResult<()>; + fn retrieve_latest_checkpoint_block_height( + &self, + ) -> DbResult>; + + fn store_latest_checkpoint_index(&self, checkpoint_index: u64) -> DbResult<()>; + fn retrieve_latest_checkpoint_index(&self) -> DbResult>; } } diff --git a/rust/main/agents/relayer/src/msg/pending_message.rs b/rust/main/agents/relayer/src/msg/pending_message.rs index 2dcb61d3ef..a73cb4ba1e 100644 --- a/rust/main/agents/relayer/src/msg/pending_message.rs +++ b/rust/main/agents/relayer/src/msg/pending_message.rs @@ -1202,6 +1202,15 @@ mod test { fn retrieve_highest_seen_message_nonce_number(&self) -> DbResult>; fn store_payload_uuids_by_message_id(&self, message_id: &H256, payload_uuids: Vec) -> DbResult<()>; fn retrieve_payload_uuids_by_message_id(&self, message_id: &H256) -> DbResult>>; + fn store_latest_checkpoint_block_height( + &self, + checkpoint_index: u64, + ) -> DbResult<()>; + fn retrieve_latest_checkpoint_block_height( + &self, + ) -> DbResult>; + fn store_latest_checkpoint_index(&self, checkpoint_index: u64) -> DbResult<()>; + fn retrieve_latest_checkpoint_index(&self) -> DbResult>; } } diff --git a/rust/main/agents/validator/src/submit.rs b/rust/main/agents/validator/src/submit.rs index 844b70fc7b..d20876adbe 100644 --- a/rust/main/agents/validator/src/submit.rs +++ b/rust/main/agents/validator/src/submit.rs @@ -16,7 +16,7 @@ use hyperlane_core::{ HyperlaneSignerExt, IncrementalMerkleAtBlock, }; use hyperlane_core::{ - ChainResult, HyperlaneSigner, MerkleTreeHook, ReorgEvent, ReorgPeriod, SignedType, + ChainResult, HyperlaneSigner, MerkleTreeHook, ReorgEvent, ReorgPeriod, SignedType, H256, }; use hyperlane_ethereum::{Signers, SingletonSignerHandle}; @@ -118,6 +118,17 @@ impl ValidatorSubmitter { true }; + let mut latest_seen_checkpoint_index = self + .db + .retrieve_latest_checkpoint_index() + .unwrap_or_default() + .unwrap_or_default() as u32; + let mut latest_seen_checkpoint_block_height = self + .db + .retrieve_latest_checkpoint_block_height() + .unwrap_or_default() + .unwrap_or_default(); + loop { // Lag by reorg period because this is our correctness checkpoint. let latest_checkpoint = call_and_retry_indefinitely(|| { @@ -127,6 +138,49 @@ impl ValidatorSubmitter { }) .await; + if latest_checkpoint.index < latest_seen_checkpoint_index { + if let Some(block_height) = latest_checkpoint.block_height { + if block_height >= latest_seen_checkpoint_block_height { + tracing::error!( + ?latest_checkpoint, + ?latest_seen_checkpoint_index, + ?latest_seen_checkpoint_block_height, + "Latest checkpoint index is lower than previously seen, but has a block height equal or greater."); + + let checkpoint = self.checkpoint(&tree); + Self::panic_with_reorg( + &self.reorg_reporter, + &self.reorg_period, + &self.checkpoint_syncer, + tree.root(), + &latest_checkpoint, + &checkpoint, + ) + .await; + } + } + } + + if latest_checkpoint.index > latest_seen_checkpoint_index { + tracing::debug!( + old_index = latest_seen_checkpoint_index, + new_index = latest_checkpoint.index, + "Updating latest seen checkpoint index" + ); + latest_seen_checkpoint_index = latest_checkpoint.index; + if let Some(block_height) = latest_checkpoint.block_height { + if block_height < latest_seen_checkpoint_block_height { + tracing::warn!( + checkpoint_index = latest_checkpoint.index, + checkpoint_block_height = block_height, + latest_seen_checkpoint_block_height, + "Receive a checkpoint with a higher index, but lower block height" + ); + } + latest_seen_checkpoint_block_height = block_height; + } + } + self.metrics .set_latest_checkpoint_observed(&latest_checkpoint); @@ -238,41 +292,15 @@ impl ValidatorSubmitter { // If the tree's checkpoint doesn't match the correctness checkpoint, something went wrong // and we bail loudly. if checkpoint != correctness_checkpoint.checkpoint { - let reorg_event = ReorgEvent::new( + Self::panic_with_reorg( + &self.reorg_reporter, + &self.reorg_period, + &self.checkpoint_syncer, tree.root(), - correctness_checkpoint.root, - checkpoint.index, - chrono::Utc::now().timestamp() as u64, - self.reorg_period.clone(), - ); - error!( - ?checkpoint, - ?correctness_checkpoint, - ?reorg_event, - "Incorrect tree root. Most likely a reorg has occurred. Please reach out for help, this is a potentially serious error impacting signed messages. Do NOT forcefully resume operation of this validator. Keep it crashlooping or shut down until you receive support." - ); - - if let Some(height) = correctness_checkpoint.block_height { - self.reorg_reporter.report_at_block(height).await; - } else { - info!("Blockchain does not support block height, reporting with reorg period"); - self.reorg_reporter - .report_with_reorg_period(&self.reorg_period) - .await; - } - - let mut panic_message = "Incorrect tree root. Most likely a reorg has occurred. Please reach out for help, this is a potentially serious error impacting signed messages. Do NOT forcefully resume operation of this validator. Keep it crashlooping or shut down until you receive support.".to_owned(); - if let Err(e) = self - .checkpoint_syncer - .write_reorg_status(&reorg_event) - .await - { - panic_message.push_str(&format!( - " Reorg troubleshooting details couldn't be written to checkpoint storage: {}", - e - )); - } - panic!("{panic_message}"); + correctness_checkpoint, + &checkpoint, + ) + .await; } tracing::info!( @@ -296,6 +324,54 @@ impl ValidatorSubmitter { } } + async fn panic_with_reorg( + reorg_reporter: &Arc, + reorg_period: &ReorgPeriod, + checkpoint_syncer: &Arc, + tree_root: H256, + correctness_checkpoint: &CheckpointAtBlock, + incorrect_checkpoint: &Checkpoint, + ) { + let reorg_event = ReorgEvent::new( + tree_root, + correctness_checkpoint.root, + incorrect_checkpoint.index, + chrono::Utc::now().timestamp() as u64, + reorg_period.clone(), + ); + error!( + ?incorrect_checkpoint, + ?correctness_checkpoint, + ?reorg_event, + "Incorrect tree root. Most likely a reorg has occurred. Please reach out for help, this is a potentially serious error impacting signed messages. Do NOT forcefully resume operation of this validator. Keep it crashlooping or shut down until you receive support." + ); + + Self::report_reorg_with_checkpoint(reorg_reporter, reorg_period, correctness_checkpoint) + .await; + + let mut panic_message = "Incorrect tree root. Most likely a reorg has occurred. Please reach out for help, this is a potentially serious error impacting signed messages. Do NOT forcefully resume operation of this validator. Keep it crashlooping or shut down until you receive support.".to_owned(); + if let Err(e) = checkpoint_syncer.write_reorg_status(&reorg_event).await { + panic_message.push_str(&format!( + " Reorg troubleshooting details couldn't be written to checkpoint storage: {}", + e + )); + } + panic!("{panic_message}"); + } + + async fn report_reorg_with_checkpoint( + reorg_reporter: &Arc, + reorg_period: &ReorgPeriod, + correctness_checkpoint: &CheckpointAtBlock, + ) { + if let Some(height) = correctness_checkpoint.block_height { + reorg_reporter.report_at_block(height).await; + } else { + info!("Blockchain does not support block height, reporting with reorg period"); + reorg_reporter.report_with_reorg_period(reorg_period).await; + } + } + async fn sign_checkpoint( &self, checkpoint: CheckpointWithMessageId, diff --git a/rust/main/agents/validator/src/submit/tests.rs b/rust/main/agents/validator/src/submit/tests.rs index 9cea33b727..5ae9bf0eef 100644 --- a/rust/main/agents/validator/src/submit/tests.rs +++ b/rust/main/agents/validator/src/submit/tests.rs @@ -130,6 +130,15 @@ mockall::mock! { fn retrieve_highest_seen_message_nonce_number(&self) -> DbResult>; fn store_payload_uuids_by_message_id(&self, message_id: &H256, payload_uuids: Vec) -> DbResult<()>; fn retrieve_payload_uuids_by_message_id(&self, message_id: &H256) -> DbResult>>; + fn store_latest_checkpoint_block_height( + &self, + checkpoint_index: u64, + ) -> DbResult<()>; + fn retrieve_latest_checkpoint_block_height( + &self, + ) -> DbResult>; + fn store_latest_checkpoint_index(&self, checkpoint_index: u64) -> DbResult<()>; + fn retrieve_latest_checkpoint_index(&self) -> DbResult>; } } diff --git a/rust/main/hyperlane-base/src/db/mod.rs b/rust/main/hyperlane-base/src/db/mod.rs index 972f9f9c20..b7e3eddbd1 100644 --- a/rust/main/hyperlane-base/src/db/mod.rs +++ b/rust/main/hyperlane-base/src/db/mod.rs @@ -173,4 +173,14 @@ pub trait HyperlaneDb: Send + Sync { &self, message_id: &H256, ) -> DbResult>>; + + /// Store latest seen checkpoint block height + fn store_latest_checkpoint_block_height(&self, checkpoint_index: u64) -> DbResult<()>; + /// Retrieve latest seen checkpoint block height + fn retrieve_latest_checkpoint_block_height(&self) -> DbResult>; + + /// Store latest seen checkpoint index + fn store_latest_checkpoint_index(&self, checkpoint_index: u64) -> DbResult<()>; + /// Store latest seen checkpoint index + fn retrieve_latest_checkpoint_index(&self) -> DbResult>; } diff --git a/rust/main/hyperlane-base/src/db/rocks/hyperlane_db.rs b/rust/main/hyperlane-base/src/db/rocks/hyperlane_db.rs index be016e218c..1e0d471bce 100644 --- a/rust/main/hyperlane-base/src/db/rocks/hyperlane_db.rs +++ b/rust/main/hyperlane-base/src/db/rocks/hyperlane_db.rs @@ -40,6 +40,8 @@ const MERKLE_TREE_INSERTION_BLOCK_NUMBER_BY_LEAF_INDEX: &str = "merkle_tree_insertion_block_number_by_leaf_index_"; const LATEST_INDEXED_GAS_PAYMENT_BLOCK: &str = "latest_indexed_gas_payment_block"; const PAYLOAD_UUIDS_BY_MESSAGE_ID: &str = "payload_uuids_by_message_id_"; +const LATEST_CHECKPOINT_BLOCK_HEIGHT: &str = "latest_checkpoint_block_height"; +const LATEST_CHECKPOINT_INDEX: &str = "latest_checkpoint_index"; /// Rocks DB result type pub type DbResult = std::result::Result; @@ -698,6 +700,24 @@ impl HyperlaneDb for HyperlaneRocksDB { ) -> DbResult>> { self.retrieve_value_by_key(PAYLOAD_UUIDS_BY_MESSAGE_ID, message_id) } + + fn store_latest_checkpoint_block_height(&self, checkpoint_block_height: u64) -> DbResult<()> { + self.store_value_by_key( + LATEST_CHECKPOINT_BLOCK_HEIGHT, + &bool::default(), + &checkpoint_block_height, + ) + } + fn retrieve_latest_checkpoint_block_height(&self) -> DbResult> { + self.retrieve_value_by_key(LATEST_CHECKPOINT_BLOCK_HEIGHT, &bool::default()) + } + + fn store_latest_checkpoint_index(&self, checkpoint_index: u64) -> DbResult<()> { + self.store_value_by_key(LATEST_CHECKPOINT_INDEX, &bool::default(), &checkpoint_index) + } + fn retrieve_latest_checkpoint_index(&self) -> DbResult> { + self.retrieve_value_by_key(LATEST_CHECKPOINT_INDEX, &bool::default()) + } } impl HyperlaneRocksDB { From a435843a62e3d660e7378bd7e96311883338dafb Mon Sep 17 00:00:00 2001 From: kamiyaa Date: Wed, 15 Oct 2025 14:45:23 -0400 Subject: [PATCH 2/8] feat: add logic to store latest chheckpoint info --- rust/main/agents/validator/src/submit.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/rust/main/agents/validator/src/submit.rs b/rust/main/agents/validator/src/submit.rs index d20876adbe..062a3b45f7 100644 --- a/rust/main/agents/validator/src/submit.rs +++ b/rust/main/agents/validator/src/submit.rs @@ -168,6 +168,12 @@ impl ValidatorSubmitter { "Updating latest seen checkpoint index" ); latest_seen_checkpoint_index = latest_checkpoint.index; + if let Err(err) = self + .db + .store_latest_checkpoint_index(latest_seen_checkpoint_index as u64) + { + tracing::error!(?err, "Failed to store latest_checkpoint_index"); + }; if let Some(block_height) = latest_checkpoint.block_height { if block_height < latest_seen_checkpoint_block_height { tracing::warn!( @@ -178,6 +184,12 @@ impl ValidatorSubmitter { ); } latest_seen_checkpoint_block_height = block_height; + if let Err(err) = self + .db + .store_latest_checkpoint_block_height(latest_seen_checkpoint_block_height) + { + tracing::error!(?err, "Failed to store latest_checkpoint_block_height"); + } } } From 73d113b441b8cd0a2cace008ff3ba8ba42180d21 Mon Sep 17 00:00:00 2001 From: kamiyaa Date: Wed, 15 Oct 2025 14:48:17 -0400 Subject: [PATCH 3/8] feat: rename var --- rust/main/agents/relayer/src/msg/db_loader.rs | 2 +- rust/main/agents/relayer/src/msg/pending_message.rs | 2 +- rust/main/agents/validator/src/submit/tests.rs | 2 +- rust/main/hyperlane-base/src/db/mod.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rust/main/agents/relayer/src/msg/db_loader.rs b/rust/main/agents/relayer/src/msg/db_loader.rs index f34feceffc..a0f44e6455 100644 --- a/rust/main/agents/relayer/src/msg/db_loader.rs +++ b/rust/main/agents/relayer/src/msg/db_loader.rs @@ -739,7 +739,7 @@ pub mod test { fn store_latest_checkpoint_block_height( &self, - checkpoint_index: u64, + checkpoint_block_height: u64, ) -> DbResult<()>; fn retrieve_latest_checkpoint_block_height( &self, diff --git a/rust/main/agents/relayer/src/msg/pending_message.rs b/rust/main/agents/relayer/src/msg/pending_message.rs index a73cb4ba1e..69da34ffa5 100644 --- a/rust/main/agents/relayer/src/msg/pending_message.rs +++ b/rust/main/agents/relayer/src/msg/pending_message.rs @@ -1204,7 +1204,7 @@ mod test { fn retrieve_payload_uuids_by_message_id(&self, message_id: &H256) -> DbResult>>; fn store_latest_checkpoint_block_height( &self, - checkpoint_index: u64, + checkpoint_block_height: u64, ) -> DbResult<()>; fn retrieve_latest_checkpoint_block_height( &self, diff --git a/rust/main/agents/validator/src/submit/tests.rs b/rust/main/agents/validator/src/submit/tests.rs index 5ae9bf0eef..6c6125bd4c 100644 --- a/rust/main/agents/validator/src/submit/tests.rs +++ b/rust/main/agents/validator/src/submit/tests.rs @@ -132,7 +132,7 @@ mockall::mock! { fn retrieve_payload_uuids_by_message_id(&self, message_id: &H256) -> DbResult>>; fn store_latest_checkpoint_block_height( &self, - checkpoint_index: u64, + checkpoint_block_height: u64, ) -> DbResult<()>; fn retrieve_latest_checkpoint_block_height( &self, diff --git a/rust/main/hyperlane-base/src/db/mod.rs b/rust/main/hyperlane-base/src/db/mod.rs index b7e3eddbd1..a1c7b127e5 100644 --- a/rust/main/hyperlane-base/src/db/mod.rs +++ b/rust/main/hyperlane-base/src/db/mod.rs @@ -175,7 +175,7 @@ pub trait HyperlaneDb: Send + Sync { ) -> DbResult>>; /// Store latest seen checkpoint block height - fn store_latest_checkpoint_block_height(&self, checkpoint_index: u64) -> DbResult<()>; + fn store_latest_checkpoint_block_height(&self, checkpoint_block_height: u64) -> DbResult<()>; /// Retrieve latest seen checkpoint block height fn retrieve_latest_checkpoint_block_height(&self) -> DbResult>; From 70675c4808cc9f92cacc320c0c9f9c8c628f5b8c Mon Sep 17 00:00:00 2001 From: kamiyaa Date: Wed, 15 Oct 2025 15:02:35 -0400 Subject: [PATCH 4/8] feat: update comment --- rust/main/hyperlane-base/src/db/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/main/hyperlane-base/src/db/mod.rs b/rust/main/hyperlane-base/src/db/mod.rs index a1c7b127e5..77f4ac43c0 100644 --- a/rust/main/hyperlane-base/src/db/mod.rs +++ b/rust/main/hyperlane-base/src/db/mod.rs @@ -181,6 +181,6 @@ pub trait HyperlaneDb: Send + Sync { /// Store latest seen checkpoint index fn store_latest_checkpoint_index(&self, checkpoint_index: u64) -> DbResult<()>; - /// Store latest seen checkpoint index + /// Retrieve latest seen checkpoint index fn retrieve_latest_checkpoint_index(&self) -> DbResult>; } From 582293220933ca38c7036ae52f90d56c20c693a9 Mon Sep 17 00:00:00 2001 From: kamiyaa Date: Thu, 16 Oct 2025 11:43:00 -0400 Subject: [PATCH 5/8] feat: combine variable --- rust/main/agents/relayer/src/msg/db_loader.rs | 14 +-- .../agents/relayer/src/msg/pending_message.rs | 12 +-- rust/main/agents/validator/src/submit.rs | 101 +++++++----------- .../main/agents/validator/src/submit/tests.rs | 27 +++-- rust/main/hyperlane-base/src/db/mod.rs | 19 ++-- .../src/db/rocks/hyperlane_db.rs | 24 ++--- .../src/settings/checkpoint_syncer.rs | 6 +- .../hyperlane-core/src/types/checkpoint.rs | 37 ++++++- rust/main/hyperlane-core/src/types/reorg.rs | 7 +- 9 files changed, 115 insertions(+), 132 deletions(-) diff --git a/rust/main/agents/relayer/src/msg/db_loader.rs b/rust/main/agents/relayer/src/msg/db_loader.rs index a0f44e6455..76379df711 100644 --- a/rust/main/agents/relayer/src/msg/db_loader.rs +++ b/rust/main/agents/relayer/src/msg/db_loader.rs @@ -422,7 +422,7 @@ pub mod test { }, }; use hyperlane_core::{ - identifiers::UniqueIdentifier, test_utils::dummy_domain, GasPaymentKey, + identifiers::UniqueIdentifier, test_utils::dummy_domain, CheckpointInfo, GasPaymentKey, InterchainGasPayment, InterchainGasPaymentMeta, MerkleTreeInsertion, PendingOperationStatus, H256, }; @@ -737,16 +737,8 @@ pub mod test { fn retrieve_payload_uuids_by_message_id(&self, message_id: &H256) -> DbResult>>; - fn store_latest_checkpoint_block_height( - &self, - checkpoint_block_height: u64, - ) -> DbResult<()>; - fn retrieve_latest_checkpoint_block_height( - &self, - ) -> DbResult>; - - fn store_latest_checkpoint_index(&self, checkpoint_index: u64) -> DbResult<()>; - fn retrieve_latest_checkpoint_index(&self) -> DbResult>; + fn store_latest_checkpoint_info(&self, checkpoint_info: &CheckpointInfo) -> DbResult<()>; + fn retrieve_latest_checkpoint_info(&self) -> DbResult>; } } diff --git a/rust/main/agents/relayer/src/msg/pending_message.rs b/rust/main/agents/relayer/src/msg/pending_message.rs index 69da34ffa5..0cafa3a3ac 100644 --- a/rust/main/agents/relayer/src/msg/pending_message.rs +++ b/rust/main/agents/relayer/src/msg/pending_message.rs @@ -1202,15 +1202,9 @@ mod test { fn retrieve_highest_seen_message_nonce_number(&self) -> DbResult>; fn store_payload_uuids_by_message_id(&self, message_id: &H256, payload_uuids: Vec) -> DbResult<()>; fn retrieve_payload_uuids_by_message_id(&self, message_id: &H256) -> DbResult>>; - fn store_latest_checkpoint_block_height( - &self, - checkpoint_block_height: u64, - ) -> DbResult<()>; - fn retrieve_latest_checkpoint_block_height( - &self, - ) -> DbResult>; - fn store_latest_checkpoint_index(&self, checkpoint_index: u64) -> DbResult<()>; - fn retrieve_latest_checkpoint_index(&self) -> DbResult>; + + fn store_latest_checkpoint_info(&self, checkpoint_info: &CheckpointInfo) -> DbResult<()>; + fn retrieve_latest_checkpoint_info(&self) -> DbResult>; } } diff --git a/rust/main/agents/validator/src/submit.rs b/rust/main/agents/validator/src/submit.rs index 062a3b45f7..4a6bf3f2ab 100644 --- a/rust/main/agents/validator/src/submit.rs +++ b/rust/main/agents/validator/src/submit.rs @@ -118,14 +118,9 @@ impl ValidatorSubmitter { true }; - let mut latest_seen_checkpoint_index = self + let mut latest_seen_checkpoint = self .db - .retrieve_latest_checkpoint_index() - .unwrap_or_default() - .unwrap_or_default() as u32; - let mut latest_seen_checkpoint_block_height = self - .db - .retrieve_latest_checkpoint_block_height() + .retrieve_latest_checkpoint_info() .unwrap_or_default() .unwrap_or_default(); @@ -138,59 +133,40 @@ impl ValidatorSubmitter { }) .await; - if latest_checkpoint.index < latest_seen_checkpoint_index { - if let Some(block_height) = latest_checkpoint.block_height { - if block_height >= latest_seen_checkpoint_block_height { - tracing::error!( - ?latest_checkpoint, - ?latest_seen_checkpoint_index, - ?latest_seen_checkpoint_block_height, - "Latest checkpoint index is lower than previously seen, but has a block height equal or greater."); - - let checkpoint = self.checkpoint(&tree); - Self::panic_with_reorg( - &self.reorg_reporter, - &self.reorg_period, - &self.checkpoint_syncer, - tree.root(), - &latest_checkpoint, - &checkpoint, - ) - .await; - } + if let Some(block_height) = latest_checkpoint.block_height { + if latest_checkpoint.index < latest_seen_checkpoint.checkpoint_index + && block_height >= latest_seen_checkpoint.block_height + { + tracing::error!( + ?latest_checkpoint, + ?latest_seen_checkpoint, + "Latest checkpoint index is lower than previously seen, but has a block height equal or greater."); + + let checkpoint = self.checkpoint(&tree); + Self::panic_with_reorg( + &self.reorg_reporter, + &self.reorg_period, + &self.checkpoint_syncer, + tree.root(), + &latest_checkpoint, + &checkpoint, + ) + .await; } - } - - if latest_checkpoint.index > latest_seen_checkpoint_index { tracing::debug!( - old_index = latest_seen_checkpoint_index, - new_index = latest_checkpoint.index, + ?latest_checkpoint, + ?latest_seen_checkpoint, "Updating latest seen checkpoint index" ); - latest_seen_checkpoint_index = latest_checkpoint.index; - if let Err(err) = self - .db - .store_latest_checkpoint_index(latest_seen_checkpoint_index as u64) - { - tracing::error!(?err, "Failed to store latest_checkpoint_index"); - }; - if let Some(block_height) = latest_checkpoint.block_height { - if block_height < latest_seen_checkpoint_block_height { - tracing::warn!( - checkpoint_index = latest_checkpoint.index, - checkpoint_block_height = block_height, - latest_seen_checkpoint_block_height, - "Receive a checkpoint with a higher index, but lower block height" - ); - } - latest_seen_checkpoint_block_height = block_height; - if let Err(err) = self - .db - .store_latest_checkpoint_block_height(latest_seen_checkpoint_block_height) - { - tracing::error!(?err, "Failed to store latest_checkpoint_block_height"); - } + if block_height < latest_seen_checkpoint.block_height { + tracing::warn!( + ?latest_checkpoint, + ?latest_seen_checkpoint, + "Receive a checkpoint with a higher index, but lower block height" + ); } + latest_seen_checkpoint.block_height = block_height; + latest_seen_checkpoint.checkpoint_index = latest_checkpoint.index; } self.metrics @@ -344,13 +320,14 @@ impl ValidatorSubmitter { correctness_checkpoint: &CheckpointAtBlock, incorrect_checkpoint: &Checkpoint, ) { - let reorg_event = ReorgEvent::new( - tree_root, - correctness_checkpoint.root, - incorrect_checkpoint.index, - chrono::Utc::now().timestamp() as u64, - reorg_period.clone(), - ); + let reorg_event = ReorgEvent { + local_merkle_root: tree_root, + local_checkpoint_index: incorrect_checkpoint.index, + canonical_merkle_root: correctness_checkpoint.root, + canonical_checkpoint_index: correctness_checkpoint.index, + unix_timestamp: chrono::Utc::now().timestamp() as u64, + reorg_period: reorg_period.clone(), + }; error!( ?incorrect_checkpoint, ?correctness_checkpoint, diff --git a/rust/main/agents/validator/src/submit/tests.rs b/rust/main/agents/validator/src/submit/tests.rs index 6c6125bd4c..3b52567e60 100644 --- a/rust/main/agents/validator/src/submit/tests.rs +++ b/rust/main/agents/validator/src/submit/tests.rs @@ -9,10 +9,11 @@ use hyperlane_base::db::{ DbResult, HyperlaneDb, InterchainGasExpenditureData, InterchainGasPaymentData, }; use hyperlane_core::{ - identifiers::UniqueIdentifier, test_utils::dummy_domain, GasPaymentKey, HyperlaneChain, - HyperlaneContract, HyperlaneDomain, HyperlaneMessage, HyperlaneProvider, InterchainGasPayment, - InterchainGasPaymentMeta, MerkleTreeHook, MerkleTreeInsertion, PendingOperationStatus, - ReorgEvent, SignedAnnouncement, SignedCheckpointWithMessageId, H160, H256, + identifiers::UniqueIdentifier, test_utils::dummy_domain, CheckpointInfo, GasPaymentKey, + HyperlaneChain, HyperlaneContract, HyperlaneDomain, HyperlaneMessage, HyperlaneProvider, + InterchainGasPayment, InterchainGasPaymentMeta, MerkleTreeHook, MerkleTreeInsertion, + PendingOperationStatus, ReorgEvent, SignedAnnouncement, SignedCheckpointWithMessageId, H160, + H256, }; use super::*; @@ -130,15 +131,9 @@ mockall::mock! { fn retrieve_highest_seen_message_nonce_number(&self) -> DbResult>; fn store_payload_uuids_by_message_id(&self, message_id: &H256, payload_uuids: Vec) -> DbResult<()>; fn retrieve_payload_uuids_by_message_id(&self, message_id: &H256) -> DbResult>>; - fn store_latest_checkpoint_block_height( - &self, - checkpoint_block_height: u64, - ) -> DbResult<()>; - fn retrieve_latest_checkpoint_block_height( - &self, - ) -> DbResult>; - fn store_latest_checkpoint_index(&self, checkpoint_index: u64) -> DbResult<()>; - fn retrieve_latest_checkpoint_index(&self) -> DbResult>; + + fn store_latest_checkpoint_info(&self, checkpoint_info: &CheckpointInfo) -> DbResult<()>; + fn retrieve_latest_checkpoint_info(&self) -> DbResult>; } } @@ -227,12 +222,16 @@ fn reorg_event_is_correct( reorg_event.canonical_merkle_root, mock_onchain_merkle_tree.root() ); + assert_eq!( + reorg_event.canonical_checkpoint_index, + mock_onchain_merkle_tree.index() + ); assert_eq!( reorg_event.local_merkle_root, expected_local_merkle_tree.root() ); assert_eq!( - reorg_event.checkpoint_index, + reorg_event.local_checkpoint_index, expected_local_merkle_tree.index() ); // timestamp diff should be less than 1 second diff --git a/rust/main/hyperlane-base/src/db/mod.rs b/rust/main/hyperlane-base/src/db/mod.rs index 77f4ac43c0..2c3a03018a 100644 --- a/rust/main/hyperlane-base/src/db/mod.rs +++ b/rust/main/hyperlane-base/src/db/mod.rs @@ -3,9 +3,9 @@ pub use error::*; pub use rocks::*; use hyperlane_core::{ - identifiers::UniqueIdentifier, GasPaymentKey, HyperlaneDomain, HyperlaneMessage, - InterchainGasPayment, InterchainGasPaymentMeta, MerkleTreeInsertion, PendingOperationStatus, - H256, + identifiers::UniqueIdentifier, CheckpointInfo, GasPaymentKey, HyperlaneDomain, + HyperlaneMessage, InterchainGasPayment, InterchainGasPaymentMeta, MerkleTreeInsertion, + PendingOperationStatus, H256, }; mod error; @@ -174,13 +174,8 @@ pub trait HyperlaneDb: Send + Sync { message_id: &H256, ) -> DbResult>>; - /// Store latest seen checkpoint block height - fn store_latest_checkpoint_block_height(&self, checkpoint_block_height: u64) -> DbResult<()>; - /// Retrieve latest seen checkpoint block height - fn retrieve_latest_checkpoint_block_height(&self) -> DbResult>; - - /// Store latest seen checkpoint index - fn store_latest_checkpoint_index(&self, checkpoint_index: u64) -> DbResult<()>; - /// Retrieve latest seen checkpoint index - fn retrieve_latest_checkpoint_index(&self) -> DbResult>; + /// Store latest seen checkpoint info + fn store_latest_checkpoint_info(&self, checkpoint_info: &CheckpointInfo) -> DbResult<()>; + /// Retrieve latest seen checkpoint info + fn retrieve_latest_checkpoint_info(&self) -> DbResult>; } diff --git a/rust/main/hyperlane-base/src/db/rocks/hyperlane_db.rs b/rust/main/hyperlane-base/src/db/rocks/hyperlane_db.rs index 1e0d471bce..3859d0a65b 100644 --- a/rust/main/hyperlane-base/src/db/rocks/hyperlane_db.rs +++ b/rust/main/hyperlane-base/src/db/rocks/hyperlane_db.rs @@ -5,7 +5,7 @@ use eyre::{bail, Result}; use tracing::{debug, instrument, trace}; use hyperlane_core::{ - identifiers::UniqueIdentifier, Decode, Encode, GasPaymentKey, HyperlaneDomain, + identifiers::UniqueIdentifier, CheckpointInfo, Decode, Encode, GasPaymentKey, HyperlaneDomain, HyperlaneLogStore, HyperlaneMessage, HyperlaneSequenceAwareIndexerStoreReader, HyperlaneWatermarkedLogStore, Indexed, InterchainGasExpenditure, InterchainGasPayment, InterchainGasPaymentMeta, LogMeta, MerkleTreeInsertion, PendingOperationStatus, H256, @@ -40,8 +40,7 @@ const MERKLE_TREE_INSERTION_BLOCK_NUMBER_BY_LEAF_INDEX: &str = "merkle_tree_insertion_block_number_by_leaf_index_"; const LATEST_INDEXED_GAS_PAYMENT_BLOCK: &str = "latest_indexed_gas_payment_block"; const PAYLOAD_UUIDS_BY_MESSAGE_ID: &str = "payload_uuids_by_message_id_"; -const LATEST_CHECKPOINT_BLOCK_HEIGHT: &str = "latest_checkpoint_block_height"; -const LATEST_CHECKPOINT_INDEX: &str = "latest_checkpoint_index"; +const LATEST_CHECKPOINT_INFO: &str = "latest_checkpoint_info"; /// Rocks DB result type pub type DbResult = std::result::Result; @@ -701,22 +700,11 @@ impl HyperlaneDb for HyperlaneRocksDB { self.retrieve_value_by_key(PAYLOAD_UUIDS_BY_MESSAGE_ID, message_id) } - fn store_latest_checkpoint_block_height(&self, checkpoint_block_height: u64) -> DbResult<()> { - self.store_value_by_key( - LATEST_CHECKPOINT_BLOCK_HEIGHT, - &bool::default(), - &checkpoint_block_height, - ) - } - fn retrieve_latest_checkpoint_block_height(&self) -> DbResult> { - self.retrieve_value_by_key(LATEST_CHECKPOINT_BLOCK_HEIGHT, &bool::default()) - } - - fn store_latest_checkpoint_index(&self, checkpoint_index: u64) -> DbResult<()> { - self.store_value_by_key(LATEST_CHECKPOINT_INDEX, &bool::default(), &checkpoint_index) + fn store_latest_checkpoint_info(&self, checkpoint_info: &CheckpointInfo) -> DbResult<()> { + self.store_value_by_key(LATEST_CHECKPOINT_INFO, &bool::default(), checkpoint_info) } - fn retrieve_latest_checkpoint_index(&self) -> DbResult> { - self.retrieve_value_by_key(LATEST_CHECKPOINT_INDEX, &bool::default()) + fn retrieve_latest_checkpoint_info(&self) -> DbResult> { + self.retrieve_value_by_key(LATEST_CHECKPOINT_INFO, &bool::default()) } } diff --git a/rust/main/hyperlane-base/src/settings/checkpoint_syncer.rs b/rust/main/hyperlane-base/src/settings/checkpoint_syncer.rs index 05c1f5719b..927014b987 100644 --- a/rust/main/hyperlane-base/src/settings/checkpoint_syncer.rs +++ b/rust/main/hyperlane-base/src/settings/checkpoint_syncer.rs @@ -207,13 +207,15 @@ mod test { let dummy_canonical_merkle_root = H256::from_str("0xb437b888332ef12f7260c7f679aad3c96b91ab81c2dc7242f8b290f0b6bba92b") .unwrap(); - let dummy_checkpoint_index = 56; + let dummy_local_checkpoint_index = 56; + let dummy_canonical_checkpoint_index = 56; let unix_timestamp = 1620000000; let reorg_period = ReorgPeriod::from_blocks(5); let dummy_reorg_event = ReorgEvent { local_merkle_root: dummy_local_merkle_root, + local_checkpoint_index: dummy_local_checkpoint_index, canonical_merkle_root: dummy_canonical_merkle_root, - checkpoint_index: dummy_checkpoint_index, + canonical_checkpoint_index: dummy_canonical_checkpoint_index, unix_timestamp, reorg_period, }; diff --git a/rust/main/hyperlane-core/src/types/checkpoint.rs b/rust/main/hyperlane-core/src/types/checkpoint.rs index 38810204c9..16fc7dc0f0 100644 --- a/rust/main/hyperlane-core/src/types/checkpoint.rs +++ b/rust/main/hyperlane-core/src/types/checkpoint.rs @@ -4,7 +4,7 @@ use derive_more::Deref; use serde::{Deserialize, Serialize}; use sha3::{digest::Update, Digest, Keccak256}; -use crate::{utils::domain_hash, Signable, Signature, SignedType, H256}; +use crate::{utils::domain_hash, Decode, Encode, Signable, Signature, SignedType, H256}; /// An Hyperlane checkpoint #[derive(Copy, Clone, Eq, PartialEq, Serialize, Deserialize, Debug)] @@ -98,3 +98,38 @@ impl TryFrom<&mut Vec> for MultisigSignedCheckpoi }) } } + +/// Information about the checkpoint +#[derive(Copy, Clone, Eq, PartialEq, Serialize, Deserialize, Debug, Default)] +pub struct CheckpointInfo { + /// block height of checkpoint + pub block_height: u64, + /// index of checkpoint + pub checkpoint_index: u32, +} + +impl Encode for CheckpointInfo { + fn write_to(&self, writer: &mut W) -> std::io::Result + where + W: std::io::Write, + { + let mut written: usize = self.checkpoint_index.write_to(writer)?; + written = written.saturating_add(self.block_height.write_to(writer)?); + Ok(written) + } +} + +impl Decode for CheckpointInfo { + fn read_from(reader: &mut R) -> Result + where + R: std::io::Read, + Self: Sized, + { + let checkpoint_index = u32::read_from(reader)?; + let block_height = u64::read_from(reader)?; + Ok(Self { + checkpoint_index, + block_height, + }) + } +} diff --git a/rust/main/hyperlane-core/src/types/reorg.rs b/rust/main/hyperlane-core/src/types/reorg.rs index 74faf8626a..4a4fd1517d 100644 --- a/rust/main/hyperlane-core/src/types/reorg.rs +++ b/rust/main/hyperlane-core/src/types/reorg.rs @@ -10,9 +10,10 @@ pub struct ReorgEvent { pub local_merkle_root: H256, /// the onchain merkle root pub canonical_merkle_root: H256, - /// the index of the checkpoint when the reorg was detected - /// (due to a mismatch between local and canonical merkle roots) - pub checkpoint_index: u32, + /// the latest local checkpoint index + pub local_checkpoint_index: u32, + /// the latest onchain checkpoint index + pub canonical_checkpoint_index: u32, /// the timestamp when the reorg was detected, in seconds since the Unix epoch pub unix_timestamp: u64, /// the reorg period configured for the agent From c88f7ce9646bd58e630ef5abe0e0535682758fe2 Mon Sep 17 00:00:00 2001 From: kamiyaa Date: Thu, 16 Oct 2025 11:45:56 -0400 Subject: [PATCH 6/8] feat: update last seen after there are no checkpoint issues --- rust/main/agents/validator/src/submit.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/rust/main/agents/validator/src/submit.rs b/rust/main/agents/validator/src/submit.rs index 4a6bf3f2ab..8224cb5e8b 100644 --- a/rust/main/agents/validator/src/submit.rs +++ b/rust/main/agents/validator/src/submit.rs @@ -165,8 +165,6 @@ impl ValidatorSubmitter { "Receive a checkpoint with a higher index, but lower block height" ); } - latest_seen_checkpoint.block_height = block_height; - latest_seen_checkpoint.checkpoint_index = latest_checkpoint.index; } self.metrics @@ -204,6 +202,12 @@ impl ValidatorSubmitter { // Set that initial consistency has been reached on first loop run. Subsequent runs are idempotent. self.metrics.reached_initial_consistency.set(1); + // Update latest seen valid checkpoint + if let Some(block_height) = latest_checkpoint.block_height { + latest_seen_checkpoint.block_height = block_height; + latest_seen_checkpoint.checkpoint_index = latest_checkpoint.index; + } + sleep(self.interval).await; } } From 898d11c4fdade5aa47f323afb4c973aba5383639 Mon Sep 17 00:00:00 2001 From: kamiyaa Date: Thu, 16 Oct 2025 12:19:14 -0400 Subject: [PATCH 7/8] feat: add unittest --- rust/main/agents/validator/src/submit.rs | 91 +++++++++------ .../main/agents/validator/src/submit/tests.rs | 108 +++++++++++++++--- 2 files changed, 149 insertions(+), 50 deletions(-) diff --git a/rust/main/agents/validator/src/submit.rs b/rust/main/agents/validator/src/submit.rs index 8224cb5e8b..d07169ca79 100644 --- a/rust/main/agents/validator/src/submit.rs +++ b/rust/main/agents/validator/src/submit.rs @@ -16,7 +16,8 @@ use hyperlane_core::{ HyperlaneSignerExt, IncrementalMerkleAtBlock, }; use hyperlane_core::{ - ChainResult, HyperlaneSigner, MerkleTreeHook, ReorgEvent, ReorgPeriod, SignedType, H256, + ChainResult, CheckpointInfo, HyperlaneSigner, MerkleTreeHook, ReorgEvent, ReorgPeriod, + SignedType, H256, }; use hyperlane_ethereum::{Signers, SingletonSignerHandle}; @@ -133,40 +134,8 @@ impl ValidatorSubmitter { }) .await; - if let Some(block_height) = latest_checkpoint.block_height { - if latest_checkpoint.index < latest_seen_checkpoint.checkpoint_index - && block_height >= latest_seen_checkpoint.block_height - { - tracing::error!( - ?latest_checkpoint, - ?latest_seen_checkpoint, - "Latest checkpoint index is lower than previously seen, but has a block height equal or greater."); - - let checkpoint = self.checkpoint(&tree); - Self::panic_with_reorg( - &self.reorg_reporter, - &self.reorg_period, - &self.checkpoint_syncer, - tree.root(), - &latest_checkpoint, - &checkpoint, - ) - .await; - } - tracing::debug!( - ?latest_checkpoint, - ?latest_seen_checkpoint, - "Updating latest seen checkpoint index" - ); - if block_height < latest_seen_checkpoint.block_height { - tracing::warn!( - ?latest_checkpoint, - ?latest_seen_checkpoint, - "Receive a checkpoint with a higher index, but lower block height" - ); - } - } - + self.verify_checkpoint(&tree, &latest_checkpoint, &latest_seen_checkpoint) + .await; self.metrics .set_latest_checkpoint_observed(&latest_checkpoint); @@ -204,6 +173,18 @@ impl ValidatorSubmitter { // Update latest seen valid checkpoint if let Some(block_height) = latest_checkpoint.block_height { + tracing::debug!( + ?latest_checkpoint, + ?latest_seen_checkpoint, + "Updating latest seen checkpoint index" + ); + if block_height < latest_seen_checkpoint.block_height { + tracing::warn!( + ?latest_checkpoint, + ?latest_seen_checkpoint, + "Receive a checkpoint with a higher index, but lower block height" + ); + } latest_seen_checkpoint.block_height = block_height; latest_seen_checkpoint.checkpoint_index = latest_checkpoint.index; } @@ -316,6 +297,46 @@ impl ValidatorSubmitter { } } + /// Verify checkpoint is valid + async fn verify_checkpoint( + &self, + tree: &IncrementalMerkle, + latest_checkpoint: &CheckpointAtBlock, + latest_seen_checkpoint: &CheckpointInfo, + ) { + // if checkpoint has an index greater than last seen, then it is valid + if latest_seen_checkpoint.checkpoint_index < latest_checkpoint.index { + return; + } + + let block_height = match latest_checkpoint.block_height { + Some(s) => s, + None => return, + }; + // if checkpoint has a block height greater than last seen, then it is valid + if latest_seen_checkpoint.block_height < block_height { + return; + } + + // otherwise, a reorg occurred when checkpoint has a lower index + // but has the same or higher block height + tracing::error!( + ?latest_checkpoint, + ?latest_seen_checkpoint, + "Latest checkpoint index is lower than previously seen, but has a block height equal or greater."); + + let checkpoint = self.checkpoint(&tree); + Self::panic_with_reorg( + &self.reorg_reporter, + &self.reorg_period, + &self.checkpoint_syncer, + tree.root(), + &latest_checkpoint, + &checkpoint, + ) + .await; + } + async fn panic_with_reorg( reorg_reporter: &Arc, reorg_period: &ReorgPeriod, diff --git a/rust/main/agents/validator/src/submit/tests.rs b/rust/main/agents/validator/src/submit/tests.rs index 3b52567e60..984c397147 100644 --- a/rust/main/agents/validator/src/submit/tests.rs +++ b/rust/main/agents/validator/src/submit/tests.rs @@ -249,20 +249,20 @@ async fn reorg_is_detected_and_persisted_to_checkpoint_storage() { let unix_timestamp = chrono::Utc::now().timestamp() as u64; let expected_reorg_period = 12; - let pre_reorg_merke_insertions = [ + let pre_reorg_merkle_insertions = [ MerkleTreeInsertion::new(0, H256::random()), MerkleTreeInsertion::new(1, H256::random()), MerkleTreeInsertion::new(2, H256::random()), ]; let mut expected_local_merkle_tree = IncrementalMerkle::default(); - for insertion in pre_reorg_merke_insertions.iter() { + for insertion in pre_reorg_merkle_insertions.iter() { expected_local_merkle_tree.ingest(insertion.message_id()); } // the last leaf is different post-reorg let post_reorg_merkle_insertions = [ - pre_reorg_merke_insertions[0], - pre_reorg_merke_insertions[1], + pre_reorg_merkle_insertions[0], + pre_reorg_merkle_insertions[1], MerkleTreeInsertion::new(2, H256::random()), ]; let mut mock_onchain_merkle_tree = IncrementalMerkle::default(); @@ -279,7 +279,7 @@ async fn reorg_is_detected_and_persisted_to_checkpoint_storage() { // the db returns the pre-reorg merkle tree insertions let mut db = MockDb::new(); db.expect_retrieve_merkle_tree_insertion_by_leaf_index() - .returning(move |sequence| Ok(Some(pre_reorg_merke_insertions[*sequence as usize]))); + .returning(move |sequence| Ok(Some(pre_reorg_merkle_insertions[*sequence as usize]))); // boilerplate mocks let mut mock_merkle_tree_hook = MockMerkleTreeHook::new(); @@ -362,20 +362,20 @@ async fn reorg_is_detected_and_persisted_to_checkpoint_storage() { async fn sign_and_submit_checkpoint_same_signature() { let expected_reorg_period = 12; - let pre_reorg_merke_insertions = [ + let pre_reorg_merkle_insertions = [ MerkleTreeInsertion::new(0, H256::random()), MerkleTreeInsertion::new(1, H256::random()), MerkleTreeInsertion::new(2, H256::random()), ]; let mut expected_local_merkle_tree = IncrementalMerkle::default(); - for insertion in pre_reorg_merke_insertions.iter() { + for insertion in pre_reorg_merkle_insertions.iter() { expected_local_merkle_tree.ingest(insertion.message_id()); } // the last leaf is different post-reorg let post_reorg_merkle_insertions = [ - pre_reorg_merke_insertions[0], - pre_reorg_merke_insertions[1], + pre_reorg_merkle_insertions[0], + pre_reorg_merkle_insertions[1], MerkleTreeInsertion::new(2, H256::random()), ]; let mut mock_onchain_merkle_tree = IncrementalMerkle::default(); @@ -392,7 +392,7 @@ async fn sign_and_submit_checkpoint_same_signature() { // the db returns the pre-reorg merkle tree insertions let mut db = MockDb::new(); db.expect_retrieve_merkle_tree_insertion_by_leaf_index() - .returning(move |sequence| Ok(Some(pre_reorg_merke_insertions[*sequence as usize]))); + .returning(move |sequence| Ok(Some(pre_reorg_merkle_insertions[*sequence as usize]))); // boilerplate mocks let mut mock_merkle_tree_hook = MockMerkleTreeHook::new(); @@ -467,20 +467,20 @@ async fn sign_and_submit_checkpoint_same_signature() { async fn sign_and_submit_checkpoint_different_signature() { let expected_reorg_period = 12; - let pre_reorg_merke_insertions = [ + let pre_reorg_merkle_insertions = [ MerkleTreeInsertion::new(0, H256::random()), MerkleTreeInsertion::new(1, H256::random()), MerkleTreeInsertion::new(2, H256::random()), ]; let mut expected_local_merkle_tree = IncrementalMerkle::default(); - for insertion in pre_reorg_merke_insertions.iter() { + for insertion in pre_reorg_merkle_insertions.iter() { expected_local_merkle_tree.ingest(insertion.message_id()); } // the last leaf is different post-reorg let post_reorg_merkle_insertions = [ - pre_reorg_merke_insertions[0], - pre_reorg_merke_insertions[1], + pre_reorg_merkle_insertions[0], + pre_reorg_merkle_insertions[1], MerkleTreeInsertion::new(2, H256::random()), ]; let mut mock_onchain_merkle_tree = IncrementalMerkle::default(); @@ -497,7 +497,7 @@ async fn sign_and_submit_checkpoint_different_signature() { // the db returns the pre-reorg merkle tree insertions let mut db = MockDb::new(); db.expect_retrieve_merkle_tree_insertion_by_leaf_index() - .returning(move |sequence| Ok(Some(pre_reorg_merke_insertions[*sequence as usize]))); + .returning(move |sequence| Ok(Some(pre_reorg_merkle_insertions[*sequence as usize]))); // boilerplate mocks let mut mock_merkle_tree_hook = MockMerkleTreeHook::new(); @@ -580,3 +580,81 @@ async fn sign_and_submit_checkpoint_different_signature() { logs_contain("Checkpoint already submitted, but with different signature, overwriting"); } + +#[should_panic] +#[tokio::test] +#[tracing_test::traced_test] +async fn verify_checkpoint() { + let expected_reorg_period = 12; + let merkle_insertions = [ + MerkleTreeInsertion::new(0, H256::random()), + MerkleTreeInsertion::new(1, H256::random()), + MerkleTreeInsertion::new(2, H256::random()), + ]; + let mut mock_onchain_merkle_tree = IncrementalMerkle::default(); + for insertion in merkle_insertions.iter() { + mock_onchain_merkle_tree.ingest(insertion.message_id()); + } + + let db = MockDb::new(); + // boilerplate mocks + let mut mock_merkle_tree_hook = MockMerkleTreeHook::new(); + mock_merkle_tree_hook + .expect_address() + .returning(|| H256::from_low_u64_be(0)); + let dummy_domain = dummy_domain(0, "dummy_domain"); + mock_merkle_tree_hook + .expect_domain() + .return_const(dummy_domain.clone()); + let mut mock_checkpoint_syncer = MockCheckpointSyncer::new(); + mock_checkpoint_syncer + .expect_write_reorg_status() + .returning(|_| Ok(())); + + let signer: Signers = "1111111111111111111111111111111111111111111111111111111111111111" + .parse::() + .unwrap() + .into(); + + let mut mock_reorg_reporter = MockReorgReporter::new(); + mock_reorg_reporter + .expect_report_at_block() + .return_const(()); + // instantiate the validator submitter + let validator_submitter = ValidatorSubmitter::new( + Duration::from_secs(1), + ReorgPeriod::from_blocks(expected_reorg_period), + Arc::new(mock_merkle_tree_hook), + dummy_singleton_handle(), + signer, + Arc::new(mock_checkpoint_syncer), + Arc::new(db), + dummy_metrics(), + 50, + Arc::new(mock_reorg_reporter), + ); + + let mock_onchain_checkpoint = Checkpoint { + root: mock_onchain_merkle_tree.root(), + index: mock_onchain_merkle_tree.index(), + merkle_tree_hook_address: H256::from_low_u64_be(0), + mailbox_domain: 100, + }; + + let checkpoint_at_block = CheckpointAtBlock { + checkpoint: mock_onchain_checkpoint, + block_height: Some(1000), + }; + + let latest_seen_checkpoint = CheckpointInfo { + block_height: 1000, + checkpoint_index: 4, + }; + validator_submitter + .verify_checkpoint( + &mock_onchain_merkle_tree, + &checkpoint_at_block, + &latest_seen_checkpoint, + ) + .await; +} From fec8f7ee3ef0a8bf8a3aa8eb3ec0f91d9fe125a4 Mon Sep 17 00:00:00 2001 From: kamiyaa Date: Thu, 16 Oct 2025 12:25:40 -0400 Subject: [PATCH 8/8] feat: clippy --- rust/main/agents/validator/src/submit.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/main/agents/validator/src/submit.rs b/rust/main/agents/validator/src/submit.rs index d07169ca79..b6e10f252d 100644 --- a/rust/main/agents/validator/src/submit.rs +++ b/rust/main/agents/validator/src/submit.rs @@ -325,13 +325,13 @@ impl ValidatorSubmitter { ?latest_seen_checkpoint, "Latest checkpoint index is lower than previously seen, but has a block height equal or greater."); - let checkpoint = self.checkpoint(&tree); + let checkpoint = self.checkpoint(tree); Self::panic_with_reorg( &self.reorg_reporter, &self.reorg_period, &self.checkpoint_syncer, tree.root(), - &latest_checkpoint, + latest_checkpoint, &checkpoint, ) .await;