From 0e764476fd85d4c4f38cd8fd9c9da180c4d8a937 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Turmel?= Date: Mon, 7 Jul 2025 18:02:41 +0200 Subject: [PATCH 01/18] feature(client-lib, client-cli): introduce a function to download digests and verified them against certificate --- .../client-cardano-database-v2/src/main.rs | 2 +- .../src/commands/cardano_db/shared_steps.rs | 20 ++- .../src/commands/cardano_db/verify.rs | 9 + .../src/cardano_database_client/api.rs | 14 ++ .../src/cardano_database_client/mod.rs | 1 + .../src/cardano_database_client/proving.rs | 159 +++++++++++++++++- mithril-client/src/message.rs | 6 +- ...no_db_snapshot_list_get_download_verify.rs | 2 +- 8 files changed, 201 insertions(+), 12 deletions(-) diff --git a/examples/client-cardano-database-v2/src/main.rs b/examples/client-cardano-database-v2/src/main.rs index 33dcd17de62..e34768b4f2b 100644 --- a/examples/client-cardano-database-v2/src/main.rs +++ b/examples/client-cardano-database-v2/src/main.rs @@ -135,7 +135,7 @@ async fn main() -> MithrilResult<()> { ); let message = wait_spinner( &progress_bar, - MessageBuilder::new().compute_cardano_database_message(&certificate, &merkle_proof), + MessageBuilder::new().compute_cardano_database_message(&certificate, merkle_proof.root()), ) .await?; diff --git a/mithril-client-cli/src/commands/cardano_db/shared_steps.rs b/mithril-client-cli/src/commands/cardano_db/shared_steps.rs index 5efdd3e675d..98531e5041f 100644 --- a/mithril-client-cli/src/commands/cardano_db/shared_steps.rs +++ b/mithril-client-cli/src/commands/cardano_db/shared_steps.rs @@ -5,7 +5,7 @@ use std::path::Path; use mithril_client::{ CardanoDatabaseSnapshot, Client, MessageBuilder, MithrilCertificate, MithrilResult, - cardano_database_client::ImmutableFileRange, + cardano_database_client::{ImmutableFileRange, VerifiedDigests}, common::{ImmutableFileNumber, MKProof, ProtocolMessage}, }; @@ -47,6 +47,22 @@ pub fn immutable_file_range( } } +pub async fn download_and_verify_digests( + step_number: u16, + progress_printer: &ProgressPrinter, + client: &Client, + certificate: &MithrilCertificate, + cardano_database_snapshot: &CardanoDatabaseSnapshot, +) -> MithrilResult { + progress_printer.report_step(step_number, "Downloading and verifying digests…")?; + let verified_digests = client + .cardano_database_v2() + .download_and_verify_digests(certificate, cardano_database_snapshot) + .await?; + + Ok(verified_digests) +} + /// Computes and verifies the Merkle proof for the given certificate and database snapshot. pub async fn compute_verify_merkle_proof( step_number: u16, @@ -85,7 +101,7 @@ pub async fn compute_cardano_db_snapshot_message( progress_printer.report_step(step_number, "Computing the cardano db snapshot message")?; let message = CardanoDbUtils::wait_spinner( progress_printer, - MessageBuilder::new().compute_cardano_database_message(certificate, merkle_proof), + MessageBuilder::new().compute_cardano_database_message(certificate, merkle_proof.root()), ) .await .with_context(|| "Can not compute the cardano db snapshot message")?; diff --git a/mithril-client-cli/src/commands/cardano_db/verify.rs b/mithril-client-cli/src/commands/cardano_db/verify.rs index 6545f693423..a3351311311 100644 --- a/mithril-client-cli/src/commands/cardano_db/verify.rs +++ b/mithril-client-cli/src/commands/cardano_db/verify.rs @@ -104,6 +104,15 @@ impl CardanoDbVerifyCommand { let immutable_file_range = shared_steps::immutable_file_range(None, None); + let verified_digests = shared_steps::download_and_verify_digests( + 2, + &progress_printer, + &client, + &certificate, + &cardano_db_message, + ) + .await?; + let merkle_proof = shared_steps::compute_verify_merkle_proof( 2, &progress_printer, diff --git a/mithril-client/src/cardano_database_client/api.rs b/mithril-client/src/cardano_database_client/api.rs index 3f643e3fa49..0583f71638e 100644 --- a/mithril-client/src/cardano_database_client/api.rs +++ b/mithril-client/src/cardano_database_client/api.rs @@ -16,6 +16,8 @@ use mithril_cardano_node_internal_database::entities::ImmutableFile; use crate::aggregator_client::AggregatorClient; #[cfg(feature = "fs")] +use crate::cardano_database_client::VerifiedDigests; +#[cfg(feature = "fs")] use crate::feedback::FeedbackSender; #[cfg(feature = "fs")] use crate::file_downloader::FileDownloader; @@ -100,6 +102,18 @@ impl CardanoDatabaseClient { .await } + /// Download and verify the digests against the certificate. + #[cfg(feature = "fs")] + pub async fn download_and_verify_digests( + &self, + certificate: &CertificateMessage, + cardano_database_snapshot: &CardanoDatabaseSnapshotMessage, + ) -> MithrilResult { + self.artifact_prover + .download_and_verify_digests(certificate, cardano_database_snapshot) + .await + } + /// Compute the Merkle proof of membership for the given immutable file range. #[cfg(feature = "fs")] pub async fn compute_merkle_proof( diff --git a/mithril-client/src/cardano_database_client/mod.rs b/mithril-client/src/cardano_database_client/mod.rs index 7a243cf3c20..34e46394acf 100644 --- a/mithril-client/src/cardano_database_client/mod.rs +++ b/mithril-client/src/cardano_database_client/mod.rs @@ -136,4 +136,5 @@ cfg_fs! { pub use download_unpack::DownloadUnpackOptions; pub use immutable_file_range::ImmutableFileRange; + pub use proving::VerifiedDigests; } diff --git a/mithril-client/src/cardano_database_client/proving.rs b/mithril-client/src/cardano_database_client/proving.rs index ff8be6882d4..792f5ca709f 100644 --- a/mithril-client/src/cardano_database_client/proving.rs +++ b/mithril-client/src/cardano_database_client/proving.rs @@ -21,7 +21,7 @@ use mithril_common::{ }; use crate::{ - MithrilResult, + MessageBuilder, MithrilResult, feedback::MithrilEvent, file_downloader::{DownloadEvent, FileDownloader, FileDownloaderUri}, utils::{create_directory_if_not_exists, delete_directory, read_files_in_directory}, @@ -29,6 +29,14 @@ use crate::{ use super::immutable_file_range::ImmutableFileRange; +/// Represents the verified digests and the Merkle tree built from them. +pub struct VerifiedDigests { + /// A map of immutable file names to their corresponding verified digests. + pub digests: BTreeMap, + /// The Merkle tree built from the digests. + pub merkle_tree: MKTree, +} + pub struct InternalArtifactProver { http_file_downloader: Arc, logger: slog::Logger, @@ -84,6 +92,52 @@ impl InternalArtifactProver { merkle_tree.compute_proof(&computed_digests) } + ///Download digests and verify its authenticity against the certificate. + pub async fn download_and_verify_digests( + &self, + certificate: &CertificateMessage, + cardano_database_snapshot: &CardanoDatabaseSnapshotMessage, + ) -> MithrilResult { + let digest_target_dir = Self::digest_target_dir(); + delete_directory(&digest_target_dir)?; + self.download_unpack_digest_file(&cardano_database_snapshot.digests, &digest_target_dir) + .await?; + let last_immutable_file_number = cardano_database_snapshot.beacon.immutable_file_number; + + let downloaded_digests = self.read_digest_file(&digest_target_dir)?; + delete_directory(&digest_target_dir)?; + + let filtered_digests = downloaded_digests + .clone() + .into_iter() + .filter(|(immutable_file_name, _)| { + match ImmutableFile::new(Path::new(immutable_file_name).to_path_buf()) { + Ok(immutable_file) => immutable_file.number <= last_immutable_file_number, + Err(_) => false, + } + }) + .collect::>(); + + let filtered_digests_values = filtered_digests.values().collect::>(); + let merkle_tree: MKTree = MKTree::new(&filtered_digests_values)?; + + let message = MessageBuilder::new() + .compute_cardano_database_message(certificate, &merkle_tree.compute_root()?) + .await?; + + if !certificate.match_message(&message) { + return Err(anyhow!( + "Certificate message does not match the computed message for certificate {}", + certificate.hash + )); + } + + Ok(VerifiedDigests { + digests: filtered_digests, + merkle_tree, + }) + } + async fn download_unpack_digest_file( &self, digests_locations: &DigestsMessagePart, @@ -198,7 +252,9 @@ mod tests { IMMUTABLE_DIR, digesters::ComputedImmutablesDigests, }; use mithril_common::{ - StdResult, entities::ImmutableFileNumber, messages::DigestsMessagePart, + StdResult, + entities::{ImmutableFileNumber, ProtocolMessage, ProtocolMessagePartKey}, + messages::DigestsMessagePart, }; use super::*; @@ -208,6 +264,7 @@ mod tests { beacon: &CardanoDbBeacon, immutable_file_range: &RangeInclusive, digests_offset: usize, + digests_location: &str, ) -> ( PathBuf, CardanoDatabaseSnapshotMessage, @@ -221,7 +278,7 @@ mod tests { digests: DigestsMessagePart { size_uncompressed: 1024, locations: vec![DigestLocation::CloudStorage { - uri: "http://whatever/digests.json".to_string(), + uri: digests_location.to_string(), compression_algorithm: None, }], }, @@ -298,6 +355,96 @@ mod tests { Ok(()) } + fn build_digests_map(size: usize) -> BTreeMap { + let mut digests = BTreeMap::new(); + for i in 1..=size { + for name in ["chunk", "primary", "secondary"] { + let immutable_file_name = format!("{i:05}.{name}"); + let immutable_file = + ImmutableFile::new(PathBuf::from(immutable_file_name)).unwrap(); + let digest = format!("digest-{i}-{name}"); + digests.insert(immutable_file, digest); + } + } + + digests + } + + #[tokio::test] + async fn download_and_verify_digest_should_return_digest_map_acording_to_beacon() { + let beacon = CardanoDbBeacon { + epoch: Epoch(123), + immutable_file_number: 42, + }; + let hightest_immutable_number_in_digest_file = + 123 + beacon.immutable_file_number as usize; + let digests_in_certificate_map = + build_digests_map(beacon.immutable_file_number as usize); + let protocol_message_merkle_root = { + let digests_in_certificate_values = + digests_in_certificate_map.values().cloned().collect::>(); + let certificate_merkle_tree: MKTree = + MKTree::new(&digests_in_certificate_values).unwrap(); + + certificate_merkle_tree.compute_root().unwrap().to_hex() + }; + let mut protocol_message = ProtocolMessage::new(); + protocol_message.set_message_part( + ProtocolMessagePartKey::CardanoDatabaseMerkleRoot, + protocol_message_merkle_root, + ); + let certificate = CertificateMessage { + protocol_message: protocol_message.clone(), + signed_message: protocol_message.compute_hash(), + ..CertificateMessage::dummy() + }; + + let digests_location = "http://whatever/digests.json"; + let cardano_database_snapshot = CardanoDatabaseSnapshotMessage { + beacon, + digests: DigestsMessagePart { + size_uncompressed: 1024, + locations: vec![DigestLocation::CloudStorage { + uri: digests_location.to_string(), + compression_algorithm: None, + }], + }, + ..CardanoDatabaseSnapshotMessage::dummy() + }; + let client = CardanoDatabaseClientDependencyInjector::new() + .with_http_file_downloader(Arc::new( + MockFileDownloaderBuilder::default() + .with_file_uri(digests_location) + .with_target_dir(InternalArtifactProver::digest_target_dir()) + .with_compression(None) + .with_returning(Box::new(move |_, _, _, _, _| { + write_digest_file( + &InternalArtifactProver::digest_target_dir(), + &build_digests_map(hightest_immutable_number_in_digest_file), + )?; + + Ok(()) + })) + .build(), + )) + .build_cardano_database_client(); + + let verified_digests = client + .download_and_verify_digests(&certificate, &cardano_database_snapshot) + .await + .unwrap(); + + let expected_digests_in_certificate = digests_in_certificate_map + .iter() + .map(|(immutable_file, digest)| { + (immutable_file.filename.clone(), digest.to_string()) + }) + .collect(); + assert_eq!(verified_digests.digests, expected_digests_in_certificate); + + assert!(!InternalArtifactProver::digest_target_dir().exists()); + } + #[tokio::test] async fn compute_merkle_proof_succeeds() { let beacon = CardanoDbBeacon { @@ -307,19 +454,21 @@ mod tests { let immutable_file_range = 1..=15; let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); let digests_offset = 3; + let digests_location = "http://whatever/digests.json"; let (database_dir, cardano_database_snapshot, certificate, merkle_tree, digests) = prepare_fake_digests( "compute_merkle_proof_succeeds", &beacon, &immutable_file_range, digests_offset, + digests_location, ) .await; let expected_merkle_root = merkle_tree.compute_root().unwrap(); let client = CardanoDatabaseClientDependencyInjector::new() .with_http_file_downloader(Arc::new( MockFileDownloaderBuilder::default() - .with_file_uri("http://whatever/digests.json") + .with_file_uri(digests_location) .with_target_dir(InternalArtifactProver::digest_target_dir()) .with_compression(None) .with_returning(Box::new(move |_, _, _, _, _| { @@ -347,7 +496,7 @@ mod tests { let merkle_proof_root = merkle_proof.root().to_owned(); assert_eq!(expected_merkle_root, merkle_proof_root); - assert!(!database_dir.join("digest").exists()); + assert!(!InternalArtifactProver::digest_target_dir().exists()); } } diff --git a/mithril-client/src/message.rs b/mithril-client/src/message.rs index 85e085ca4e6..4c9699eca62 100644 --- a/mithril-client/src/message.rs +++ b/mithril-client/src/message.rs @@ -13,7 +13,7 @@ use mithril_common::logging::LoggerExtensions; use mithril_common::protocol::SignerBuilder; use mithril_common::signable_builder::CardanoStakeDistributionSignableBuilder; #[cfg(feature = "fs")] -use mithril_common::{crypto_helper::MKProof, entities::SignedEntityType}; +use mithril_common::{crypto_helper::MKTreeNode, entities::SignedEntityType}; use crate::{ CardanoStakeDistribution, MithrilCertificate, MithrilResult, MithrilSigner, @@ -105,12 +105,12 @@ impl MessageBuilder { pub async fn compute_cardano_database_message( &self, certificate: &MithrilCertificate, - merkle_proof: &MKProof, + merkle_root: &MKTreeNode, ) -> MithrilResult { let mut message = certificate.protocol_message.clone(); message.set_message_part( ProtocolMessagePartKey::CardanoDatabaseMerkleRoot, - merkle_proof.root().to_hex(), + merkle_root.to_hex(), ); Ok(message) diff --git a/mithril-client/tests/cardano_db_snapshot_list_get_download_verify.rs b/mithril-client/tests/cardano_db_snapshot_list_get_download_verify.rs index 6058fd96af4..d092d8d3138 100644 --- a/mithril-client/tests/cardano_db_snapshot_list_get_download_verify.rs +++ b/mithril-client/tests/cardano_db_snapshot_list_get_download_verify.rs @@ -145,7 +145,7 @@ async fn cardano_db_snapshot_list_get_download_verify() { merkle_proof.verify().expect("Merkle proof should be valid"); let message = MessageBuilder::new() - .compute_cardano_database_message(&certificate, &merkle_proof) + .compute_cardano_database_message(&certificate, merkle_proof.root()) .await .expect("Computing cardano database snapshot message should not fail"); From ccd859516d80b32339821121966089a936a0da28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Turmel?= Date: Wed, 9 Jul 2025 15:53:18 +0200 Subject: [PATCH 02/18] refactor(client-lib): move merkleproof creation and verification inside the snapshotmessage computation --- .../client-cardano-database-v2/src/main.rs | 9 +- .../src/commands/cardano_db/download/v2.rs | 29 ++- .../src/commands/cardano_db/shared_steps.rs | 17 +- .../src/commands/cardano_db/verify.rs | 26 +-- .../src/cardano_database_client/api.rs | 7 +- .../src/cardano_database_client/mod.rs | 9 +- .../src/cardano_database_client/proving.rs | 165 +++++++----------- mithril-client/src/message.rs | 139 ++++++++++++++- ...no_db_snapshot_list_get_download_verify.rs | 29 ++- 9 files changed, 285 insertions(+), 145 deletions(-) diff --git a/examples/client-cardano-database-v2/src/main.rs b/examples/client-cardano-database-v2/src/main.rs index e34768b4f2b..9ab47e5df8a 100644 --- a/examples/client-cardano-database-v2/src/main.rs +++ b/examples/client-cardano-database-v2/src/main.rs @@ -98,14 +98,21 @@ async fn main() -> MithrilResult<()> { ) .await?; + println!("Downloading and verifying digests file authenticity..."); + let verified_digest = client + .cardano_database_v2() + .download_and_verify_digests(&certificate, &cardano_database_snapshot) + .await?; + println!("Computing Cardano database Merkle proof...",); let merkle_proof = client .cardano_database_v2() .compute_merkle_proof( &certificate, - &cardano_database_snapshot, + cardano_database_snapshot.beacon.immutable_file_number, &immutable_file_range, &unpacked_dir, + &verified_digest, ) .await?; merkle_proof diff --git a/mithril-client-cli/src/commands/cardano_db/download/v2.rs b/mithril-client-cli/src/commands/cardano_db/download/v2.rs index 9c604b8a281..b51a4a16692 100644 --- a/mithril-client-cli/src/commands/cardano_db/download/v2.rs +++ b/mithril-client-cli/src/commands/cardano_db/download/v2.rs @@ -124,22 +124,41 @@ impl PreparedCardanoDbV2Download { ) })?; - let merkle_proof = shared_steps::compute_verify_merkle_proof( + let verified_digests = shared_steps::download_and_verify_digests( 4, &progress_printer, &client, &certificate, &cardano_db_message, - &restoration_options.immutable_file_range, - &restoration_options.db_dir, ) - .await?; + .await + .with_context(|| { + format!( + "Can not download and verify digests file for cardano db snapshot with hash: '{}'", + self.hash + ) + })?; + + // let merkle_proof = shared_steps::compute_verify_merkle_proof( + // 5, + // &progress_printer, + // &client, + // &certificate, + // &cardano_db_message, + // &restoration_options.immutable_file_range, + // &restoration_options.db_dir, + // &verified_digests, + // ) + // .await?; let message = shared_steps::compute_cardano_db_snapshot_message( 5, &progress_printer, &certificate, - &merkle_proof, + &cardano_db_message, + &restoration_options.immutable_file_range, + &restoration_options.db_dir, + &verified_digests, ) .await?; diff --git a/mithril-client-cli/src/commands/cardano_db/shared_steps.rs b/mithril-client-cli/src/commands/cardano_db/shared_steps.rs index 98531e5041f..b0e8b2cb3a4 100644 --- a/mithril-client-cli/src/commands/cardano_db/shared_steps.rs +++ b/mithril-client-cli/src/commands/cardano_db/shared_steps.rs @@ -72,15 +72,17 @@ pub async fn compute_verify_merkle_proof( cardano_database_snapshot: &CardanoDatabaseSnapshot, immutable_file_range: &ImmutableFileRange, unpacked_dir: &Path, + verified_digests: &VerifiedDigests, ) -> MithrilResult { progress_printer.report_step(step_number, "Computing and verifying the Merkle proof…")?; let merkle_proof = client .cardano_database_v2() .compute_merkle_proof( certificate, - cardano_database_snapshot, + cardano_database_snapshot.beacon.immutable_file_number, immutable_file_range, Path::new(&unpacked_dir), + verified_digests, ) .await?; @@ -96,12 +98,21 @@ pub async fn compute_cardano_db_snapshot_message( step_number: u16, progress_printer: &ProgressPrinter, certificate: &MithrilCertificate, - merkle_proof: &MKProof, + cardano_database_snapshot: &CardanoDatabaseSnapshot, + immutable_file_range: &ImmutableFileRange, + database_dir: &Path, + verified_digest: &VerifiedDigests, ) -> MithrilResult { progress_printer.report_step(step_number, "Computing the cardano db snapshot message")?; let message = CardanoDbUtils::wait_spinner( progress_printer, - MessageBuilder::new().compute_cardano_database_message(certificate, merkle_proof.root()), + MessageBuilder::new().compute_cardano_database_message( + certificate, + cardano_database_snapshot, + immutable_file_range, + database_dir, + verified_digest, + ), ) .await .with_context(|| "Can not compute the cardano db snapshot message")?; diff --git a/mithril-client-cli/src/commands/cardano_db/verify.rs b/mithril-client-cli/src/commands/cardano_db/verify.rs index a3351311311..89b9255fdf0 100644 --- a/mithril-client-cli/src/commands/cardano_db/verify.rs +++ b/mithril-client-cli/src/commands/cardano_db/verify.rs @@ -113,22 +113,26 @@ impl CardanoDbVerifyCommand { ) .await?; - let merkle_proof = shared_steps::compute_verify_merkle_proof( - 2, - &progress_printer, - &client, - &certificate, - &cardano_db_message, - &immutable_file_range, - db_dir, - ) - .await?; + // let merkle_proof = shared_steps::compute_verify_merkle_proof( + // 3, + // &progress_printer, + // &client, + // &certificate, + // &cardano_db_message, + // &immutable_file_range, + // db_dir, + // &verified_digests, + // ) + // .await?; let message = shared_steps::compute_cardano_db_snapshot_message( 3, &progress_printer, &certificate, - &merkle_proof, + &cardano_db_message, + &immutable_file_range, + db_dir, + &verified_digests, ) .await?; diff --git a/mithril-client/src/cardano_database_client/api.rs b/mithril-client/src/cardano_database_client/api.rs index 0583f71638e..0d222a1a8e2 100644 --- a/mithril-client/src/cardano_database_client/api.rs +++ b/mithril-client/src/cardano_database_client/api.rs @@ -8,6 +8,7 @@ use slog::Logger; #[cfg(feature = "fs")] use mithril_common::{ crypto_helper::MKProof, + entities::ImmutableFileNumber, messages::{CardanoDatabaseSnapshotMessage, CertificateMessage}, }; @@ -119,16 +120,18 @@ impl CardanoDatabaseClient { pub async fn compute_merkle_proof( &self, certificate: &CertificateMessage, - cardano_database_snapshot: &CardanoDatabaseSnapshotMessage, + last_immutable_file_number_from_beacon: ImmutableFileNumber, immutable_file_range: &ImmutableFileRange, database_dir: &Path, + verified_digests: &VerifiedDigests, ) -> MithrilResult { self.artifact_prover .compute_merkle_proof( certificate, - cardano_database_snapshot, + last_immutable_file_number_from_beacon, immutable_file_range, database_dir, + verified_digests, ) .await } diff --git a/mithril-client/src/cardano_database_client/mod.rs b/mithril-client/src/cardano_database_client/mod.rs index 34e46394acf..dd82784caec 100644 --- a/mithril-client/src/cardano_database_client/mod.rs +++ b/mithril-client/src/cardano_database_client/mod.rs @@ -113,9 +113,16 @@ //! ) //! .await?; //! +//! let verified_digests = client +//! .cardano_database_v2() +//! .download_and_verify_digests( +//! &certificate, +//! &cardano_database_snapshot) +//! .await?; +//! //! let merkle_proof = client //! .cardano_database_v2() -//! .compute_merkle_proof(&certificate, &cardano_database_snapshot, &immutable_file_range, &target_directory) +//! .compute_merkle_proof(&certificate, cardano_database_snapshot.beacon.immutable_file_number, &immutable_file_range, &target_directory, &verified_digests) //! .await?; //! # //! # Ok(()) diff --git a/mithril-client/src/cardano_database_client/proving.rs b/mithril-client/src/cardano_database_client/proving.rs index 792f5ca709f..b9ff88a2cce 100644 --- a/mithril-client/src/cardano_database_client/proving.rs +++ b/mithril-client/src/cardano_database_client/proving.rs @@ -13,7 +13,10 @@ use mithril_cardano_node_internal_database::{ }; use mithril_common::{ crypto_helper::{MKProof, MKTree, MKTreeNode, MKTreeStoreInMemory}, - entities::{DigestLocation, HexEncodedDigest, ImmutableFileName}, + entities::{ + DigestLocation, HexEncodedDigest, ImmutableFileName, ImmutableFileNumber, + ProtocolMessagePartKey, + }, messages::{ CardanoDatabaseDigestListItemMessage, CardanoDatabaseSnapshotMessage, CertificateMessage, DigestsMessagePart, @@ -21,7 +24,7 @@ use mithril_common::{ }; use crate::{ - MessageBuilder, MithrilResult, + MithrilResult, feedback::MithrilEvent, file_downloader::{DownloadEvent, FileDownloader, FileDownloaderUri}, utils::{create_directory_if_not_exists, delete_directory, read_files_in_directory}, @@ -55,30 +58,14 @@ impl InternalArtifactProver { pub async fn compute_merkle_proof( &self, certificate: &CertificateMessage, - cardano_database_snapshot: &CardanoDatabaseSnapshotMessage, + last_immutable_file_number_from_beacon: ImmutableFileNumber, immutable_file_range: &ImmutableFileRange, database_dir: &Path, + verified_digests: &VerifiedDigests, ) -> MithrilResult { - let digest_target_dir = Self::digest_target_dir(); - delete_directory(&digest_target_dir)?; - self.download_unpack_digest_file(&cardano_database_snapshot.digests, &digest_target_dir) - .await?; let network = certificate.metadata.network.clone(); - let last_immutable_file_number = cardano_database_snapshot.beacon.immutable_file_number; let immutable_file_number_range = - immutable_file_range.to_range_inclusive(last_immutable_file_number)?; - let downloaded_digests = self.read_digest_file(&digest_target_dir)?; - let downloaded_digests_values = downloaded_digests - .into_iter() - .filter(|(immutable_file_name, _)| { - match ImmutableFile::new(Path::new(immutable_file_name).to_path_buf()) { - Ok(immutable_file) => immutable_file.number <= last_immutable_file_number, - Err(_) => false, - } - }) - .map(|(_immutable_file_name, digest)| digest) - .collect::>(); - let merkle_tree: MKTree = MKTree::new(&downloaded_digests_values)?; + immutable_file_range.to_range_inclusive(last_immutable_file_number_from_beacon)?; let immutable_digester = CardanoImmutableDigester::new(network, None, self.logger.clone()); let computed_digests = immutable_digester .compute_digests_for_range(database_dir, &immutable_file_number_range) @@ -87,9 +74,29 @@ impl InternalArtifactProver { .values() .map(MKTreeNode::from) .collect::>(); - delete_directory(&digest_target_dir)?; - merkle_tree.compute_proof(&computed_digests) + verified_digests.merkle_tree.compute_proof(&computed_digests) + // merkle_tree.compute_proof(&computed_digests) + } + + fn check_merkle_root_is_signed_by_certificate( + certificate: &CertificateMessage, + merkle_root: &MKTreeNode, + ) -> MithrilResult<()> { + let mut message = certificate.protocol_message.clone(); + message.set_message_part( + ProtocolMessagePartKey::CardanoDatabaseMerkleRoot, + merkle_root.to_hex(), + ); + + if !certificate.match_message(&message) { + return Err(anyhow!( + "Certificate message does not match the computed message for certificate {}", + certificate.hash + )); + } + + Ok(()) } ///Download digests and verify its authenticity against the certificate. @@ -121,16 +128,10 @@ impl InternalArtifactProver { let filtered_digests_values = filtered_digests.values().collect::>(); let merkle_tree: MKTree = MKTree::new(&filtered_digests_values)?; - let message = MessageBuilder::new() - .compute_cardano_database_message(certificate, &merkle_tree.compute_root()?) - .await?; - - if !certificate.match_message(&message) { - return Err(anyhow!( - "Certificate message does not match the computed message for certificate {}", - certificate.hash - )); - } + Self::check_merkle_root_is_signed_by_certificate( + certificate, + &merkle_tree.compute_root()?, + )?; Ok(VerifiedDigests { digests: filtered_digests, @@ -248,9 +249,6 @@ mod tests { use std::ops::RangeInclusive; - use mithril_cardano_node_internal_database::{ - IMMUTABLE_DIR, digesters::ComputedImmutablesDigests, - }; use mithril_common::{ StdResult, entities::{ImmutableFileNumber, ProtocolMessage, ProtocolMessagePartKey}, @@ -259,31 +257,11 @@ mod tests { use super::*; - async fn prepare_fake_digests( + async fn prepare_db_and_verified_digests( dir_name: &str, beacon: &CardanoDbBeacon, immutable_file_range: &RangeInclusive, - digests_offset: usize, - digests_location: &str, - ) -> ( - PathBuf, - CardanoDatabaseSnapshotMessage, - CertificateMessage, - MKTree, - ComputedImmutablesDigests, - ) { - let cardano_database_snapshot = CardanoDatabaseSnapshotMessage { - hash: "hash-123".to_string(), - beacon: beacon.clone(), - digests: DigestsMessagePart { - size_uncompressed: 1024, - locations: vec![DigestLocation::CloudStorage { - uri: digests_location.to_string(), - compression_algorithm: None, - }], - }, - ..CardanoDatabaseSnapshotMessage::dummy() - }; + ) -> (PathBuf, CertificateMessage, VerifiedDigests) { let certificate = CertificateMessage { hash: "cert-hash-123".to_string(), ..CertificateMessage::dummy() @@ -302,31 +280,24 @@ mod tests { .compute_digests_for_range(database_dir, immutable_file_range) .await .unwrap(); - // We remove the last digests_offset digests to simulate receiving - // a digest file with more immutable files than downloaded - for (immutable_file, _digest) in - computed_digests.entries.iter().rev().take(digests_offset) - { - fs::remove_file( - database_dir.join( - database_dir.join(IMMUTABLE_DIR).join(immutable_file.filename.clone()), - ), - ) - .unwrap(); - } + + let digests = computed_digests + .entries + .iter() + .map(|(immutable_file, digest)| (immutable_file.filename.clone(), digest.clone())) + .collect::>(); let merkle_tree = immutable_digester .compute_merkle_tree(database_dir, beacon) .await .unwrap(); - ( - database_dir.to_owned(), - cardano_database_snapshot, - certificate, + let verified_digests = VerifiedDigests { + digests, merkle_tree, - computed_digests, - ) + }; + + (database_dir.to_owned(), certificate, verified_digests) } fn write_digest_file( @@ -453,50 +424,32 @@ mod tests { }; let immutable_file_range = 1..=15; let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); - let digests_offset = 3; - let digests_location = "http://whatever/digests.json"; - let (database_dir, cardano_database_snapshot, certificate, merkle_tree, digests) = - prepare_fake_digests( - "compute_merkle_proof_succeeds", - &beacon, - &immutable_file_range, - digests_offset, - digests_location, - ) - .await; - let expected_merkle_root = merkle_tree.compute_root().unwrap(); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_file_downloader(Arc::new( - MockFileDownloaderBuilder::default() - .with_file_uri(digests_location) - .with_target_dir(InternalArtifactProver::digest_target_dir()) - .with_compression(None) - .with_returning(Box::new(move |_, _, _, _, _| { - let digests = digests.entries.clone(); - let digest_dir = InternalArtifactProver::digest_target_dir(); - write_digest_file(digest_dir.as_path(), &digests)?; + let (database_dir, certificate, verified_digests) = prepare_db_and_verified_digests( + "compute_merkle_proof_succeeds", + &beacon, + &immutable_file_range, + ) + .await; + let expected_merkle_root = verified_digests.merkle_tree.compute_root().unwrap(); - Ok(()) - })) - .build(), - )) - .build_cardano_database_client(); + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); let merkle_proof = client .compute_merkle_proof( &certificate, - &cardano_database_snapshot, + beacon.immutable_file_number, &immutable_file_range_to_prove, &database_dir, + &verified_digests, ) .await .unwrap(); + merkle_proof.verify().unwrap(); let merkle_proof_root = merkle_proof.root().to_owned(); assert_eq!(expected_merkle_root, merkle_proof_root); - - assert!(!InternalArtifactProver::digest_target_dir().exists()); } } diff --git a/mithril-client/src/message.rs b/mithril-client/src/message.rs index 4c9699eca62..92c398da16b 100644 --- a/mithril-client/src/message.rs +++ b/mithril-client/src/message.rs @@ -5,15 +5,19 @@ use std::path::Path; #[cfg(feature = "fs")] use std::sync::Arc; +#[cfg(feature = "fs")] +use crate::cardano_database_client::{ImmutableFileRange, VerifiedDigests}; #[cfg(feature = "fs")] use mithril_cardano_node_internal_database::digesters::{ CardanoImmutableDigester, ImmutableDigester, }; -use mithril_common::logging::LoggerExtensions; use mithril_common::protocol::SignerBuilder; use mithril_common::signable_builder::CardanoStakeDistributionSignableBuilder; #[cfg(feature = "fs")] -use mithril_common::{crypto_helper::MKTreeNode, entities::SignedEntityType}; +use mithril_common::{ + crypto_helper::MKTreeNode, entities::SignedEntityType, messages::CertificateMessage, +}; +use mithril_common::{logging::LoggerExtensions, messages::CardanoDatabaseSnapshotMessage}; use crate::{ CardanoStakeDistribution, MithrilCertificate, MithrilResult, MithrilSigner, @@ -78,9 +82,9 @@ impl MessageBuilder { SignedEntityType::CardanoImmutableFilesFull(beacon) => {Ok(beacon)}, other => { Err(anyhow::anyhow!( - "Can't compute message: Given certificate `{}` does not certify a snapshot, certificate signed entity: {:?}", - snapshot_certificate.hash, - other + "Can't compute message: Given certificate `{}` does not certify a snapshot, certificate signed entity: {:?}", + snapshot_certificate.hash, + other ) )}, }?; @@ -104,13 +108,33 @@ impl MessageBuilder { /// Compute message for a Cardano database. pub async fn compute_cardano_database_message( &self, - certificate: &MithrilCertificate, - merkle_root: &MKTreeNode, + certificate: &CertificateMessage, + cardano_database_snapshot: &CardanoDatabaseSnapshotMessage, + immutable_file_range: &ImmutableFileRange, + database_dir: &Path, + verified_digests: &VerifiedDigests, ) -> MithrilResult { + let network = certificate.metadata.network.clone(); + let immutable_file_number_range = immutable_file_range + .to_range_inclusive(cardano_database_snapshot.beacon.immutable_file_number)?; + let immutable_digester = CardanoImmutableDigester::new(network, None, self.logger.clone()); + let computed_digests = immutable_digester + .compute_digests_for_range(database_dir, &immutable_file_number_range) + .await? + .entries + .values() + .map(MKTreeNode::from) + .collect::>(); + + let merkle_proof = verified_digests.merkle_tree.compute_proof(&computed_digests)?; + merkle_proof + .verify() + .with_context(|| "Merkle proof verification failed")?; + let mut message = certificate.protocol_message.clone(); message.set_message_part( ProtocolMessagePartKey::CardanoDatabaseMerkleRoot, - merkle_root.to_hex(), + merkle_proof.root().to_hex(), ); Ok(message) @@ -187,3 +211,102 @@ impl Default for MessageBuilder { Self::new() } } + +#[cfg(test)] +mod tests { + + use std::ops::RangeInclusive; + use std::{collections::BTreeMap, path::PathBuf}; + + use mithril_cardano_node_internal_database::test::DummyCardanoDbBuilder; + use mithril_common::{ + entities::{CardanoDbBeacon, Epoch, ImmutableFileNumber}, + messages::CertificateMessage, + }; + + use crate::cardano_database_client::ImmutableFileRange; + use crate::{cardano_database_client::VerifiedDigests, test_utils::TestLogger}; + + use super::*; + + async fn prepare_db_and_verified_digests( + dir_name: &str, + beacon: &CardanoDbBeacon, + immutable_file_range: &RangeInclusive, + ) -> (PathBuf, CertificateMessage, VerifiedDigests) { + let cardano_db = DummyCardanoDbBuilder::new(dir_name) + .with_immutables(&immutable_file_range.clone().collect::>()) + .append_immutable_trio() + .build(); + let database_dir = cardano_db.get_dir(); + let immutable_digester = + CardanoImmutableDigester::new("whatever".to_string(), None, TestLogger::stdout()); + let computed_digests = immutable_digester + .compute_digests_for_range(database_dir, immutable_file_range) + .await + .unwrap(); + + let digests = computed_digests + .entries + .iter() + .map(|(immutable_file, digest)| (immutable_file.filename.clone(), digest.clone())) + .collect::>(); + + let merkle_tree = immutable_digester + .compute_merkle_tree(database_dir, beacon) + .await + .unwrap(); + + let verified_digests = VerifiedDigests { + digests, + merkle_tree, + }; + + let certificate = { + let protocol_message_merkle_root = + verified_digests.merkle_tree.compute_root().unwrap().to_hex(); + let mut protocol_message = ProtocolMessage::new(); + protocol_message.set_message_part( + ProtocolMessagePartKey::CardanoDatabaseMerkleRoot, + protocol_message_merkle_root, + ); + + CertificateMessage { + protocol_message: protocol_message.clone(), + signed_message: protocol_message.compute_hash(), + ..CertificateMessage::dummy() + } + }; + + (database_dir.to_owned(), certificate, verified_digests) + } + + #[tokio::test] + async fn compute_cardano_database_message_succeeds() { + let beacon = CardanoDbBeacon { + epoch: Epoch(123), + immutable_file_number: 10, + }; + let immutable_file_range = 1..=15; + let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); + let (database_dir, certificate, verified_digests) = prepare_db_and_verified_digests( + "compute_cardano_database_message_succeeds", + &beacon, + &immutable_file_range, + ) + .await; + + let message = MessageBuilder::new() + .compute_cardano_database_message( + &certificate, + &CardanoDatabaseSnapshotMessage::dummy(), + &immutable_file_range_to_prove, + &database_dir, + &verified_digests, + ) + .await + .unwrap(); + + assert!(certificate.match_message(&message)); + } +} diff --git a/mithril-client/tests/cardano_db_snapshot_list_get_download_verify.rs b/mithril-client/tests/cardano_db_snapshot_list_get_download_verify.rs index d092d8d3138..4a08f70afbf 100644 --- a/mithril-client/tests/cardano_db_snapshot_list_get_download_verify.rs +++ b/mithril-client/tests/cardano_db_snapshot_list_get_download_verify.rs @@ -132,21 +132,34 @@ async fn cardano_db_snapshot_list_get_download_verify() { .route() ))); - let merkle_proof = client + let verified_digests = client .cardano_database_v2() - .compute_merkle_proof( + .download_and_verify_digests(&certificate, &cardano_db_snapshot) + .await + .unwrap(); + + // let merkle_proof = client + // .cardano_database_v2() + // .compute_merkle_proof( + // &certificate, + // cardano_db_snapshot.beacon.immutable_file_number, + // &immutable_file_range, + // &unpacked_dir, + // &verified_digests, + // ) + // .await + // .expect("Computing merkle proof should not fail"); + // merkle_proof.verify().expect("Merkle proof should be valid"); //TODO verify? + + let message = MessageBuilder::new() + .compute_cardano_database_message( &certificate, &cardano_db_snapshot, &immutable_file_range, &unpacked_dir, + &verified_digests, ) .await - .expect("Computing merkle proof should not fail"); - merkle_proof.verify().expect("Merkle proof should be valid"); - - let message = MessageBuilder::new() - .compute_cardano_database_message(&certificate, merkle_proof.root()) - .await .expect("Computing cardano database snapshot message should not fail"); assert!( From 0898dc1454325830e1148f2c59407ae40b13dcb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Turmel?= Date: Wed, 9 Jul 2025 17:38:00 +0200 Subject: [PATCH 03/18] refactor(client-lib): remove compute_merkle_proof since logic is moved to compute_cardano_database_message --- .../src/commands/cardano_db/download/v2.rs | 12 -- .../src/commands/cardano_db/shared_steps.rs | 32 +---- .../src/commands/cardano_db/verify.rs | 12 -- .../src/cardano_database_client/api.rs | 27 +--- .../src/cardano_database_client/mod.rs | 4 +- .../src/cardano_database_client/proving.rs | 126 +----------------- ...no_db_snapshot_list_get_download_verify.rs | 13 -- 7 files changed, 9 insertions(+), 217 deletions(-) diff --git a/mithril-client-cli/src/commands/cardano_db/download/v2.rs b/mithril-client-cli/src/commands/cardano_db/download/v2.rs index b51a4a16692..b7c728c4c1a 100644 --- a/mithril-client-cli/src/commands/cardano_db/download/v2.rs +++ b/mithril-client-cli/src/commands/cardano_db/download/v2.rs @@ -139,18 +139,6 @@ impl PreparedCardanoDbV2Download { ) })?; - // let merkle_proof = shared_steps::compute_verify_merkle_proof( - // 5, - // &progress_printer, - // &client, - // &certificate, - // &cardano_db_message, - // &restoration_options.immutable_file_range, - // &restoration_options.db_dir, - // &verified_digests, - // ) - // .await?; - let message = shared_steps::compute_cardano_db_snapshot_message( 5, &progress_printer, diff --git a/mithril-client-cli/src/commands/cardano_db/shared_steps.rs b/mithril-client-cli/src/commands/cardano_db/shared_steps.rs index b0e8b2cb3a4..cbdff83987d 100644 --- a/mithril-client-cli/src/commands/cardano_db/shared_steps.rs +++ b/mithril-client-cli/src/commands/cardano_db/shared_steps.rs @@ -6,7 +6,7 @@ use std::path::Path; use mithril_client::{ CardanoDatabaseSnapshot, Client, MessageBuilder, MithrilCertificate, MithrilResult, cardano_database_client::{ImmutableFileRange, VerifiedDigests}, - common::{ImmutableFileNumber, MKProof, ProtocolMessage}, + common::{ImmutableFileNumber, ProtocolMessage}, }; use crate::utils::{CardanoDbUtils, ProgressPrinter}; @@ -63,36 +63,6 @@ pub async fn download_and_verify_digests( Ok(verified_digests) } -/// Computes and verifies the Merkle proof for the given certificate and database snapshot. -pub async fn compute_verify_merkle_proof( - step_number: u16, - progress_printer: &ProgressPrinter, - client: &Client, - certificate: &MithrilCertificate, - cardano_database_snapshot: &CardanoDatabaseSnapshot, - immutable_file_range: &ImmutableFileRange, - unpacked_dir: &Path, - verified_digests: &VerifiedDigests, -) -> MithrilResult { - progress_printer.report_step(step_number, "Computing and verifying the Merkle proof…")?; - let merkle_proof = client - .cardano_database_v2() - .compute_merkle_proof( - certificate, - cardano_database_snapshot.beacon.immutable_file_number, - immutable_file_range, - Path::new(&unpacked_dir), - verified_digests, - ) - .await?; - - merkle_proof - .verify() - .with_context(|| "Merkle proof verification failed")?; - - Ok(merkle_proof) -} - /// Computes the Cardano database snapshot message using the provided certificate and Merkle proof. pub async fn compute_cardano_db_snapshot_message( step_number: u16, diff --git a/mithril-client-cli/src/commands/cardano_db/verify.rs b/mithril-client-cli/src/commands/cardano_db/verify.rs index 89b9255fdf0..53d52ca3b2b 100644 --- a/mithril-client-cli/src/commands/cardano_db/verify.rs +++ b/mithril-client-cli/src/commands/cardano_db/verify.rs @@ -113,18 +113,6 @@ impl CardanoDbVerifyCommand { ) .await?; - // let merkle_proof = shared_steps::compute_verify_merkle_proof( - // 3, - // &progress_printer, - // &client, - // &certificate, - // &cardano_db_message, - // &immutable_file_range, - // db_dir, - // &verified_digests, - // ) - // .await?; - let message = shared_steps::compute_cardano_db_snapshot_message( 3, &progress_printer, diff --git a/mithril-client/src/cardano_database_client/api.rs b/mithril-client/src/cardano_database_client/api.rs index 0d222a1a8e2..3f0b59b9937 100644 --- a/mithril-client/src/cardano_database_client/api.rs +++ b/mithril-client/src/cardano_database_client/api.rs @@ -6,11 +6,7 @@ use std::sync::Arc; use slog::Logger; #[cfg(feature = "fs")] -use mithril_common::{ - crypto_helper::MKProof, - entities::ImmutableFileNumber, - messages::{CardanoDatabaseSnapshotMessage, CertificateMessage}, -}; +use mithril_common::messages::{CardanoDatabaseSnapshotMessage, CertificateMessage}; #[cfg(feature = "fs")] use mithril_cardano_node_internal_database::entities::ImmutableFile; @@ -115,27 +111,6 @@ impl CardanoDatabaseClient { .await } - /// Compute the Merkle proof of membership for the given immutable file range. - #[cfg(feature = "fs")] - pub async fn compute_merkle_proof( - &self, - certificate: &CertificateMessage, - last_immutable_file_number_from_beacon: ImmutableFileNumber, - immutable_file_range: &ImmutableFileRange, - database_dir: &Path, - verified_digests: &VerifiedDigests, - ) -> MithrilResult { - self.artifact_prover - .compute_merkle_proof( - certificate, - last_immutable_file_number_from_beacon, - immutable_file_range, - database_dir, - verified_digests, - ) - .await - } - /// Checks if immutable directory exists with at least one immutable in it #[cfg(feature = "fs")] pub fn check_has_immutables(&self, database_dir: &Path) -> MithrilResult<()> { diff --git a/mithril-client/src/cardano_database_client/mod.rs b/mithril-client/src/cardano_database_client/mod.rs index dd82784caec..85f50840fb3 100644 --- a/mithril-client/src/cardano_database_client/mod.rs +++ b/mithril-client/src/cardano_database_client/mod.rs @@ -4,8 +4,8 @@ //! - [get][CardanoDatabaseClient::get]: get a Cardano database data from its hash //! - [list][CardanoDatabaseClient::list]: get the list of available Cardano database //! - [download_unpack][CardanoDatabaseClient::download_unpack]: download and unpack a Cardano database snapshot for a given immutable files range -//! - [compute_merkle_proof][CardanoDatabaseClient::compute_merkle_proof]: compute a Merkle proof for a given Cardano database snapshot and a given immutable files range -//! +//! - [download_and_verify_digests][CardanoDatabaseClient::download_and_verify_digests]: download and verify the digests of the immutable files in a Cardano database snapshot + //! # Get a Cardano database //! //! To get a Cardano database using the [ClientBuilder][crate::client::ClientBuilder]. diff --git a/mithril-client/src/cardano_database_client/proving.rs b/mithril-client/src/cardano_database_client/proving.rs index b9ff88a2cce..4651216299c 100644 --- a/mithril-client/src/cardano_database_client/proving.rs +++ b/mithril-client/src/cardano_database_client/proving.rs @@ -7,16 +7,10 @@ use std::{ use anyhow::{Context, anyhow}; -use mithril_cardano_node_internal_database::{ - digesters::{CardanoImmutableDigester, ImmutableDigester}, - entities::ImmutableFile, -}; +use mithril_cardano_node_internal_database::entities::ImmutableFile; use mithril_common::{ - crypto_helper::{MKProof, MKTree, MKTreeNode, MKTreeStoreInMemory}, - entities::{ - DigestLocation, HexEncodedDigest, ImmutableFileName, ImmutableFileNumber, - ProtocolMessagePartKey, - }, + crypto_helper::{MKTree, MKTreeNode, MKTreeStoreInMemory}, + entities::{DigestLocation, HexEncodedDigest, ImmutableFileName, ProtocolMessagePartKey}, messages::{ CardanoDatabaseDigestListItemMessage, CardanoDatabaseSnapshotMessage, CertificateMessage, DigestsMessagePart, @@ -30,8 +24,6 @@ use crate::{ utils::{create_directory_if_not_exists, delete_directory, read_files_in_directory}, }; -use super::immutable_file_range::ImmutableFileRange; - /// Represents the verified digests and the Merkle tree built from them. pub struct VerifiedDigests { /// A map of immutable file names to their corresponding verified digests. @@ -54,31 +46,6 @@ impl InternalArtifactProver { } } - /// Compute the Merkle proof of membership for the given immutable file range. - pub async fn compute_merkle_proof( - &self, - certificate: &CertificateMessage, - last_immutable_file_number_from_beacon: ImmutableFileNumber, - immutable_file_range: &ImmutableFileRange, - database_dir: &Path, - verified_digests: &VerifiedDigests, - ) -> MithrilResult { - let network = certificate.metadata.network.clone(); - let immutable_file_number_range = - immutable_file_range.to_range_inclusive(last_immutable_file_number_from_beacon)?; - let immutable_digester = CardanoImmutableDigester::new(network, None, self.logger.clone()); - let computed_digests = immutable_digester - .compute_digests_for_range(database_dir, &immutable_file_number_range) - .await? - .entries - .values() - .map(MKTreeNode::from) - .collect::>(); - - verified_digests.merkle_tree.compute_proof(&computed_digests) - // merkle_tree.compute_proof(&computed_digests) - } - fn check_merkle_root_is_signed_by_certificate( certificate: &CertificateMessage, merkle_root: &MKTreeNode, @@ -231,7 +198,6 @@ mod tests { use std::path::Path; use std::sync::Arc; - use mithril_cardano_node_internal_database::test::DummyCardanoDbBuilder; use mithril_common::{ entities::{CardanoDbBeacon, Epoch, HexEncodedDigest}, messages::CardanoDatabaseDigestListItemMessage, @@ -245,61 +211,15 @@ mod tests { use super::*; - mod compute_merkle_proof { - - use std::ops::RangeInclusive; - + mod download_and_verify_digests { use mithril_common::{ StdResult, - entities::{ImmutableFileNumber, ProtocolMessage, ProtocolMessagePartKey}, + entities::{ProtocolMessage, ProtocolMessagePartKey}, messages::DigestsMessagePart, }; use super::*; - async fn prepare_db_and_verified_digests( - dir_name: &str, - beacon: &CardanoDbBeacon, - immutable_file_range: &RangeInclusive, - ) -> (PathBuf, CertificateMessage, VerifiedDigests) { - let certificate = CertificateMessage { - hash: "cert-hash-123".to_string(), - ..CertificateMessage::dummy() - }; - let cardano_db = DummyCardanoDbBuilder::new(dir_name) - .with_immutables(&immutable_file_range.clone().collect::>()) - .append_immutable_trio() - .build(); - let database_dir = cardano_db.get_dir(); - let immutable_digester = CardanoImmutableDigester::new( - certificate.metadata.network.to_string(), - None, - TestLogger::stdout(), - ); - let computed_digests = immutable_digester - .compute_digests_for_range(database_dir, immutable_file_range) - .await - .unwrap(); - - let digests = computed_digests - .entries - .iter() - .map(|(immutable_file, digest)| (immutable_file.filename.clone(), digest.clone())) - .collect::>(); - - let merkle_tree = immutable_digester - .compute_merkle_tree(database_dir, beacon) - .await - .unwrap(); - - let verified_digests = VerifiedDigests { - digests, - merkle_tree, - }; - - (database_dir.to_owned(), certificate, verified_digests) - } - fn write_digest_file( digest_dir: &Path, digests: &BTreeMap, @@ -415,42 +335,6 @@ mod tests { assert!(!InternalArtifactProver::digest_target_dir().exists()); } - - #[tokio::test] - async fn compute_merkle_proof_succeeds() { - let beacon = CardanoDbBeacon { - epoch: Epoch(123), - immutable_file_number: 10, - }; - let immutable_file_range = 1..=15; - let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); - let (database_dir, certificate, verified_digests) = prepare_db_and_verified_digests( - "compute_merkle_proof_succeeds", - &beacon, - &immutable_file_range, - ) - .await; - let expected_merkle_root = verified_digests.merkle_tree.compute_root().unwrap(); - - let client = - CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); - - let merkle_proof = client - .compute_merkle_proof( - &certificate, - beacon.immutable_file_number, - &immutable_file_range_to_prove, - &database_dir, - &verified_digests, - ) - .await - .unwrap(); - - merkle_proof.verify().unwrap(); - - let merkle_proof_root = merkle_proof.root().to_owned(); - assert_eq!(expected_merkle_root, merkle_proof_root); - } } mod download_unpack_digest_file { diff --git a/mithril-client/tests/cardano_db_snapshot_list_get_download_verify.rs b/mithril-client/tests/cardano_db_snapshot_list_get_download_verify.rs index 4a08f70afbf..514af718e55 100644 --- a/mithril-client/tests/cardano_db_snapshot_list_get_download_verify.rs +++ b/mithril-client/tests/cardano_db_snapshot_list_get_download_verify.rs @@ -138,19 +138,6 @@ async fn cardano_db_snapshot_list_get_download_verify() { .await .unwrap(); - // let merkle_proof = client - // .cardano_database_v2() - // .compute_merkle_proof( - // &certificate, - // cardano_db_snapshot.beacon.immutable_file_number, - // &immutable_file_range, - // &unpacked_dir, - // &verified_digests, - // ) - // .await - // .expect("Computing merkle proof should not fail"); - // merkle_proof.verify().expect("Merkle proof should be valid"); //TODO verify? - let message = MessageBuilder::new() .compute_cardano_database_message( &certificate, From 2992846d917de4f19b7d7593be2c06ba51b2ea14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Turmel?= Date: Thu, 10 Jul 2025 12:12:59 +0200 Subject: [PATCH 04/18] feature(client-lib): list missing immutable files when computing cardano database message with a structured error --- mithril-client/src/message.rs | 410 ++++++++++++++++++++++++++-------- 1 file changed, 318 insertions(+), 92 deletions(-) diff --git a/mithril-client/src/message.rs b/mithril-client/src/message.rs index 92c398da16b..e48acab9d07 100644 --- a/mithril-client/src/message.rs +++ b/mithril-client/src/message.rs @@ -1,30 +1,79 @@ use anyhow::Context; use slog::{Logger, o}; #[cfg(feature = "fs")] +use std::fmt; +#[cfg(feature = "fs")] +use std::ops::RangeInclusive; +#[cfg(feature = "fs")] use std::path::Path; +use std::path::PathBuf; #[cfg(feature = "fs")] use std::sync::Arc; +#[cfg(feature = "fs")] +use thiserror::Error; #[cfg(feature = "fs")] use crate::cardano_database_client::{ImmutableFileRange, VerifiedDigests}; #[cfg(feature = "fs")] -use mithril_cardano_node_internal_database::digesters::{ - CardanoImmutableDigester, ImmutableDigester, +use mithril_cardano_node_internal_database::{ + IMMUTABLE_DIR, + digesters::{CardanoImmutableDigester, ImmutableDigester, ImmutableDigesterError}, }; use mithril_common::protocol::SignerBuilder; use mithril_common::signable_builder::CardanoStakeDistributionSignableBuilder; #[cfg(feature = "fs")] use mithril_common::{ - crypto_helper::MKTreeNode, entities::SignedEntityType, messages::CertificateMessage, + crypto_helper::MKTreeNode, + entities::{ImmutableFileName, ImmutableFileNumber, SignedEntityType}, + messages::CertificateMessage, }; use mithril_common::{logging::LoggerExtensions, messages::CardanoDatabaseSnapshotMessage}; +#[cfg(feature = "fs")] +use crate::MithrilError; use crate::{ CardanoStakeDistribution, MithrilCertificate, MithrilResult, MithrilSigner, MithrilStakeDistribution, VerifiedCardanoTransactions, common::{ProtocolMessage, ProtocolMessagePartKey}, }; +#[derive(Debug)] +pub struct ImmutableFilesLists { + pub dir_path: PathBuf, + pub missing: Vec, + pub tampered: Vec, +} + +#[derive(Error, Debug)] +pub enum ComputeCardanoDatabaseMessageError { + ImmutableFiles(ImmutableFilesLists), + + ImmutableFilesDigester(#[from] ImmutableDigesterError), + + MerkleProofVerification(#[source] MithrilError), + + ImmutableFilesRange(#[source] MithrilError), +} + +impl fmt::Display for ComputeCardanoDatabaseMessageError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ComputeCardanoDatabaseMessageError::ImmutableFiles(files) => { + write!(f, "immutable files: {:?}", files) + } + ComputeCardanoDatabaseMessageError::ImmutableFilesDigester(e) => { + write!(f, "Immutable files digester error: {}", e) + } + ComputeCardanoDatabaseMessageError::MerkleProofVerification(e) => { + write!(f, "Merkle proof verification error: {}", e) + } + ComputeCardanoDatabaseMessageError::ImmutableFilesRange(e) => { + write!(f, "Immutable files range error: {}", e) + } + } + } +} + /// A [MessageBuilder] can be used to compute the message of Mithril artifacts. pub struct MessageBuilder { #[cfg(feature = "fs")] @@ -105,6 +154,29 @@ impl MessageBuilder { Ok(message) } + fn immutable_dir(db_dir: &Path) -> PathBuf { + db_dir.join(IMMUTABLE_DIR) + } + + fn list_missing_immutable_files( + database_dir: &Path, + immutable_file_number_range: &RangeInclusive, + ) -> Vec { + let immutable_dir = Self::immutable_dir(database_dir); + let mut missing_files = Vec::new(); + + for immutable_file_number in immutable_file_number_range.clone() { + for immutable_type in ["chunk", "primary", "secondary"] { + let file_name = format!("{immutable_file_number:05}.{immutable_type}"); + if !immutable_dir.join(&file_name).exists() { + missing_files.push(ImmutableFileName::from(file_name)); + } + } + } + + missing_files + } + /// Compute message for a Cardano database. pub async fn compute_cardano_database_message( &self, @@ -113,10 +185,15 @@ impl MessageBuilder { immutable_file_range: &ImmutableFileRange, database_dir: &Path, verified_digests: &VerifiedDigests, - ) -> MithrilResult { + ) -> Result { let network = certificate.metadata.network.clone(); let immutable_file_number_range = immutable_file_range - .to_range_inclusive(cardano_database_snapshot.beacon.immutable_file_number)?; + .to_range_inclusive(cardano_database_snapshot.beacon.immutable_file_number) + .map_err(ComputeCardanoDatabaseMessageError::ImmutableFilesRange)?; + + let missing_immutable_files = + Self::list_missing_immutable_files(database_dir, &immutable_file_number_range); + let immutable_digester = CardanoImmutableDigester::new(network, None, self.logger.clone()); let computed_digests = immutable_digester .compute_digests_for_range(database_dir, &immutable_file_number_range) @@ -126,10 +203,10 @@ impl MessageBuilder { .map(MKTreeNode::from) .collect::>(); - let merkle_proof = verified_digests.merkle_tree.compute_proof(&computed_digests)?; + let merkle_proof = verified_digests.merkle_tree.compute_proof(&computed_digests).unwrap(); merkle_proof .verify() - .with_context(|| "Merkle proof verification failed")?; + .map_err(ComputeCardanoDatabaseMessageError::MerkleProofVerification)?; let mut message = certificate.protocol_message.clone(); message.set_message_part( @@ -137,6 +214,16 @@ impl MessageBuilder { merkle_proof.root().to_hex(), ); + if !missing_immutable_files.is_empty() { + return Err(ComputeCardanoDatabaseMessageError::ImmutableFiles( + ImmutableFilesLists { + dir_path: Self::immutable_dir(database_dir), + missing: missing_immutable_files, + tampered: vec![], + }, + )); + } + Ok(message) } } @@ -214,99 +301,238 @@ impl Default for MessageBuilder { #[cfg(test)] mod tests { + use super::*; + + #[cfg(feature = "fs")] + mod fs { + use mithril_cardano_node_internal_database::test::DummyCardanoDbBuilder; + + use super::*; - use std::ops::RangeInclusive; - use std::{collections::BTreeMap, path::PathBuf}; + fn remove_immutable_files>(database_dir: &Path, immutable_file_names: &[T]) { + for immutable_file_name in immutable_file_names { + let immutable_file_path = + MessageBuilder::immutable_dir(database_dir).join(immutable_file_name.as_ref()); + std::fs::remove_file(immutable_file_path).unwrap(); + } + } - use mithril_cardano_node_internal_database::test::DummyCardanoDbBuilder; - use mithril_common::{ - entities::{CardanoDbBeacon, Epoch, ImmutableFileNumber}, - messages::CertificateMessage, - }; + mod list_missing_immutable_files { + use mithril_common::temp_dir_create; - use crate::cardano_database_client::ImmutableFileRange; - use crate::{cardano_database_client::VerifiedDigests, test_utils::TestLogger}; + use super::*; - use super::*; + #[test] + fn list_missing_immutable_files_should_return_empty_list_if_no_missing_files() { + let immutable_files_in_db = 1..=10; + let range_to_verify = 3..=5; + let cardano_db = + DummyCardanoDbBuilder::new(&format!("{}", temp_dir_create!().display())) + .with_immutables(&immutable_files_in_db.collect::>()) + .append_immutable_trio() + .build(); - async fn prepare_db_and_verified_digests( - dir_name: &str, - beacon: &CardanoDbBeacon, - immutable_file_range: &RangeInclusive, - ) -> (PathBuf, CertificateMessage, VerifiedDigests) { - let cardano_db = DummyCardanoDbBuilder::new(dir_name) - .with_immutables(&immutable_file_range.clone().collect::>()) - .append_immutable_trio() - .build(); - let database_dir = cardano_db.get_dir(); - let immutable_digester = - CardanoImmutableDigester::new("whatever".to_string(), None, TestLogger::stdout()); - let computed_digests = immutable_digester - .compute_digests_for_range(database_dir, immutable_file_range) - .await - .unwrap(); - - let digests = computed_digests - .entries - .iter() - .map(|(immutable_file, digest)| (immutable_file.filename.clone(), digest.clone())) - .collect::>(); - - let merkle_tree = immutable_digester - .compute_merkle_tree(database_dir, beacon) - .await - .unwrap(); - - let verified_digests = VerifiedDigests { - digests, - merkle_tree, - }; - - let certificate = { - let protocol_message_merkle_root = - verified_digests.merkle_tree.compute_root().unwrap().to_hex(); - let mut protocol_message = ProtocolMessage::new(); - protocol_message.set_message_part( - ProtocolMessagePartKey::CardanoDatabaseMerkleRoot, - protocol_message_merkle_root, - ); + let missing_files = MessageBuilder::list_missing_immutable_files( + cardano_db.get_dir(), + &range_to_verify, + ); - CertificateMessage { - protocol_message: protocol_message.clone(), - signed_message: protocol_message.compute_hash(), - ..CertificateMessage::dummy() + assert!(missing_files.is_empty()); } - }; - (database_dir.to_owned(), certificate, verified_digests) - } + #[test] + fn list_missing_immutable_files_should_return_empty_list_if_missing_files_outside_range() + { + let immutable_files_in_db = 1..=10; + let range_to_verify = 3..=5; + let cardano_db = + DummyCardanoDbBuilder::new(&format!("{}", temp_dir_create!().display())) + .with_immutables(&immutable_files_in_db.collect::>()) + .append_immutable_trio() + .build(); + let files_to_remove = vec!["00002.chunk", "00006.primary"]; + remove_immutable_files(cardano_db.get_dir(), &files_to_remove); + + let missing_files = MessageBuilder::list_missing_immutable_files( + cardano_db.get_dir(), + &range_to_verify, + ); + + assert!(missing_files.is_empty()); + } - #[tokio::test] - async fn compute_cardano_database_message_succeeds() { - let beacon = CardanoDbBeacon { - epoch: Epoch(123), - immutable_file_number: 10, - }; - let immutable_file_range = 1..=15; - let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); - let (database_dir, certificate, verified_digests) = prepare_db_and_verified_digests( - "compute_cardano_database_message_succeeds", - &beacon, - &immutable_file_range, - ) - .await; - - let message = MessageBuilder::new() - .compute_cardano_database_message( - &certificate, - &CardanoDatabaseSnapshotMessage::dummy(), - &immutable_file_range_to_prove, - &database_dir, - &verified_digests, - ) - .await - .unwrap(); - - assert!(certificate.match_message(&message)); + #[test] + fn list_missing_immutable_files_should_return_list_of_missing_files_inside_range() { + let immutable_files_in_db = 1..=10; + let range_to_verify = 3..=5; + let cardano_db = + DummyCardanoDbBuilder::new(&format!("{}", temp_dir_create!().display())) + .with_immutables(&immutable_files_in_db.collect::>()) + .append_immutable_trio() + .build(); + let files_to_remove = vec!["00004.chunk", "00005.primary"]; + remove_immutable_files(cardano_db.get_dir(), &files_to_remove); + + let missing_files = MessageBuilder::list_missing_immutable_files( + cardano_db.get_dir(), + &range_to_verify, + ); + + assert_eq!(missing_files, files_to_remove); + } + } + + mod cardano_database_message { + + use std::{collections::BTreeMap, path::PathBuf}; + + use mithril_common::{ + entities::{CardanoDbBeacon, Epoch}, + messages::CertificateMessage, + }; + + use crate::cardano_database_client::ImmutableFileRange; + use crate::{cardano_database_client::VerifiedDigests, test_utils::TestLogger}; + + use super::*; + + async fn prepare_db_and_verified_digests( + dir_name: &str, + beacon: &CardanoDbBeacon, + immutable_file_range: &RangeInclusive, + ) -> (PathBuf, CertificateMessage, VerifiedDigests) { + let cardano_db = DummyCardanoDbBuilder::new(dir_name) + .with_immutables(&immutable_file_range.clone().collect::>()) + .append_immutable_trio() + .build(); + let database_dir = cardano_db.get_dir(); + let immutable_digester = CardanoImmutableDigester::new( + "whatever".to_string(), + None, + TestLogger::stdout(), + ); + let computed_digests = immutable_digester + .compute_digests_for_range(database_dir, immutable_file_range) + .await + .unwrap(); + + let digests = computed_digests + .entries + .iter() + .map(|(immutable_file, digest)| { + (immutable_file.filename.clone(), digest.clone()) + }) + .collect::>(); + + let merkle_tree = immutable_digester + .compute_merkle_tree(database_dir, beacon) + .await + .unwrap(); + + let verified_digests = VerifiedDigests { + digests, + merkle_tree, + }; + + let certificate = { + let protocol_message_merkle_root = + verified_digests.merkle_tree.compute_root().unwrap().to_hex(); + let mut protocol_message = ProtocolMessage::new(); + protocol_message.set_message_part( + ProtocolMessagePartKey::CardanoDatabaseMerkleRoot, + protocol_message_merkle_root, + ); + + CertificateMessage { + protocol_message: protocol_message.clone(), + signed_message: protocol_message.compute_hash(), + ..CertificateMessage::dummy() + } + }; + + (database_dir.to_owned(), certificate, verified_digests) + } + + #[tokio::test] + async fn compute_cardano_database_message_succeeds() { + let beacon = CardanoDbBeacon { + epoch: Epoch(123), + immutable_file_number: 10, + }; + let immutable_file_range = 1..=15; + let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); + let (database_dir, certificate, verified_digests) = + prepare_db_and_verified_digests( + "compute_cardano_database_message_succeeds", + &beacon, + &immutable_file_range, + ) + .await; + + let message = MessageBuilder::new() + .compute_cardano_database_message( + &certificate, + &CardanoDatabaseSnapshotMessage::dummy(), + &immutable_file_range_to_prove, + &database_dir, + &verified_digests, + ) + .await + .unwrap(); + + assert!(certificate.match_message(&message)); + } + + #[tokio::test] + async fn compute_cardano_database_message_should_fail_if_immutable_is_missing() { + let beacon = CardanoDbBeacon { + epoch: Epoch(123), + immutable_file_number: 10, + }; + let immutable_file_range = 1..=15; + let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); + let (database_dir, certificate, verified_digests) = + prepare_db_and_verified_digests( + "compute_cardano_database_message_should_fail_if_immutable_is_missing", + &beacon, + &immutable_file_range, + ) + .await; + + let files_to_remove = vec!["00003.chunk", "00004.primary"]; + remove_immutable_files(&database_dir, &files_to_remove); + + let error = MessageBuilder::new() + .compute_cardano_database_message( + &certificate, + &CardanoDatabaseSnapshotMessage::dummy(), + &immutable_file_range_to_prove, + &database_dir, + &verified_digests, + ) + .await + .expect_err( + "compute_cardano_database_message should fail if a immutable is missing", + ); + + let expected_missing = files_to_remove + .into_iter() + .map(ImmutableFileName::from) + .collect::>(); + let expected_lists = ImmutableFilesLists { + dir_path: MessageBuilder::immutable_dir(&database_dir), + missing: expected_missing, + tampered: vec![], + }; + + assert!(matches!( + error, + ComputeCardanoDatabaseMessageError::ImmutableFiles(..) + )); + } + } } + //test with missing immutable and no tampered immutable + //test with tampered immutable and no missing immutable + //test with missing and tampered immutable } From b07677c0bd84f437d56a250d4ce7442ff8d47f1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Turmel?= Date: Thu, 10 Jul 2025 16:36:24 +0200 Subject: [PATCH 05/18] feature(client-lib): add function in VerifiedDigests to list tampered immutables files --- .../src/cardano_database_client/proving.rs | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/mithril-client/src/cardano_database_client/proving.rs b/mithril-client/src/cardano_database_client/proving.rs index 4651216299c..c68e6f4790e 100644 --- a/mithril-client/src/cardano_database_client/proving.rs +++ b/mithril-client/src/cardano_database_client/proving.rs @@ -32,6 +32,37 @@ pub struct VerifiedDigests { pub merkle_tree: MKTree, } +#[derive(PartialEq, Debug)] +pub(crate) struct InvalidImmutableFiles { + pub tampered_files: Vec, + pub non_verifiable_files: Vec, +} + +impl VerifiedDigests { + pub(crate) fn list_tampered_immutable_files( + &self, + computed_digests: &BTreeMap, + ) -> MithrilResult { + let mut tampered_files = vec![]; + let mut non_verifiable_files = vec![]; + + computed_digests.iter().for_each(|(immutable_file_name, digest)| { + if let Some(verified_digest) = self.digests.get(immutable_file_name) { + if verified_digest != digest { + tampered_files.push(immutable_file_name.clone()); + } + } else { + non_verifiable_files.push(immutable_file_name.clone()); + } + }); + + Ok(InvalidImmutableFiles { + tampered_files, + non_verifiable_files, + }) + } +} + pub struct InternalArtifactProver { http_file_downloader: Arc, logger: slog::Logger, @@ -211,6 +242,92 @@ mod tests { use super::*; + mod list_tampered_immutable_files { + + use super::*; + + #[test] + fn should_return_empty_list_when_no_tampered_files() { + let digests_to_verify = BTreeMap::from([ + ("00001.chunk".to_string(), "digest-1".to_string()), + ("00002.chunk".to_string(), "digest-2".to_string()), + ]); + + let verified_digests = VerifiedDigests { + digests: BTreeMap::from([ + ("00001.chunk".to_string(), "digest-1".to_string()), + ("00002.chunk".to_string(), "digest-2".to_string()), + ]), + merkle_tree: MKTree::new(&["whatever"]).unwrap(), + }; + + let invalid_files = verified_digests + .list_tampered_immutable_files(&digests_to_verify) + .unwrap(); + + assert_eq!( + invalid_files, + InvalidImmutableFiles { + tampered_files: vec![], + non_verifiable_files: vec![], + } + ); + } + + #[test] + fn should_return_list_with_tampered_files() { + let digests_to_verify = BTreeMap::from([ + ("00001.chunk".to_string(), "digest-1".to_string()), + ("00002.chunk".to_string(), "digest-2".to_string()), + ]); + + let verified_digests = VerifiedDigests { + digests: BTreeMap::from([ + ("00001.chunk".to_string(), "digest-1".to_string()), + ("00002.chunk".to_string(), "INVALID".to_string()), + ]), + merkle_tree: MKTree::new(&["whatever"]).unwrap(), + }; + + let invalid_files = verified_digests + .list_tampered_immutable_files(&digests_to_verify) + .unwrap(); + + assert_eq!( + invalid_files, + InvalidImmutableFiles { + tampered_files: vec!["00002.chunk".to_string()], + non_verifiable_files: vec![], + } + ); + } + + #[test] + fn should_return_list_with_non_verifiable() { + let digests_to_verify = BTreeMap::from([ + ("00001.chunk".to_string(), "digest-1".to_string()), + ("00002.not.verifiable".to_string(), "digest-2".to_string()), + ]); + + let verified_digests = VerifiedDigests { + digests: BTreeMap::from([("00001.chunk".to_string(), "digest-1".to_string())]), + merkle_tree: MKTree::new(&["whatever"]).unwrap(), + }; + + let invalid_files = verified_digests + .list_tampered_immutable_files(&digests_to_verify) + .unwrap(); + + assert_eq!( + invalid_files, + InvalidImmutableFiles { + tampered_files: vec![], + non_verifiable_files: vec!["00002.not.verifiable".to_string()], + } + ); + } + } + mod download_and_verify_digests { use mithril_common::{ StdResult, From 8fd5c7ab38c5889627695e27acc2a2623906ab66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Turmel?= Date: Thu, 10 Jul 2025 17:33:33 +0200 Subject: [PATCH 06/18] feature(client-lib): compute cardano database message verify tampered files --- .../src/cardano_database_client/proving.rs | 80 ++++--- mithril-client/src/message.rs | 197 ++++++++++++++---- 2 files changed, 203 insertions(+), 74 deletions(-) diff --git a/mithril-client/src/cardano_database_client/proving.rs b/mithril-client/src/cardano_database_client/proving.rs index c68e6f4790e..1ee4838594d 100644 --- a/mithril-client/src/cardano_database_client/proving.rs +++ b/mithril-client/src/cardano_database_client/proving.rs @@ -32,34 +32,40 @@ pub struct VerifiedDigests { pub merkle_tree: MKTree, } +/// Represents the immutable files that were not verified during the digest verification process. #[derive(PartialEq, Debug)] -pub(crate) struct InvalidImmutableFiles { +pub(crate) struct ImmutableFilesNotVerified { + /// List of immutable files that were tampered (i.e. their digest does not match the verified digest) pub tampered_files: Vec, + /// List of immutable files that could not be verified (i.e., not present in the digests) pub non_verifiable_files: Vec, } impl VerifiedDigests { - pub(crate) fn list_tampered_immutable_files( + pub(crate) fn list_immutable_files_not_verified( &self, - computed_digests: &BTreeMap, - ) -> MithrilResult { + computed_digests: &BTreeMap, + ) -> ImmutableFilesNotVerified { let mut tampered_files = vec![]; let mut non_verifiable_files = vec![]; - computed_digests.iter().for_each(|(immutable_file_name, digest)| { - if let Some(verified_digest) = self.digests.get(immutable_file_name) { - if verified_digest != digest { - tampered_files.push(immutable_file_name.clone()); + for (immutable_file, digest) in computed_digests.iter() { + let immutable_file_name_to_verify = immutable_file.filename.clone(); + match self.digests.get(&immutable_file_name_to_verify) { + Some(verified_digest) if verified_digest != digest => { + tampered_files.push(immutable_file_name_to_verify); } - } else { - non_verifiable_files.push(immutable_file_name.clone()); + None => { + non_verifiable_files.push(immutable_file_name_to_verify); + } + _ => {} } - }); + } - Ok(InvalidImmutableFiles { + ImmutableFilesNotVerified { tampered_files, non_verifiable_files, - }) + } } } @@ -97,7 +103,7 @@ impl InternalArtifactProver { Ok(()) } - ///Download digests and verify its authenticity against the certificate. + /// Download digests and verify its authenticity against the certificate. pub async fn download_and_verify_digests( &self, certificate: &CertificateMessage, @@ -242,15 +248,23 @@ mod tests { use super::*; - mod list_tampered_immutable_files { + mod list_immutable_files_not_verified { use super::*; + fn fake_immutable(filename: &str) -> ImmutableFile { + ImmutableFile { + path: PathBuf::from("whatever"), + number: 1, + filename: filename.to_string(), + } + } + #[test] fn should_return_empty_list_when_no_tampered_files() { let digests_to_verify = BTreeMap::from([ - ("00001.chunk".to_string(), "digest-1".to_string()), - ("00002.chunk".to_string(), "digest-2".to_string()), + (fake_immutable("00001.chunk"), "digest-1".to_string()), + (fake_immutable("00002.chunk"), "digest-2".to_string()), ]); let verified_digests = VerifiedDigests { @@ -261,13 +275,12 @@ mod tests { merkle_tree: MKTree::new(&["whatever"]).unwrap(), }; - let invalid_files = verified_digests - .list_tampered_immutable_files(&digests_to_verify) - .unwrap(); + let invalid_files = + verified_digests.list_immutable_files_not_verified(&digests_to_verify); assert_eq!( invalid_files, - InvalidImmutableFiles { + ImmutableFilesNotVerified { tampered_files: vec![], non_verifiable_files: vec![], } @@ -277,8 +290,8 @@ mod tests { #[test] fn should_return_list_with_tampered_files() { let digests_to_verify = BTreeMap::from([ - ("00001.chunk".to_string(), "digest-1".to_string()), - ("00002.chunk".to_string(), "digest-2".to_string()), + (fake_immutable("00001.chunk"), "digest-1".to_string()), + (fake_immutable("00002.chunk"), "digest-2".to_string()), ]); let verified_digests = VerifiedDigests { @@ -289,13 +302,12 @@ mod tests { merkle_tree: MKTree::new(&["whatever"]).unwrap(), }; - let invalid_files = verified_digests - .list_tampered_immutable_files(&digests_to_verify) - .unwrap(); + let invalid_files = + verified_digests.list_immutable_files_not_verified(&digests_to_verify); assert_eq!( invalid_files, - InvalidImmutableFiles { + ImmutableFilesNotVerified { tampered_files: vec!["00002.chunk".to_string()], non_verifiable_files: vec![], } @@ -305,8 +317,11 @@ mod tests { #[test] fn should_return_list_with_non_verifiable() { let digests_to_verify = BTreeMap::from([ - ("00001.chunk".to_string(), "digest-1".to_string()), - ("00002.not.verifiable".to_string(), "digest-2".to_string()), + (fake_immutable("00001.chunk"), "digest-1".to_string()), + ( + fake_immutable("00002.not.verifiable"), + "digest-2".to_string(), + ), ]); let verified_digests = VerifiedDigests { @@ -314,13 +329,12 @@ mod tests { merkle_tree: MKTree::new(&["whatever"]).unwrap(), }; - let invalid_files = verified_digests - .list_tampered_immutable_files(&digests_to_verify) - .unwrap(); + let invalid_files = + verified_digests.list_immutable_files_not_verified(&digests_to_verify); assert_eq!( invalid_files, - InvalidImmutableFiles { + ImmutableFilesNotVerified { tampered_files: vec![], non_verifiable_files: vec!["00002.not.verifiable".to_string()], } diff --git a/mithril-client/src/message.rs b/mithril-client/src/message.rs index e48acab9d07..e482a8a2b04 100644 --- a/mithril-client/src/message.rs +++ b/mithril-client/src/message.rs @@ -37,7 +37,7 @@ use crate::{ common::{ProtocolMessage, ProtocolMessagePartKey}, }; -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub struct ImmutableFilesLists { pub dir_path: PathBuf, pub missing: Vec, @@ -46,7 +46,7 @@ pub struct ImmutableFilesLists { #[derive(Error, Debug)] pub enum ComputeCardanoDatabaseMessageError { - ImmutableFiles(ImmutableFilesLists), + ImmutableFilesVerification(ImmutableFilesLists), ImmutableFilesDigester(#[from] ImmutableDigesterError), @@ -58,7 +58,7 @@ pub enum ComputeCardanoDatabaseMessageError { impl fmt::Display for ComputeCardanoDatabaseMessageError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - ComputeCardanoDatabaseMessageError::ImmutableFiles(files) => { + ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(files) => { write!(f, "immutable files: {:?}", files) } ComputeCardanoDatabaseMessageError::ImmutableFilesDigester(e) => { @@ -190,41 +190,53 @@ impl MessageBuilder { let immutable_file_number_range = immutable_file_range .to_range_inclusive(cardano_database_snapshot.beacon.immutable_file_number) .map_err(ComputeCardanoDatabaseMessageError::ImmutableFilesRange)?; - let missing_immutable_files = Self::list_missing_immutable_files(database_dir, &immutable_file_number_range); - let immutable_digester = CardanoImmutableDigester::new(network, None, self.logger.clone()); - let computed_digests = immutable_digester + let computed_digest_entries = immutable_digester .compute_digests_for_range(database_dir, &immutable_file_number_range) .await? - .entries + .entries; + let computed_digests = computed_digest_entries .values() .map(MKTreeNode::from) .collect::>(); - let merkle_proof = verified_digests.merkle_tree.compute_proof(&computed_digests).unwrap(); - merkle_proof - .verify() - .map_err(ComputeCardanoDatabaseMessageError::MerkleProofVerification)?; - - let mut message = certificate.protocol_message.clone(); - message.set_message_part( - ProtocolMessagePartKey::CardanoDatabaseMerkleRoot, - merkle_proof.root().to_hex(), - ); + let proof_result = verified_digests.merkle_tree.compute_proof(&computed_digests); + if let Ok(ref merkle_proof) = proof_result + && missing_immutable_files.is_empty() + { + merkle_proof + .verify() + .map_err(ComputeCardanoDatabaseMessageError::MerkleProofVerification)?; + + let mut message = certificate.protocol_message.clone(); + message.set_message_part( + ProtocolMessagePartKey::CardanoDatabaseMerkleRoot, + merkle_proof.root().to_hex(), + ); - if !missing_immutable_files.is_empty() { - return Err(ComputeCardanoDatabaseMessageError::ImmutableFiles( - ImmutableFilesLists { - dir_path: Self::immutable_dir(database_dir), - missing: missing_immutable_files, - tampered: vec![], - }, - )); + return Ok(message); } - Ok(message) + let tampered_files = match proof_result { + // TODO: we need to handle the error properly. + // It's not possible yet since we cannot filter/match on a returned error type since the `compute_proof` method returns an `anyhow::Error` + Err(_e) => { + verified_digests + .list_immutable_files_not_verified(&computed_digest_entries) + .tampered_files + } + Ok(_) => vec![], + }; + + Err( + ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(ImmutableFilesLists { + dir_path: Self::immutable_dir(database_dir), + missing: missing_immutable_files, + tampered: tampered_files, + }), + ) } } @@ -317,6 +329,14 @@ mod tests { } } + fn tamper_immutable_files>(database_dir: &Path, immutable_file_names: &[T]) { + for immutable_file_name in immutable_file_names { + let immutable_file_path = + MessageBuilder::immutable_dir(database_dir).join(immutable_file_name.as_ref()); + std::fs::write(immutable_file_path, "tampered content").unwrap(); + } + } + mod list_missing_immutable_files { use mithril_common::temp_dir_create; @@ -453,6 +473,10 @@ mod tests { (database_dir.to_owned(), certificate, verified_digests) } + fn to_vec_immutable_file_name(list: &[&str]) -> Vec { + list.iter().map(|s| ImmutableFileName::from(*s)).collect() + } + #[tokio::test] async fn compute_cardano_database_message_succeeds() { let beacon = CardanoDbBeacon { @@ -515,24 +539,115 @@ mod tests { "compute_cardano_database_message should fail if a immutable is missing", ); - let expected_missing = files_to_remove - .into_iter() - .map(ImmutableFileName::from) - .collect::>(); - let expected_lists = ImmutableFilesLists { - dir_path: MessageBuilder::immutable_dir(&database_dir), - missing: expected_missing, - tampered: vec![], + let error_lists = match error { + ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(lists) => lists, + _ => panic!("Expected ImmutableFilesVerification error, got: {error}"), + }; + + assert_eq!( + error_lists, + ImmutableFilesLists { + dir_path: MessageBuilder::immutable_dir(&database_dir), + missing: to_vec_immutable_file_name(&files_to_remove), + tampered: vec![], + } + ); + } + + #[tokio::test] + async fn compute_cardano_database_message_should_fail_if_immutable_is_tampered() { + let beacon = CardanoDbBeacon { + epoch: Epoch(123), + immutable_file_number: 10, + }; + let immutable_file_range = 1..=15; + let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); + let (database_dir, certificate, verified_digests) = + prepare_db_and_verified_digests( + "compute_cardano_database_message_should_fail_if_immutable_is_tampered", + &beacon, + &immutable_file_range, + ) + .await; + + let files_to_tamper = vec!["00003.chunk", "00004.primary"]; + tamper_immutable_files(&database_dir, &files_to_tamper); + + let error = MessageBuilder::new() + .compute_cardano_database_message( + &certificate, + &CardanoDatabaseSnapshotMessage::dummy(), + &immutable_file_range_to_prove, + &database_dir, + &verified_digests, + ) + .await + .expect_err( + "compute_cardano_database_message should fail if a immutable is missing", + ); + + let error_lists = match error { + ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(lists) => lists, + _ => panic!("Expected ImmutableFilesVerification error, got: {error}"), }; + assert_eq!( + error_lists, + ImmutableFilesLists { + dir_path: MessageBuilder::immutable_dir(&database_dir), + missing: vec![], + tampered: to_vec_immutable_file_name(&files_to_tamper), + } + ) + } - assert!(matches!( - error, - ComputeCardanoDatabaseMessageError::ImmutableFiles(..) - )); + #[tokio::test] + async fn compute_cardano_database_message_should_fail_if_immutables_are_missing_and_tampered() + { + let beacon = CardanoDbBeacon { + epoch: Epoch(123), + immutable_file_number: 10, + }; + let immutable_file_range = 1..=15; + let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); + let (database_dir, certificate, verified_digests) = + prepare_db_and_verified_digests( + "compute_cardano_database_message_should_fail_if_immutables_are_missing_and_tampered", + &beacon, + &immutable_file_range, + ) + .await; + + let files_to_remove = vec!["00003.chunk"]; + let files_to_tamper = vec!["00004.primary"]; + remove_immutable_files(&database_dir, &files_to_remove); + tamper_immutable_files(&database_dir, &files_to_tamper); + + let error = MessageBuilder::new() + .compute_cardano_database_message( + &certificate, + &CardanoDatabaseSnapshotMessage::dummy(), + &immutable_file_range_to_prove, + &database_dir, + &verified_digests, + ) + .await + .expect_err( + "compute_cardano_database_message should fail if a immutable is missing", + ); + + let error_lists = match error { + ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(lists) => lists, + _ => panic!("Expected ImmutableFilesVerification error, got: {error}"), + }; + assert_eq!( + error_lists, + ImmutableFilesLists { + dir_path: MessageBuilder::immutable_dir(&database_dir), + missing: to_vec_immutable_file_name(&files_to_remove), + tampered: to_vec_immutable_file_name(&files_to_tamper), + } + ) } } } - //test with missing immutable and no tampered immutable - //test with tampered immutable and no missing immutable - //test with missing and tampered immutable } From 3b4bb5eef6ef93f83012426cf803991ba1184dec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Turmel?= Date: Fri, 11 Jul 2025 12:25:46 +0200 Subject: [PATCH 07/18] feature(client-lib): display 10 elements of each missing and/or tampered immutables files in case of ImmutableFilesVerification error --- mithril-client/src/message.rs | 802 ++++++++++++++++++++-------------- 1 file changed, 475 insertions(+), 327 deletions(-) diff --git a/mithril-client/src/message.rs b/mithril-client/src/message.rs index e482a8a2b04..f7c394af11d 100644 --- a/mithril-client/src/message.rs +++ b/mithril-client/src/message.rs @@ -1,12 +1,13 @@ use anyhow::Context; +#[cfg(feature = "fs")] +use slog::warn; use slog::{Logger, o}; #[cfg(feature = "fs")] use std::fmt; #[cfg(feature = "fs")] use std::ops::RangeInclusive; #[cfg(feature = "fs")] -use std::path::Path; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; #[cfg(feature = "fs")] use std::sync::Arc; #[cfg(feature = "fs")] @@ -19,15 +20,15 @@ use mithril_cardano_node_internal_database::{ IMMUTABLE_DIR, digesters::{CardanoImmutableDigester, ImmutableDigester, ImmutableDigesterError}, }; +use mithril_common::logging::LoggerExtensions; use mithril_common::protocol::SignerBuilder; use mithril_common::signable_builder::CardanoStakeDistributionSignableBuilder; #[cfg(feature = "fs")] use mithril_common::{ crypto_helper::MKTreeNode, entities::{ImmutableFileName, ImmutableFileNumber, SignedEntityType}, - messages::CertificateMessage, + messages::{CardanoDatabaseSnapshotMessage, CertificateMessage}, }; -use mithril_common::{logging::LoggerExtensions, messages::CardanoDatabaseSnapshotMessage}; #[cfg(feature = "fs")] use crate::MithrilError; @@ -37,38 +38,83 @@ use crate::{ common::{ProtocolMessage, ProtocolMessagePartKey}, }; -#[derive(Debug, PartialEq)] -pub struct ImmutableFilesLists { - pub dir_path: PathBuf, - pub missing: Vec, - pub tampered: Vec, -} +cfg_fs! { + const MERKLE_PROOF_COMPUTATION_ERROR:&str = "Merkle proof computation failed"; + + /// Type containing the lists of immutable files that are missing or tampered. + #[derive(Debug, PartialEq)] + pub struct ImmutableFilesLists { + /// The immutables files directory. + pub immutables_dir: PathBuf, + /// List of missing immutable files. + pub missing: Vec, + /// List of tampered immutable files. + pub tampered: Vec, + } -#[derive(Error, Debug)] -pub enum ComputeCardanoDatabaseMessageError { - ImmutableFilesVerification(ImmutableFilesLists), + /// Compute Cardano database message related errors. + #[derive(Error, Debug)] + pub enum ComputeCardanoDatabaseMessageError { + /// Error related to the verification of immutable files. + ImmutableFilesVerification(ImmutableFilesLists), - ImmutableFilesDigester(#[from] ImmutableDigesterError), + /// Error related to the immutable files digests computation. + ImmutableFilesDigester(#[from] ImmutableDigesterError), - MerkleProofVerification(#[source] MithrilError), + /// Error related to the Merkle proof verification. + MerkleProofVerification(#[source] MithrilError), - ImmutableFilesRange(#[source] MithrilError), -} + /// Error related to the immutable files range. + ImmutableFilesRange(#[source] MithrilError), + } -impl fmt::Display for ComputeCardanoDatabaseMessageError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(files) => { - write!(f, "immutable files: {:?}", files) - } - ComputeCardanoDatabaseMessageError::ImmutableFilesDigester(e) => { - write!(f, "Immutable files digester error: {}", e) - } - ComputeCardanoDatabaseMessageError::MerkleProofVerification(e) => { - write!(f, "Merkle proof verification error: {}", e) - } - ComputeCardanoDatabaseMessageError::ImmutableFilesRange(e) => { - write!(f, "Immutable files range error: {}", e) + impl fmt::Display for ComputeCardanoDatabaseMessageError { + + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(lists) => { + fn get_first_10_files_path( + files: &[ImmutableFileName], + immutables_dir: &Path, + ) -> String { + files + .iter() + .take(10) + .map(|file| immutables_dir.join(file).to_string_lossy().to_string()) + .collect::>() + .join("\n") + } + + let missing_files_subset = get_first_10_files_path(&lists.missing, &lists.immutables_dir); + let tampered_files_subset = get_first_10_files_path(&lists.tampered, &lists.immutables_dir); + if !lists.missing.is_empty() { + writeln!( + f, + "Number of missing immutable files: {}", + lists.missing.len() + )?; + writeln!(f, "First 10 missing immutable files paths:")?; + writeln!(f, "{missing_files_subset}")?; + } + if !lists.missing.is_empty() && !lists.tampered.is_empty() { + writeln!(f)?; + } + if !lists.tampered.is_empty() { + writeln!(f,"Number of tampered immutable files: {}",lists.tampered.len())?; + writeln!(f, "First 10 tampered immutable files paths:")?; + writeln!(f, "{tampered_files_subset}")?; + } + Ok(()) + } + ComputeCardanoDatabaseMessageError::ImmutableFilesDigester(e) => { + write!(f, "Immutable files digester error: {e:?}") + } + ComputeCardanoDatabaseMessageError::MerkleProofVerification(e) => { + write!(f, "Merkle proof verification error: {e:?}") + } + ComputeCardanoDatabaseMessageError::ImmutableFilesRange(e) => { + write!(f, "Immutable files range error: {e:?}") + } } } } @@ -220,19 +266,17 @@ impl MessageBuilder { } let tampered_files = match proof_result { - // TODO: we need to handle the error properly. - // It's not possible yet since we cannot filter/match on a returned error type since the `compute_proof` method returns an `anyhow::Error` - Err(_e) => { + Err(e) => { + warn!(self.logger, "{MERKLE_PROOF_COMPUTATION_ERROR}: {e:}"); verified_digests .list_immutable_files_not_verified(&computed_digest_entries) .tampered_files } Ok(_) => vec![], }; - Err( ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(ImmutableFilesLists { - dir_path: Self::immutable_dir(database_dir), + immutables_dir: Self::immutable_dir(database_dir), missing: missing_immutable_files, tampered: tampered_files, }), @@ -312,304 +356,297 @@ impl Default for MessageBuilder { } #[cfg(test)] +#[cfg(feature = "fs")] mod tests { + use mithril_cardano_node_internal_database::test::DummyCardanoDbBuilder; + use super::*; - #[cfg(feature = "fs")] - mod fs { - use mithril_cardano_node_internal_database::test::DummyCardanoDbBuilder; + fn remove_immutable_files>(database_dir: &Path, immutable_file_names: &[T]) { + for immutable_file_name in immutable_file_names { + let immutable_file_path = + MessageBuilder::immutable_dir(database_dir).join(immutable_file_name.as_ref()); + std::fs::remove_file(immutable_file_path).unwrap(); + } + } + + fn tamper_immutable_files>(database_dir: &Path, immutable_file_names: &[T]) { + for immutable_file_name in immutable_file_names { + let immutable_file_path = + MessageBuilder::immutable_dir(database_dir).join(immutable_file_name.as_ref()); + std::fs::write(immutable_file_path, "tampered content").unwrap(); + } + } + + mod list_missing_immutable_files { + use mithril_common::temp_dir_create; use super::*; - fn remove_immutable_files>(database_dir: &Path, immutable_file_names: &[T]) { - for immutable_file_name in immutable_file_names { - let immutable_file_path = - MessageBuilder::immutable_dir(database_dir).join(immutable_file_name.as_ref()); - std::fs::remove_file(immutable_file_path).unwrap(); - } - } + #[test] + fn list_missing_immutable_files_should_return_empty_list_if_no_missing_files() { + let immutable_files_in_db = 1..=10; + let range_to_verify = 3..=5; + let cardano_db = + DummyCardanoDbBuilder::new(&format!("{}", temp_dir_create!().display())) + .with_immutables(&immutable_files_in_db.collect::>()) + .append_immutable_trio() + .build(); - fn tamper_immutable_files>(database_dir: &Path, immutable_file_names: &[T]) { - for immutable_file_name in immutable_file_names { - let immutable_file_path = - MessageBuilder::immutable_dir(database_dir).join(immutable_file_name.as_ref()); - std::fs::write(immutable_file_path, "tampered content").unwrap(); - } + let missing_files = MessageBuilder::list_missing_immutable_files( + cardano_db.get_dir(), + &range_to_verify, + ); + + assert!(missing_files.is_empty()); } - mod list_missing_immutable_files { - use mithril_common::temp_dir_create; + #[test] + fn list_missing_immutable_files_should_return_empty_list_if_missing_files_outside_range() { + let immutable_files_in_db = 1..=10; + let range_to_verify = 3..=5; + let cardano_db = + DummyCardanoDbBuilder::new(&format!("{}", temp_dir_create!().display())) + .with_immutables(&immutable_files_in_db.collect::>()) + .append_immutable_trio() + .build(); + let files_to_remove = vec!["00002.chunk", "00006.primary"]; + remove_immutable_files(cardano_db.get_dir(), &files_to_remove); - use super::*; + let missing_files = MessageBuilder::list_missing_immutable_files( + cardano_db.get_dir(), + &range_to_verify, + ); - #[test] - fn list_missing_immutable_files_should_return_empty_list_if_no_missing_files() { - let immutable_files_in_db = 1..=10; - let range_to_verify = 3..=5; - let cardano_db = - DummyCardanoDbBuilder::new(&format!("{}", temp_dir_create!().display())) - .with_immutables(&immutable_files_in_db.collect::>()) - .append_immutable_trio() - .build(); + assert!(missing_files.is_empty()); + } - let missing_files = MessageBuilder::list_missing_immutable_files( - cardano_db.get_dir(), - &range_to_verify, - ); + #[test] + fn list_missing_immutable_files_should_return_list_of_missing_files_inside_range() { + let immutable_files_in_db = 1..=10; + let range_to_verify = 3..=5; + let cardano_db = + DummyCardanoDbBuilder::new(&format!("{}", temp_dir_create!().display())) + .with_immutables(&immutable_files_in_db.collect::>()) + .append_immutable_trio() + .build(); + let files_to_remove = vec!["00004.chunk", "00005.primary"]; + remove_immutable_files(cardano_db.get_dir(), &files_to_remove); - assert!(missing_files.is_empty()); - } + let missing_files = MessageBuilder::list_missing_immutable_files( + cardano_db.get_dir(), + &range_to_verify, + ); - #[test] - fn list_missing_immutable_files_should_return_empty_list_if_missing_files_outside_range() - { - let immutable_files_in_db = 1..=10; - let range_to_verify = 3..=5; - let cardano_db = - DummyCardanoDbBuilder::new(&format!("{}", temp_dir_create!().display())) - .with_immutables(&immutable_files_in_db.collect::>()) - .append_immutable_trio() - .build(); - let files_to_remove = vec!["00002.chunk", "00006.primary"]; - remove_immutable_files(cardano_db.get_dir(), &files_to_remove); - - let missing_files = MessageBuilder::list_missing_immutable_files( - cardano_db.get_dir(), - &range_to_verify, - ); + assert_eq!(missing_files, files_to_remove); + } + } - assert!(missing_files.is_empty()); - } + mod cardano_database_message { - #[test] - fn list_missing_immutable_files_should_return_list_of_missing_files_inside_range() { - let immutable_files_in_db = 1..=10; - let range_to_verify = 3..=5; - let cardano_db = - DummyCardanoDbBuilder::new(&format!("{}", temp_dir_create!().display())) - .with_immutables(&immutable_files_in_db.collect::>()) - .append_immutable_trio() - .build(); - let files_to_remove = vec!["00004.chunk", "00005.primary"]; - remove_immutable_files(cardano_db.get_dir(), &files_to_remove); - - let missing_files = MessageBuilder::list_missing_immutable_files( - cardano_db.get_dir(), - &range_to_verify, - ); + use std::{collections::BTreeMap, path::PathBuf}; - assert_eq!(missing_files, files_to_remove); - } - } + use mithril_common::{ + entities::{CardanoDbBeacon, Epoch}, + messages::CertificateMessage, + }; - mod cardano_database_message { + use crate::cardano_database_client::ImmutableFileRange; + use crate::{cardano_database_client::VerifiedDigests, test_utils::TestLogger}; - use std::{collections::BTreeMap, path::PathBuf}; + use super::*; - use mithril_common::{ - entities::{CardanoDbBeacon, Epoch}, - messages::CertificateMessage, - }; + async fn prepare_db_and_verified_digests( + dir_name: &str, + beacon: &CardanoDbBeacon, + immutable_file_range: &RangeInclusive, + ) -> (PathBuf, CertificateMessage, VerifiedDigests) { + let cardano_db = DummyCardanoDbBuilder::new(dir_name) + .with_immutables(&immutable_file_range.clone().collect::>()) + .append_immutable_trio() + .build(); + let database_dir = cardano_db.get_dir(); + let immutable_digester = + CardanoImmutableDigester::new("whatever".to_string(), None, TestLogger::stdout()); + let computed_digests = immutable_digester + .compute_digests_for_range(database_dir, immutable_file_range) + .await + .unwrap(); - use crate::cardano_database_client::ImmutableFileRange; - use crate::{cardano_database_client::VerifiedDigests, test_utils::TestLogger}; + let digests = computed_digests + .entries + .iter() + .map(|(immutable_file, digest)| (immutable_file.filename.clone(), digest.clone())) + .collect::>(); - use super::*; + let merkle_tree = immutable_digester + .compute_merkle_tree(database_dir, beacon) + .await + .unwrap(); - async fn prepare_db_and_verified_digests( - dir_name: &str, - beacon: &CardanoDbBeacon, - immutable_file_range: &RangeInclusive, - ) -> (PathBuf, CertificateMessage, VerifiedDigests) { - let cardano_db = DummyCardanoDbBuilder::new(dir_name) - .with_immutables(&immutable_file_range.clone().collect::>()) - .append_immutable_trio() - .build(); - let database_dir = cardano_db.get_dir(); - let immutable_digester = CardanoImmutableDigester::new( - "whatever".to_string(), - None, - TestLogger::stdout(), + let verified_digests = VerifiedDigests { + digests, + merkle_tree, + }; + + let certificate = { + let protocol_message_merkle_root = + verified_digests.merkle_tree.compute_root().unwrap().to_hex(); + let mut protocol_message = ProtocolMessage::new(); + protocol_message.set_message_part( + ProtocolMessagePartKey::CardanoDatabaseMerkleRoot, + protocol_message_merkle_root, ); - let computed_digests = immutable_digester - .compute_digests_for_range(database_dir, immutable_file_range) - .await - .unwrap(); - - let digests = computed_digests - .entries - .iter() - .map(|(immutable_file, digest)| { - (immutable_file.filename.clone(), digest.clone()) - }) - .collect::>(); - - let merkle_tree = immutable_digester - .compute_merkle_tree(database_dir, beacon) - .await - .unwrap(); - - let verified_digests = VerifiedDigests { - digests, - merkle_tree, - }; - - let certificate = { - let protocol_message_merkle_root = - verified_digests.merkle_tree.compute_root().unwrap().to_hex(); - let mut protocol_message = ProtocolMessage::new(); - protocol_message.set_message_part( - ProtocolMessagePartKey::CardanoDatabaseMerkleRoot, - protocol_message_merkle_root, - ); - - CertificateMessage { - protocol_message: protocol_message.clone(), - signed_message: protocol_message.compute_hash(), - ..CertificateMessage::dummy() - } - }; - (database_dir.to_owned(), certificate, verified_digests) - } + CertificateMessage { + protocol_message: protocol_message.clone(), + signed_message: protocol_message.compute_hash(), + ..CertificateMessage::dummy() + } + }; - fn to_vec_immutable_file_name(list: &[&str]) -> Vec { - list.iter().map(|s| ImmutableFileName::from(*s)).collect() - } + (database_dir.to_owned(), certificate, verified_digests) + } - #[tokio::test] - async fn compute_cardano_database_message_succeeds() { - let beacon = CardanoDbBeacon { - epoch: Epoch(123), - immutable_file_number: 10, - }; - let immutable_file_range = 1..=15; - let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); - let (database_dir, certificate, verified_digests) = - prepare_db_and_verified_digests( - "compute_cardano_database_message_succeeds", - &beacon, - &immutable_file_range, - ) - .await; + fn to_vec_immutable_file_name(list: &[&str]) -> Vec { + list.iter().map(|s| ImmutableFileName::from(*s)).collect() + } - let message = MessageBuilder::new() - .compute_cardano_database_message( - &certificate, - &CardanoDatabaseSnapshotMessage::dummy(), - &immutable_file_range_to_prove, - &database_dir, - &verified_digests, - ) - .await - .unwrap(); + #[tokio::test] + async fn compute_cardano_database_message_succeeds() { + let beacon = CardanoDbBeacon { + epoch: Epoch(123), + immutable_file_number: 10, + }; + let immutable_file_range = 1..=15; + let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); + let (database_dir, certificate, verified_digests) = prepare_db_and_verified_digests( + "compute_cardano_database_message_succeeds", + &beacon, + &immutable_file_range, + ) + .await; + + let message = MessageBuilder::new() + .compute_cardano_database_message( + &certificate, + &CardanoDatabaseSnapshotMessage::dummy(), + &immutable_file_range_to_prove, + &database_dir, + &verified_digests, + ) + .await + .unwrap(); - assert!(certificate.match_message(&message)); - } + assert!(certificate.match_message(&message)); + } - #[tokio::test] - async fn compute_cardano_database_message_should_fail_if_immutable_is_missing() { - let beacon = CardanoDbBeacon { - epoch: Epoch(123), - immutable_file_number: 10, - }; - let immutable_file_range = 1..=15; - let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); - let (database_dir, certificate, verified_digests) = - prepare_db_and_verified_digests( - "compute_cardano_database_message_should_fail_if_immutable_is_missing", - &beacon, - &immutable_file_range, - ) - .await; + #[tokio::test] + async fn compute_cardano_database_message_should_fail_if_immutable_is_missing() { + let beacon = CardanoDbBeacon { + epoch: Epoch(123), + immutable_file_number: 10, + }; + let immutable_file_range = 1..=15; + let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); + let (database_dir, certificate, verified_digests) = prepare_db_and_verified_digests( + "compute_cardano_database_message_should_fail_if_immutable_is_missing", + &beacon, + &immutable_file_range, + ) + .await; + + let files_to_remove = vec!["00003.chunk", "00004.primary"]; + remove_immutable_files(&database_dir, &files_to_remove); + + let error = MessageBuilder::new() + .compute_cardano_database_message( + &certificate, + &CardanoDatabaseSnapshotMessage::dummy(), + &immutable_file_range_to_prove, + &database_dir, + &verified_digests, + ) + .await + .expect_err( + "compute_cardano_database_message should fail if a immutable is missing", + ); + + let error_lists = match error { + ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(lists) => lists, + _ => panic!("Expected ImmutableFilesVerification error, got: {error}"), + }; - let files_to_remove = vec!["00003.chunk", "00004.primary"]; - remove_immutable_files(&database_dir, &files_to_remove); + assert_eq!( + error_lists, + ImmutableFilesLists { + immutables_dir: MessageBuilder::immutable_dir(&database_dir), + missing: to_vec_immutable_file_name(&files_to_remove), + tampered: vec![], + } + ); + } - let error = MessageBuilder::new() - .compute_cardano_database_message( - &certificate, - &CardanoDatabaseSnapshotMessage::dummy(), - &immutable_file_range_to_prove, - &database_dir, - &verified_digests, - ) - .await - .expect_err( - "compute_cardano_database_message should fail if a immutable is missing", - ); - - let error_lists = match error { - ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(lists) => lists, - _ => panic!("Expected ImmutableFilesVerification error, got: {error}"), - }; - - assert_eq!( - error_lists, - ImmutableFilesLists { - dir_path: MessageBuilder::immutable_dir(&database_dir), - missing: to_vec_immutable_file_name(&files_to_remove), - tampered: vec![], - } - ); - } + #[tokio::test] + async fn compute_cardano_database_message_should_fail_if_immutable_is_tampered() { + let beacon = CardanoDbBeacon { + epoch: Epoch(123), + immutable_file_number: 10, + }; + let immutable_file_range = 1..=15; + let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); + let (database_dir, certificate, verified_digests) = prepare_db_and_verified_digests( + "compute_cardano_database_message_should_fail_if_immutable_is_tampered", + &beacon, + &immutable_file_range, + ) + .await; - #[tokio::test] - async fn compute_cardano_database_message_should_fail_if_immutable_is_tampered() { - let beacon = CardanoDbBeacon { - epoch: Epoch(123), - immutable_file_number: 10, - }; - let immutable_file_range = 1..=15; - let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); - let (database_dir, certificate, verified_digests) = - prepare_db_and_verified_digests( - "compute_cardano_database_message_should_fail_if_immutable_is_tampered", - &beacon, - &immutable_file_range, - ) - .await; + let files_to_tamper = vec!["00003.chunk", "00004.primary"]; + tamper_immutable_files(&database_dir, &files_to_tamper); - let files_to_tamper = vec!["00003.chunk", "00004.primary"]; - tamper_immutable_files(&database_dir, &files_to_tamper); + let (logger, log_inspector) = TestLogger::memory(); - let error = MessageBuilder::new() - .compute_cardano_database_message( - &certificate, - &CardanoDatabaseSnapshotMessage::dummy(), - &immutable_file_range_to_prove, - &database_dir, - &verified_digests, - ) - .await - .expect_err( - "compute_cardano_database_message should fail if a immutable is missing", - ); - - let error_lists = match error { - ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(lists) => lists, - _ => panic!("Expected ImmutableFilesVerification error, got: {error}"), - }; - assert_eq!( - error_lists, - ImmutableFilesLists { - dir_path: MessageBuilder::immutable_dir(&database_dir), - missing: vec![], - tampered: to_vec_immutable_file_name(&files_to_tamper), - } + let error = MessageBuilder::new() + .with_logger(logger) + .compute_cardano_database_message( + &certificate, + &CardanoDatabaseSnapshotMessage::dummy(), + &immutable_file_range_to_prove, + &database_dir, + &verified_digests, ) - } + .await + .expect_err( + "compute_cardano_database_message should fail if a immutable is missing", + ); + + assert!(log_inspector.contains_log(MERKLE_PROOF_COMPUTATION_ERROR)); - #[tokio::test] - async fn compute_cardano_database_message_should_fail_if_immutables_are_missing_and_tampered() - { - let beacon = CardanoDbBeacon { - epoch: Epoch(123), - immutable_file_number: 10, - }; - let immutable_file_range = 1..=15; - let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); - let (database_dir, certificate, verified_digests) = + let error_lists = match error { + ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(lists) => lists, + _ => panic!("Expected ImmutableFilesVerification error, got: {error}"), + }; + assert_eq!( + error_lists, + ImmutableFilesLists { + immutables_dir: MessageBuilder::immutable_dir(&database_dir), + missing: vec![], + tampered: to_vec_immutable_file_name(&files_to_tamper), + } + ) + } + + #[tokio::test] + async fn compute_cardano_database_message_should_fail_if_immutables_are_missing_and_tampered() + { + let beacon = CardanoDbBeacon { + epoch: Epoch(123), + immutable_file_number: 10, + }; + let immutable_file_range = 1..=15; + let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); + let (database_dir, certificate, verified_digests) = prepare_db_and_verified_digests( "compute_cardano_database_message_should_fail_if_immutables_are_missing_and_tampered", &beacon, @@ -617,37 +654,148 @@ mod tests { ) .await; - let files_to_remove = vec!["00003.chunk"]; - let files_to_tamper = vec!["00004.primary"]; - remove_immutable_files(&database_dir, &files_to_remove); - tamper_immutable_files(&database_dir, &files_to_tamper); - - let error = MessageBuilder::new() - .compute_cardano_database_message( - &certificate, - &CardanoDatabaseSnapshotMessage::dummy(), - &immutable_file_range_to_prove, - &database_dir, - &verified_digests, - ) - .await - .expect_err( - "compute_cardano_database_message should fail if a immutable is missing", - ); - - let error_lists = match error { - ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(lists) => lists, - _ => panic!("Expected ImmutableFilesVerification error, got: {error}"), - }; - assert_eq!( - error_lists, - ImmutableFilesLists { - dir_path: MessageBuilder::immutable_dir(&database_dir), - missing: to_vec_immutable_file_name(&files_to_remove), - tampered: to_vec_immutable_file_name(&files_to_tamper), - } + let files_to_remove = vec!["00003.chunk"]; + let files_to_tamper = vec!["00004.primary"]; + remove_immutable_files(&database_dir, &files_to_remove); + tamper_immutable_files(&database_dir, &files_to_tamper); + + let error = MessageBuilder::new() + .compute_cardano_database_message( + &certificate, + &CardanoDatabaseSnapshotMessage::dummy(), + &immutable_file_range_to_prove, + &database_dir, + &verified_digests, ) - } + .await + .expect_err( + "compute_cardano_database_message should fail if a immutable is missing", + ); + + let error_lists = match error { + ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(lists) => lists, + _ => panic!("Expected ImmutableFilesVerification error, got: {error}"), + }; + assert_eq!( + error_lists, + ImmutableFilesLists { + immutables_dir: MessageBuilder::immutable_dir(&database_dir), + missing: to_vec_immutable_file_name(&files_to_remove), + tampered: to_vec_immutable_file_name(&files_to_tamper), + } + ) + } + } + + mod compute_cardano_database_message_error { + use super::*; + + fn generate_immutable_files_verification_error( + missing_range: Option>, + tampered_range: Option>, + immutable_path: &str, + ) -> ComputeCardanoDatabaseMessageError { + let missing: Vec = match missing_range { + Some(range) => range + .map(|i| ImmutableFileName::from(format!("{i:05}.chunk"))) + .collect(), + None => vec![], + }; + let tampered: Vec = match tampered_range { + Some(range) => range + .map(|i| ImmutableFileName::from(format!("{i:05}.chunk"))) + .collect(), + None => vec![], + }; + + ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(ImmutableFilesLists { + immutables_dir: PathBuf::from(immutable_path), + missing, + tampered, + }) + } + + fn normalize_path_separators(s: &str) -> String { + s.replace('\\', "/") + } + + #[test] + fn display_immutable_files_verification_error_should_displayed_lists_with_10_elements() { + let error = generate_immutable_files_verification_error( + Some(1..=15), + Some(20..=31), + "/path/to/immutables", + ); + + let display = normalize_path_separators(&format!("{error}")); + + assert_eq!( + display, + r###"Number of missing immutable files: 15 +First 10 missing immutable files paths: +/path/to/immutables/00001.chunk +/path/to/immutables/00002.chunk +/path/to/immutables/00003.chunk +/path/to/immutables/00004.chunk +/path/to/immutables/00005.chunk +/path/to/immutables/00006.chunk +/path/to/immutables/00007.chunk +/path/to/immutables/00008.chunk +/path/to/immutables/00009.chunk +/path/to/immutables/00010.chunk + +Number of tampered immutable files: 12 +First 10 tampered immutable files paths: +/path/to/immutables/00020.chunk +/path/to/immutables/00021.chunk +/path/to/immutables/00022.chunk +/path/to/immutables/00023.chunk +/path/to/immutables/00024.chunk +/path/to/immutables/00025.chunk +/path/to/immutables/00026.chunk +/path/to/immutables/00027.chunk +/path/to/immutables/00028.chunk +/path/to/immutables/00029.chunk +"### + ); + } + + #[test] + fn display_immutable_files_should_not_display_error_list_if_missing_is_empty() { + let error = generate_immutable_files_verification_error( + None, + Some(1..=1), + "/path/to/immutables", + ); + + let display = normalize_path_separators(&format!("{error}")); + + assert_eq!( + display, + r###"Number of tampered immutable files: 1 +First 10 tampered immutable files paths: +/path/to/immutables/00001.chunk +"### + ); + } + + #[test] + fn display_immutable_files_should_not_display_error_list_if_tampered_is_empty() { + let error = generate_immutable_files_verification_error( + Some(1..=1), + None, + "/path/to/immutables", + ); + + let display = normalize_path_separators(&format!("{error}")); + + assert_eq!( + display, + r###"Number of missing immutable files: 1 +First 10 missing immutable files paths: +/path/to/immutables/00001.chunk +"### + ); } } } From e9796322877f05cd28a087b58b0d06d9e014c130 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Turmel?= Date: Fri, 11 Jul 2025 16:34:39 +0200 Subject: [PATCH 08/18] feature(client-cli): adapt call of compute_cardano_database_message to handle the new typed error --- .../src/commands/cardano_db/verify.rs | 55 +++++++++++-------- mithril-client-cli/src/utils/cardano_db.rs | 11 ++-- 2 files changed, 40 insertions(+), 26 deletions(-) diff --git a/mithril-client-cli/src/commands/cardano_db/verify.rs b/mithril-client-cli/src/commands/cardano_db/verify.rs index 53d52ca3b2b..01c37dec21b 100644 --- a/mithril-client-cli/src/commands/cardano_db/verify.rs +++ b/mithril-client-cli/src/commands/cardano_db/verify.rs @@ -4,10 +4,10 @@ use std::{ sync::Arc, }; -use anyhow::Context; +use anyhow::{Context, anyhow}; use chrono::Utc; use clap::Parser; -use mithril_client::MithrilResult; +use mithril_client::{ComputeCardanoDatabaseMessageError, MithrilResult}; use crate::{ CommandContext, @@ -122,26 +122,37 @@ impl CardanoDbVerifyCommand { db_dir, &verified_digests, ) - .await?; - - shared_steps::verify_message_matches_certificate( - &context.logger().clone(), - 4, - &progress_printer, - &certificate, - &message, - &cardano_db_message, - db_dir, - ) - .await?; - - Self::log_verified_information( - db_dir, - &cardano_db_message.hash, - context.is_json_output_enabled(), - )?; - - Ok(()) + .await; + + match message { + Err(e) => match e.downcast_ref::() { + Some(ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(lists)) => { + // let missing_files = lists.missing; + Ok(()) + } + _ => Err(e), + }, + Ok(message) => { + shared_steps::verify_message_matches_certificate( + &context.logger().clone(), + 4, + &progress_printer, + &certificate, + &message, + &cardano_db_message, + db_dir, + ) + .await?; + + Self::log_verified_information( + db_dir, + &cardano_db_message.hash, + context.is_json_output_enabled(), + )?; + + Ok(()) + } + } } fn log_verified_information( diff --git a/mithril-client-cli/src/utils/cardano_db.rs b/mithril-client-cli/src/utils/cardano_db.rs index 2632ab073ad..fcae05d3ddb 100644 --- a/mithril-client-cli/src/utils/cardano_db.rs +++ b/mithril-client-cli/src/utils/cardano_db.rs @@ -22,10 +22,13 @@ impl CardanoDbUtils { } /// Display a spinner while waiting for the result of a future - pub async fn wait_spinner( + pub async fn wait_spinner( progress_bar: &MultiProgress, - future: impl Future>, - ) -> MithrilResult { + future: impl Future>, + ) -> MithrilResult + where + MithrilError: From, + { let pb = progress_bar.add(ProgressBar::new_spinner()); let spinner = async move { loop { @@ -36,7 +39,7 @@ impl CardanoDbUtils { tokio::select! { _ = spinner => Err(anyhow!("timeout")), - res = future => res, + res = future => res.map_err(Into::into), } } From d9a936713112f426e55b760bc7faddcf1abc87b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Turmel?= Date: Fri, 11 Jul 2025 17:04:20 +0200 Subject: [PATCH 09/18] feature(client-cli): display missing and/or tampered immutables when verify command fail --- .../src/commands/cardano_db/verify.rs | 116 +++++++++++++++++- 1 file changed, 110 insertions(+), 6 deletions(-) diff --git a/mithril-client-cli/src/commands/cardano_db/verify.rs b/mithril-client-cli/src/commands/cardano_db/verify.rs index 01c37dec21b..2eedbb6c8b2 100644 --- a/mithril-client-cli/src/commands/cardano_db/verify.rs +++ b/mithril-client-cli/src/commands/cardano_db/verify.rs @@ -4,10 +4,13 @@ use std::{ sync::Arc, }; -use anyhow::{Context, anyhow}; -use chrono::Utc; +use anyhow::Context; +use chrono::{DateTime, Utc}; use clap::Parser; -use mithril_client::{ComputeCardanoDatabaseMessageError, MithrilResult}; +use mithril_client::{ + CardanoDatabaseSnapshot, ComputeCardanoDatabaseMessageError, ImmutableFilesLists, + MithrilResult, cardano_database_client::ImmutableFileRange, common::ImmutableFileNumber, +}; use crate::{ CommandContext, @@ -38,6 +41,18 @@ pub struct CardanoDbVerifyCommand { /// Genesis verification key to check the certificate chain. #[clap(long, env = "GENESIS_VERIFICATION_KEY")] genesis_verification_key: Option, + + /// The first immutable file number to verify. + /// + /// If not set, the verify process will start from the first immutable file. + #[clap(long)] + start: Option, + + /// The last immutable file number to verify. + /// + /// If not set, the verify will continue until the last certified immutable file. + #[clap(long)] + end: Option, } impl CardanoDbVerifyCommand { @@ -94,6 +109,14 @@ impl CardanoDbVerifyCommand { .await? .with_context(|| format!("Can not get the cardano db for hash: '{}'", self.digest))?; + let immutable_file_range = shared_steps::immutable_file_range(self.start, self.end); + + print_immutables_range_to_verify( + &cardano_db_message, + &immutable_file_range, + context.is_json_output_enabled(), + )?; + let certificate = shared_steps::fetch_certificate_and_verifying_chain( 1, &progress_printer, @@ -102,8 +125,6 @@ impl CardanoDbVerifyCommand { ) .await?; - let immutable_file_range = shared_steps::immutable_file_range(None, None); - let verified_digests = shared_steps::download_and_verify_digests( 2, &progress_printer, @@ -127,7 +148,10 @@ impl CardanoDbVerifyCommand { match message { Err(e) => match e.downcast_ref::() { Some(ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(lists)) => { - // let missing_files = lists.missing; + Self::print_immutables_verification_error( + lists, + context.is_json_output_enabled(), + ); Ok(()) } _ => Err(e), @@ -176,6 +200,86 @@ impl CardanoDbVerifyCommand { } Ok(()) } + + fn print_immutables_verification_error(lists: &ImmutableFilesLists, json_output: bool) { + let utc_now = Utc::now(); + let json_file_path = write_json_file_error(utc_now, lists); + let error_message = "Verifying immutables files has failed"; + if json_output { + let json = serde_json::json!({ + "timestamp": utc_now.to_rfc3339(), + "verify_error" : { + "message": error_message, + "immutables_verification_error_file": json_file_path, + "immutables_dir": lists.immutables_dir, + "missing_files_count": lists.missing.len(), + "tampered_files_count": lists.tampered.len() + } + }); + + println!("{json}"); + } else { + println!("{error_message}"); + println!( + "See the lists of all missing and tampered files in {}", + json_file_path.display() + ); + if !lists.missing.is_empty() { + println!("Number of missing immutable files: {}", lists.missing.len()); + } + if !lists.tampered.is_empty() { + println!( + "Number of tampered immutable files: {:?}", + lists.tampered.len() + ); + } + } + } +} + +fn write_json_file_error(date: DateTime, lists: &ImmutableFilesLists) -> PathBuf { + let file_path = PathBuf::from(format!( + "immutables_verification_error-{}.json", + date.timestamp() + )); + std::fs::write( + &file_path, + serde_json::to_string_pretty(&serde_json::json!({ + "timestamp": date.to_rfc3339(), + "immutables_dir": lists.immutables_dir, + "missing-files": lists.missing, + "tampered-files": lists.tampered, + })) + .unwrap(), + ) + .expect("Could not write immutables verification error to file"); + file_path +} + +fn print_immutables_range_to_verify( + cardano_db_message: &CardanoDatabaseSnapshot, + immutable_file_range: &ImmutableFileRange, + json_output: bool, +) -> Result<(), anyhow::Error> { + let range_to_verify = + immutable_file_range.to_range_inclusive(cardano_db_message.beacon.immutable_file_number)?; + if json_output { + let json = serde_json::json!({ + "timestamp": Utc::now().to_rfc3339(), + "local_immutable_range_to_verify": { + "start": range_to_verify.start(), + "end": range_to_verify.end(), + }, + }); + println!("{json}"); + } else { + eprintln!( + "Verifying local immutable files from number {} to {}", + range_to_verify.start(), + range_to_verify.end() + ); + } + Ok(()) } impl ConfigSource for CardanoDbVerifyCommand { From 29831c039490f3a267c58204a8f9e44f454d1d95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Turmel?= Date: Wed, 16 Jul 2025 11:41:53 +0200 Subject: [PATCH 10/18] feature(client-lib,client-cli): add arg --allow-missing in cardano-db verify command --- .../src/commands/cardano_db/download/v2.rs | 13 +++-- .../src/commands/cardano_db/shared_steps.rs | 16 ++++-- .../src/commands/cardano_db/verify.rs | 17 ++++-- mithril-client/src/message.rs | 53 +++++++++++++++++-- ...no_db_snapshot_list_get_download_verify.rs | 1 + 5 files changed, 85 insertions(+), 15 deletions(-) diff --git a/mithril-client-cli/src/commands/cardano_db/download/v2.rs b/mithril-client-cli/src/commands/cardano_db/download/v2.rs index b7c728c4c1a..d08ae8af7a5 100644 --- a/mithril-client-cli/src/commands/cardano_db/download/v2.rs +++ b/mithril-client-cli/src/commands/cardano_db/download/v2.rs @@ -15,7 +15,10 @@ use mithril_client::{ use crate::{ CommandContext, commands::{ - cardano_db::{download::DB_DIRECTORY_NAME, shared_steps}, + cardano_db::{ + download::DB_DIRECTORY_NAME, + shared_steps::{self, ComputeCardanoDatabaseMessageOptions}, + }, client_builder, }, utils::{ @@ -139,13 +142,17 @@ impl PreparedCardanoDbV2Download { ) })?; + let options = ComputeCardanoDatabaseMessageOptions { + db_dir: restoration_options.db_dir.clone(), + immutable_file_range: restoration_options.immutable_file_range, + allow_missing: false, + }; let message = shared_steps::compute_cardano_db_snapshot_message( 5, &progress_printer, &certificate, &cardano_db_message, - &restoration_options.immutable_file_range, - &restoration_options.db_dir, + &options, &verified_digests, ) .await?; diff --git a/mithril-client-cli/src/commands/cardano_db/shared_steps.rs b/mithril-client-cli/src/commands/cardano_db/shared_steps.rs index cbdff83987d..56e1d55fd51 100644 --- a/mithril-client-cli/src/commands/cardano_db/shared_steps.rs +++ b/mithril-client-cli/src/commands/cardano_db/shared_steps.rs @@ -1,7 +1,7 @@ use anyhow::{Context, anyhow}; use chrono::Utc; use slog::{Logger, debug, warn}; -use std::path::Path; +use std::path::{Path, PathBuf}; use mithril_client::{ CardanoDatabaseSnapshot, Client, MessageBuilder, MithrilCertificate, MithrilResult, @@ -11,6 +11,12 @@ use mithril_client::{ use crate::utils::{CardanoDbUtils, ProgressPrinter}; +pub struct ComputeCardanoDatabaseMessageOptions { + pub db_dir: PathBuf, + pub immutable_file_range: ImmutableFileRange, + pub allow_missing: bool, +} + pub async fn fetch_certificate_and_verifying_chain( step_number: u16, progress_printer: &ProgressPrinter, @@ -69,8 +75,7 @@ pub async fn compute_cardano_db_snapshot_message( progress_printer: &ProgressPrinter, certificate: &MithrilCertificate, cardano_database_snapshot: &CardanoDatabaseSnapshot, - immutable_file_range: &ImmutableFileRange, - database_dir: &Path, + options: &ComputeCardanoDatabaseMessageOptions, verified_digest: &VerifiedDigests, ) -> MithrilResult { progress_printer.report_step(step_number, "Computing the cardano db snapshot message")?; @@ -79,8 +84,9 @@ pub async fn compute_cardano_db_snapshot_message( MessageBuilder::new().compute_cardano_database_message( certificate, cardano_database_snapshot, - immutable_file_range, - database_dir, + &options.immutable_file_range, + options.allow_missing, + &options.db_dir, verified_digest, ), ) diff --git a/mithril-client-cli/src/commands/cardano_db/verify.rs b/mithril-client-cli/src/commands/cardano_db/verify.rs index 2eedbb6c8b2..303e5846c44 100644 --- a/mithril-client-cli/src/commands/cardano_db/verify.rs +++ b/mithril-client-cli/src/commands/cardano_db/verify.rs @@ -15,7 +15,10 @@ use mithril_client::{ use crate::{ CommandContext, commands::{ - cardano_db::{CardanoDbCommandsBackend, shared_steps}, + cardano_db::{ + CardanoDbCommandsBackend, + shared_steps::{self, ComputeCardanoDatabaseMessageOptions}, + }, client_builder, }, configuration::{ConfigError, ConfigSource}, @@ -53,6 +56,10 @@ pub struct CardanoDbVerifyCommand { /// If not set, the verify will continue until the last certified immutable file. #[clap(long)] end: Option, + + /// If set, the verification will not fail if some immutable files are missing. + #[clap(long)] + allow_missing: bool, } impl CardanoDbVerifyCommand { @@ -134,13 +141,17 @@ impl CardanoDbVerifyCommand { ) .await?; + let options = ComputeCardanoDatabaseMessageOptions { + db_dir: db_dir.to_path_buf(), + immutable_file_range, + allow_missing: self.allow_missing, + }; let message = shared_steps::compute_cardano_db_snapshot_message( 3, &progress_printer, &certificate, &cardano_db_message, - &immutable_file_range, - db_dir, + &options, &verified_digests, ) .await; diff --git a/mithril-client/src/message.rs b/mithril-client/src/message.rs index f7c394af11d..f1ab7d168ae 100644 --- a/mithril-client/src/message.rs +++ b/mithril-client/src/message.rs @@ -229,6 +229,7 @@ impl MessageBuilder { certificate: &CertificateMessage, cardano_database_snapshot: &CardanoDatabaseSnapshotMessage, immutable_file_range: &ImmutableFileRange, + allow_missing: bool, database_dir: &Path, verified_digests: &VerifiedDigests, ) -> Result { @@ -236,8 +237,11 @@ impl MessageBuilder { let immutable_file_number_range = immutable_file_range .to_range_inclusive(cardano_database_snapshot.beacon.immutable_file_number) .map_err(ComputeCardanoDatabaseMessageError::ImmutableFilesRange)?; - let missing_immutable_files = - Self::list_missing_immutable_files(database_dir, &immutable_file_number_range); + let missing_immutable_files = if allow_missing { + vec![] + } else { + Self::list_missing_immutable_files(database_dir, &immutable_file_number_range) + }; let immutable_digester = CardanoImmutableDigester::new(network, None, self.logger.clone()); let computed_digest_entries = immutable_digester .compute_digests_for_range(database_dir, &immutable_file_number_range) @@ -532,6 +536,7 @@ mod tests { &certificate, &CardanoDatabaseSnapshotMessage::dummy(), &immutable_file_range_to_prove, + false, &database_dir, &verified_digests, ) @@ -542,7 +547,8 @@ mod tests { } #[tokio::test] - async fn compute_cardano_database_message_should_fail_if_immutable_is_missing() { + async fn compute_cardano_database_message_should_fail_if_immutable_is_missing_and_allow_missing_not_set() + { let beacon = CardanoDbBeacon { epoch: Epoch(123), immutable_file_number: 10, @@ -550,7 +556,7 @@ mod tests { let immutable_file_range = 1..=15; let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); let (database_dir, certificate, verified_digests) = prepare_db_and_verified_digests( - "compute_cardano_database_message_should_fail_if_immutable_is_missing", + "compute_cardano_database_message_should_fail_if_immutable_is_missing_and_allow_missing_not_set", &beacon, &immutable_file_range, ) @@ -559,11 +565,13 @@ mod tests { let files_to_remove = vec!["00003.chunk", "00004.primary"]; remove_immutable_files(&database_dir, &files_to_remove); + let allow_missing = false; let error = MessageBuilder::new() .compute_cardano_database_message( &certificate, &CardanoDatabaseSnapshotMessage::dummy(), &immutable_file_range_to_prove, + allow_missing, &database_dir, &verified_digests, ) @@ -587,6 +595,41 @@ mod tests { ); } + #[tokio::test] + async fn compute_cardano_database_message_should_success_if_immutable_is_missing_and_allow_missing_is_set() + { + let beacon = CardanoDbBeacon { + epoch: Epoch(123), + immutable_file_number: 10, + }; + let immutable_file_range = 1..=15; + let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); + let (database_dir, certificate, verified_digests) = prepare_db_and_verified_digests( + "compute_cardano_database_message_should_success_if_immutable_is_missing_and_allow_missing_is_set", + &beacon, + &immutable_file_range, + ) + .await; + + let files_to_remove = vec!["00003.chunk", "00004.primary"]; + remove_immutable_files(&database_dir, &files_to_remove); + + let allow_missing = true; + MessageBuilder::new() + .compute_cardano_database_message( + &certificate, + &CardanoDatabaseSnapshotMessage::dummy(), + &immutable_file_range_to_prove, + allow_missing, + &database_dir, + &verified_digests, + ) + .await + .expect( + "compute_cardano_database_message should succeed if a immutable is missing but 'allow_missing' is set", + ); + } + #[tokio::test] async fn compute_cardano_database_message_should_fail_if_immutable_is_tampered() { let beacon = CardanoDbBeacon { @@ -613,6 +656,7 @@ mod tests { &certificate, &CardanoDatabaseSnapshotMessage::dummy(), &immutable_file_range_to_prove, + false, &database_dir, &verified_digests, ) @@ -664,6 +708,7 @@ mod tests { &certificate, &CardanoDatabaseSnapshotMessage::dummy(), &immutable_file_range_to_prove, + false, &database_dir, &verified_digests, ) diff --git a/mithril-client/tests/cardano_db_snapshot_list_get_download_verify.rs b/mithril-client/tests/cardano_db_snapshot_list_get_download_verify.rs index 514af718e55..6ad72758a91 100644 --- a/mithril-client/tests/cardano_db_snapshot_list_get_download_verify.rs +++ b/mithril-client/tests/cardano_db_snapshot_list_get_download_verify.rs @@ -143,6 +143,7 @@ async fn cardano_db_snapshot_list_get_download_verify() { &certificate, &cardano_db_snapshot, &immutable_file_range, + false, &unpacked_dir, &verified_digests, ) From 266b2a37e36aa77de33662a3fc284e18bd5acefb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Turmel?= Date: Wed, 16 Jul 2025 16:58:55 +0200 Subject: [PATCH 11/18] fix(examples): realign immutables range with release-preprod environment --- examples/client-cardano-database-v2/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/client-cardano-database-v2/src/main.rs b/examples/client-cardano-database-v2/src/main.rs index 9ab47e5df8a..7ce87836a27 100644 --- a/examples/client-cardano-database-v2/src/main.rs +++ b/examples/client-cardano-database-v2/src/main.rs @@ -82,7 +82,7 @@ async fn main() -> MithrilResult<()> { .verify_chain(&cardano_database_snapshot.certificate_hash) .await?; - let immutable_file_range = ImmutableFileRange::From(15000); + let immutable_file_range = ImmutableFileRange::From(4000); let download_unpack_options = DownloadUnpackOptions { allow_override: true, include_ancillary: true, From 0b3de45b5d74d17569b86b6b5a1afb77a961ca80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Turmel?= Date: Wed, 16 Jul 2025 17:00:33 +0200 Subject: [PATCH 12/18] feature(client-lib, examples): adapt examples with the new compute_cardano_database_message function --- .../client-cardano-database-v2/src/main.rs | 40 +++++++++---------- .../src/cardano_database_client/mod.rs | 16 +++++--- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/examples/client-cardano-database-v2/src/main.rs b/examples/client-cardano-database-v2/src/main.rs index 7ce87836a27..c380a5adad6 100644 --- a/examples/client-cardano-database-v2/src/main.rs +++ b/examples/client-cardano-database-v2/src/main.rs @@ -17,7 +17,7 @@ use std::time::Duration; use tokio::sync::RwLock; use mithril_client::feedback::{FeedbackReceiver, MithrilEvent, MithrilEventCardanoDatabase}; -use mithril_client::{ClientBuilder, MessageBuilder, MithrilResult}; +use mithril_client::{ClientBuilder, MessageBuilder, MithrilError, MithrilResult}; #[derive(Parser, Debug)] #[command(version)] @@ -99,26 +99,11 @@ async fn main() -> MithrilResult<()> { .await?; println!("Downloading and verifying digests file authenticity..."); - let verified_digest = client + let verified_digests = client .cardano_database_v2() .download_and_verify_digests(&certificate, &cardano_database_snapshot) .await?; - println!("Computing Cardano database Merkle proof...",); - let merkle_proof = client - .cardano_database_v2() - .compute_merkle_proof( - &certificate, - cardano_database_snapshot.beacon.immutable_file_number, - &immutable_file_range, - &unpacked_dir, - &verified_digest, - ) - .await?; - merkle_proof - .verify() - .with_context(|| "Merkle proof verification failed")?; - println!("Sending usage statistics to the aggregator..."); let full_restoration = immutable_file_range == ImmutableFileRange::Full; let include_ancillary = download_unpack_options.include_ancillary; @@ -140,9 +125,17 @@ async fn main() -> MithrilResult<()> { "Computing Cardano database snapshot '{}' message...", cardano_database_snapshot.hash ); + let allow_missing_immutables_files = false; let message = wait_spinner( &progress_bar, - MessageBuilder::new().compute_cardano_database_message(&certificate, merkle_proof.root()), + MessageBuilder::new().compute_cardano_database_message( + &certificate, + &cardano_database_snapshot, + &immutable_file_range, + allow_missing_immutables_files, + &unpacked_dir, + &verified_digests, + ), ) .await?; @@ -268,10 +261,13 @@ fn get_temp_dir() -> MithrilResult { Ok(dir) } -async fn wait_spinner( +pub async fn wait_spinner( progress_bar: &MultiProgress, - future: impl Future>, -) -> MithrilResult { + future: impl Future>, +) -> MithrilResult +where + MithrilError: From, +{ let pb = progress_bar.add(ProgressBar::new_spinner()); let spinner = async move { loop { @@ -282,6 +278,6 @@ async fn wait_spinner( tokio::select! { _ = spinner => Err(anyhow!("timeout")), - res = future => res, + res = future => res.map_err(Into::into), } } diff --git a/mithril-client/src/cardano_database_client/mod.rs b/mithril-client/src/cardano_database_client/mod.rs index 85f50840fb3..bbd0adc2e56 100644 --- a/mithril-client/src/cardano_database_client/mod.rs +++ b/mithril-client/src/cardano_database_client/mod.rs @@ -5,7 +5,6 @@ //! - [list][CardanoDatabaseClient::list]: get the list of available Cardano database //! - [download_unpack][CardanoDatabaseClient::download_unpack]: download and unpack a Cardano database snapshot for a given immutable files range //! - [download_and_verify_digests][CardanoDatabaseClient::download_and_verify_digests]: download and verify the digests of the immutable files in a Cardano database snapshot - //! # Get a Cardano database //! //! To get a Cardano database using the [ClientBuilder][crate::client::ClientBuilder]. @@ -88,7 +87,7 @@ //! ```no_run //! # #[cfg(feature = "fs")] //! # async fn run() -> mithril_client::MithrilResult<()> { -//! use mithril_client::{ClientBuilder, cardano_database_client::{ImmutableFileRange, DownloadUnpackOptions}}; +//! use mithril_client::{ClientBuilder, MessageBuilder, cardano_database_client::{ImmutableFileRange, DownloadUnpackOptions}}; //! use std::path::Path; //! //! let client = ClientBuilder::aggregator("YOUR_AGGREGATOR_ENDPOINT", "YOUR_GENESIS_VERIFICATION_KEY").build()?; @@ -120,10 +119,15 @@ //! &cardano_database_snapshot) //! .await?; //! -//! let merkle_proof = client -//! .cardano_database_v2() -//! .compute_merkle_proof(&certificate, cardano_database_snapshot.beacon.immutable_file_number, &immutable_file_range, &target_directory, &verified_digests) -//! .await?; +//! let allow_missing_immutables_files = false; +//! let message = MessageBuilder::new().compute_cardano_database_message( +//! &certificate, +//! &cardano_database_snapshot, +//! &immutable_file_range, +//! allow_missing_immutables_files, +//! &target_directory, +//! &verified_digests, +//! ).await?; //! # //! # Ok(()) //! # } From da1ee2c88940f2454334e6e9a39b39d53c80e9b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Turmel?= Date: Fri, 18 Jul 2025 11:37:55 +0200 Subject: [PATCH 13/18] docs: update mithril client doc with new verify command arguments --- docs/website/root/manual/develop/nodes/mithril-client.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/website/root/manual/develop/nodes/mithril-client.md b/docs/website/root/manual/develop/nodes/mithril-client.md index 75a8b7ecfb7..9ee51d14d2b 100644 --- a/docs/website/root/manual/develop/nodes/mithril-client.md +++ b/docs/website/root/manual/develop/nodes/mithril-client.md @@ -618,6 +618,9 @@ Here is a list of the available parameters: | `digest` | - | - | - | Digest of the Cardano db snapshot to verify or `latest` for the latest artifact | - | - | :heavy_check_mark: | | `db_dir` | `--db-dir` | - | - | Directory from where the immutable will be verified | - | - | - | | `genesis_verification_key` | `--genesis-verification-key` | - | `GENESIS_VERIFICATION_KEY` | Genesis verification key to check the certificate chain | - | - | - | +| `start` | `--start` | - | - | The first immutable file number to verify | - | - | - | +| `end` | `--end` | - | - | The last immutable file number to verify | - | - | - | +| `allow_missing` | `--allow-missing` | - | - | If set, the verification will not fail if some immutable files are missing | `false` | - | - | | `run_mode` | `--run-mode` | - | `RUN_MODE` | Run Mode | `dev` | - | - | | `verbose` | `--verbose` | `-v` | - | Verbosity level (-v=warning, -vv=info, -vvv=debug, -vvvv=trace) | `0` | - | - | | `config_directory` | `--config-directory` | - | - | Directory where configuration file is located | `./config` | - | - | From 0fca1a1cbce3e488a3de8b300218107647fbe78a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Turmel?= Date: Fri, 18 Jul 2025 16:55:44 +0200 Subject: [PATCH 14/18] feature(client-lib): return also list of non verified files in case of proof computation error --- mithril-client/src/message.rs | 144 +++++++++++++++++++++++++++++++--- 1 file changed, 134 insertions(+), 10 deletions(-) diff --git a/mithril-client/src/message.rs b/mithril-client/src/message.rs index f1ab7d168ae..7d075a61dec 100644 --- a/mithril-client/src/message.rs +++ b/mithril-client/src/message.rs @@ -50,6 +50,8 @@ cfg_fs! { pub missing: Vec, /// List of tampered immutable files. pub tampered: Vec, + /// List of non-verifiable immutable files. + pub non_verifiable: Vec, } /// Compute Cardano database message related errors. @@ -85,9 +87,8 @@ cfg_fs! { .join("\n") } - let missing_files_subset = get_first_10_files_path(&lists.missing, &lists.immutables_dir); - let tampered_files_subset = get_first_10_files_path(&lists.tampered, &lists.immutables_dir); if !lists.missing.is_empty() { + let missing_files_subset = get_first_10_files_path(&lists.missing, &lists.immutables_dir); writeln!( f, "Number of missing immutable files: {}", @@ -100,10 +101,20 @@ cfg_fs! { writeln!(f)?; } if !lists.tampered.is_empty() { + let tampered_files_subset = get_first_10_files_path(&lists.tampered, &lists.immutables_dir); writeln!(f,"Number of tampered immutable files: {}",lists.tampered.len())?; writeln!(f, "First 10 tampered immutable files paths:")?; writeln!(f, "{tampered_files_subset}")?; } + if (!lists.missing.is_empty() || !lists.tampered.is_empty()) && !lists.non_verifiable.is_empty() { + writeln!(f)?; + } + if !lists.non_verifiable.is_empty() { + let non_verifiable_files_subset = get_first_10_files_path(&lists.non_verifiable, &lists.immutables_dir); + writeln!(f, "Number of non verifiable immutable files: {}", lists.non_verifiable.len())?; + writeln!(f, "First 10 non verifiable immutable files paths:")?; + writeln!(f, "{non_verifiable_files_subset}")?; + } Ok(()) } ComputeCardanoDatabaseMessageError::ImmutableFilesDigester(e) => { @@ -269,20 +280,22 @@ impl MessageBuilder { return Ok(message); } - let tampered_files = match proof_result { + let (tampered, non_verifiable) = match proof_result { Err(e) => { warn!(self.logger, "{MERKLE_PROOF_COMPUTATION_ERROR}: {e:}"); - verified_digests - .list_immutable_files_not_verified(&computed_digest_entries) - .tampered_files + let verified_digests = verified_digests + .list_immutable_files_not_verified(&computed_digest_entries); + + (verified_digests.tampered_files, verified_digests.non_verifiable_files) } - Ok(_) => vec![], + Ok(_) => (vec![], vec![]), }; Err( ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(ImmutableFilesLists { immutables_dir: Self::immutable_dir(database_dir), missing: missing_immutable_files, - tampered: tampered_files, + tampered, + non_verifiable, }), ) } @@ -591,6 +604,7 @@ mod tests { immutables_dir: MessageBuilder::immutable_dir(&database_dir), missing: to_vec_immutable_file_name(&files_to_remove), tampered: vec![], + non_verifiable: vec![], } ); } @@ -677,6 +691,7 @@ mod tests { immutables_dir: MessageBuilder::immutable_dir(&database_dir), missing: vec![], tampered: to_vec_immutable_file_name(&files_to_tamper), + non_verifiable: vec![], } ) } @@ -727,9 +742,77 @@ mod tests { immutables_dir: MessageBuilder::immutable_dir(&database_dir), missing: to_vec_immutable_file_name(&files_to_remove), tampered: to_vec_immutable_file_name(&files_to_tamper), + non_verifiable: vec![], } ) } + + #[tokio::test] + async fn compute_cardano_database_message_should_fail_if_there_is_more_local_immutable_than_verified_digest() + { + let last_verified_digest_number = 10; + let last_local_immutable_file_number = 15; + let range_of_non_verifiable_files = + last_verified_digest_number + 1..=last_local_immutable_file_number; + + let expected_non_verifiable_files: Vec = + (range_of_non_verifiable_files) + .flat_map(|i| { + [ + format!("{i:05}.chunk"), + format!("{i:05}.primary"), + format!("{i:05}.secondary"), + ] + }) + .collect(); + + let beacon = CardanoDbBeacon { + epoch: Epoch(123), + immutable_file_number: last_verified_digest_number, + }; + //create verified digests for immutable files 1 to 10 + let (_, certificate, verified_digests) = prepare_db_and_verified_digests( + "database_dir_for_verified_digests", + &beacon, + &(1..=last_verified_digest_number), + ) + .await; + //create a local database with immutable files 1 to 15 + let (database_dir, _, _) = prepare_db_and_verified_digests( + "database_dir_for_local_immutables", + &beacon, + &(1..=last_local_immutable_file_number), + ) + .await; + + let error = MessageBuilder::new() + .compute_cardano_database_message( + &certificate, + &CardanoDatabaseSnapshotMessage::dummy(), + &ImmutableFileRange::Range(1, 15), + false, + &database_dir, + &verified_digests, + ) + .await + .expect_err( + "compute_cardano_database_message should fail if there is more local immutable than verified digest", + ); + + let error_lists = match error { + ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(lists) => lists, + _ => panic!("Expected ImmutableFilesVerification error, got: {error}"), + }; + assert_eq!( + error_lists, + ImmutableFilesLists { + immutables_dir: MessageBuilder::immutable_dir(&database_dir), + missing: vec![], + tampered: vec![], + non_verifiable: expected_non_verifiable_files, + } + ); + } } mod compute_cardano_database_message_error { @@ -738,6 +821,7 @@ mod tests { fn generate_immutable_files_verification_error( missing_range: Option>, tampered_range: Option>, + non_verifiable_range: Option>, immutable_path: &str, ) -> ComputeCardanoDatabaseMessageError { let missing: Vec = match missing_range { @@ -753,10 +837,18 @@ mod tests { None => vec![], }; + let non_verifiable: Vec = match non_verifiable_range { + Some(range) => range + .map(|i| ImmutableFileName::from(format!("{i:05}.chunk"))) + .collect(), + None => vec![], + }; + ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(ImmutableFilesLists { immutables_dir: PathBuf::from(immutable_path), missing, tampered, + non_verifiable, }) } @@ -769,6 +861,7 @@ mod tests { let error = generate_immutable_files_verification_error( Some(1..=15), Some(20..=31), + Some(40..=41), "/path/to/immutables", ); @@ -801,15 +894,21 @@ First 10 tampered immutable files paths: /path/to/immutables/00027.chunk /path/to/immutables/00028.chunk /path/to/immutables/00029.chunk + +Number of non verifiable immutable files: 2 +First 10 non verifiable immutable files paths: +/path/to/immutables/00040.chunk +/path/to/immutables/00041.chunk "### ); } #[test] - fn display_immutable_files_should_not_display_error_list_if_missing_is_empty() { + fn display_immutable_files_should_display_tampered_files_only() { let error = generate_immutable_files_verification_error( None, Some(1..=1), + None, "/path/to/immutables", ); @@ -825,10 +924,11 @@ First 10 tampered immutable files paths: } #[test] - fn display_immutable_files_should_not_display_error_list_if_tampered_is_empty() { + fn display_immutable_files_should_display_missing_files_only() { let error = generate_immutable_files_verification_error( Some(1..=1), None, + None, "/path/to/immutables", ); @@ -839,6 +939,30 @@ First 10 tampered immutable files paths: r###"Number of missing immutable files: 1 First 10 missing immutable files paths: /path/to/immutables/00001.chunk +"### + ); + } + + #[test] + fn display_immutable_files_should_display_non_verifiable_files_only() { + let error = generate_immutable_files_verification_error( + None, + None, + Some(1..=5), + "/path/to/immutables", + ); + + let display = normalize_path_separators(&format!("{error}")); + + assert_eq!( + display, + r###"Number of non verifiable immutable files: 5 +First 10 non verifiable immutable files paths: +/path/to/immutables/00001.chunk +/path/to/immutables/00002.chunk +/path/to/immutables/00003.chunk +/path/to/immutables/00004.chunk +/path/to/immutables/00005.chunk "### ); } From 857cc80d8bcc40df162d33aaec4e9036881af508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Turmel?= Date: Mon, 21 Jul 2025 15:18:13 +0200 Subject: [PATCH 15/18] feature(client-cli): improve verify command output with non verifiable files --- mithril-client-cli/src/commands/cardano_db/verify.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mithril-client-cli/src/commands/cardano_db/verify.rs b/mithril-client-cli/src/commands/cardano_db/verify.rs index 303e5846c44..68b806e9806 100644 --- a/mithril-client-cli/src/commands/cardano_db/verify.rs +++ b/mithril-client-cli/src/commands/cardano_db/verify.rs @@ -224,7 +224,8 @@ impl CardanoDbVerifyCommand { "immutables_verification_error_file": json_file_path, "immutables_dir": lists.immutables_dir, "missing_files_count": lists.missing.len(), - "tampered_files_count": lists.tampered.len() + "tampered_files_count": lists.tampered.len(), + "non_verifiable_files_count": lists.non_verifiable.len(), } }); @@ -232,7 +233,7 @@ impl CardanoDbVerifyCommand { } else { println!("{error_message}"); println!( - "See the lists of all missing and tampered files in {}", + "See the lists of all missing, tampered and non verifiable files in {}", json_file_path.display() ); if !lists.missing.is_empty() { @@ -244,6 +245,12 @@ impl CardanoDbVerifyCommand { lists.tampered.len() ); } + if !lists.non_verifiable.is_empty() { + println!( + "Number of non verifiable immutable files: {:?}", + lists.non_verifiable.len() + ); + } } } } @@ -260,6 +267,7 @@ fn write_json_file_error(date: DateTime, lists: &ImmutableFilesLists) -> Pa "immutables_dir": lists.immutables_dir, "missing-files": lists.missing, "tampered-files": lists.tampered, + "non-verifiable-files": lists.non_verifiable, })) .unwrap(), ) From ee2869053bf60ca7ab8c15e5aa9fa13d23c5eafc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Turmel?= Date: Tue, 22 Jul 2025 09:14:07 +0200 Subject: [PATCH 16/18] refactoring(client-lib,client-cli): improve name of cardano database verification result --- .../src/commands/cardano_db/verify.rs | 6 ++-- mithril-client/src/message.rs | 28 ++++++++++--------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/mithril-client-cli/src/commands/cardano_db/verify.rs b/mithril-client-cli/src/commands/cardano_db/verify.rs index 68b806e9806..20e43dec10f 100644 --- a/mithril-client-cli/src/commands/cardano_db/verify.rs +++ b/mithril-client-cli/src/commands/cardano_db/verify.rs @@ -8,7 +8,7 @@ use anyhow::Context; use chrono::{DateTime, Utc}; use clap::Parser; use mithril_client::{ - CardanoDatabaseSnapshot, ComputeCardanoDatabaseMessageError, ImmutableFilesLists, + CardanoDatabaseSnapshot, ComputeCardanoDatabaseMessageError, ImmutableVerificationResult, MithrilResult, cardano_database_client::ImmutableFileRange, common::ImmutableFileNumber, }; @@ -212,7 +212,7 @@ impl CardanoDbVerifyCommand { Ok(()) } - fn print_immutables_verification_error(lists: &ImmutableFilesLists, json_output: bool) { + fn print_immutables_verification_error(lists: &ImmutableVerificationResult, json_output: bool) { let utc_now = Utc::now(); let json_file_path = write_json_file_error(utc_now, lists); let error_message = "Verifying immutables files has failed"; @@ -255,7 +255,7 @@ impl CardanoDbVerifyCommand { } } -fn write_json_file_error(date: DateTime, lists: &ImmutableFilesLists) -> PathBuf { +fn write_json_file_error(date: DateTime, lists: &ImmutableVerificationResult) -> PathBuf { let file_path = PathBuf::from(format!( "immutables_verification_error-{}.json", date.timestamp() diff --git a/mithril-client/src/message.rs b/mithril-client/src/message.rs index 7d075a61dec..eabe6de689a 100644 --- a/mithril-client/src/message.rs +++ b/mithril-client/src/message.rs @@ -43,7 +43,7 @@ cfg_fs! { /// Type containing the lists of immutable files that are missing or tampered. #[derive(Debug, PartialEq)] - pub struct ImmutableFilesLists { + pub struct ImmutableVerificationResult { /// The immutables files directory. pub immutables_dir: PathBuf, /// List of missing immutable files. @@ -58,7 +58,7 @@ cfg_fs! { #[derive(Error, Debug)] pub enum ComputeCardanoDatabaseMessageError { /// Error related to the verification of immutable files. - ImmutableFilesVerification(ImmutableFilesLists), + ImmutableFilesVerification(ImmutableVerificationResult), /// Error related to the immutable files digests computation. ImmutableFilesDigester(#[from] ImmutableDigesterError), @@ -291,7 +291,7 @@ impl MessageBuilder { Ok(_) => (vec![], vec![]), }; Err( - ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(ImmutableFilesLists { + ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(ImmutableVerificationResult { immutables_dir: Self::immutable_dir(database_dir), missing: missing_immutable_files, tampered, @@ -600,7 +600,7 @@ mod tests { assert_eq!( error_lists, - ImmutableFilesLists { + ImmutableVerificationResult { immutables_dir: MessageBuilder::immutable_dir(&database_dir), missing: to_vec_immutable_file_name(&files_to_remove), tampered: vec![], @@ -687,7 +687,7 @@ mod tests { }; assert_eq!( error_lists, - ImmutableFilesLists { + ImmutableVerificationResult { immutables_dir: MessageBuilder::immutable_dir(&database_dir), missing: vec![], tampered: to_vec_immutable_file_name(&files_to_tamper), @@ -738,7 +738,7 @@ mod tests { }; assert_eq!( error_lists, - ImmutableFilesLists { + ImmutableVerificationResult { immutables_dir: MessageBuilder::immutable_dir(&database_dir), missing: to_vec_immutable_file_name(&files_to_remove), tampered: to_vec_immutable_file_name(&files_to_tamper), @@ -805,7 +805,7 @@ mod tests { }; assert_eq!( error_lists, - ImmutableFilesLists { + ImmutableVerificationResult { immutables_dir: MessageBuilder::immutable_dir(&database_dir), missing: vec![], tampered: vec![], @@ -844,12 +844,14 @@ mod tests { None => vec![], }; - ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(ImmutableFilesLists { - immutables_dir: PathBuf::from(immutable_path), - missing, - tampered, - non_verifiable, - }) + ComputeCardanoDatabaseMessageError::ImmutableFilesVerification( + ImmutableVerificationResult { + immutables_dir: PathBuf::from(immutable_path), + missing, + tampered, + non_verifiable, + }, + ) } fn normalize_path_separators(s: &str) -> String { From 2605f9496b2ce8ae15e5e8ff0ead4cbd003b30cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Turmel?= Date: Tue, 22 Jul 2025 10:39:44 +0200 Subject: [PATCH 17/18] refactoring(client-lib): reorganize imports --- mithril-client/src/message.rs | 37 ++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/mithril-client/src/message.rs b/mithril-client/src/message.rs index eabe6de689a..52efeae98da 100644 --- a/mithril-client/src/message.rs +++ b/mithril-client/src/message.rs @@ -1,42 +1,43 @@ use anyhow::Context; -#[cfg(feature = "fs")] -use slog::warn; use slog::{Logger, o}; #[cfg(feature = "fs")] -use std::fmt; -#[cfg(feature = "fs")] -use std::ops::RangeInclusive; -#[cfg(feature = "fs")] -use std::path::{Path, PathBuf}; -#[cfg(feature = "fs")] -use std::sync::Arc; -#[cfg(feature = "fs")] -use thiserror::Error; +use { + slog::warn, + std::{ + fmt, + ops::RangeInclusive, + path::{Path, PathBuf}, + sync::Arc, + }, + thiserror::Error, +}; -#[cfg(feature = "fs")] -use crate::cardano_database_client::{ImmutableFileRange, VerifiedDigests}; #[cfg(feature = "fs")] use mithril_cardano_node_internal_database::{ IMMUTABLE_DIR, digesters::{CardanoImmutableDigester, ImmutableDigester, ImmutableDigesterError}, }; -use mithril_common::logging::LoggerExtensions; -use mithril_common::protocol::SignerBuilder; -use mithril_common::signable_builder::CardanoStakeDistributionSignableBuilder; #[cfg(feature = "fs")] use mithril_common::{ crypto_helper::MKTreeNode, entities::{ImmutableFileName, ImmutableFileNumber, SignedEntityType}, messages::{CardanoDatabaseSnapshotMessage, CertificateMessage}, }; +use mithril_common::{ + logging::LoggerExtensions, protocol::SignerBuilder, + signable_builder::CardanoStakeDistributionSignableBuilder, +}; -#[cfg(feature = "fs")] -use crate::MithrilError; use crate::{ CardanoStakeDistribution, MithrilCertificate, MithrilResult, MithrilSigner, MithrilStakeDistribution, VerifiedCardanoTransactions, common::{ProtocolMessage, ProtocolMessagePartKey}, }; +#[cfg(feature = "fs")] +use crate::{ + MithrilError, + cardano_database_client::{ImmutableFileRange, VerifiedDigests}, +}; cfg_fs! { const MERKLE_PROOF_COMPUTATION_ERROR:&str = "Merkle proof computation failed"; From 9eaf920920213517a4db09346a82bfa57fa11271 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Turmel?= Date: Tue, 22 Jul 2025 14:42:31 +0200 Subject: [PATCH 18/18] feature(client-lib): rename ComputeCardanoDatabaseMessageError errors, fix dummy tool import --- mithril-client/src/message.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mithril-client/src/message.rs b/mithril-client/src/message.rs index 52efeae98da..ad8b2a733a5 100644 --- a/mithril-client/src/message.rs +++ b/mithril-client/src/message.rs @@ -62,13 +62,13 @@ cfg_fs! { ImmutableFilesVerification(ImmutableVerificationResult), /// Error related to the immutable files digests computation. - ImmutableFilesDigester(#[from] ImmutableDigesterError), + DigestsComputation(#[from] ImmutableDigesterError), /// Error related to the Merkle proof verification. MerkleProofVerification(#[source] MithrilError), /// Error related to the immutable files range. - ImmutableFilesRange(#[source] MithrilError), + ImmutableFilesRangeCreation(#[source] MithrilError), } impl fmt::Display for ComputeCardanoDatabaseMessageError { @@ -118,13 +118,13 @@ cfg_fs! { } Ok(()) } - ComputeCardanoDatabaseMessageError::ImmutableFilesDigester(e) => { + ComputeCardanoDatabaseMessageError::DigestsComputation(e) => { write!(f, "Immutable files digester error: {e:?}") } ComputeCardanoDatabaseMessageError::MerkleProofVerification(e) => { write!(f, "Merkle proof verification error: {e:?}") } - ComputeCardanoDatabaseMessageError::ImmutableFilesRange(e) => { + ComputeCardanoDatabaseMessageError::ImmutableFilesRangeCreation(e) => { write!(f, "Immutable files range error: {e:?}") } } @@ -248,7 +248,7 @@ impl MessageBuilder { let network = certificate.metadata.network.clone(); let immutable_file_number_range = immutable_file_range .to_range_inclusive(cardano_database_snapshot.beacon.immutable_file_number) - .map_err(ComputeCardanoDatabaseMessageError::ImmutableFilesRange)?; + .map_err(ComputeCardanoDatabaseMessageError::ImmutableFilesRangeCreation)?; let missing_immutable_files = if allow_missing { vec![] } else { @@ -467,6 +467,7 @@ mod tests { use mithril_common::{ entities::{CardanoDbBeacon, Epoch}, messages::CertificateMessage, + test::double::Dummy, }; use crate::cardano_database_client::ImmutableFileRange;