diff --git a/docs/website/root/manual/develop/nodes/mithril-client.md b/docs/website/root/manual/develop/nodes/mithril-client.md index 75a8b7ecfb7..9ee51d14d2b 100644 --- a/docs/website/root/manual/develop/nodes/mithril-client.md +++ b/docs/website/root/manual/develop/nodes/mithril-client.md @@ -618,6 +618,9 @@ Here is a list of the available parameters: | `digest` | - | - | - | Digest of the Cardano db snapshot to verify or `latest` for the latest artifact | - | - | :heavy_check_mark: | | `db_dir` | `--db-dir` | - | - | Directory from where the immutable will be verified | - | - | - | | `genesis_verification_key` | `--genesis-verification-key` | - | `GENESIS_VERIFICATION_KEY` | Genesis verification key to check the certificate chain | - | - | - | +| `start` | `--start` | - | - | The first immutable file number to verify | - | - | - | +| `end` | `--end` | - | - | The last immutable file number to verify | - | - | - | +| `allow_missing` | `--allow-missing` | - | - | If set, the verification will not fail if some immutable files are missing | `false` | - | - | | `run_mode` | `--run-mode` | - | `RUN_MODE` | Run Mode | `dev` | - | - | | `verbose` | `--verbose` | `-v` | - | Verbosity level (-v=warning, -vv=info, -vvv=debug, -vvvv=trace) | `0` | - | - | | `config_directory` | `--config-directory` | - | - | Directory where configuration file is located | `./config` | - | - | diff --git a/examples/client-cardano-database-v2/src/main.rs b/examples/client-cardano-database-v2/src/main.rs index 33dcd17de62..c380a5adad6 100644 --- a/examples/client-cardano-database-v2/src/main.rs +++ b/examples/client-cardano-database-v2/src/main.rs @@ -17,7 +17,7 @@ use std::time::Duration; use tokio::sync::RwLock; use mithril_client::feedback::{FeedbackReceiver, MithrilEvent, MithrilEventCardanoDatabase}; -use mithril_client::{ClientBuilder, MessageBuilder, MithrilResult}; +use mithril_client::{ClientBuilder, MessageBuilder, MithrilError, MithrilResult}; #[derive(Parser, Debug)] #[command(version)] @@ -82,7 +82,7 @@ async fn main() -> MithrilResult<()> { .verify_chain(&cardano_database_snapshot.certificate_hash) .await?; - let immutable_file_range = ImmutableFileRange::From(15000); + let immutable_file_range = ImmutableFileRange::From(4000); let download_unpack_options = DownloadUnpackOptions { allow_override: true, include_ancillary: true, @@ -98,19 +98,11 @@ async fn main() -> MithrilResult<()> { ) .await?; - println!("Computing Cardano database Merkle proof...",); - let merkle_proof = client + println!("Downloading and verifying digests file authenticity..."); + let verified_digests = client .cardano_database_v2() - .compute_merkle_proof( - &certificate, - &cardano_database_snapshot, - &immutable_file_range, - &unpacked_dir, - ) + .download_and_verify_digests(&certificate, &cardano_database_snapshot) .await?; - merkle_proof - .verify() - .with_context(|| "Merkle proof verification failed")?; println!("Sending usage statistics to the aggregator..."); let full_restoration = immutable_file_range == ImmutableFileRange::Full; @@ -133,9 +125,17 @@ async fn main() -> MithrilResult<()> { "Computing Cardano database snapshot '{}' message...", cardano_database_snapshot.hash ); + let allow_missing_immutables_files = false; let message = wait_spinner( &progress_bar, - MessageBuilder::new().compute_cardano_database_message(&certificate, &merkle_proof), + MessageBuilder::new().compute_cardano_database_message( + &certificate, + &cardano_database_snapshot, + &immutable_file_range, + allow_missing_immutables_files, + &unpacked_dir, + &verified_digests, + ), ) .await?; @@ -261,10 +261,13 @@ fn get_temp_dir() -> MithrilResult { Ok(dir) } -async fn wait_spinner( +pub async fn wait_spinner( progress_bar: &MultiProgress, - future: impl Future>, -) -> MithrilResult { + future: impl Future>, +) -> MithrilResult +where + MithrilError: From, +{ let pb = progress_bar.add(ProgressBar::new_spinner()); let spinner = async move { loop { @@ -275,6 +278,6 @@ async fn wait_spinner( tokio::select! { _ = spinner => Err(anyhow!("timeout")), - res = future => res, + res = future => res.map_err(Into::into), } } diff --git a/mithril-client-cli/src/commands/cardano_db/download/v2.rs b/mithril-client-cli/src/commands/cardano_db/download/v2.rs index 9c604b8a281..d08ae8af7a5 100644 --- a/mithril-client-cli/src/commands/cardano_db/download/v2.rs +++ b/mithril-client-cli/src/commands/cardano_db/download/v2.rs @@ -15,7 +15,10 @@ use mithril_client::{ use crate::{ CommandContext, commands::{ - cardano_db::{download::DB_DIRECTORY_NAME, shared_steps}, + cardano_db::{ + download::DB_DIRECTORY_NAME, + shared_steps::{self, ComputeCardanoDatabaseMessageOptions}, + }, client_builder, }, utils::{ @@ -124,22 +127,33 @@ impl PreparedCardanoDbV2Download { ) })?; - let merkle_proof = shared_steps::compute_verify_merkle_proof( + let verified_digests = shared_steps::download_and_verify_digests( 4, &progress_printer, &client, &certificate, &cardano_db_message, - &restoration_options.immutable_file_range, - &restoration_options.db_dir, ) - .await?; + .await + .with_context(|| { + format!( + "Can not download and verify digests file for cardano db snapshot with hash: '{}'", + self.hash + ) + })?; + let options = ComputeCardanoDatabaseMessageOptions { + db_dir: restoration_options.db_dir.clone(), + immutable_file_range: restoration_options.immutable_file_range, + allow_missing: false, + }; let message = shared_steps::compute_cardano_db_snapshot_message( 5, &progress_printer, &certificate, - &merkle_proof, + &cardano_db_message, + &options, + &verified_digests, ) .await?; diff --git a/mithril-client-cli/src/commands/cardano_db/shared_steps.rs b/mithril-client-cli/src/commands/cardano_db/shared_steps.rs index 5efdd3e675d..56e1d55fd51 100644 --- a/mithril-client-cli/src/commands/cardano_db/shared_steps.rs +++ b/mithril-client-cli/src/commands/cardano_db/shared_steps.rs @@ -1,16 +1,22 @@ use anyhow::{Context, anyhow}; use chrono::Utc; use slog::{Logger, debug, warn}; -use std::path::Path; +use std::path::{Path, PathBuf}; use mithril_client::{ CardanoDatabaseSnapshot, Client, MessageBuilder, MithrilCertificate, MithrilResult, - cardano_database_client::ImmutableFileRange, - common::{ImmutableFileNumber, MKProof, ProtocolMessage}, + cardano_database_client::{ImmutableFileRange, VerifiedDigests}, + common::{ImmutableFileNumber, ProtocolMessage}, }; use crate::utils::{CardanoDbUtils, ProgressPrinter}; +pub struct ComputeCardanoDatabaseMessageOptions { + pub db_dir: PathBuf, + pub immutable_file_range: ImmutableFileRange, + pub allow_missing: bool, +} + pub async fn fetch_certificate_and_verifying_chain( step_number: u16, progress_printer: &ProgressPrinter, @@ -47,32 +53,20 @@ pub fn immutable_file_range( } } -/// Computes and verifies the Merkle proof for the given certificate and database snapshot. -pub async fn compute_verify_merkle_proof( +pub async fn download_and_verify_digests( step_number: u16, progress_printer: &ProgressPrinter, client: &Client, certificate: &MithrilCertificate, cardano_database_snapshot: &CardanoDatabaseSnapshot, - immutable_file_range: &ImmutableFileRange, - unpacked_dir: &Path, -) -> MithrilResult { - progress_printer.report_step(step_number, "Computing and verifying the Merkle proof…")?; - let merkle_proof = client +) -> MithrilResult { + progress_printer.report_step(step_number, "Downloading and verifying digests…")?; + let verified_digests = client .cardano_database_v2() - .compute_merkle_proof( - certificate, - cardano_database_snapshot, - immutable_file_range, - Path::new(&unpacked_dir), - ) + .download_and_verify_digests(certificate, cardano_database_snapshot) .await?; - merkle_proof - .verify() - .with_context(|| "Merkle proof verification failed")?; - - Ok(merkle_proof) + Ok(verified_digests) } /// Computes the Cardano database snapshot message using the provided certificate and Merkle proof. @@ -80,12 +74,21 @@ pub async fn compute_cardano_db_snapshot_message( step_number: u16, progress_printer: &ProgressPrinter, certificate: &MithrilCertificate, - merkle_proof: &MKProof, + cardano_database_snapshot: &CardanoDatabaseSnapshot, + options: &ComputeCardanoDatabaseMessageOptions, + verified_digest: &VerifiedDigests, ) -> MithrilResult { progress_printer.report_step(step_number, "Computing the cardano db snapshot message")?; let message = CardanoDbUtils::wait_spinner( progress_printer, - MessageBuilder::new().compute_cardano_database_message(certificate, merkle_proof), + MessageBuilder::new().compute_cardano_database_message( + certificate, + cardano_database_snapshot, + &options.immutable_file_range, + options.allow_missing, + &options.db_dir, + verified_digest, + ), ) .await .with_context(|| "Can not compute the cardano db snapshot message")?; diff --git a/mithril-client-cli/src/commands/cardano_db/verify.rs b/mithril-client-cli/src/commands/cardano_db/verify.rs index 6545f693423..20e43dec10f 100644 --- a/mithril-client-cli/src/commands/cardano_db/verify.rs +++ b/mithril-client-cli/src/commands/cardano_db/verify.rs @@ -5,14 +5,20 @@ use std::{ }; use anyhow::Context; -use chrono::Utc; +use chrono::{DateTime, Utc}; use clap::Parser; -use mithril_client::MithrilResult; +use mithril_client::{ + CardanoDatabaseSnapshot, ComputeCardanoDatabaseMessageError, ImmutableVerificationResult, + MithrilResult, cardano_database_client::ImmutableFileRange, common::ImmutableFileNumber, +}; use crate::{ CommandContext, commands::{ - cardano_db::{CardanoDbCommandsBackend, shared_steps}, + cardano_db::{ + CardanoDbCommandsBackend, + shared_steps::{self, ComputeCardanoDatabaseMessageOptions}, + }, client_builder, }, configuration::{ConfigError, ConfigSource}, @@ -38,6 +44,22 @@ pub struct CardanoDbVerifyCommand { /// Genesis verification key to check the certificate chain. #[clap(long, env = "GENESIS_VERIFICATION_KEY")] genesis_verification_key: Option, + + /// The first immutable file number to verify. + /// + /// If not set, the verify process will start from the first immutable file. + #[clap(long)] + start: Option, + + /// The last immutable file number to verify. + /// + /// If not set, the verify will continue until the last certified immutable file. + #[clap(long)] + end: Option, + + /// If set, the verification will not fail if some immutable files are missing. + #[clap(long)] + allow_missing: bool, } impl CardanoDbVerifyCommand { @@ -94,6 +116,14 @@ impl CardanoDbVerifyCommand { .await? .with_context(|| format!("Can not get the cardano db for hash: '{}'", self.digest))?; + let immutable_file_range = shared_steps::immutable_file_range(self.start, self.end); + + print_immutables_range_to_verify( + &cardano_db_message, + &immutable_file_range, + context.is_json_output_enabled(), + )?; + let certificate = shared_steps::fetch_certificate_and_verifying_chain( 1, &progress_printer, @@ -102,45 +132,62 @@ impl CardanoDbVerifyCommand { ) .await?; - let immutable_file_range = shared_steps::immutable_file_range(None, None); - - let merkle_proof = shared_steps::compute_verify_merkle_proof( + let verified_digests = shared_steps::download_and_verify_digests( 2, &progress_printer, &client, &certificate, &cardano_db_message, - &immutable_file_range, - db_dir, ) .await?; + let options = ComputeCardanoDatabaseMessageOptions { + db_dir: db_dir.to_path_buf(), + immutable_file_range, + allow_missing: self.allow_missing, + }; let message = shared_steps::compute_cardano_db_snapshot_message( 3, &progress_printer, &certificate, - &merkle_proof, - ) - .await?; - - shared_steps::verify_message_matches_certificate( - &context.logger().clone(), - 4, - &progress_printer, - &certificate, - &message, &cardano_db_message, - db_dir, + &options, + &verified_digests, ) - .await?; + .await; - Self::log_verified_information( - db_dir, - &cardano_db_message.hash, - context.is_json_output_enabled(), - )?; + match message { + Err(e) => match e.downcast_ref::() { + Some(ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(lists)) => { + Self::print_immutables_verification_error( + lists, + context.is_json_output_enabled(), + ); + Ok(()) + } + _ => Err(e), + }, + Ok(message) => { + shared_steps::verify_message_matches_certificate( + &context.logger().clone(), + 4, + &progress_printer, + &certificate, + &message, + &cardano_db_message, + db_dir, + ) + .await?; - Ok(()) + Self::log_verified_information( + db_dir, + &cardano_db_message.hash, + context.is_json_output_enabled(), + )?; + + Ok(()) + } + } } fn log_verified_information( @@ -164,6 +211,94 @@ impl CardanoDbVerifyCommand { } Ok(()) } + + fn print_immutables_verification_error(lists: &ImmutableVerificationResult, json_output: bool) { + let utc_now = Utc::now(); + let json_file_path = write_json_file_error(utc_now, lists); + let error_message = "Verifying immutables files has failed"; + if json_output { + let json = serde_json::json!({ + "timestamp": utc_now.to_rfc3339(), + "verify_error" : { + "message": error_message, + "immutables_verification_error_file": json_file_path, + "immutables_dir": lists.immutables_dir, + "missing_files_count": lists.missing.len(), + "tampered_files_count": lists.tampered.len(), + "non_verifiable_files_count": lists.non_verifiable.len(), + } + }); + + println!("{json}"); + } else { + println!("{error_message}"); + println!( + "See the lists of all missing, tampered and non verifiable files in {}", + json_file_path.display() + ); + if !lists.missing.is_empty() { + println!("Number of missing immutable files: {}", lists.missing.len()); + } + if !lists.tampered.is_empty() { + println!( + "Number of tampered immutable files: {:?}", + lists.tampered.len() + ); + } + if !lists.non_verifiable.is_empty() { + println!( + "Number of non verifiable immutable files: {:?}", + lists.non_verifiable.len() + ); + } + } + } +} + +fn write_json_file_error(date: DateTime, lists: &ImmutableVerificationResult) -> PathBuf { + let file_path = PathBuf::from(format!( + "immutables_verification_error-{}.json", + date.timestamp() + )); + std::fs::write( + &file_path, + serde_json::to_string_pretty(&serde_json::json!({ + "timestamp": date.to_rfc3339(), + "immutables_dir": lists.immutables_dir, + "missing-files": lists.missing, + "tampered-files": lists.tampered, + "non-verifiable-files": lists.non_verifiable, + })) + .unwrap(), + ) + .expect("Could not write immutables verification error to file"); + file_path +} + +fn print_immutables_range_to_verify( + cardano_db_message: &CardanoDatabaseSnapshot, + immutable_file_range: &ImmutableFileRange, + json_output: bool, +) -> Result<(), anyhow::Error> { + let range_to_verify = + immutable_file_range.to_range_inclusive(cardano_db_message.beacon.immutable_file_number)?; + if json_output { + let json = serde_json::json!({ + "timestamp": Utc::now().to_rfc3339(), + "local_immutable_range_to_verify": { + "start": range_to_verify.start(), + "end": range_to_verify.end(), + }, + }); + println!("{json}"); + } else { + eprintln!( + "Verifying local immutable files from number {} to {}", + range_to_verify.start(), + range_to_verify.end() + ); + } + Ok(()) } impl ConfigSource for CardanoDbVerifyCommand { diff --git a/mithril-client-cli/src/utils/cardano_db.rs b/mithril-client-cli/src/utils/cardano_db.rs index 2632ab073ad..fcae05d3ddb 100644 --- a/mithril-client-cli/src/utils/cardano_db.rs +++ b/mithril-client-cli/src/utils/cardano_db.rs @@ -22,10 +22,13 @@ impl CardanoDbUtils { } /// Display a spinner while waiting for the result of a future - pub async fn wait_spinner( + pub async fn wait_spinner( progress_bar: &MultiProgress, - future: impl Future>, - ) -> MithrilResult { + future: impl Future>, + ) -> MithrilResult + where + MithrilError: From, + { let pb = progress_bar.add(ProgressBar::new_spinner()); let spinner = async move { loop { @@ -36,7 +39,7 @@ impl CardanoDbUtils { tokio::select! { _ = spinner => Err(anyhow!("timeout")), - res = future => res, + res = future => res.map_err(Into::into), } } diff --git a/mithril-client/src/cardano_database_client/api.rs b/mithril-client/src/cardano_database_client/api.rs index 3f643e3fa49..3f0b59b9937 100644 --- a/mithril-client/src/cardano_database_client/api.rs +++ b/mithril-client/src/cardano_database_client/api.rs @@ -6,16 +6,15 @@ use std::sync::Arc; use slog::Logger; #[cfg(feature = "fs")] -use mithril_common::{ - crypto_helper::MKProof, - messages::{CardanoDatabaseSnapshotMessage, CertificateMessage}, -}; +use mithril_common::messages::{CardanoDatabaseSnapshotMessage, CertificateMessage}; #[cfg(feature = "fs")] use mithril_cardano_node_internal_database::entities::ImmutableFile; use crate::aggregator_client::AggregatorClient; #[cfg(feature = "fs")] +use crate::cardano_database_client::VerifiedDigests; +#[cfg(feature = "fs")] use crate::feedback::FeedbackSender; #[cfg(feature = "fs")] use crate::file_downloader::FileDownloader; @@ -100,22 +99,15 @@ impl CardanoDatabaseClient { .await } - /// Compute the Merkle proof of membership for the given immutable file range. + /// Download and verify the digests against the certificate. #[cfg(feature = "fs")] - pub async fn compute_merkle_proof( + pub async fn download_and_verify_digests( &self, certificate: &CertificateMessage, cardano_database_snapshot: &CardanoDatabaseSnapshotMessage, - immutable_file_range: &ImmutableFileRange, - database_dir: &Path, - ) -> MithrilResult { + ) -> MithrilResult { self.artifact_prover - .compute_merkle_proof( - certificate, - cardano_database_snapshot, - immutable_file_range, - database_dir, - ) + .download_and_verify_digests(certificate, cardano_database_snapshot) .await } diff --git a/mithril-client/src/cardano_database_client/mod.rs b/mithril-client/src/cardano_database_client/mod.rs index 7a243cf3c20..bbd0adc2e56 100644 --- a/mithril-client/src/cardano_database_client/mod.rs +++ b/mithril-client/src/cardano_database_client/mod.rs @@ -4,8 +4,7 @@ //! - [get][CardanoDatabaseClient::get]: get a Cardano database data from its hash //! - [list][CardanoDatabaseClient::list]: get the list of available Cardano database //! - [download_unpack][CardanoDatabaseClient::download_unpack]: download and unpack a Cardano database snapshot for a given immutable files range -//! - [compute_merkle_proof][CardanoDatabaseClient::compute_merkle_proof]: compute a Merkle proof for a given Cardano database snapshot and a given immutable files range -//! +//! - [download_and_verify_digests][CardanoDatabaseClient::download_and_verify_digests]: download and verify the digests of the immutable files in a Cardano database snapshot //! # Get a Cardano database //! //! To get a Cardano database using the [ClientBuilder][crate::client::ClientBuilder]. @@ -88,7 +87,7 @@ //! ```no_run //! # #[cfg(feature = "fs")] //! # async fn run() -> mithril_client::MithrilResult<()> { -//! use mithril_client::{ClientBuilder, cardano_database_client::{ImmutableFileRange, DownloadUnpackOptions}}; +//! use mithril_client::{ClientBuilder, MessageBuilder, cardano_database_client::{ImmutableFileRange, DownloadUnpackOptions}}; //! use std::path::Path; //! //! let client = ClientBuilder::aggregator("YOUR_AGGREGATOR_ENDPOINT", "YOUR_GENESIS_VERIFICATION_KEY").build()?; @@ -113,10 +112,22 @@ //! ) //! .await?; //! -//! let merkle_proof = client +//! let verified_digests = client //! .cardano_database_v2() -//! .compute_merkle_proof(&certificate, &cardano_database_snapshot, &immutable_file_range, &target_directory) +//! .download_and_verify_digests( +//! &certificate, +//! &cardano_database_snapshot) //! .await?; +//! +//! let allow_missing_immutables_files = false; +//! let message = MessageBuilder::new().compute_cardano_database_message( +//! &certificate, +//! &cardano_database_snapshot, +//! &immutable_file_range, +//! allow_missing_immutables_files, +//! &target_directory, +//! &verified_digests, +//! ).await?; //! # //! # Ok(()) //! # } @@ -136,4 +147,5 @@ cfg_fs! { pub use download_unpack::DownloadUnpackOptions; pub use immutable_file_range::ImmutableFileRange; + pub use proving::VerifiedDigests; } diff --git a/mithril-client/src/cardano_database_client/proving.rs b/mithril-client/src/cardano_database_client/proving.rs index ff8be6882d4..1ee4838594d 100644 --- a/mithril-client/src/cardano_database_client/proving.rs +++ b/mithril-client/src/cardano_database_client/proving.rs @@ -7,13 +7,10 @@ use std::{ use anyhow::{Context, anyhow}; -use mithril_cardano_node_internal_database::{ - digesters::{CardanoImmutableDigester, ImmutableDigester}, - entities::ImmutableFile, -}; +use mithril_cardano_node_internal_database::entities::ImmutableFile; use mithril_common::{ - crypto_helper::{MKProof, MKTree, MKTreeNode, MKTreeStoreInMemory}, - entities::{DigestLocation, HexEncodedDigest, ImmutableFileName}, + crypto_helper::{MKTree, MKTreeNode, MKTreeStoreInMemory}, + entities::{DigestLocation, HexEncodedDigest, ImmutableFileName, ProtocolMessagePartKey}, messages::{ CardanoDatabaseDigestListItemMessage, CardanoDatabaseSnapshotMessage, CertificateMessage, DigestsMessagePart, @@ -27,7 +24,50 @@ use crate::{ utils::{create_directory_if_not_exists, delete_directory, read_files_in_directory}, }; -use super::immutable_file_range::ImmutableFileRange; +/// Represents the verified digests and the Merkle tree built from them. +pub struct VerifiedDigests { + /// A map of immutable file names to their corresponding verified digests. + pub digests: BTreeMap, + /// The Merkle tree built from the digests. + pub merkle_tree: MKTree, +} + +/// Represents the immutable files that were not verified during the digest verification process. +#[derive(PartialEq, Debug)] +pub(crate) struct ImmutableFilesNotVerified { + /// List of immutable files that were tampered (i.e. their digest does not match the verified digest) + pub tampered_files: Vec, + /// List of immutable files that could not be verified (i.e., not present in the digests) + pub non_verifiable_files: Vec, +} + +impl VerifiedDigests { + pub(crate) fn list_immutable_files_not_verified( + &self, + computed_digests: &BTreeMap, + ) -> ImmutableFilesNotVerified { + let mut tampered_files = vec![]; + let mut non_verifiable_files = vec![]; + + for (immutable_file, digest) in computed_digests.iter() { + let immutable_file_name_to_verify = immutable_file.filename.clone(); + match self.digests.get(&immutable_file_name_to_verify) { + Some(verified_digest) if verified_digest != digest => { + tampered_files.push(immutable_file_name_to_verify); + } + None => { + non_verifiable_files.push(immutable_file_name_to_verify); + } + _ => {} + } + } + + ImmutableFilesNotVerified { + tampered_files, + non_verifiable_files, + } + } +} pub struct InternalArtifactProver { http_file_downloader: Arc, @@ -43,24 +83,43 @@ impl InternalArtifactProver { } } - /// Compute the Merkle proof of membership for the given immutable file range. - pub async fn compute_merkle_proof( + fn check_merkle_root_is_signed_by_certificate( + certificate: &CertificateMessage, + merkle_root: &MKTreeNode, + ) -> MithrilResult<()> { + let mut message = certificate.protocol_message.clone(); + message.set_message_part( + ProtocolMessagePartKey::CardanoDatabaseMerkleRoot, + merkle_root.to_hex(), + ); + + if !certificate.match_message(&message) { + return Err(anyhow!( + "Certificate message does not match the computed message for certificate {}", + certificate.hash + )); + } + + Ok(()) + } + + /// Download digests and verify its authenticity against the certificate. + pub async fn download_and_verify_digests( &self, certificate: &CertificateMessage, cardano_database_snapshot: &CardanoDatabaseSnapshotMessage, - immutable_file_range: &ImmutableFileRange, - database_dir: &Path, - ) -> MithrilResult { + ) -> MithrilResult { let digest_target_dir = Self::digest_target_dir(); delete_directory(&digest_target_dir)?; self.download_unpack_digest_file(&cardano_database_snapshot.digests, &digest_target_dir) .await?; - let network = certificate.metadata.network.clone(); let last_immutable_file_number = cardano_database_snapshot.beacon.immutable_file_number; - let immutable_file_number_range = - immutable_file_range.to_range_inclusive(last_immutable_file_number)?; + let downloaded_digests = self.read_digest_file(&digest_target_dir)?; - let downloaded_digests_values = downloaded_digests + delete_directory(&digest_target_dir)?; + + let filtered_digests = downloaded_digests + .clone() .into_iter() .filter(|(immutable_file_name, _)| { match ImmutableFile::new(Path::new(immutable_file_name).to_path_buf()) { @@ -68,20 +127,20 @@ impl InternalArtifactProver { Err(_) => false, } }) - .map(|(_immutable_file_name, digest)| digest) - .collect::>(); - let merkle_tree: MKTree = MKTree::new(&downloaded_digests_values)?; - let immutable_digester = CardanoImmutableDigester::new(network, None, self.logger.clone()); - let computed_digests = immutable_digester - .compute_digests_for_range(database_dir, &immutable_file_number_range) - .await? - .entries - .values() - .map(MKTreeNode::from) - .collect::>(); - delete_directory(&digest_target_dir)?; + .collect::>(); + + let filtered_digests_values = filtered_digests.values().collect::>(); + let merkle_tree: MKTree = MKTree::new(&filtered_digests_values)?; - merkle_tree.compute_proof(&computed_digests) + Self::check_merkle_root_is_signed_by_certificate( + certificate, + &merkle_tree.compute_root()?, + )?; + + Ok(VerifiedDigests { + digests: filtered_digests, + merkle_tree, + }) } async fn download_unpack_digest_file( @@ -176,7 +235,6 @@ mod tests { use std::path::Path; use std::sync::Arc; - use mithril_cardano_node_internal_database::test::DummyCardanoDbBuilder; use mithril_common::{ entities::{CardanoDbBeacon, Epoch, HexEncodedDigest}, messages::CardanoDatabaseDigestListItemMessage, @@ -190,87 +248,108 @@ mod tests { use super::*; - mod compute_merkle_proof { - - use std::ops::RangeInclusive; - - use mithril_cardano_node_internal_database::{ - IMMUTABLE_DIR, digesters::ComputedImmutablesDigests, - }; - use mithril_common::{ - StdResult, entities::ImmutableFileNumber, messages::DigestsMessagePart, - }; + mod list_immutable_files_not_verified { use super::*; - async fn prepare_fake_digests( - dir_name: &str, - beacon: &CardanoDbBeacon, - immutable_file_range: &RangeInclusive, - digests_offset: usize, - ) -> ( - PathBuf, - CardanoDatabaseSnapshotMessage, - CertificateMessage, - MKTree, - ComputedImmutablesDigests, - ) { - let cardano_database_snapshot = CardanoDatabaseSnapshotMessage { - hash: "hash-123".to_string(), - beacon: beacon.clone(), - digests: DigestsMessagePart { - size_uncompressed: 1024, - locations: vec![DigestLocation::CloudStorage { - uri: "http://whatever/digests.json".to_string(), - compression_algorithm: None, - }], - }, - ..CardanoDatabaseSnapshotMessage::dummy() + fn fake_immutable(filename: &str) -> ImmutableFile { + ImmutableFile { + path: PathBuf::from("whatever"), + number: 1, + filename: filename.to_string(), + } + } + + #[test] + fn should_return_empty_list_when_no_tampered_files() { + let digests_to_verify = BTreeMap::from([ + (fake_immutable("00001.chunk"), "digest-1".to_string()), + (fake_immutable("00002.chunk"), "digest-2".to_string()), + ]); + + let verified_digests = VerifiedDigests { + digests: BTreeMap::from([ + ("00001.chunk".to_string(), "digest-1".to_string()), + ("00002.chunk".to_string(), "digest-2".to_string()), + ]), + merkle_tree: MKTree::new(&["whatever"]).unwrap(), }; - let certificate = CertificateMessage { - hash: "cert-hash-123".to_string(), - ..CertificateMessage::dummy() + + let invalid_files = + verified_digests.list_immutable_files_not_verified(&digests_to_verify); + + assert_eq!( + invalid_files, + ImmutableFilesNotVerified { + tampered_files: vec![], + non_verifiable_files: vec![], + } + ); + } + + #[test] + fn should_return_list_with_tampered_files() { + let digests_to_verify = BTreeMap::from([ + (fake_immutable("00001.chunk"), "digest-1".to_string()), + (fake_immutable("00002.chunk"), "digest-2".to_string()), + ]); + + let verified_digests = VerifiedDigests { + digests: BTreeMap::from([ + ("00001.chunk".to_string(), "digest-1".to_string()), + ("00002.chunk".to_string(), "INVALID".to_string()), + ]), + merkle_tree: MKTree::new(&["whatever"]).unwrap(), }; - let cardano_db = DummyCardanoDbBuilder::new(dir_name) - .with_immutables(&immutable_file_range.clone().collect::>()) - .append_immutable_trio() - .build(); - let database_dir = cardano_db.get_dir(); - let immutable_digester = CardanoImmutableDigester::new( - certificate.metadata.network.to_string(), - None, - TestLogger::stdout(), + + let invalid_files = + verified_digests.list_immutable_files_not_verified(&digests_to_verify); + + assert_eq!( + invalid_files, + ImmutableFilesNotVerified { + tampered_files: vec!["00002.chunk".to_string()], + non_verifiable_files: vec![], + } ); - let computed_digests = immutable_digester - .compute_digests_for_range(database_dir, immutable_file_range) - .await - .unwrap(); - // We remove the last digests_offset digests to simulate receiving - // a digest file with more immutable files than downloaded - for (immutable_file, _digest) in - computed_digests.entries.iter().rev().take(digests_offset) - { - fs::remove_file( - database_dir.join( - database_dir.join(IMMUTABLE_DIR).join(immutable_file.filename.clone()), - ), - ) - .unwrap(); - } + } - let merkle_tree = immutable_digester - .compute_merkle_tree(database_dir, beacon) - .await - .unwrap(); + #[test] + fn should_return_list_with_non_verifiable() { + let digests_to_verify = BTreeMap::from([ + (fake_immutable("00001.chunk"), "digest-1".to_string()), + ( + fake_immutable("00002.not.verifiable"), + "digest-2".to_string(), + ), + ]); - ( - database_dir.to_owned(), - cardano_database_snapshot, - certificate, - merkle_tree, - computed_digests, - ) + let verified_digests = VerifiedDigests { + digests: BTreeMap::from([("00001.chunk".to_string(), "digest-1".to_string())]), + merkle_tree: MKTree::new(&["whatever"]).unwrap(), + }; + + let invalid_files = + verified_digests.list_immutable_files_not_verified(&digests_to_verify); + + assert_eq!( + invalid_files, + ImmutableFilesNotVerified { + tampered_files: vec![], + non_verifiable_files: vec!["00002.not.verifiable".to_string()], + } + ); } + } + + mod download_and_verify_digests { + use mithril_common::{ + StdResult, + entities::{ProtocolMessage, ProtocolMessagePartKey}, + messages::DigestsMessagePart, + }; + + use super::*; fn write_digest_file( digest_dir: &Path, @@ -298,34 +377,73 @@ mod tests { Ok(()) } + fn build_digests_map(size: usize) -> BTreeMap { + let mut digests = BTreeMap::new(); + for i in 1..=size { + for name in ["chunk", "primary", "secondary"] { + let immutable_file_name = format!("{i:05}.{name}"); + let immutable_file = + ImmutableFile::new(PathBuf::from(immutable_file_name)).unwrap(); + let digest = format!("digest-{i}-{name}"); + digests.insert(immutable_file, digest); + } + } + + digests + } + #[tokio::test] - async fn compute_merkle_proof_succeeds() { + async fn download_and_verify_digest_should_return_digest_map_acording_to_beacon() { let beacon = CardanoDbBeacon { epoch: Epoch(123), - immutable_file_number: 10, + immutable_file_number: 42, + }; + let hightest_immutable_number_in_digest_file = + 123 + beacon.immutable_file_number as usize; + let digests_in_certificate_map = + build_digests_map(beacon.immutable_file_number as usize); + let protocol_message_merkle_root = { + let digests_in_certificate_values = + digests_in_certificate_map.values().cloned().collect::>(); + let certificate_merkle_tree: MKTree = + MKTree::new(&digests_in_certificate_values).unwrap(); + + certificate_merkle_tree.compute_root().unwrap().to_hex() + }; + let mut protocol_message = ProtocolMessage::new(); + protocol_message.set_message_part( + ProtocolMessagePartKey::CardanoDatabaseMerkleRoot, + protocol_message_merkle_root, + ); + let certificate = CertificateMessage { + protocol_message: protocol_message.clone(), + signed_message: protocol_message.compute_hash(), + ..CertificateMessage::dummy() + }; + + let digests_location = "http://whatever/digests.json"; + let cardano_database_snapshot = CardanoDatabaseSnapshotMessage { + beacon, + digests: DigestsMessagePart { + size_uncompressed: 1024, + locations: vec![DigestLocation::CloudStorage { + uri: digests_location.to_string(), + compression_algorithm: None, + }], + }, + ..CardanoDatabaseSnapshotMessage::dummy() }; - let immutable_file_range = 1..=15; - let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); - let digests_offset = 3; - let (database_dir, cardano_database_snapshot, certificate, merkle_tree, digests) = - prepare_fake_digests( - "compute_merkle_proof_succeeds", - &beacon, - &immutable_file_range, - digests_offset, - ) - .await; - let expected_merkle_root = merkle_tree.compute_root().unwrap(); let client = CardanoDatabaseClientDependencyInjector::new() .with_http_file_downloader(Arc::new( MockFileDownloaderBuilder::default() - .with_file_uri("http://whatever/digests.json") + .with_file_uri(digests_location) .with_target_dir(InternalArtifactProver::digest_target_dir()) .with_compression(None) .with_returning(Box::new(move |_, _, _, _, _| { - let digests = digests.entries.clone(); - let digest_dir = InternalArtifactProver::digest_target_dir(); - write_digest_file(digest_dir.as_path(), &digests)?; + write_digest_file( + &InternalArtifactProver::digest_target_dir(), + &build_digests_map(hightest_immutable_number_in_digest_file), + )?; Ok(()) })) @@ -333,21 +451,20 @@ mod tests { )) .build_cardano_database_client(); - let merkle_proof = client - .compute_merkle_proof( - &certificate, - &cardano_database_snapshot, - &immutable_file_range_to_prove, - &database_dir, - ) + let verified_digests = client + .download_and_verify_digests(&certificate, &cardano_database_snapshot) .await .unwrap(); - merkle_proof.verify().unwrap(); - let merkle_proof_root = merkle_proof.root().to_owned(); - assert_eq!(expected_merkle_root, merkle_proof_root); + let expected_digests_in_certificate = digests_in_certificate_map + .iter() + .map(|(immutable_file, digest)| { + (immutable_file.filename.clone(), digest.to_string()) + }) + .collect(); + assert_eq!(verified_digests.digests, expected_digests_in_certificate); - assert!(!database_dir.join("digest").exists()); + assert!(!InternalArtifactProver::digest_target_dir().exists()); } } diff --git a/mithril-client/src/message.rs b/mithril-client/src/message.rs index 85e085ca4e6..ad8b2a733a5 100644 --- a/mithril-client/src/message.rs +++ b/mithril-client/src/message.rs @@ -1,25 +1,136 @@ use anyhow::Context; use slog::{Logger, o}; #[cfg(feature = "fs")] -use std::path::Path; -#[cfg(feature = "fs")] -use std::sync::Arc; +use { + slog::warn, + std::{ + fmt, + ops::RangeInclusive, + path::{Path, PathBuf}, + sync::Arc, + }, + thiserror::Error, +}; #[cfg(feature = "fs")] -use mithril_cardano_node_internal_database::digesters::{ - CardanoImmutableDigester, ImmutableDigester, +use mithril_cardano_node_internal_database::{ + IMMUTABLE_DIR, + digesters::{CardanoImmutableDigester, ImmutableDigester, ImmutableDigesterError}, }; -use mithril_common::logging::LoggerExtensions; -use mithril_common::protocol::SignerBuilder; -use mithril_common::signable_builder::CardanoStakeDistributionSignableBuilder; #[cfg(feature = "fs")] -use mithril_common::{crypto_helper::MKProof, entities::SignedEntityType}; +use mithril_common::{ + crypto_helper::MKTreeNode, + entities::{ImmutableFileName, ImmutableFileNumber, SignedEntityType}, + messages::{CardanoDatabaseSnapshotMessage, CertificateMessage}, +}; +use mithril_common::{ + logging::LoggerExtensions, protocol::SignerBuilder, + signable_builder::CardanoStakeDistributionSignableBuilder, +}; use crate::{ CardanoStakeDistribution, MithrilCertificate, MithrilResult, MithrilSigner, MithrilStakeDistribution, VerifiedCardanoTransactions, common::{ProtocolMessage, ProtocolMessagePartKey}, }; +#[cfg(feature = "fs")] +use crate::{ + MithrilError, + cardano_database_client::{ImmutableFileRange, VerifiedDigests}, +}; + +cfg_fs! { + const MERKLE_PROOF_COMPUTATION_ERROR:&str = "Merkle proof computation failed"; + + /// Type containing the lists of immutable files that are missing or tampered. + #[derive(Debug, PartialEq)] + pub struct ImmutableVerificationResult { + /// The immutables files directory. + pub immutables_dir: PathBuf, + /// List of missing immutable files. + pub missing: Vec, + /// List of tampered immutable files. + pub tampered: Vec, + /// List of non-verifiable immutable files. + pub non_verifiable: Vec, + } + + /// Compute Cardano database message related errors. + #[derive(Error, Debug)] + pub enum ComputeCardanoDatabaseMessageError { + /// Error related to the verification of immutable files. + ImmutableFilesVerification(ImmutableVerificationResult), + + /// Error related to the immutable files digests computation. + DigestsComputation(#[from] ImmutableDigesterError), + + /// Error related to the Merkle proof verification. + MerkleProofVerification(#[source] MithrilError), + + /// Error related to the immutable files range. + ImmutableFilesRangeCreation(#[source] MithrilError), + } + + impl fmt::Display for ComputeCardanoDatabaseMessageError { + + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(lists) => { + fn get_first_10_files_path( + files: &[ImmutableFileName], + immutables_dir: &Path, + ) -> String { + files + .iter() + .take(10) + .map(|file| immutables_dir.join(file).to_string_lossy().to_string()) + .collect::>() + .join("\n") + } + + if !lists.missing.is_empty() { + let missing_files_subset = get_first_10_files_path(&lists.missing, &lists.immutables_dir); + writeln!( + f, + "Number of missing immutable files: {}", + lists.missing.len() + )?; + writeln!(f, "First 10 missing immutable files paths:")?; + writeln!(f, "{missing_files_subset}")?; + } + if !lists.missing.is_empty() && !lists.tampered.is_empty() { + writeln!(f)?; + } + if !lists.tampered.is_empty() { + let tampered_files_subset = get_first_10_files_path(&lists.tampered, &lists.immutables_dir); + writeln!(f,"Number of tampered immutable files: {}",lists.tampered.len())?; + writeln!(f, "First 10 tampered immutable files paths:")?; + writeln!(f, "{tampered_files_subset}")?; + } + if (!lists.missing.is_empty() || !lists.tampered.is_empty()) && !lists.non_verifiable.is_empty() { + writeln!(f)?; + } + if !lists.non_verifiable.is_empty() { + let non_verifiable_files_subset = get_first_10_files_path(&lists.non_verifiable, &lists.immutables_dir); + writeln!(f, "Number of non verifiable immutable files: {}", lists.non_verifiable.len())?; + writeln!(f, "First 10 non verifiable immutable files paths:")?; + writeln!(f, "{non_verifiable_files_subset}")?; + } + Ok(()) + } + ComputeCardanoDatabaseMessageError::DigestsComputation(e) => { + write!(f, "Immutable files digester error: {e:?}") + } + ComputeCardanoDatabaseMessageError::MerkleProofVerification(e) => { + write!(f, "Merkle proof verification error: {e:?}") + } + ComputeCardanoDatabaseMessageError::ImmutableFilesRangeCreation(e) => { + write!(f, "Immutable files range error: {e:?}") + } + } + } + } +} /// A [MessageBuilder] can be used to compute the message of Mithril artifacts. pub struct MessageBuilder { @@ -78,9 +189,9 @@ impl MessageBuilder { SignedEntityType::CardanoImmutableFilesFull(beacon) => {Ok(beacon)}, other => { Err(anyhow::anyhow!( - "Can't compute message: Given certificate `{}` does not certify a snapshot, certificate signed entity: {:?}", - snapshot_certificate.hash, - other + "Can't compute message: Given certificate `{}` does not certify a snapshot, certificate signed entity: {:?}", + snapshot_certificate.hash, + other ) )}, }?; @@ -101,19 +212,93 @@ impl MessageBuilder { Ok(message) } + fn immutable_dir(db_dir: &Path) -> PathBuf { + db_dir.join(IMMUTABLE_DIR) + } + + fn list_missing_immutable_files( + database_dir: &Path, + immutable_file_number_range: &RangeInclusive, + ) -> Vec { + let immutable_dir = Self::immutable_dir(database_dir); + let mut missing_files = Vec::new(); + + for immutable_file_number in immutable_file_number_range.clone() { + for immutable_type in ["chunk", "primary", "secondary"] { + let file_name = format!("{immutable_file_number:05}.{immutable_type}"); + if !immutable_dir.join(&file_name).exists() { + missing_files.push(ImmutableFileName::from(file_name)); + } + } + } + + missing_files + } + /// Compute message for a Cardano database. pub async fn compute_cardano_database_message( &self, - certificate: &MithrilCertificate, - merkle_proof: &MKProof, - ) -> MithrilResult { - let mut message = certificate.protocol_message.clone(); - message.set_message_part( - ProtocolMessagePartKey::CardanoDatabaseMerkleRoot, - merkle_proof.root().to_hex(), - ); + certificate: &CertificateMessage, + cardano_database_snapshot: &CardanoDatabaseSnapshotMessage, + immutable_file_range: &ImmutableFileRange, + allow_missing: bool, + database_dir: &Path, + verified_digests: &VerifiedDigests, + ) -> Result { + let network = certificate.metadata.network.clone(); + let immutable_file_number_range = immutable_file_range + .to_range_inclusive(cardano_database_snapshot.beacon.immutable_file_number) + .map_err(ComputeCardanoDatabaseMessageError::ImmutableFilesRangeCreation)?; + let missing_immutable_files = if allow_missing { + vec![] + } else { + Self::list_missing_immutable_files(database_dir, &immutable_file_number_range) + }; + let immutable_digester = CardanoImmutableDigester::new(network, None, self.logger.clone()); + let computed_digest_entries = immutable_digester + .compute_digests_for_range(database_dir, &immutable_file_number_range) + .await? + .entries; + let computed_digests = computed_digest_entries + .values() + .map(MKTreeNode::from) + .collect::>(); - Ok(message) + let proof_result = verified_digests.merkle_tree.compute_proof(&computed_digests); + if let Ok(ref merkle_proof) = proof_result + && missing_immutable_files.is_empty() + { + merkle_proof + .verify() + .map_err(ComputeCardanoDatabaseMessageError::MerkleProofVerification)?; + + let mut message = certificate.protocol_message.clone(); + message.set_message_part( + ProtocolMessagePartKey::CardanoDatabaseMerkleRoot, + merkle_proof.root().to_hex(), + ); + + return Ok(message); + } + + let (tampered, non_verifiable) = match proof_result { + Err(e) => { + warn!(self.logger, "{MERKLE_PROOF_COMPUTATION_ERROR}: {e:}"); + let verified_digests = verified_digests + .list_immutable_files_not_verified(&computed_digest_entries); + + (verified_digests.tampered_files, verified_digests.non_verifiable_files) + } + Ok(_) => (vec![], vec![]), + }; + Err( + ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(ImmutableVerificationResult { + immutables_dir: Self::immutable_dir(database_dir), + missing: missing_immutable_files, + tampered, + non_verifiable, + }), + ) } } @@ -187,3 +372,603 @@ impl Default for MessageBuilder { Self::new() } } + +#[cfg(test)] +#[cfg(feature = "fs")] +mod tests { + use mithril_cardano_node_internal_database::test::DummyCardanoDbBuilder; + + use super::*; + + fn remove_immutable_files>(database_dir: &Path, immutable_file_names: &[T]) { + for immutable_file_name in immutable_file_names { + let immutable_file_path = + MessageBuilder::immutable_dir(database_dir).join(immutable_file_name.as_ref()); + std::fs::remove_file(immutable_file_path).unwrap(); + } + } + + fn tamper_immutable_files>(database_dir: &Path, immutable_file_names: &[T]) { + for immutable_file_name in immutable_file_names { + let immutable_file_path = + MessageBuilder::immutable_dir(database_dir).join(immutable_file_name.as_ref()); + std::fs::write(immutable_file_path, "tampered content").unwrap(); + } + } + + mod list_missing_immutable_files { + use mithril_common::temp_dir_create; + + use super::*; + + #[test] + fn list_missing_immutable_files_should_return_empty_list_if_no_missing_files() { + let immutable_files_in_db = 1..=10; + let range_to_verify = 3..=5; + let cardano_db = + DummyCardanoDbBuilder::new(&format!("{}", temp_dir_create!().display())) + .with_immutables(&immutable_files_in_db.collect::>()) + .append_immutable_trio() + .build(); + + let missing_files = MessageBuilder::list_missing_immutable_files( + cardano_db.get_dir(), + &range_to_verify, + ); + + assert!(missing_files.is_empty()); + } + + #[test] + fn list_missing_immutable_files_should_return_empty_list_if_missing_files_outside_range() { + let immutable_files_in_db = 1..=10; + let range_to_verify = 3..=5; + let cardano_db = + DummyCardanoDbBuilder::new(&format!("{}", temp_dir_create!().display())) + .with_immutables(&immutable_files_in_db.collect::>()) + .append_immutable_trio() + .build(); + let files_to_remove = vec!["00002.chunk", "00006.primary"]; + remove_immutable_files(cardano_db.get_dir(), &files_to_remove); + + let missing_files = MessageBuilder::list_missing_immutable_files( + cardano_db.get_dir(), + &range_to_verify, + ); + + assert!(missing_files.is_empty()); + } + + #[test] + fn list_missing_immutable_files_should_return_list_of_missing_files_inside_range() { + let immutable_files_in_db = 1..=10; + let range_to_verify = 3..=5; + let cardano_db = + DummyCardanoDbBuilder::new(&format!("{}", temp_dir_create!().display())) + .with_immutables(&immutable_files_in_db.collect::>()) + .append_immutable_trio() + .build(); + let files_to_remove = vec!["00004.chunk", "00005.primary"]; + remove_immutable_files(cardano_db.get_dir(), &files_to_remove); + + let missing_files = MessageBuilder::list_missing_immutable_files( + cardano_db.get_dir(), + &range_to_verify, + ); + + assert_eq!(missing_files, files_to_remove); + } + } + + mod cardano_database_message { + + use std::{collections::BTreeMap, path::PathBuf}; + + use mithril_common::{ + entities::{CardanoDbBeacon, Epoch}, + messages::CertificateMessage, + test::double::Dummy, + }; + + use crate::cardano_database_client::ImmutableFileRange; + use crate::{cardano_database_client::VerifiedDigests, test_utils::TestLogger}; + + use super::*; + + async fn prepare_db_and_verified_digests( + dir_name: &str, + beacon: &CardanoDbBeacon, + immutable_file_range: &RangeInclusive, + ) -> (PathBuf, CertificateMessage, VerifiedDigests) { + let cardano_db = DummyCardanoDbBuilder::new(dir_name) + .with_immutables(&immutable_file_range.clone().collect::>()) + .append_immutable_trio() + .build(); + let database_dir = cardano_db.get_dir(); + let immutable_digester = + CardanoImmutableDigester::new("whatever".to_string(), None, TestLogger::stdout()); + let computed_digests = immutable_digester + .compute_digests_for_range(database_dir, immutable_file_range) + .await + .unwrap(); + + let digests = computed_digests + .entries + .iter() + .map(|(immutable_file, digest)| (immutable_file.filename.clone(), digest.clone())) + .collect::>(); + + let merkle_tree = immutable_digester + .compute_merkle_tree(database_dir, beacon) + .await + .unwrap(); + + let verified_digests = VerifiedDigests { + digests, + merkle_tree, + }; + + let certificate = { + let protocol_message_merkle_root = + verified_digests.merkle_tree.compute_root().unwrap().to_hex(); + let mut protocol_message = ProtocolMessage::new(); + protocol_message.set_message_part( + ProtocolMessagePartKey::CardanoDatabaseMerkleRoot, + protocol_message_merkle_root, + ); + + CertificateMessage { + protocol_message: protocol_message.clone(), + signed_message: protocol_message.compute_hash(), + ..CertificateMessage::dummy() + } + }; + + (database_dir.to_owned(), certificate, verified_digests) + } + + fn to_vec_immutable_file_name(list: &[&str]) -> Vec { + list.iter().map(|s| ImmutableFileName::from(*s)).collect() + } + + #[tokio::test] + async fn compute_cardano_database_message_succeeds() { + let beacon = CardanoDbBeacon { + epoch: Epoch(123), + immutable_file_number: 10, + }; + let immutable_file_range = 1..=15; + let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); + let (database_dir, certificate, verified_digests) = prepare_db_and_verified_digests( + "compute_cardano_database_message_succeeds", + &beacon, + &immutable_file_range, + ) + .await; + + let message = MessageBuilder::new() + .compute_cardano_database_message( + &certificate, + &CardanoDatabaseSnapshotMessage::dummy(), + &immutable_file_range_to_prove, + false, + &database_dir, + &verified_digests, + ) + .await + .unwrap(); + + assert!(certificate.match_message(&message)); + } + + #[tokio::test] + async fn compute_cardano_database_message_should_fail_if_immutable_is_missing_and_allow_missing_not_set() + { + let beacon = CardanoDbBeacon { + epoch: Epoch(123), + immutable_file_number: 10, + }; + let immutable_file_range = 1..=15; + let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); + let (database_dir, certificate, verified_digests) = prepare_db_and_verified_digests( + "compute_cardano_database_message_should_fail_if_immutable_is_missing_and_allow_missing_not_set", + &beacon, + &immutable_file_range, + ) + .await; + + let files_to_remove = vec!["00003.chunk", "00004.primary"]; + remove_immutable_files(&database_dir, &files_to_remove); + + let allow_missing = false; + let error = MessageBuilder::new() + .compute_cardano_database_message( + &certificate, + &CardanoDatabaseSnapshotMessage::dummy(), + &immutable_file_range_to_prove, + allow_missing, + &database_dir, + &verified_digests, + ) + .await + .expect_err( + "compute_cardano_database_message should fail if a immutable is missing", + ); + + let error_lists = match error { + ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(lists) => lists, + _ => panic!("Expected ImmutableFilesVerification error, got: {error}"), + }; + + assert_eq!( + error_lists, + ImmutableVerificationResult { + immutables_dir: MessageBuilder::immutable_dir(&database_dir), + missing: to_vec_immutable_file_name(&files_to_remove), + tampered: vec![], + non_verifiable: vec![], + } + ); + } + + #[tokio::test] + async fn compute_cardano_database_message_should_success_if_immutable_is_missing_and_allow_missing_is_set() + { + let beacon = CardanoDbBeacon { + epoch: Epoch(123), + immutable_file_number: 10, + }; + let immutable_file_range = 1..=15; + let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); + let (database_dir, certificate, verified_digests) = prepare_db_and_verified_digests( + "compute_cardano_database_message_should_success_if_immutable_is_missing_and_allow_missing_is_set", + &beacon, + &immutable_file_range, + ) + .await; + + let files_to_remove = vec!["00003.chunk", "00004.primary"]; + remove_immutable_files(&database_dir, &files_to_remove); + + let allow_missing = true; + MessageBuilder::new() + .compute_cardano_database_message( + &certificate, + &CardanoDatabaseSnapshotMessage::dummy(), + &immutable_file_range_to_prove, + allow_missing, + &database_dir, + &verified_digests, + ) + .await + .expect( + "compute_cardano_database_message should succeed if a immutable is missing but 'allow_missing' is set", + ); + } + + #[tokio::test] + async fn compute_cardano_database_message_should_fail_if_immutable_is_tampered() { + let beacon = CardanoDbBeacon { + epoch: Epoch(123), + immutable_file_number: 10, + }; + let immutable_file_range = 1..=15; + let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); + let (database_dir, certificate, verified_digests) = prepare_db_and_verified_digests( + "compute_cardano_database_message_should_fail_if_immutable_is_tampered", + &beacon, + &immutable_file_range, + ) + .await; + + let files_to_tamper = vec!["00003.chunk", "00004.primary"]; + tamper_immutable_files(&database_dir, &files_to_tamper); + + let (logger, log_inspector) = TestLogger::memory(); + + let error = MessageBuilder::new() + .with_logger(logger) + .compute_cardano_database_message( + &certificate, + &CardanoDatabaseSnapshotMessage::dummy(), + &immutable_file_range_to_prove, + false, + &database_dir, + &verified_digests, + ) + .await + .expect_err( + "compute_cardano_database_message should fail if a immutable is missing", + ); + + assert!(log_inspector.contains_log(MERKLE_PROOF_COMPUTATION_ERROR)); + + let error_lists = match error { + ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(lists) => lists, + _ => panic!("Expected ImmutableFilesVerification error, got: {error}"), + }; + assert_eq!( + error_lists, + ImmutableVerificationResult { + immutables_dir: MessageBuilder::immutable_dir(&database_dir), + missing: vec![], + tampered: to_vec_immutable_file_name(&files_to_tamper), + non_verifiable: vec![], + } + ) + } + + #[tokio::test] + async fn compute_cardano_database_message_should_fail_if_immutables_are_missing_and_tampered() + { + let beacon = CardanoDbBeacon { + epoch: Epoch(123), + immutable_file_number: 10, + }; + let immutable_file_range = 1..=15; + let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); + let (database_dir, certificate, verified_digests) = + prepare_db_and_verified_digests( + "compute_cardano_database_message_should_fail_if_immutables_are_missing_and_tampered", + &beacon, + &immutable_file_range, + ) + .await; + + let files_to_remove = vec!["00003.chunk"]; + let files_to_tamper = vec!["00004.primary"]; + remove_immutable_files(&database_dir, &files_to_remove); + tamper_immutable_files(&database_dir, &files_to_tamper); + + let error = MessageBuilder::new() + .compute_cardano_database_message( + &certificate, + &CardanoDatabaseSnapshotMessage::dummy(), + &immutable_file_range_to_prove, + false, + &database_dir, + &verified_digests, + ) + .await + .expect_err( + "compute_cardano_database_message should fail if a immutable is missing", + ); + + let error_lists = match error { + ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(lists) => lists, + _ => panic!("Expected ImmutableFilesVerification error, got: {error}"), + }; + assert_eq!( + error_lists, + ImmutableVerificationResult { + immutables_dir: MessageBuilder::immutable_dir(&database_dir), + missing: to_vec_immutable_file_name(&files_to_remove), + tampered: to_vec_immutable_file_name(&files_to_tamper), + non_verifiable: vec![], + } + ) + } + + #[tokio::test] + async fn compute_cardano_database_message_should_fail_if_there_is_more_local_immutable_than_verified_digest() + { + let last_verified_digest_number = 10; + let last_local_immutable_file_number = 15; + let range_of_non_verifiable_files = + last_verified_digest_number + 1..=last_local_immutable_file_number; + + let expected_non_verifiable_files: Vec = + (range_of_non_verifiable_files) + .flat_map(|i| { + [ + format!("{i:05}.chunk"), + format!("{i:05}.primary"), + format!("{i:05}.secondary"), + ] + }) + .collect(); + + let beacon = CardanoDbBeacon { + epoch: Epoch(123), + immutable_file_number: last_verified_digest_number, + }; + //create verified digests for immutable files 1 to 10 + let (_, certificate, verified_digests) = prepare_db_and_verified_digests( + "database_dir_for_verified_digests", + &beacon, + &(1..=last_verified_digest_number), + ) + .await; + //create a local database with immutable files 1 to 15 + let (database_dir, _, _) = prepare_db_and_verified_digests( + "database_dir_for_local_immutables", + &beacon, + &(1..=last_local_immutable_file_number), + ) + .await; + + let error = MessageBuilder::new() + .compute_cardano_database_message( + &certificate, + &CardanoDatabaseSnapshotMessage::dummy(), + &ImmutableFileRange::Range(1, 15), + false, + &database_dir, + &verified_digests, + ) + .await + .expect_err( + "compute_cardano_database_message should fail if there is more local immutable than verified digest", + ); + + let error_lists = match error { + ComputeCardanoDatabaseMessageError::ImmutableFilesVerification(lists) => lists, + _ => panic!("Expected ImmutableFilesVerification error, got: {error}"), + }; + assert_eq!( + error_lists, + ImmutableVerificationResult { + immutables_dir: MessageBuilder::immutable_dir(&database_dir), + missing: vec![], + tampered: vec![], + non_verifiable: expected_non_verifiable_files, + } + ); + } + } + + mod compute_cardano_database_message_error { + use super::*; + + fn generate_immutable_files_verification_error( + missing_range: Option>, + tampered_range: Option>, + non_verifiable_range: Option>, + immutable_path: &str, + ) -> ComputeCardanoDatabaseMessageError { + let missing: Vec = match missing_range { + Some(range) => range + .map(|i| ImmutableFileName::from(format!("{i:05}.chunk"))) + .collect(), + None => vec![], + }; + let tampered: Vec = match tampered_range { + Some(range) => range + .map(|i| ImmutableFileName::from(format!("{i:05}.chunk"))) + .collect(), + None => vec![], + }; + + let non_verifiable: Vec = match non_verifiable_range { + Some(range) => range + .map(|i| ImmutableFileName::from(format!("{i:05}.chunk"))) + .collect(), + None => vec![], + }; + + ComputeCardanoDatabaseMessageError::ImmutableFilesVerification( + ImmutableVerificationResult { + immutables_dir: PathBuf::from(immutable_path), + missing, + tampered, + non_verifiable, + }, + ) + } + + fn normalize_path_separators(s: &str) -> String { + s.replace('\\', "/") + } + + #[test] + fn display_immutable_files_verification_error_should_displayed_lists_with_10_elements() { + let error = generate_immutable_files_verification_error( + Some(1..=15), + Some(20..=31), + Some(40..=41), + "/path/to/immutables", + ); + + let display = normalize_path_separators(&format!("{error}")); + + assert_eq!( + display, + r###"Number of missing immutable files: 15 +First 10 missing immutable files paths: +/path/to/immutables/00001.chunk +/path/to/immutables/00002.chunk +/path/to/immutables/00003.chunk +/path/to/immutables/00004.chunk +/path/to/immutables/00005.chunk +/path/to/immutables/00006.chunk +/path/to/immutables/00007.chunk +/path/to/immutables/00008.chunk +/path/to/immutables/00009.chunk +/path/to/immutables/00010.chunk + +Number of tampered immutable files: 12 +First 10 tampered immutable files paths: +/path/to/immutables/00020.chunk +/path/to/immutables/00021.chunk +/path/to/immutables/00022.chunk +/path/to/immutables/00023.chunk +/path/to/immutables/00024.chunk +/path/to/immutables/00025.chunk +/path/to/immutables/00026.chunk +/path/to/immutables/00027.chunk +/path/to/immutables/00028.chunk +/path/to/immutables/00029.chunk + +Number of non verifiable immutable files: 2 +First 10 non verifiable immutable files paths: +/path/to/immutables/00040.chunk +/path/to/immutables/00041.chunk +"### + ); + } + + #[test] + fn display_immutable_files_should_display_tampered_files_only() { + let error = generate_immutable_files_verification_error( + None, + Some(1..=1), + None, + "/path/to/immutables", + ); + + let display = normalize_path_separators(&format!("{error}")); + + assert_eq!( + display, + r###"Number of tampered immutable files: 1 +First 10 tampered immutable files paths: +/path/to/immutables/00001.chunk +"### + ); + } + + #[test] + fn display_immutable_files_should_display_missing_files_only() { + let error = generate_immutable_files_verification_error( + Some(1..=1), + None, + None, + "/path/to/immutables", + ); + + let display = normalize_path_separators(&format!("{error}")); + + assert_eq!( + display, + r###"Number of missing immutable files: 1 +First 10 missing immutable files paths: +/path/to/immutables/00001.chunk +"### + ); + } + + #[test] + fn display_immutable_files_should_display_non_verifiable_files_only() { + let error = generate_immutable_files_verification_error( + None, + None, + Some(1..=5), + "/path/to/immutables", + ); + + let display = normalize_path_separators(&format!("{error}")); + + assert_eq!( + display, + r###"Number of non verifiable immutable files: 5 +First 10 non verifiable immutable files paths: +/path/to/immutables/00001.chunk +/path/to/immutables/00002.chunk +/path/to/immutables/00003.chunk +/path/to/immutables/00004.chunk +/path/to/immutables/00005.chunk +"### + ); + } + } +} diff --git a/mithril-client/tests/cardano_db_snapshot_list_get_download_verify.rs b/mithril-client/tests/cardano_db_snapshot_list_get_download_verify.rs index 6058fd96af4..6ad72758a91 100644 --- a/mithril-client/tests/cardano_db_snapshot_list_get_download_verify.rs +++ b/mithril-client/tests/cardano_db_snapshot_list_get_download_verify.rs @@ -132,21 +132,22 @@ async fn cardano_db_snapshot_list_get_download_verify() { .route() ))); - let merkle_proof = client + let verified_digests = client .cardano_database_v2() - .compute_merkle_proof( + .download_and_verify_digests(&certificate, &cardano_db_snapshot) + .await + .unwrap(); + + let message = MessageBuilder::new() + .compute_cardano_database_message( &certificate, &cardano_db_snapshot, &immutable_file_range, + false, &unpacked_dir, + &verified_digests, ) .await - .expect("Computing merkle proof should not fail"); - merkle_proof.verify().expect("Merkle proof should be valid"); - - let message = MessageBuilder::new() - .compute_cardano_database_message(&certificate, &merkle_proof) - .await .expect("Computing cardano database snapshot message should not fail"); assert!(