Merge pull request #1560 from input-output-hk/jpraynaud/1533-block-range-merkle-tree-cardano-tx

Implement Block range merkelization for Cardano transactions

Author: jpraynaud

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-aggregator"
-version = "0.4.47"
+version = "0.4.48"
 description = "A Mithril Aggregator server"
 authors = { workspace = true }
 edition = { workspace = true }

mithril-aggregator/Cargo.toml

@@ -31,11 +31,9 @@ use mithril_common::{
         EraChecker, EraMarker, EraReader, EraReaderAdapter, SupportedEra,
     },
     signable_builder::{
-        CardanoImmutableFilesFullSignableBuilder, MithrilStakeDistributionSignableBuilder,
-    },
-    signable_builder::{
-        CardanoTransactionsSignableBuilder, MithrilSignableBuilderService, SignableBuilderService,
-        TransactionStore,
+        CardanoImmutableFilesFullSignableBuilder, CardanoTransactionsSignableBuilder,
+        MithrilSignableBuilderService, MithrilStakeDistributionSignableBuilder,
+        SignableBuilderService, TransactionStore,
     },
     BeaconProvider, BeaconProviderImpl,
 };

mithril-aggregator/src/dependency_injection/builder.rs

@@ -1,11 +1,13 @@
-use std::sync::Arc;
+use std::{collections::HashMap, sync::Arc};
 
 use anyhow::Context;
 use async_trait::async_trait;
 
 use mithril_common::{
-    crypto_helper::{MKTree, MKTreeNode, MKTreeStore},
-    entities::{Beacon, CardanoTransaction, CardanoTransactionsSetProof, TransactionHash},
+    crypto_helper::{MKMap, MKMapNode, MKTree, MKTreeNode},
+    entities::{
+        Beacon, BlockRange, CardanoTransaction, CardanoTransactionsSetProof, TransactionHash,
+    },
     StdResult,
 };
 
@@ -44,6 +46,36 @@ impl MithrilProverService {
             transaction_retriever,
         }
     }
+
+    fn compute_merkle_map_from_transactions(
+        &self,
+        transactions: &[CardanoTransaction],
+    ) -> StdResult<MKMap<BlockRange, MKMapNode<BlockRange>>> {
+        let mut transactions_by_block_ranges: HashMap<BlockRange, Vec<TransactionHash>> =
+            HashMap::new();
+        for transaction in transactions {
+            let block_range = BlockRange::from_block_number(transaction.block_number);
+            transactions_by_block_ranges
+                .entry(block_range)
+                .or_default()
+                .push(transaction.transaction_hash.to_owned());
+        }
+        let mk_hash_map = MKMap::new(
+            transactions_by_block_ranges
+                .into_iter()
+                .try_fold(
+                    vec![],
+                    |mut acc, (block_range, transactions)| -> StdResult<Vec<(_, MKMapNode<_>)>> {
+                        acc.push((block_range, MKTree::new(&transactions)?.into()));
+                        Ok(acc)
+                    },
+                )?
+                .as_slice(),
+        )
+        .with_context(|| "ProverService failed to compute the merkelized structure that proves ownership of the transaction")?;
+
+        Ok(mk_hash_map)
+    }
 }
 
 #[async_trait]
@@ -54,26 +86,33 @@ impl ProverService for MithrilProverService {
         transaction_hashes: &[TransactionHash],
     ) -> StdResult<Vec<CardanoTransactionsSetProof>> {
         let transactions = self.transaction_retriever.get_up_to(up_to).await?;
-        let mk_leaves_all: Vec<MKTreeNode> =
-            transactions.iter().map(|t| t.to_owned().into()).collect();
-        let store = MKTreeStore::default();
-        let mktree = MKTree::new(&mk_leaves_all, &store)
-            .with_context(|| "MKTree creation should not fail")?;
-
+        let mk_map = self.compute_merkle_map_from_transactions(&transactions)?;
+        let transactions_to_prove = transactions
+            .iter()
+            .filter_map(|transaction| {
+                let block_range = BlockRange::from_block_number(transaction.block_number);
+                transaction_hashes
+                    .contains(&transaction.transaction_hash)
+                    .then(|| (block_range, transaction.to_owned()))
+            })
+            .collect::<Vec<_>>();
         let mut transaction_hashes_certified = vec![];
-        for transaction_hash in transaction_hashes {
-            let mk_leaf = transaction_hash.to_string().into();
-            if mktree.compute_proof(&[mk_leaf]).is_ok() {
-                transaction_hashes_certified.push(transaction_hash.to_string());
+        for (_block_range, transaction) in transactions_to_prove {
+            let mk_tree_node_transaction_hash: MKTreeNode =
+                transaction.transaction_hash.to_owned().into();
+            if mk_map
+                .compute_proof(&[mk_tree_node_transaction_hash])
+                .is_ok()
+            {
+                transaction_hashes_certified.push(transaction.transaction_hash.to_string());
             }
         }
-
         if !transaction_hashes_certified.is_empty() {
             let mk_leaves: Vec<MKTreeNode> = transaction_hashes_certified
                 .iter()
                 .map(|h| h.to_owned().into())
                 .collect();
-            let mk_proof = mktree.compute_proof(&mk_leaves)?;
+            let mk_proof = mk_map.compute_proof(&mk_leaves)?;
             let transactions_set_proof_batch =
                 CardanoTransactionsSetProof::new(transaction_hashes_certified, mk_proof);
 
@@ -171,9 +210,6 @@ mod tests {
         transactions_set_proof[0].verify().unwrap();
     }
 
-    // this one can't be done right now because we don't have a merkle tree of merkle tree yet
-    // todo: compute_proof_for_multiple_set_with_multiple_transactions
-
     #[tokio::test]
     async fn cant_compute_proof_if_retriever_fail() {
         let (transaction_hashes, _transactions) = generate_transactions(3);

mithril-aggregator/src/services/prover.rs

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-client"
-version = "0.6.7"
+version = "0.6.8"
 description = "Mithril client library"
 authors = { workspace = true }
 edition = { workspace = true }

mithril-client/Cargo.toml

@@ -160,7 +160,9 @@ mod proof {
                 certificate_hash: certificate_hash.to_string(),
                 certified_transactions: vec![CardanoTransactionsSetProof {
                     transactions_hashes: tx_hashes.iter().map(|h| h.to_string()).collect(),
-                    proof: ProtocolMkProof::new(proof.clone()).to_json_hex().unwrap(),
+                    proof: ProtocolMkProof::new(proof.to_owned().into())
+                        .to_json_hex()
+                        .unwrap(),
                 }],
                 non_certified_transactions: vec![],
                 latest_immutable_file_number: 9999,

mithril-client/tests/extensions/fake.rs

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-common"
-version = "0.3.19"
+version = "0.3.20"
 description = "Common types, interfaces, and utilities for Mithril nodes."
 authors = { workspace = true }
 edition = { workspace = true }
@@ -15,6 +15,15 @@ crate-type = ["lib", "cdylib", "staticlib"]
 [[bench]]
 name = "digester"
 harness = false
+required-features = ["fs"]
+
+[[bench]]
+name = "merkle_tree"
+harness = false
+
+[[bench]]
+name = "merkle_map"
+harness = false
 
 [dependencies]
 anyhow = "1.0.79"
@@ -91,7 +100,7 @@ reqwest = { version = "0.11.23", features = ["json"] }
 slog-async = "2.8.0"
 slog-scope = "4.4.0"
 slog-term = "2.9.0"
-tokio = { version = "1.35.1", features = ["macros", "time"] }
+tokio = { version = "1.35.1", features = ["macros", "rt-multi-thread", "time"] }
 
 [build-dependencies]
 mithril-build-script = { path = "../internal/mithril-build-script", version = "=0.1.3" }

mithril-common/Cargo.toml

@@ -24,6 +24,9 @@ check:
 doc:
 	${CARGO} doc --no-deps --open --features full
 
+bench:
+	${CARGO} bench --features full --verbose
+
 # Compute the powerset of all the given features and save it to a file
 .feature-sets:
 	powerset() { [ $$# -eq 0 ] && echo || (shift; powerset "$$@") | while read r ; do echo "$$1 $$r"; echo "$$r"; done };\
@@ -35,5 +38,7 @@ check-all-features-set: .feature-sets
 		echo "Clippy common with feature '$$features_set''"; \
 		${CARGO} clippy -p mithril-common --features "$$features_set"; \
 	done
+	echo "Clippy common without features"; \
+	${CARGO} clippy -p mithril-common
 
 	rm .features-sets