Merge pull request #1564 from input-output-hk/ensemble/1557-use-pallas-hardano-transactions-parser

Use pallas hardano transactions parser

Author: sfauvel

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-common"
-version = "0.3.16"
+version = "0.3.17"
 description = "Common types, interfaces, and utilities for Mithril nodes."
 authors = { workspace = true }
 edition = { workspace = true }
@@ -42,6 +42,7 @@ minicbor = { version = "0.20", features = [
 nom = "7.1.3"
 pallas-addresses = { version = "0.24.0", optional = true }
 pallas-codec = { version = "0.24.0", optional = true }
+pallas-hardano = { version = "0.24.0", optional = true }
 pallas-network = { version = "0.24.0", optional = true }
 pallas-primitives = { version = "0.24.0", optional = true }
 pallas-traverse = { version = "0.24.0", optional = true }
@@ -107,6 +108,7 @@ fs = [
     "dep:minicbor",
     "dep:pallas-addresses",
     "dep:pallas-codec",
+    "dep:pallas-hardano",
     "dep:pallas-network",
     "dep:pallas-primitives",
     "dep:pallas-traverse",

mithril-common/Cargo.toml

@@ -1,17 +1,14 @@
 //! The module used for parsing Cardano transactions
-
 use crate::{
     digesters::ImmutableFile,
     entities::{Beacon, BlockNumber, CardanoTransaction, ImmutableFileNumber, TransactionHash},
     StdResult,
 };
-use anyhow::Context;
+use anyhow::{anyhow, Context};
 use async_trait::async_trait;
-use pallas_traverse::{
-    probe::{block_era, Outcome},
-    MultiEraBlock,
-};
-use std::{cmp::min, fs, path::Path};
+use pallas_hardano::storage::immutable::chunk::{read_blocks, Reader};
+use pallas_traverse::MultiEraBlock;
+use std::path::Path;
 use tokio::sync::RwLock;
 
 /// A parser that can read cardano transactions in a cardano database
@@ -119,43 +116,54 @@ impl CardanoTransactionParser {
 
     /// Read blocks from immutable file
     fn read_blocks_from_immutable_file(immutable_file: &ImmutableFile) -> StdResult<Vec<Block>> {
-        let cbor = fs::read(&immutable_file.path).with_context(|| {
+        let cardano_blocks_reader =
+            CardanoTransactionParser::cardano_blocks_reader(immutable_file)?;
+
+        let mut blocks = Vec::new();
+        for parsed_block in cardano_blocks_reader {
+            let block = parsed_block.with_context(|| {
+                format!(
+                    "Error while reading block in immutable file: '{:?}'",
+                    immutable_file.path
+                )
+            })?;
+            blocks.push(CardanoTransactionParser::convert_to_block(
+                &block,
+                immutable_file,
+            )?);
+        }
+        Ok(blocks)
+    }
+
+    fn convert_to_block(block: &[u8], immutable_file: &ImmutableFile) -> StdResult<Block> {
+        let multi_era_block = MultiEraBlock::decode(block).with_context(|| {
             format!(
-                "CardanoTransactionParser could not read from immutable file: {:?}",
+                "Error while decoding block in immutable file: '{:?}'",
                 immutable_file.path
             )
         })?;
 
-        let mut blocks_start_byte_index: Vec<_> = (0..cbor.len())
-            .filter(|&byte_index| {
-                let cbor_header_maybe = &cbor[byte_index..min(byte_index + 2, cbor.len())];
-                match block_era(cbor_header_maybe) {
-                    Outcome::Matched(_) | Outcome::EpochBoundary => true,
-                    Outcome::Inconclusive => false,
-                }
-            })
-            .collect();
-
-        blocks_start_byte_index.push(cbor.len() + 1);
+        Block::try_convert(multi_era_block, immutable_file.number).with_context(|| {
+            format!(
+                "CardanoTransactionParser could not read data from block in immutable file: {:?}",
+                immutable_file.path
+            )
+        })
+    }
 
-        let mut blocks = Vec::new();
-        let mut last_start_byte_index = 0;
-        for block_start_index in blocks_start_byte_index.into_iter().skip(1) {
-            let maybe_end_byte_index = min(block_start_index, cbor.len());
-            if let Ok(multi_era_block) =
-                MultiEraBlock::decode(&cbor[last_start_byte_index..maybe_end_byte_index])
-            {
-                let block = Block::try_convert(multi_era_block, immutable_file.number)
-                    .with_context(|| {
-                        format!(
-                        "CardanoTransactionParser could not read data from block in immutable file: {:?}",
-                        immutable_file.path
-                    )
-                    })?;
-                blocks.push(block);
-                last_start_byte_index = block_start_index;
-            }
-        }
+    fn cardano_blocks_reader(immutable_file: &ImmutableFile) -> StdResult<Reader> {
+        let dir_path = immutable_file.path.parent().ok_or(anyhow!(format!(
+            "Could not retrieve immutable file directory with immutable file path: '{:?}'",
+            immutable_file.path
+        )))?;
+        let file_name = &Path::new(&immutable_file.filename)
+            .file_stem()
+            .ok_or(anyhow!(format!(
+                "Could not extract immutable file name from file: '{}'",
+                immutable_file.filename
+            )))?
+            .to_string_lossy();
+        let blocks = read_blocks(dir_path, file_name)?;
 
         Ok(blocks)
     }
@@ -222,7 +230,7 @@ mod tests {
     #[tokio::test]
     async fn test_parse_expected_number_of_transactions() {
         // We known the number of transactions in those prebuilt immutables
-        let immutable_files = [("00000", 20usize), ("00001", 9), ("00002", 0)];
+        let immutable_files = [("00000", 0usize), ("00001", 2), ("00002", 3)];
         let db_path = Path::new("../mithril-test-lab/test_data/immutable/");
         assert!(get_number_of_immutable_chunk_in_dir(db_path) >= 3);
 
@@ -244,12 +252,12 @@ mod tests {
     #[tokio::test]
     async fn test_parse_up_to_given_beacon() {
         // We known the number of transactions in those prebuilt immutables
-        let immutable_files = [("00000", 20usize)];
+        let immutable_files = [("00000", 0usize), ("00001", 2)];
         let db_path = Path::new("../mithril-test-lab/test_data/immutable/");
         assert!(get_number_of_immutable_chunk_in_dir(db_path) >= 2);
 
         let beacon = Beacon {
-            immutable_file_number: 0,
+            immutable_file_number: 1,
             ..Beacon::default()
         };
         let tx_count: usize = immutable_files.iter().map(|(_, count)| *count).sum();

mithril-common/src/cardano_transaction_parser.rs

@@ -17,6 +17,12 @@
 # Download sources from github
 git clone https://github.com/input-output-hk/mithril
 
+# Build nodes
+make -C mithril-aggregator build
+make -C mithril-signer build
+make -C mithril-client-cli build
+make -C mithril-relay build
+
 # Go to sources directory
 cd mithril-test-lab/mithril-end-to-end
 ```
@@ -31,7 +37,7 @@ make build
 ./mithril-end-to-end --help
 
 # Run
-./mithril-end-to-end -vvv --db-directory db/ --bin-directory ../../target/release
+./mithril-end-to-end -vvv --work-directory db/ --bin-directory ../../target/release --devnet-scripts-directory=../mithril-devnet
 ```
 
 To run `mithril-end-to-end` command, you must first compile the Mithril nodes:

mithril-test-lab/mithril-end-to-end/README.md

@@ -5,69 +5,5 @@
 > immutable db directory by our db parser (see ([immutable_file.rs](./../../mithril-common/src/digesters/immutable_file.rs))).
 
 > [!NOTE]
-> The `.primary` and `.secondary` files are not real data. They're empty files that are
-> here in order to simulate a real cardano immutable db structure.
-
-These files were generated manually from the blocks in the sibling folder:
-
-| Imput Block file | Number of transactions | Output immutable file |
-|------------------|------------------------|-----------------------|
-| [byron2.block](./blocks/byron2.block) | `2` | `00000.chunk`         |
-| [shelley1.block](./blocks/shelley1.block) | `4` | `00000.chunk`         |
-| [mary1.block](./blocks/mary1.block) | `14` | `00000.chunk`         |
-| [allegra1.block](./blocks/allegra1.block) | `3` | `00001.chunk`         |
-| [alonzo1.block](./blocks/alonzo1.block) | `5` | `00001.chunk`         |
-| [conway1.block](./blocks/conway1.block) | `1` | `00001.chunk`         |
-
-The following code was used to generate the immutable files:
-```rust
-use std::io::Write;
-
-let immutable_files = vec![
-    (
-        "./00000.chunk",
-        vec![
-            (
-                include_str!("./blocks/byron2.block"),
-                2usize,
-            ),
-            (
-                include_str!("./blocks/shelley1.block"),
-                4,
-            ),
-            (
-                include_str!("./blocks/mary1.block"),
-                14,
-            ),
-        ],
-    ),
-    (
-        "./00001.chunk",
-        vec![
-            (
-                include_str!("./blocks/allegra1.block"),
-                3,
-            ),
-            (
-                include_str!("./blocks/alonzo1.block"),
-                5,
-            ),
-            (
-                include_str!("./blocks/conway1.block"),
-                5,
-            ),
-        ],
-    ),
-];
-
-for immutable_file_definition in immutable_files {
-    let immutable_file_path = immutable_file_definition.0;
-    let mut immutable_file = fs::File::create(immutable_file_path).unwrap();
-
-    for (block_str, _tx_count) in immutable_file_definition.1.into_iter() {
-        let cbor = hex::decode(block_str).expect("invalid hex");
-        immutable_file.write_all(&cbor).unwrap();
-    }
-}
-```
-
+> The immutable files (`.chunk`, `.primary` and `.secondary`) files are data that are the result of
+> the `mithril-end-to-end` test command execution.