Change BlockScanner to return a list of Blocks instead of a list of Transactions

Author: Alenar

mithril-aggregator/src/services/cardano_transactions_importer.rs

@@ -79,7 +79,13 @@ impl CardanoTransactionsImporter {
             return Ok(());
         }
 
-        let parsed_transactions = self.block_scanner.parse(&self.dirpath, from, until).await?;
+        // todo: temp algorithm, should be optimized to avoid loading all blocks & transactions
+        // at once in memory (probably using iterators)
+        let scanned_blocks = self.block_scanner.scan(&self.dirpath, from, until).await?;
+        let parsed_transactions: Vec<CardanoTransaction> = scanned_blocks
+            .into_iter()
+            .flat_map(|b| b.into_transactions())
+            .collect();
         debug!(
             self.logger,
             "TransactionsImporter retrieved '{}' Cardano transactions between immutables '{}' and '{until}'",
@@ -114,6 +120,8 @@ mod tests {
     use mockall::mock;
     use mockall::predicate::eq;
 
+    use mithril_common::cardano_block_scanner::ScannedBlock;
+
     use crate::database::repository::CardanoTransactionRepository;
     use crate::database::test_helper::cardano_tx_db_connection;
 
@@ -124,19 +132,23 @@ mod tests {
 
         #[async_trait]
         impl BlockScanner for BlockScannerImpl {
-            async fn parse(
+            async fn scan(
               &self,
               dirpath: &Path,
               from_immutable: Option<ImmutableFileNumber>,
               until_immutable: ImmutableFileNumber,
-            ) -> StdResult<Vec<CardanoTransaction>>;
+            ) -> StdResult<Vec<ScannedBlock>>;
         }
     }
 
-    fn build_importer(
-        scanner_mock_config: &dyn Fn(&mut MockBlockScannerImpl),
-        store_mock_config: &dyn Fn(&mut MockTransactionStore),
-    ) -> CardanoTransactionsImporter {
+    fn build_importer<TParser, TStore>(
+        scanner_mock_config: TParser,
+        store_mock_config: TStore,
+    ) -> CardanoTransactionsImporter
+    where
+        TParser: FnOnce(&mut MockBlockScannerImpl),
+        TStore: FnOnce(&mut MockTransactionStore),
+    {
         let db_path = Path::new("");
         let mut scanner = MockBlockScannerImpl::new();
         scanner_mock_config(&mut scanner);
@@ -156,23 +168,24 @@ mod tests {
 
     #[tokio::test]
     async fn if_nothing_stored_parse_and_store_all_transactions() {
-        let transactions = vec![
-            CardanoTransaction::new("tx_hash-1", 10, 15, "block_hash-1", 11),
-            CardanoTransaction::new("tx_hash-2", 10, 20, "block_hash-1", 11),
-            CardanoTransaction::new("tx_hash-3", 20, 25, "block_hash-2", 12),
-            CardanoTransaction::new("tx_hash-4", 20, 30, "block_hash-2", 12),
+        let blocks = vec![
+            ScannedBlock::new("block_hash-1", 10, 15, 11, vec!["tx_hash-1", "tx_hash-2"]),
+            ScannedBlock::new("block_hash-2", 20, 25, 12, vec!["tx_hash-3", "tx_hash-4"]),
         ];
+        let transactions: Vec<CardanoTransaction> = blocks
+            .iter()
+            .flat_map(|b| b.clone().into_transactions())
+            .collect();
         let up_to_beacon = 12;
 
         let importer = build_importer(
-            &|scanner_mock| {
-                let parsed_transactions = transactions.clone();
+            |scanner_mock| {
                 scanner_mock
-                    .expect_parse()
+                    .expect_scan()
                     .withf(move |_, from, until| from.is_none() && until == &up_to_beacon)
-                    .return_once(move |_, _, _| Ok(parsed_transactions));
+                    .return_once(move |_, _, _| Ok(blocks));
             },
-            &|store_mock| {
+            |store_mock| {
                 let expected_stored_transactions = transactions.clone();
                 store_mock
                     .expect_get_highest_beacon()
@@ -196,10 +209,10 @@ mod tests {
         let up_to_beacon = 12;
 
         let importer = build_importer(
-            &|scanner_mock| {
-                scanner_mock.expect_parse().never();
+            |scanner_mock| {
+                scanner_mock.expect_scan().never();
             },
-            &|store_mock| {
+            |store_mock| {
                 store_mock
                     .expect_get_highest_beacon()
                     .returning(|| Ok(Some(12)));
@@ -215,23 +228,25 @@ mod tests {
 
     #[tokio::test]
     async fn if_all_half_are_stored_the_other_half_is_parsed_and_stored() {
-        let transactions = vec![
-            CardanoTransaction::new("tx_hash-1", 10, 15, "block_hash-10", 11),
-            CardanoTransaction::new("tx_hash-2", 20, 20, "block_hash-20", 12),
-            CardanoTransaction::new("tx_hash-3", 30, 25, "block_hash-30", 13),
-            CardanoTransaction::new("tx_hash-4", 40, 30, "block_hash-40", 14),
+        let blocks = [
+            ScannedBlock::new("block_hash-1", 10, 15, 11, vec!["tx_hash-1", "tx_hash-2"]),
+            ScannedBlock::new("block_hash-2", 20, 25, 12, vec!["tx_hash-3", "tx_hash-4"]),
         ];
+        let transactions: Vec<CardanoTransaction> = blocks
+            .iter()
+            .flat_map(|b| b.clone().into_transactions())
+            .collect();
         let up_to_beacon = 14;
 
         let importer = build_importer(
-            &|scanner_mock| {
-                let parsed_transactions = transactions[2..=3].to_vec();
+            |scanner_mock| {
+                let scanned_blocks = vec![blocks[1].clone()];
                 scanner_mock
-                    .expect_parse()
+                    .expect_scan()
                     .withf(move |_, from, until| from == &Some(13) && until == &up_to_beacon)
-                    .return_once(move |_, _, _| Ok(parsed_transactions));
+                    .return_once(move |_, _, _| Ok(scanned_blocks));
             },
-            &|store_mock| {
+            |store_mock| {
                 store_mock
                     .expect_get_highest_beacon()
                     .returning(|| Ok(Some(12)));
@@ -253,19 +268,18 @@ mod tests {
     #[tokio::test]
     async fn importing_twice_starting_with_nothing_in_a_real_db_should_yield_the_transactions_in_same_order(
     ) {
-        let transactions = vec![
-            CardanoTransaction::new("tx_hash-1", 10, 15, "block_hash-1", 11),
-            CardanoTransaction::new("tx_hash-2", 10, 20, "block_hash-1", 11),
-            CardanoTransaction::new("tx_hash-3", 20, 25, "block_hash-2", 12),
-            CardanoTransaction::new("tx_hash-4", 20, 30, "block_hash-2", 12),
+        let blocks = vec![
+            ScannedBlock::new("block_hash-1", 10, 15, 11, vec!["tx_hash-1", "tx_hash-2"]),
+            ScannedBlock::new("block_hash-2", 20, 25, 12, vec!["tx_hash-3", "tx_hash-4"]),
         ];
+        let transactions: Vec<CardanoTransaction> = blocks
+            .iter()
+            .flat_map(|b| b.clone().into_transactions())
+            .collect();
         let importer = {
             let connection = cardano_tx_db_connection().unwrap();
-            let parsed_transactions = transactions.clone();
             let mut scanner = MockBlockScannerImpl::new();
-            scanner
-                .expect_parse()
-                .return_once(move |_, _, _| Ok(parsed_transactions));
+            scanner.expect_scan().return_once(move |_, _, _| Ok(blocks));
 
             CardanoTransactionsImporter::new(
                 Arc::new(scanner),