Implement database layer for block_range_root

Alenar · Alenar · commit f2f37bfb9cd3 · 2024-04-26T11:37:48.000+02:00
diff --git a/mithril-signer/src/cardano_transactions_importer.rs b/mithril-signer/src/cardano_transactions_importer.rs
@@ -19,9 +19,9 @@ pub trait TransactionStore: Send + Sync {
     async fn get_highest_beacon(&self) -> StdResult<Option<ImmutableFileNumber>>;
 
     /// Get the interval of blocks whose merkle root has yet to be computed
-    async fn get_block_interval_without_associated_block_range_and_merkle_root(
+    async fn get_block_interval_without_block_range_root(
         &self,
-    ) -> StdResult<Option<(BlockNumber, BlockNumber)>>;
+    ) -> StdResult<Option<RangeInclusive<BlockNumber>>>;
 
     /// Get transactions between two block numbers
     async fn get_transactions_between(
@@ -125,16 +125,16 @@ impl CardanoTransactionsImporter {
     async fn import_block_ranges(&self) -> StdResult<()> {
         match self
             .transaction_store
-            .get_block_interval_without_associated_block_range_and_merkle_root()
+            .get_block_interval_without_block_range_root()
             .await?
         {
             None => {
                 // Nothing to do
                 Ok(())
             }
-            Some((start_block, end_block)) => {
+            Some(range) => {
                 let block_ranges =
-                    BlockRange::all_ranges_in(BlockRange::start(start_block)..=end_block);
+                    BlockRange::all_ranges_in(BlockRange::start(*range.start())..=*range.end());
 
                 if block_ranges.is_empty() {
                     return Ok(());
@@ -311,8 +311,8 @@ mod tests {
             |store_mock| {
                 let expected_stored_transactions = transactions.clone();
                 store_mock
-                    .expect_get_block_interval_without_associated_block_range_and_merkle_root()
-                    .returning(|| Ok(Some((0, BlockRange::LENGTH * 5 + 1))));
+                    .expect_get_block_interval_without_block_range_root()
+                    .returning(|| Ok(Some(0..=(BlockRange::LENGTH * 5 + 1))));
                 store_mock
                     .expect_get_transactions_between()
                     .withf(|range| range == &(0..=(BlockRange::LENGTH * 5)))
@@ -351,7 +351,7 @@ mod tests {
             |_scanner_mock| {},
             |store_mock| {
                 store_mock
-                    .expect_get_block_interval_without_associated_block_range_and_merkle_root()
+                    .expect_get_block_interval_without_block_range_root()
                     .returning(|| Ok(None));
                 store_mock.expect_store_block_ranges().never();
             },
@@ -437,8 +437,12 @@ mod tests {
             |store_mock| {
                 let expected_stored_transactions = transactions.clone();
                 store_mock
-                    .expect_get_block_interval_without_associated_block_range_and_merkle_root()
-                    .returning(|| Ok(Some((BlockRange::LENGTH + 2, BlockRange::LENGTH * 5 + 1))));
+                    .expect_get_block_interval_without_block_range_root()
+                    .returning(|| {
+                        Ok(Some(
+                            (BlockRange::LENGTH + 2)..=(BlockRange::LENGTH * 5 + 1),
+                        ))
+                    });
                 store_mock
                     .expect_get_transactions_between()
                     .withf(|range| range == &(BlockRange::LENGTH..=(BlockRange::LENGTH * 5)))
@@ -496,8 +500,8 @@ mod tests {
             |store_mock| {
                 let expected_stored_transactions = transactions.clone();
                 store_mock
-                    .expect_get_block_interval_without_associated_block_range_and_merkle_root()
-                    .returning(|| Ok(Some((0, BlockRange::LENGTH * 2))));
+                    .expect_get_block_interval_without_block_range_root()
+                    .returning(|| Ok(Some(0..=(BlockRange::LENGTH * 2))));
                 store_mock
                     .expect_get_transactions_between()
                     .return_once(move |_| Ok(expected_stored_transactions))
diff --git a/mithril-signer/src/database/cardano_transaction_migration.rs b/mithril-signer/src/database/cardano_transaction_migration.rs
@@ -57,6 +57,19 @@ vacuum;
             4,
             r#"
 create index block_number_index on cardano_tx(block_number);
+"#,
+        ),
+        // Migration 5
+        // Add `block_range_root` table
+        SqlMigration::new(
+            5,
+            r#"
+create table block_range_root (
+    start         integer   not null,
+    end           integer   not null,
+    merkle_root   text      not null,
+    primary key (start, end)
+);
 "#,
         ),
     ]
diff --git a/mithril-signer/src/database/provider/block_range_root/get_block_range_root.rs b/mithril-signer/src/database/provider/block_range_root/get_block_range_root.rs
@@ -0,0 +1,34 @@
+use mithril_persistence::sqlite::{Provider, SourceAlias, SqLiteEntity, SqliteConnection};
+
+use crate::database::record::BlockRangeRootRecord;
+
+/// Simple queries to retrieve [BlockRangeRootRecord] from the sqlite database.
+pub struct GetBlockRangeRootProvider<'client> {
+    connection: &'client SqliteConnection,
+}
+
+impl<'client> GetBlockRangeRootProvider<'client> {
+    #[cfg(test)]
+    /// Create a new instance
+    pub fn new(connection: &'client SqliteConnection) -> Self {
+        Self { connection }
+    }
+}
+
+#[cfg(test)]
+impl mithril_persistence::sqlite::GetAllCondition for GetBlockRangeRootProvider<'_> {}
+
+impl<'client> Provider<'client> for GetBlockRangeRootProvider<'client> {
+    type Entity = BlockRangeRootRecord;
+
+    fn get_connection(&'client self) -> &'client SqliteConnection {
+        self.connection
+    }
+
+    fn get_definition(&self, condition: &str) -> String {
+        let aliases = SourceAlias::new(&[("{:block_range_root:}", "block_range_root")]);
+        let projection = Self::Entity::get_projection().expand(aliases);
+
+        format!("select {projection} from block_range_root where {condition} order by rowid")
+    }
+}
diff --git a/mithril-signer/src/database/provider/block_range_root/get_interval_without_block_range_provider.rs b/mithril-signer/src/database/provider/block_range_root/get_interval_without_block_range_provider.rs
@@ -0,0 +1,43 @@
+use mithril_persistence::sqlite::{
+    Provider, SourceAlias, SqLiteEntity, SqliteConnection, WhereCondition,
+};
+
+use crate::database::record::IntervalWithoutBlockRangeRootRecord;
+
+/// Query that return the interval of block numbers that does not have a block range root.
+pub struct GetIntervalWithoutBlockRangeRootProvider<'client> {
+    connection: &'client SqliteConnection,
+}
+
+impl<'client> GetIntervalWithoutBlockRangeRootProvider<'client> {
+    /// Create a new instance
+    pub fn new(connection: &'client SqliteConnection) -> Self {
+        Self { connection }
+    }
+
+    pub fn get_interval_without_block_range_condition(&self) -> WhereCondition {
+        WhereCondition::default()
+    }
+}
+
+impl<'client> Provider<'client> for GetIntervalWithoutBlockRangeRootProvider<'client> {
+    type Entity = IntervalWithoutBlockRangeRootRecord;
+
+    fn get_connection(&'client self) -> &'client SqliteConnection {
+        self.connection
+    }
+
+    fn get_definition(&self, condition: &str) -> String {
+        let aliases = SourceAlias::new(&[
+            ("{:interval_start:}", "interval_start"),
+            ("{:interval_end:}", "interval_end"),
+        ]);
+        let projection = Self::Entity::get_projection().expand(aliases);
+
+        format!(
+            "select {projection} from \
+        (select max(end) as start from block_range_root where {condition}) as interval_start, \
+        (select max(block_number) as end from cardano_tx where {condition}) as interval_end"
+        )
+    }
+}
diff --git a/mithril-signer/src/database/provider/block_range_root/insert_block_range.rs b/mithril-signer/src/database/provider/block_range_root/insert_block_range.rs
@@ -0,0 +1,65 @@
+use std::iter::repeat;
+
+use sqlite::Value;
+
+use mithril_common::StdResult;
+use mithril_persistence::sqlite::{
+    Provider, SourceAlias, SqLiteEntity, SqliteConnection, WhereCondition,
+};
+
+use crate::database::record::BlockRangeRootRecord;
+
+/// Query to insert [BlockRangeRootRecord] in the sqlite database
+pub struct InsertBlockRangeRootProvider<'client> {
+    connection: &'client SqliteConnection,
+}
+
+impl<'client> InsertBlockRangeRootProvider<'client> {
+    /// Create a new instance
+    pub fn new(connection: &'client SqliteConnection) -> Self {
+        Self { connection }
+    }
+
+    /// Condition to insert multiples records.
+    pub fn get_insert_many_condition(
+        &self,
+        block_range_records: Vec<BlockRangeRootRecord>,
+    ) -> StdResult<WhereCondition> {
+        let columns = "(start, end, merkle_root)";
+        let values_columns: Vec<&str> = repeat("(?*, ?*, ?*)")
+            .take(block_range_records.len())
+            .collect();
+
+        let values: StdResult<Vec<Value>> =
+            block_range_records
+                .into_iter()
+                .try_fold(vec![], |mut vec, record| {
+                    vec.append(&mut vec![
+                        Value::Integer(record.range.start.try_into()?),
+                        Value::Integer(record.range.end.try_into()?),
+                        Value::String(record.merkle_root.to_hex()),
+                    ]);
+                    Ok(vec)
+                });
+
+        Ok(WhereCondition::new(
+            format!("{columns} values {}", values_columns.join(", ")).as_str(),
+            values?,
+        ))
+    }
+}
+
+impl<'client> Provider<'client> for InsertBlockRangeRootProvider<'client> {
+    type Entity = BlockRangeRootRecord;
+
+    fn get_connection(&'client self) -> &'client SqliteConnection {
+        self.connection
+    }
+
+    fn get_definition(&self, condition: &str) -> String {
+        let aliases = SourceAlias::new(&[("{:block_range_root:}", "block_range_root")]);
+        let projection = Self::Entity::get_projection().expand(aliases);
+
+        format!("insert or ignore into block_range_root {condition} returning {projection}")
+    }
+}
diff --git a/mithril-signer/src/database/provider/block_range_root/mod.rs b/mithril-signer/src/database/provider/block_range_root/mod.rs
@@ -0,0 +1,8 @@
+mod get_block_range_root;
+mod get_interval_without_block_range_provider;
+mod insert_block_range;
+
+#[cfg(test)]
+pub use get_block_range_root::*;
+pub use get_interval_without_block_range_provider::*;
+pub use insert_block_range::*;
diff --git a/mithril-signer/src/database/provider/mod.rs b/mithril-signer/src/database/provider/mod.rs
@@ -1,5 +1,7 @@
 //! Signer related database providers
 
+mod block_range_root;
 mod cardano_transaction;
 
+pub use block_range_root::*;
 pub use cardano_transaction::*;
diff --git a/mithril-signer/src/database/record/block_range_root.rs b/mithril-signer/src/database/record/block_range_root.rs
@@ -0,0 +1,61 @@
+use sqlite::Row;
+
+use mithril_common::crypto_helper::MKTreeNode;
+use mithril_common::entities::BlockRange;
+use mithril_persistence::sqlite::{HydrationError, Projection, SqLiteEntity};
+
+use crate::database::record::hydrator::try_to_u64;
+
+/// Block range root record is the representation of block range with its merkle root precomputed.
+#[derive(Debug, PartialEq, Clone)]
+pub struct BlockRangeRootRecord {
+    /// Range of block numbers covered
+    pub range: BlockRange,
+    /// Merkle root of the block range, computed from the list of all transactions that are
+    /// included in the range
+    pub merkle_root: MKTreeNode,
+}
+
+impl From<(BlockRange, MKTreeNode)> for BlockRangeRootRecord {
+    fn from(value: (BlockRange, MKTreeNode)) -> Self {
+        Self {
+            range: value.0,
+            merkle_root: value.1,
+        }
+    }
+}
+
+impl From<BlockRangeRootRecord> for (BlockRange, MKTreeNode) {
+    fn from(value: BlockRangeRootRecord) -> Self {
+        (value.range, value.merkle_root)
+    }
+}
+
+impl SqLiteEntity for BlockRangeRootRecord {
+    fn hydrate(row: Row) -> Result<Self, HydrationError>
+    where
+        Self: Sized,
+    {
+        let start = try_to_u64("block_range.start", row.read::<i64, _>(0))?;
+        let _end = try_to_u64("block_range.end", row.read::<i64, _>(1))?;
+        let merkle_root = row.read::<&str, _>(2);
+
+        Ok(Self {
+            range: BlockRange::from_block_number(start),
+            merkle_root: MKTreeNode::from_hex(merkle_root)
+                .map_err(|e| HydrationError::InvalidData(
+                    format!(
+                        "Field block_range.merkle_root (value={merkle_root}) is incompatible with hex representation. Error = {e}")
+                )
+            )?,
+        })
+    }
+
+    fn get_projection() -> Projection {
+        Projection::from(&[
+            ("start", "{:block_range_root:}.start", "int"),
+            ("end", "{:block_range_root:}.end", "int"),
+            ("merkle_root", "{:block_range_root:}.merkle_root", "text"),
+        ])
+    }
+}
diff --git a/mithril-signer/src/database/record/cardano_transaction.rs b/mithril-signer/src/database/record/cardano_transaction.rs
@@ -5,6 +5,8 @@ use mithril_common::entities::{
 };
 use mithril_persistence::sqlite::{HydrationError, Projection, SqLiteEntity};
 
+use crate::database::record::hydrator::try_to_u64;
+
 /// Cardano Transaction record is the representation of a cardano transaction.
 #[derive(Debug, PartialEq, Clone)]
 pub struct CardanoTransactionRecord {
@@ -53,17 +55,12 @@ impl SqLiteEntity for CardanoTransactionRecord {
     where
         Self: Sized,
     {
-        // TODO: generalize this method to other hydrator
-        fn try_to_u64(field: &str, value: i64) -> Result<u64, HydrationError> {
-            u64::try_from(value)
-                .map_err(|e| HydrationError::InvalidData(format!("Integer field cardano_tx.{field} (value={value}) is incompatible with u64 representation. Error = {e}")))
-        }
-
         let transaction_hash = row.read::<&str, _>(0);
-        let block_number = try_to_u64("block_number", row.read::<i64, _>(1))?;
-        let slot_number = try_to_u64("slot_number", row.read::<i64, _>(2))?;
+        let block_number = try_to_u64("cardano_tx.block_number", row.read::<i64, _>(1))?;
+        let slot_number = try_to_u64("cardano_tx.slot_number", row.read::<i64, _>(2))?;
         let block_hash = row.read::<&str, _>(3);
-        let immutable_file_number = try_to_u64("immutable_file_number", row.read::<i64, _>(4))?;
+        let immutable_file_number =
+            try_to_u64("cardano_tx.immutable_file_number", row.read::<i64, _>(4))?;
 
         Ok(Self {
             transaction_hash: transaction_hash.to_string(),
diff --git a/mithril-signer/src/database/record/interval_without_block_range_root.rs b/mithril-signer/src/database/record/interval_without_block_range_root.rs
@@ -0,0 +1,39 @@
+use sqlite::Row;
+
+use mithril_common::entities::BlockNumber;
+use mithril_persistence::sqlite::{HydrationError, Projection, SqLiteEntity};
+
+use crate::database::record::hydrator::try_to_u64;
+
+/// Interval of block numbers without block ranges root.
+pub struct IntervalWithoutBlockRangeRootRecord {
+    /// Start of the interval
+    pub start: Option<BlockNumber>,
+    /// End of the interval
+    pub end: Option<BlockNumber>,
+}
+
+impl SqLiteEntity for IntervalWithoutBlockRangeRootRecord {
+    fn hydrate(row: Row) -> Result<Self, HydrationError>
+    where
+        Self: Sized,
+    {
+        let start = row
+            .read::<Option<i64>, _>(0)
+            .map(|v| try_to_u64("interval_start.start", v))
+            .transpose()?;
+        let end = row
+            .read::<Option<i64>, _>(1)
+            .map(|v| try_to_u64("interval_end.end", v))
+            .transpose()?;
+
+        Ok(Self { start, end })
+    }
+
+    fn get_projection() -> Projection {
+        Projection::from(&[
+            ("start", "{:interval_start:}.start", "int"),
+            ("end", "{:interval_end:}.end", "int"),
+        ])
+    }
+}
diff --git a/mithril-signer/src/database/record/mod.rs b/mithril-signer/src/database/record/mod.rs
diff --git a/mithril-signer/src/database/repository/cardano_transaction_repository.rs b/mithril-signer/src/database/repository/cardano_transaction_repository.rs
