Merge pull request #1650 from input-output-hk/ensemble/1633/store-block-ranges-in-db

Store block range roots in db

Author: Alenar

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-persistence"
-version = "0.1.6"
+version = "0.1.7"
 description = "Common types, interfaces, and utilities to persist data for Mithril nodes."
 authors = { workspace = true }
 edition = { workspace = true }

internal/mithril-persistence/Cargo.toml

@@ -137,6 +137,14 @@ impl WhereCondition {
     }
 }
 
+/// Get all condition builder.
+pub trait GetAllCondition {
+    /// Get the condition for a get all query.
+    fn get_all_condition() -> WhereCondition {
+        WhereCondition::default()
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -349,4 +357,15 @@ mod tests {
         assert_eq!("a = ?1", &sql);
         assert_eq!(1, params.len());
     }
+
+    #[test]
+    fn expression_get_all_default() {
+        impl GetAllCondition for String {}
+
+        let expression = String::get_all_condition();
+        let (sql, params) = expression.expand();
+
+        assert_eq!("true".to_string(), sql);
+        assert!(params.is_empty());
+    }
 }

internal/mithril-persistence/src/sqlite/condition.rs

@@ -10,12 +10,12 @@ mod projection;
 mod provider;
 mod source_alias;
 
-pub use condition::WhereCondition;
+pub use condition::{GetAllCondition, WhereCondition};
 pub use connection_builder::{ConnectionBuilder, ConnectionOptions};
 pub use cursor::EntityCursor;
 pub use entity::{HydrationError, SqLiteEntity};
 pub use projection::{Projection, ProjectionField};
-pub use provider::Provider;
+pub use provider::{GetAllProvider, Provider};
 pub use source_alias::SourceAlias;
 
 use mithril_common::StdResult;

internal/mithril-persistence/src/sqlite/mod.rs

@@ -1,9 +1,10 @@
+use crate::sqlite::condition::GetAllCondition;
 use anyhow::Context;
 use mithril_common::StdResult;
 
 use super::{EntityCursor, SqLiteEntity, SqliteConnection, WhereCondition};
 
-/// A Provider is able to performe queries on a database and return iterator of a defined entity.
+/// A Provider is able to perform queries on a database and return iterator of a defined entity.
 /// It aims at being easily testable and adaptable.
 pub trait Provider<'conn> {
     /// Entity type returned by the result cursor.
@@ -38,6 +39,22 @@ pub trait Provider<'conn> {
     fn get_definition(&self, condition: &str) -> String;
 }
 
+/// An extension of the [Provider] that can return all the entities of a given type.
+pub trait GetAllProvider<'conn, E: SqLiteEntity> {
+    /// Return all the entities of the given type.
+    fn get_all(&'conn self) -> StdResult<EntityCursor<'conn, E>>;
+}
+
+impl<'conn, P, E> GetAllProvider<'conn, E> for P
+where
+    P: Provider<'conn, Entity = E> + GetAllCondition,
+    E: SqLiteEntity,
+{
+    fn get_all(&'conn self) -> StdResult<EntityCursor<'conn, E>> {
+        self.find(P::get_all_condition())
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use sqlite::{Connection, Value};
@@ -136,6 +153,8 @@ returning {projection}
         }
     }
 
+    impl GetAllCondition for TestEntityProvider<'_> {}
+
     fn init_database() -> SqliteConnection {
         let connection = Connection::open_thread_safe(":memory:").unwrap();
         connection
@@ -231,6 +250,7 @@ returning {projection}
         );
         assert!(cursor.next().is_none());
     }
+
     #[test]
     fn test_upsertion() {
         let connection = init_database();
@@ -258,4 +278,30 @@ returning {projection}
         );
         assert!(cursor.next().is_none());
     }
+
+    #[test]
+    pub fn test_blanket_get_all() {
+        let connection = init_database();
+        let provider = TestEntityProvider::new(&connection);
+        let cursor = provider.get_all().unwrap();
+        let entities: Vec<TestEntity> = cursor.collect();
+
+        assert_eq!(
+            vec![
+                TestEntity {
+                    text_data: "row 1".to_string(),
+                    real_data: 1.23,
+                    integer_data: -52,
+                    maybe_null: None
+                },
+                TestEntity {
+                    text_data: "row 2".to_string(),
+                    real_data: 2.34,
+                    integer_data: 1789,
+                    maybe_null: Some(0)
+                }
+            ],
+            entities
+        );
+    }
 }

internal/mithril-persistence/src/sqlite/provider.rs

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-aggregator"
-version = "0.4.62"
+version = "0.4.63"
 description = "A Mithril Aggregator server"
 authors = { workspace = true }
 edition = { workspace = true }

mithril-aggregator/Cargo.toml

@@ -57,6 +57,19 @@ vacuum;
             4,
             r#"
 create index block_number_index on cardano_tx(block_number);
+"#,
+        ),
+        // Migration 5
+        // Add `block_range_root` table
+        SqlMigration::new(
+            5,
+            r#"
+create table block_range_root (
+    start         integer   not null,
+    end           integer   not null,
+    merkle_root   text      not null,
+    primary key (start, end)
+);
 "#,
         ),
     ]

mithril-aggregator/src/database/cardano_transaction_migration.rs

@@ -0,0 +1,34 @@
+use mithril_persistence::sqlite::{Provider, SourceAlias, SqLiteEntity, SqliteConnection};
+
+use crate::database::record::BlockRangeRootRecord;
+
+/// Simple queries to retrieve [BlockRangeRootRecord] from the sqlite database.
+pub struct GetBlockRangeRootProvider<'client> {
+    connection: &'client SqliteConnection,
+}
+
+impl<'client> GetBlockRangeRootProvider<'client> {
+    #[cfg(test)]
+    /// Create a new instance
+    pub fn new(connection: &'client SqliteConnection) -> Self {
+        Self { connection }
+    }
+}
+
+#[cfg(test)]
+impl mithril_persistence::sqlite::GetAllCondition for GetBlockRangeRootProvider<'_> {}
+
+impl<'client> Provider<'client> for GetBlockRangeRootProvider<'client> {
+    type Entity = BlockRangeRootRecord;
+
+    fn get_connection(&'client self) -> &'client SqliteConnection {
+        self.connection
+    }
+
+    fn get_definition(&self, condition: &str) -> String {
+        let aliases = SourceAlias::new(&[("{:block_range_root:}", "block_range_root")]);
+        let projection = Self::Entity::get_projection().expand(aliases);
+
+        format!("select {projection} from block_range_root where {condition} order by start, end")
+    }
+}

mithril-aggregator/src/database/provider/block_range_root/get_block_range_root.rs

@@ -0,0 +1,43 @@
+use mithril_persistence::sqlite::{
+    Provider, SourceAlias, SqLiteEntity, SqliteConnection, WhereCondition,
+};
+
+use crate::database::record::IntervalWithoutBlockRangeRootRecord;
+
+/// Query that return the interval of block numbers that does not have a block range root.
+pub struct GetIntervalWithoutBlockRangeRootProvider<'client> {
+    connection: &'client SqliteConnection,
+}
+
+impl<'client> GetIntervalWithoutBlockRangeRootProvider<'client> {
+    /// Create a new instance
+    pub fn new(connection: &'client SqliteConnection) -> Self {
+        Self { connection }
+    }
+
+    pub fn get_interval_without_block_range_condition(&self) -> WhereCondition {
+        WhereCondition::default()
+    }
+}
+
+impl<'client> Provider<'client> for GetIntervalWithoutBlockRangeRootProvider<'client> {
+    type Entity = IntervalWithoutBlockRangeRootRecord;
+
+    fn get_connection(&'client self) -> &'client SqliteConnection {
+        self.connection
+    }
+
+    fn get_definition(&self, condition: &str) -> String {
+        let aliases = SourceAlias::new(&[
+            ("{:interval_start:}", "interval_start"),
+            ("{:interval_end:}", "interval_end"),
+        ]);
+        let projection = Self::Entity::get_projection().expand(aliases);
+
+        format!(
+            "select {projection} from \
+        (select max(end) as start from block_range_root where {condition}) as interval_start, \
+        (select max(block_number) as end from cardano_tx where {condition}) as interval_end"
+        )
+    }
+}

mithril-aggregator/src/database/provider/block_range_root/get_interval_without_block_range_provider.rs

@@ -0,0 +1,65 @@
+use std::iter::repeat;
+
+use sqlite::Value;
+
+use mithril_common::StdResult;
+use mithril_persistence::sqlite::{
+    Provider, SourceAlias, SqLiteEntity, SqliteConnection, WhereCondition,
+};
+
+use crate::database::record::BlockRangeRootRecord;
+
+/// Query to insert [BlockRangeRootRecord] in the sqlite database
+pub struct InsertBlockRangeRootProvider<'client> {
+    connection: &'client SqliteConnection,
+}
+
+impl<'client> InsertBlockRangeRootProvider<'client> {
+    /// Create a new instance
+    pub fn new(connection: &'client SqliteConnection) -> Self {
+        Self { connection }
+    }
+
+    /// Condition to insert multiples records.
+    pub fn get_insert_many_condition(
+        &self,
+        block_range_records: Vec<BlockRangeRootRecord>,
+    ) -> StdResult<WhereCondition> {
+        let columns = "(start, end, merkle_root)";
+        let values_columns: Vec<&str> = repeat("(?*, ?*, ?*)")
+            .take(block_range_records.len())
+            .collect();
+
+        let values: StdResult<Vec<Value>> =
+            block_range_records
+                .into_iter()
+                .try_fold(vec![], |mut vec, record| {
+                    vec.append(&mut vec![
+                        Value::Integer(record.range.start.try_into()?),
+                        Value::Integer(record.range.end.try_into()?),
+                        Value::String(record.merkle_root.to_hex()),
+                    ]);
+                    Ok(vec)
+                });
+
+        Ok(WhereCondition::new(
+            format!("{columns} values {}", values_columns.join(", ")).as_str(),
+            values?,
+        ))
+    }
+}
+
+impl<'client> Provider<'client> for InsertBlockRangeRootProvider<'client> {
+    type Entity = BlockRangeRootRecord;
+
+    fn get_connection(&'client self) -> &'client SqliteConnection {
+        self.connection
+    }
+
+    fn get_definition(&self, condition: &str) -> String {
+        let aliases = SourceAlias::new(&[("{:block_range_root:}", "block_range_root")]);
+        let projection = Self::Entity::get_projection().expand(aliases);
+
+        format!("insert or ignore into block_range_root {condition} returning {projection}")
+    }
+}