Feat: header metadata (#154)

* feat: decompressing + downloading file

* feat: add file download

* feat: update block data provider interface

* feat: add migration info

* fix: migration

* feat: refactor code

* feat: remove migration data

* fix: disappearing progress bar

* fix: doc

* feat: use FuturesUnordered for a better view of progress

* feat: implement ScrollMigrator on NamedChain

* feat: insert data in sqlite by batch

* fix: lint

Author: greged93

Cargo.lock

@@ -134,12 +134,13 @@ mod test {
     }
 
     #[tokio::test]
+    #[ignore]
     async fn test_database_block_data_seed() {
         // Setup the test database.
         let db = setup_test_db().await;
 
         // db should contain the seeded data after migration.
-        let data = db.get_l2_block_data_hint(0.into()).await.unwrap();
+        let data = db.get_l2_block_data_hint(0).await.unwrap();
         assert!(data.is_some());
     }
 

crates/database/db/src/db.rs

@@ -8,8 +8,8 @@ use sea_orm::entity::prelude::*;
 pub struct Model {
     #[sea_orm(primary_key)]
     number: i64,
-    hash: Vec<u8>,
     extra_data: Vec<u8>,
+    state_root: Vec<u8>,
     difficulty: Vec<u8>,
 }
 

crates/database/db/src/models/block_data.rs

@@ -1,6 +1,5 @@
 use super::{models, DatabaseError};
 use crate::DatabaseConnectionProvider;
-use alloy_eips::{BlockId, BlockNumberOrTag};
 use alloy_primitives::B256;
 use futures::{Stream, StreamExt};
 use rollup_node_primitives::{
@@ -180,20 +179,13 @@ pub trait DatabaseOperations: DatabaseConnectionProvider {
             .map(|res| Ok(res.map(Into::into)?)))
     }
 
-    /// Get the extra data for the provided [`BlockId`].
+    /// Get the extra data for the provided block number.
     async fn get_l2_block_data_hint(
         &self,
-        block_id: BlockId,
+        block_number: u64,
     ) -> Result<Option<BlockDataHint>, DatabaseError> {
-        let filter = match block_id {
-            BlockId::Hash(hash) => models::block_data::Column::Hash.eq(hash.block_hash.to_vec()),
-            BlockId::Number(BlockNumberOrTag::Number(number)) => {
-                models::block_data::Column::Number.eq(number as i64)
-            }
-            x => return Err(DatabaseError::BlockNotFound(x)),
-        };
         Ok(models::block_data::Entity::find()
-            .filter(filter)
+            .filter(models::block_data::Column::Number.eq(block_number as i64))
             .one(self.get_connection())
             .await
             .map(|x| x.map(Into::into))?)

crates/database/db/src/operations.rs

@@ -8,6 +8,6 @@ use scroll_migration::{Migrator, MigratorTrait};
 pub async fn setup_test_db() -> Database {
     let database_url = "sqlite::memory:";
     let connection = sea_orm::Database::connect(database_url).await.unwrap();
-    Migrator::up(&connection, None).await.unwrap();
+    Migrator::<()>::up(&connection, None).await.unwrap();
     connection.into()
 }

crates/database/db/src/test_utils.rs

@@ -6,10 +6,15 @@ publish = false
 
 [dependencies]
 alloy-primitives = { workspace = true, features = ["serde"] }
-async-std = { version = "1", features = ["attributes", "tokio1"] }
-csv = "1.3.1"
+eyre.workspace = true
+futures.workspace = true
+indicatif = "0.17.11"
+reqwest.workspace = true
+reqwest-retry = "0.7.0"
+reqwest-middleware = "0.4.2"
+reth-chainspec.workspace = true
 sea-orm = { workspace = true, features = ["sqlx-sqlite", "runtime-tokio-native-tls", "macros"] }
-serde = { workspace = true, features = ["derive"] }
+sha2 = "0.10.9"
 tracing.workspace = true
 
 [dependencies.sea-orm-migration]

crates/database/migration/Cargo.toml

@@ -2,21 +2,54 @@ pub use sea_orm_migration::prelude::*;
 
 mod m20220101_000001_create_batch_commit_table;
 mod m20250304_125946_add_l1_msg_table;
-mod m20250408_132123_add_block_data_table;
-mod m20250408_150338_seed_block_data_table;
+mod m20250408_132123_add_header_metadata;
+mod m20250408_150338_load_header_metadata;
 mod m20250411_072004_add_l2_block;
+mod migration_info;
+pub use migration_info::{MigrationInfo, ScrollMainnetMigrationInfo, ScrollSepoliaMigrationInfo};
 
-pub struct Migrator;
+pub struct Migrator<MI>(pub std::marker::PhantomData<MI>);
 
 #[async_trait::async_trait]
-impl MigratorTrait for Migrator {
+impl<MI: MigrationInfo + Send + Sync + 'static> MigratorTrait for Migrator<MI> {
     fn migrations() -> Vec<Box<dyn MigrationTrait>> {
         vec![
             Box::new(m20220101_000001_create_batch_commit_table::Migration),
             Box::new(m20250304_125946_add_l1_msg_table::Migration),
-            Box::new(m20250408_132123_add_block_data_table::Migration),
-            Box::new(m20250408_150338_seed_block_data_table::Migration),
+            Box::new(m20250408_132123_add_header_metadata::Migration),
+            Box::new(m20250408_150338_load_header_metadata::Migration::<MI>(Default::default())),
             Box::new(m20250411_072004_add_l2_block::Migration),
         ]
     }
 }
+
+pub mod traits {
+    use crate::{ScrollMainnetMigrationInfo, ScrollSepoliaMigrationInfo};
+    use reth_chainspec::NamedChain;
+    use sea_orm::{prelude::async_trait::async_trait, DatabaseConnection, DbErr};
+    use sea_orm_migration::MigratorTrait;
+
+    /// An instance of the trait can perform the migration for Scroll.
+    #[async_trait]
+    pub trait ScrollMigrator {
+        /// Migrates the tables.
+        async fn migrate(&self, conn: &DatabaseConnection) -> Result<(), DbErr>;
+    }
+
+    #[async_trait]
+    impl ScrollMigrator for NamedChain {
+        async fn migrate(&self, conn: &DatabaseConnection) -> Result<(), DbErr> {
+            match self {
+                NamedChain::Scroll => {
+                    Ok(super::Migrator::<ScrollMainnetMigrationInfo>::up(conn, None))
+                }
+                NamedChain::ScrollSepolia => {
+                    Ok(super::Migrator::<ScrollSepoliaMigrationInfo>::up(conn, None))
+                }
+                NamedChain::Dev => Ok(super::Migrator::<()>::up(conn, None)),
+                _ => Err(DbErr::Custom("expected Scroll Mainnet, Sepolia or Dev".into())),
+            }?
+            .await
+        }
+    }
+}

crates/database/migration/migration-data/block_data.csv

@@ -13,8 +13,8 @@ impl MigrationTrait for Migration {
                     .table(BlockData::Table)
                     .if_not_exists()
                     .col(big_unsigned(BlockData::Number).primary_key())
-                    .col(binary_len(BlockData::Hash, 32).unique_key())
                     .col(binary(BlockData::ExtraData))
+                    .col(binary_len(BlockData::StateRoot, 32))
                     .col(binary_len(BlockData::Difficulty, 32))
                     .to_owned(),
             )
@@ -30,7 +30,7 @@ impl MigrationTrait for Migration {
 enum BlockData {
     Table,
     Number,
-    Hash,
     ExtraData,
+    StateRoot,
     Difficulty,
 }