cleanup and optimise database

Author: frisitano

crates/database/db/src/db.rs

@@ -862,7 +862,7 @@ mod test {
         BatchCommitData, BatchInfo, BlockInfo, L1MessageEnvelope, L2BlockInfoWithL1Messages,
     };
     use scroll_alloy_consensus::TxL1Message;
-    use sea_orm::{ColumnTrait, EntityTrait, QueryFilter};
+    use sea_orm::EntityTrait;
 
     #[tokio::test]
     async fn test_database_round_trip_batch_commit() {
@@ -1227,11 +1227,13 @@ mod test {
 
     #[tokio::test]
     async fn test_delete_l1_messages_gt() {
+        reth_tracing::init_test_tracing();
+
         // Set up the test database.
         let db = setup_test_db().await;
 
         // Generate unstructured bytes.
-        let mut bytes = [0u8; 1024];
+        let mut bytes = [0u8; 10240];
         rand::rng().fill(bytes.as_mut_slice());
         let mut u = Unstructured::new(&bytes);
 
@@ -1276,9 +1278,13 @@ mod test {
 
     #[tokio::test]
     async fn test_get_l2_block_info_by_number() {
+        reth_tracing::init_test_tracing();
+
         // Set up the test database.
         let db = setup_test_db().await;
 
+        println!("im here");
+
         // Generate unstructured bytes.
         let mut bytes = [0u8; 1024];
         rand::rng().fill(bytes.as_mut_slice());
@@ -1289,6 +1295,8 @@ mod test {
         let batch_info: BatchInfo = batch_data.clone().into();
         db.insert_batch(batch_data).await.unwrap();
 
+        println!("inserted batch");
+
         // Generate and insert multiple L2 blocks
         let mut block_infos = Vec::new();
         for i in 200..205 {
@@ -1297,6 +1305,8 @@ mod test {
         }
         db.insert_blocks(block_infos.clone(), batch_info).await.unwrap();
 
+        println!("inserted blocks");
+
         // Test getting existing blocks
         for expected_block in block_infos {
             let retrieved_block =
@@ -1475,67 +1485,6 @@ mod test {
         }
     }
 
-    #[tokio::test]
-    async fn test_insert_block_upsert_behavior() {
-        reth_tracing::init_test_tracing();
-
-        // Set up the test database.
-        let db = setup_test_db().await;
-
-        // Generate unstructured bytes.
-        let mut bytes = [0u8; 1024];
-        rand::rng().fill(bytes.as_mut_slice());
-        let mut u = Unstructured::new(&bytes);
-
-        // Generate batches
-        let batch_data_1 = BatchCommitData { index: 100, ..Arbitrary::arbitrary(&mut u).unwrap() };
-        let batch_info_1: BatchInfo = batch_data_1.clone().into();
-        let batch_data_2 = BatchCommitData { index: 200, ..Arbitrary::arbitrary(&mut u).unwrap() };
-        let batch_info_2: BatchInfo = batch_data_2.clone().into();
-
-        db.insert_batch(batch_data_1).await.unwrap();
-        db.insert_batch(batch_data_2).await.unwrap();
-
-        // Insert initial block
-        let block_info = BlockInfo { number: 600, hash: B256::arbitrary(&mut u).unwrap() };
-        db.insert_blocks(vec![block_info], batch_info_1).await.unwrap();
-
-        // Verify initial insertion
-        let retrieved_block = db.get_l2_block_info_by_number(600).await.unwrap();
-        assert_eq!(retrieved_block, Some(block_info));
-
-        // Verify initial batch association using model conversion
-        let initial_l2_block_model = models::l2_block::Entity::find()
-            .filter(models::l2_block::Column::BlockNumber.eq(600))
-            .one(db.inner().get_connection())
-            .await
-            .unwrap()
-            .unwrap();
-        let (initial_block_info, initial_batch_info): (BlockInfo, BatchInfo) =
-            initial_l2_block_model.into();
-        assert_eq!(initial_block_info, block_info);
-        assert_eq!(initial_batch_info, batch_info_1);
-
-        // Update the same block with different batch info (upsert)
-        db.insert_blocks(vec![block_info], batch_info_2).await.unwrap();
-
-        // Verify the block still exists and was updated
-        let retrieved_block = db.get_l2_block_info_by_number(600).await.unwrap().unwrap();
-        assert_eq!(retrieved_block, block_info);
-
-        // Verify batch association was updated using model conversion
-        let updated_l2_block_model = models::l2_block::Entity::find()
-            .filter(models::l2_block::Column::BlockNumber.eq(600))
-            .one(db.inner().get_connection())
-            .await
-            .unwrap()
-            .unwrap();
-        let (updated_block_info, updated_batch_info): (BlockInfo, BatchInfo) =
-            updated_l2_block_model.into();
-        assert_eq!(updated_block_info, block_info);
-        assert_eq!(updated_batch_info, batch_info_2);
-    }
-
     #[tokio::test]
     async fn test_prepare_on_startup() {
         let db = setup_test_db().await;

crates/database/db/src/lib.rs

@@ -9,6 +9,9 @@ pub use db::Database;
 mod error;
 pub use error::DatabaseError;
 
+mod maintenance;
+pub use maintenance::DatabaseMaintenance;
+
 mod metrics;
 
 mod models;

crates/database/db/src/maintenance.rs

@@ -0,0 +1,68 @@
+use sea_orm::ConnectionTrait;
+
+use crate::DatabaseConnectionProvider;
+
+use super::Database;
+use std::sync::Arc;
+
+/// The interval in seconds between optimization runs in seconds.
+const PERIODIC_MAINTENANCE_INTERVAL_SECS: u64 = 600;
+
+/// Provides maintenance operations for the database.
+#[derive(Debug)]
+pub struct DatabaseMaintenance {
+    db: Arc<Database>,
+}
+
+impl DatabaseMaintenance {
+    /// Creates a new `DatabaseMaintenance` instance.
+    pub const fn new(db: Arc<Database>) -> Self {
+        Self { db }
+    }
+
+    /// Runs the maintenance tasks in a loop.
+    pub async fn run(self) {
+        self.startup_maintenance().await;
+
+        loop {
+            tokio::time::sleep(std::time::Duration::from_secs(PERIODIC_MAINTENANCE_INTERVAL_SECS))
+                .await;
+            self.periodic_maintenance().await;
+        }
+    }
+
+    /// Runs maintenance tasks at startup.
+    ///
+    /// This includes running `ANALYZE` and `PRAGMA optimize`.
+    async fn startup_maintenance(&self) {
+        let db = self.db.inner();
+        let conn = db.get_connection();
+
+        tracing::info!(target: "scroll::db::maintenance", "running startup ANALYZE...");
+        if let Err(err) = conn.execute_unprepared("ANALYZE;").await {
+            tracing::warn!(target: "scroll::db::maintenance", "ANALYZE failed: {:?}", err);
+        }
+
+        tracing::info!(target: "scroll::db::maintenance", "running PRAGMA optimize at startup...");
+        if let Err(err) = conn.execute_unprepared("PRAGMA optimize;").await {
+            tracing::warn!(target: "scroll::db::maintenance", "PRAGMA optimize failed: {:?}", err);
+        }
+
+        tracing::info!(target: "scroll::db::maintenance", "startup maintenance complete.");
+    }
+
+    /// Runs periodic maintenance tasks.
+    ///
+    /// This includes running `PRAGMA optimize`.
+    async fn periodic_maintenance(&self) {
+        let db = self.db.inner();
+        let conn = db.get_connection();
+
+        tracing::info!(target: "scroll::db::maintenance", "running periodic PRAGMA optimize...");
+        if let Err(err) = conn.execute_unprepared("PRAGMA optimize;").await {
+            tracing::warn!(target: "scroll::db::maintenance", "PRAGMA optimize failed: {:?}", err);
+        } else {
+            tracing::info!(target: "scroll::db::maintenance", "periodic PRAGMA optimize complete.");
+        }
+    }
+}

crates/database/db/src/operations.rs

@@ -708,13 +708,15 @@ impl<T: WriteConnectionProvider + ?Sized + Sync> DatabaseWriteOperations for T {
             blocks.into_iter().map(|b| (b, batch_info).into()).collect();
         models::l2_block::Entity::insert_many(l2_blocks)
             .on_conflict(
-                OnConflict::column(models::l2_block::Column::BlockHash)
-                    .update_columns([
-                        models::l2_block::Column::BlockNumber,
-                        models::l2_block::Column::BatchHash,
-                        models::l2_block::Column::BatchIndex,
-                    ])
-                    .to_owned(),
+                OnConflict::columns([
+                    models::l2_block::Column::BlockHash,
+                    models::l2_block::Column::BatchHash,
+                ])
+                .update_columns([
+                    models::l2_block::Column::BlockNumber,
+                    models::l2_block::Column::BatchIndex,
+                ])
+                .to_owned(),
             )
             .on_empty_do_nothing()
             .exec_without_returning(self.get_connection())

crates/database/migration/src/lib.rs

@@ -6,17 +6,7 @@ mod m20250408_132123_add_header_metadata;
 mod m20250408_150338_load_header_metadata;
 mod m20250411_072004_add_l2_block;
 mod m20250616_223947_add_metadata;
-mod m20250825_093350_remove_unsafe_l2_blocks;
-mod m20250829_042803_add_table_indexes;
 mod m20250904_175949_block_signature;
-mod m20250923_135359_add_index_block_hash;
-mod m20250929_161536_add_additional_indexes;
-mod m20251001_125444_add_index_status;
-mod m20251005_160938_add_initial_l1_block_numbers;
-mod m20251013_140946_add_initial_l1_processed_block_number;
-mod m20251021_070729_add_skipped_column;
-mod m20251021_144852_add_queue_index_index;
-mod m20251027_090416_add_table_statistics;
 mod m20251028_110719_add_l1_block_table;
 
 mod migration_info;
@@ -36,17 +26,7 @@ impl<MI: MigrationInfo + Send + Sync + 'static> MigratorTrait for Migrator<MI> {
             Box::new(m20250408_150338_load_header_metadata::Migration::<MI>(Default::default())),
             Box::new(m20250411_072004_add_l2_block::Migration::<MI>(Default::default())),
             Box::new(m20250616_223947_add_metadata::Migration),
-            Box::new(m20250825_093350_remove_unsafe_l2_blocks::Migration),
-            Box::new(m20250829_042803_add_table_indexes::Migration),
             Box::new(m20250904_175949_block_signature::Migration),
-            Box::new(m20250923_135359_add_index_block_hash::Migration),
-            Box::new(m20250929_161536_add_additional_indexes::Migration),
-            Box::new(m20251001_125444_add_index_status::Migration),
-            Box::new(m20251005_160938_add_initial_l1_block_numbers::Migration),
-            Box::new(m20251013_140946_add_initial_l1_processed_block_number::Migration),
-            Box::new(m20251021_070729_add_skipped_column::Migration),
-            Box::new(m20251021_144852_add_queue_index_index::Migration),
-            Box::new(m20251027_090416_add_table_statistics::Migration),
             Box::new(m20251028_110719_add_l1_block_table::Migration),
         ]
     }

crates/database/migration/src/m20220101_000001_create_batch_commit_table.rs

@@ -15,8 +15,9 @@ impl MigrationTrait for Migration {
                 Table::create()
                     .table(BatchCommit::Table)
                     .if_not_exists()
-                    .col(big_unsigned(BatchCommit::Index))
-                    .col(binary_len(BatchCommit::Hash, HASH_LENGTH).primary_key())
+                    .col(pk_auto(BatchCommit::Id))
+                    .col(big_unsigned(BatchCommit::Index).not_null())
+                    .col(binary_len(BatchCommit::Hash, HASH_LENGTH).not_null().unique_key())
                     .col(big_unsigned(BatchCommit::BlockNumber))
                     .col(big_unsigned(BatchCommit::BlockTimestamp))
                     .col(binary(BatchCommit::Calldata))
@@ -50,6 +51,64 @@ impl MigrationTrait for Migration {
             ))
             .await?;
 
+        // Indexes:
+        // ------------------------------------------------------------
+
+        // Add index on "index" column in batch_commit table.
+        manager
+            .create_index(
+                Index::create()
+                    .name("idx_batch_commit_batch_index")
+                    .col(BatchCommit::Index)
+                    .table(BatchCommit::Table)
+                    .to_owned(),
+            )
+            .await?;
+
+        // Add index on "block number" column in batch_commit table.
+        manager
+            .create_index(
+                Index::create()
+                    .name("idx_batch_commit_block_number")
+                    .col(BatchCommit::BlockNumber)
+                    .table(BatchCommit::Table)
+                    .to_owned(),
+            )
+            .await?;
+
+        // Add index on "finalized block number" column in batch_commit table.
+        manager
+            .create_index(
+                Index::create()
+                    .name("idx_batch_commit_finalized_block_number")
+                    .col(BatchCommit::FinalizedBlockNumber)
+                    .table(BatchCommit::Table)
+                    .to_owned(),
+            )
+            .await?;
+
+        // Add index on "reverted block number" column in batch_commit table.
+        manager
+            .create_index(
+                Index::create()
+                    .name("idx_batch_commit_reverted_block_number")
+                    .col(BatchCommit::RevertedBlockNumber)
+                    .table(BatchCommit::Table)
+                    .to_owned(),
+            )
+            .await?;
+
+        // Add index on `status` for the `batch_commit` table.
+        manager
+            .create_index(
+                Index::create()
+                    .name("idx_batch_commit_status")
+                    .col(BatchCommit::Status)
+                    .table(BatchCommit::Table)
+                    .to_owned(),
+            )
+            .await?;
+
         Ok(())
     }
 
@@ -61,6 +120,7 @@ impl MigrationTrait for Migration {
 #[derive(DeriveIden)]
 pub(crate) enum BatchCommit {
     Table,
+    Id,
     Index,
     Hash,
     BlockNumber,