test: increase coverage (#218)

* test: revert batch

* test: derivation pipeline retry

* test: derive blob batch

* test: reorg e2e test

* feat: force always_process_payload_attributes_on_canonical_head in node binary

* fix: lints

* merge diff

---------

Co-authored-by: frisitano <[email protected]>

Author: greged93

Cargo.lock

@@ -154,6 +154,7 @@ reth-primitives-traits = { git = "https://github.com/scroll-tech/reth.git", defa
 reth-provider = { git = "https://github.com/scroll-tech/reth.git", default-features = false }
 reth-rpc-builder = { git = "https://github.com/scroll-tech/reth.git", default-features = false }
 reth-rpc-server-types = { git = "https://github.com/scroll-tech/reth.git", default-features = false }
+reth-storage-api = { git = "https://github.com/scroll-tech/reth.git", default-features = false }
 reth-tasks = { git = "https://github.com/scroll-tech/reth.git", default-features = false }
 reth-tokio-util = { git = "https://github.com/scroll-tech/reth.git", default-features = false }
 reth-tracing = { git = "https://github.com/scroll-tech/reth.git", default-features = false }

Cargo.toml

@@ -38,6 +38,7 @@ async-trait.workspace = true
 alloy-primitives = { workspace = true, features = ["getrandom"] }
 criterion = { version = "0.6", features = ["async", "async_tokio"] }
 eyre.workspace = true
+rollup-node-providers = { workspace = true, features = ["test-utils"] }
 scroll-db = { workspace = true, features = ["test-utils"] }
 scroll-codec = { workspace = true, features = ["test-utils"] }
 tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }

crates/derivation-pipeline/Cargo.toml

@@ -2,21 +2,21 @@
 
 #![allow(missing_docs)]
 
-use std::sync::Arc;
+use std::{collections::HashMap, sync::Arc};
 
 use alloy_primitives::{address, b256, bytes, U256};
 use criterion::{criterion_group, criterion_main, Criterion};
 use futures::StreamExt;
 use rollup_node_primitives::{BatchCommitData, BatchInfo, L1MessageEnvelope};
-use rollup_node_providers::{test_utils::NoBlobProvider, DatabaseL1MessageProvider};
+use rollup_node_providers::{test_utils::MockL1Provider, DatabaseL1MessageProvider};
 use scroll_alloy_consensus::TxL1Message;
 use scroll_codec::decoding::test_utils::read_to_bytes;
 use scroll_db::{test_utils::setup_test_db, Database, DatabaseOperations};
 use scroll_derivation_pipeline::DerivationPipeline;
 use tokio::runtime::{Handle, Runtime};
 
 async fn setup_pipeline(
-) -> DerivationPipeline<NoBlobProvider<DatabaseL1MessageProvider<Arc<Database>>>> {
+) -> DerivationPipeline<MockL1Provider<DatabaseL1MessageProvider<Arc<Database>>>> {
     // load batch data in the db.
     let db = Arc::new(setup_test_db().await);
     let raw_calldata = read_to_bytes("./testdata/calldata_v0.bin").unwrap();
@@ -66,7 +66,7 @@ async fn setup_pipeline(
 
     // construct the pipeline.
     let l1_messages_provider = DatabaseL1MessageProvider::new(db.clone(), 0);
-    let mock_l1_provider = NoBlobProvider { l1_messages_provider };
+    let mock_l1_provider = MockL1Provider { l1_messages_provider, blobs: HashMap::new() };
     DerivationPipeline::new(mock_l1_provider, db)
 }
 

crates/derivation-pipeline/benches/pipeline.rs

@@ -0,0 +1 @@
+86b053a9000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000001a000000000000000000000000000000000000000000000000000000000000018c0000000000000000000000000000000000000000000000000000000000000190000000000000000000000000000000000000000000000000000000000000000c104000000000004cb4c000000000000000000000000000e3c2ec12a808c7097fd258c6fbaf7522274b0f872a347c3ad57b92ec08132a237253501f71b48cb81e381f052c23f0036b4625d0cc9bc003ea821f5a143d4b838faa86529a8718e42ee479ecddb356875ac3fcbfc52ea5df1f0489666200e886fc46d0000000066d85a6c0bd522bcaf28f037b8f59e1df03bb3dba8981efa98dfd4875ed8eefc7bbd112764f0b033e86cecf5e5047b15930b44920c7de1b558ab78f6ee5927e582b47a8b00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002d00000000000000000000000000000000000000000000000000000000000005a0000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000006a00000000000000000000000000000000000000000000000000000000000000700000000000000000000000000000000000000000000000000000000000000076000000000000000000000000000000000000000000000000000000000000007c00000000000000000000000000000000000000000000000000000000000000820000000000000000000000000000000000000000000000000000000000000088000000000000000000000000000000000000000000000000000000000000008e0000000000000000000000000000000000000000000000000000000000000094000000000000000000000000000000000000000000000000000000000000009a00000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000a600000000000000000000000000000000000000000000000000000000000000ac00000000000000000000000000000000000000000000000000000000000000b200000000000000000000000000000000000000000000000000000000000000b800000000000000000000000000000000000000000000000000000000000000be00000000000000000000000000000000000000000000000000000000000000c400000000000000000000000000000000000000000000000000000000000000ca00000000000000000000000000000000000000000000000000000000000000d000000000000000000000000000000000000000000000000000000000000000d600000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000000e600000000000000000000000000000000000000000000000000000000000000ec00000000000000000000000000000000000000000000000000000000000000f200000000000000000000000000000000000000000000000000000000000000f800000000000000000000000000000000000000000000000000000000000000fe0000000000000000000000000000000000000000000000000000000000000104000000000000000000000000000000000000000000000000000000000000010a00000000000000000000000000000000000000000000000000000000000001100000000000000000000000000000000000000000000000000000000000000116000000000000000000000000000000000000000000000000000000000000011c00000000000000000000000000000000000000000000000000000000000001220000000000000000000000000000000000000000000000000000000000000128000000000000000000000000000000000000000000000000000000000000012e0000000000000000000000000000000000000000000000000000000000000134000000000000000000000000000000000000000000000000000000000000013a00000000000000000000000000000000000000000000000000000000000001400000000000000000000000000000000000000000000000000000000000000146000000000000000000000000000000000000000000000000000000000000014c00000000000000000000000000000000000000000000000000000000000001520000000000000000000000000000000000000000000000000000000000000158000000000000000000000000000000000000000000000000000000000000015e0000000000000000000000000000000000000000000000000000000000000164000000000000000000000000000000000000000000000000000000000000016a0000000000000000000000000000000000000000000000000000000000000003d0100000000008930600000000066d85a6e0000000000000000000000000000000000000000000000000000000002b7dbdb00000000009896800010000000000000000000000000000000000000000000000000000000000000000000000000790200000000008930610000000066d85a700000000000000000000000000000000000000000000000000000000002b7dbdb0000000000989680000e000000000000008930620000000066d85a730000000000000000000000000000000000000000000000000000000002cf159a0000000000989680000b000000000000000000000000000000000000000000000000000000000000000000000000000000003d0100000000008930630000000066d85a760000000000000000000000000000000000000000000000000000000002cf159a000000000098968000090000000000000000000000000000000000000000000000000000000000000000000000003d0100000000008930640000000066d85a780000000000000000000000000000000000000000000000000000000002cf159a0000000000989680000f0000000000000000000000000000000000000000000000000000000000000000000000003d0100000000008930650000000066d85a7b0000000000000000000000000000000000000000000000000000000002cf159a0000000000989680000b0000000000000000000000000000000000000000000000000000000000000000000000003d0100000000008930660000000066d85a7e0000000000000000000000000000000000000000000000000000000002cf159a0000000000989680000f0000000000000000000000000000000000000000000000000000000000000000000000003d0100000000008930670000000066d85a800000000000000000000000000000000000000000000000000000000002cf159a000000000098968000120000000000000000000000000000000000000000000000000000000000000000000000003d0100000000008930680000000066d85a820000000000000000000000000000000000000000000000000000000002cf159a0000000000989680000d0000000000000000000000000000000000000000000000000000000000000000000000003d0100000000008930690000000066d85a850000000000000000000000000000000000000000000000000000000002cf159a000000000098968000120000000000000000000000000000000000000000000000000000000000000000000000003d01000000000089306a0000000066d85a880000000000000000000000000000000000000000000000000000000002cf159a0000000000989680000b0000000000000000000000000000000000000000000000000000000000000000000000003d01000000000089306b0000000066d85a8b0000000000000000000000000000000000000000000000000000000002cf159a000000000098968000110000000000000000000000000000000000000000000000000000000000000000000000003d01000000000089306c0000000066d85a8d0000000000000000000000000000000000000000000000000000000002cf159a000000000098968000090000000000000000000000000000000000000000000000000000000000000000000000003d01000000000089306d0000000066d85a900000000000000000000000000000000000000000000000000000000002cf159a0000000000989680000f0000000000000000000000000000000000000000000000000000000000000000000000003d01000000000089306e0000000066d85a930000000000000000000000000000000000000000000000000000000002cf159a000000000098968000070000000000000000000000000000000000000000000000000000000000000000000000003d01000000000089306f0000000066d85a940000000000000000000000000000000000000000000000000000000002cf159a0000000000989680000b0000000000000000000000000000000000000000000000000000000000000000000000003d0100000000008930700000000066d85a960000000000000000000000000000000000000000000000000000000002cf159a0000000000989680000e0000000000000000000000000000000000000000000000000000000000000000000000003d0100000000008930710000000066d85a990000000000000000000000000000000000000000000000000000000002bc00640000000000989680000d0000000000000000000000000000000000000000000000000000000000000000000000003d0100000000008930720000000066d85a9c0000000000000000000000000000000000000000000000000000000002bc00640000000000989680000e0000000000000000000000000000000000000000000000000000000000000000000000003d0100000000008930730000000066d85a9e0000000000000000000000000000000000000000000000000000000002bc00640000000000989680000d0000000000000000000000000000000000000000000000000000000000000000000000003d0100000000008930740000000066d85aa10000000000000000000000000000000000000000000000000000000002bc00640000000000989680000e000000000000000000000000000000000000000000000000000000000000000000000000790200000000008930750000000066d85aa40000000000000000000000000000000000000000000000000000000002ab1e1200000000009896800008000000000000008930760000000066d85aa70000000000000000000000000000000000000000000000000000000002ab1e120000000000989680000a000000000000000000000000000000000000000000000000000000000000000000000000000000003d0100000000008930770000000066d85aaa0000000000000000000000000000000000000000000000000000000002ab1e12000000000098968000070000000000000000000000000000000000000000000000000000000000000000000000003d0100000000008930780000000066d85aad0000000000000000000000000000000000000000000000000000000002be50ab0000000000989680000c0000000000000000000000000000000000000000000000000000000000000000000000003d0100000000008930790000000066d85ab00000000000000000000000000000000000000000000000000000000002be50ab000000000098968000080000000000000000000000000000000000000000000000000000000000000000000000003d01000000000089307a0000000066d85ab30000000000000000000000000000000000000000000000000000000002be50ab0000000000989680000a0000000000000000000000000000000000000000000000000000000000000000000000003d01000000000089307b0000000066d85ab60000000000000000000000000000000000000000000000000000000002be50ab000000000098968000100000000000000000000000000000000000000000000000000000000000000000000000003d01000000000089307c0000000066d85ab80000000000000000000000000000000000000000000000000000000002add58e0000000000989680000c0000000000000000000000000000000000000000000000000000000000000000000000003d01000000000089307d0000000066d85abb0000000000000000000000000000000000000000000000000000000002add58e000000000098968000160000000000000000000000000000000000000000000000000000000000000000000000003d01000000000089307e0000000066d85abd0000000000000000000000000000000000000000000000000000000002add58e0000000000989680000f0000000000000000000000000000000000000000000000000000000000000000000000003d01000000000089307f0000000066d85ac00000000000000000000000000000000000000000000000000000000002add58e000000000098968000120000000000000000000000000000000000000000000000000000000000000000000000003d0100000000008930800000000066d85ac30000000000000000000000000000000000000000000000000000000002c15db0000000000098968000110000000000000000000000000000000000000000000000000000000000000000000000003d0100000000008930810000000066d85ac60000000000000000000000000000000000000000000000000000000002c15db00000000000989680000e0000000000000000000000000000000000000000000000000000000000000000000000003d0100000000008930820000000066d85ac90000000000000000000000000000000000000000000000000000000002c15db0000000000098968000150000000000000000000000000000000000000000000000000000000000000000000000003d0100000000008930830000000066d85acb0000000000000000000000000000000000000000000000000000000002c15db0000000000098968000110000000000000000000000000000000000000000000000000000000000000000000000003d0100000000008930840000000066d85acd0000000000000000000000000000000000000000000000000000000002c15db0000000000098968000090000000000000000000000000000000000000000000000000000000000000000000000003d0100000000008930850000000066d85acf0000000000000000000000000000000000000000000000000000000002c15db00000000000989680000a0000000000000000000000000000000000000000000000000000000000000000000000003d0100000000008930860000000066d85ad00000000000000000000000000000000000000000000000000000000002c15db0000000000098968000100000000000000000000000000000000000000000000000000000000000000000000000003d0100000000008930870000000066d85ad20000000000000000000000000000000000000000000000000000000002c15db0000000000098968000040000000000000000000000000000000000000000000000000000000000000000000000003d0100000000008930880000000066d85ad50000000000000000000000000000000000000000000000000000000002c15db00000000000989680000f0000000000000000000000000000000000000000000000000000000000000000000000003d0100000000008930890000000066d85ad70000000000000000000000000000000000000000000000000000000002c15db0000000000098968000110000000000000000000000000000000000000000000000000000000000000000000000003d01000000000089308a0000000066d85ada0000000000000000000000000000000000000000000000000000000002c15db00000000000989680000d0000000000000000000000000000000000000000000000000000000000000000000000003d01000000000089308b0000000066d85adc0000000000000000000000000000000000000000000000000000000002c15db0000000000098968000110004000000000000000000000000000000000000000000000000000000000000000000003d01000000000089308c0000000066d85adf0000000000000000000000000000000000000000000000000000000002c15db00000000000989680000c0000000000000000000000000000000000000000000000000000000000000000000000003d01000000000089308d0000000066d85ae20000000000000000000000000000000000000000000000000000000002c15db00000000000989680000a0000000000000000000000000000000000000000000000000000000000000000000000003d01000000000089308e0000000066d85ae50000000000000000000000000000000000000000000000000000000002c15db00000000000989680000e00000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a00f3f5413eab47885ada130f2523ec523ff02aa2b3fa73ba6e74557aaac242d460bc6e2a083263e89f86fd6566320d2a54713e49d63e57c012c0e2846596bbd8a8110babdb1cec32754c2a39603937b357f37e86c3a2232f17bc71f8c88069b2126b89fe03ff36a43d34f435012e84f84ad7bb34db367ffea6e69ba9bb889b667381fbe4f9f0ca183969069ca6bd6402a426cac2de711bc26935afb95338c5040

crates/derivation-pipeline/src/lib.rs

@@ -259,6 +259,11 @@ where
         self.syncing
     }
 
+    /// Returns the forkchoice state.
+    pub fn forkchoice_state(&self) -> ForkchoiceState {
+        self.fcs.clone()
+    }
+
     /// Returns the alloy forkchoice state.
     pub fn alloy_forkchoice_state(&self) -> alloy_rpc_types_engine::ForkchoiceState {
         if self.is_syncing() {

crates/derivation-pipeline/testdata/blob_v4_compressed.bin

@@ -18,4 +18,6 @@ pub enum RollupManagerEvent {
     L1MessageIndexed(u64),
     /// A new event from the signer.
     SignerEvent(SignerEvent),
+    /// A reorg event.
+    Reorg(u64),
 }

crates/derivation-pipeline/testdata/calldata_v4_compressed.bin

@@ -19,7 +19,7 @@ use rollup_node_watcher::L1Notification;
 use scroll_alloy_hardforks::ScrollHardforks;
 use scroll_alloy_network::Scroll;
 use scroll_alloy_provider::ScrollEngineApi;
-use scroll_engine::{EngineDriver, EngineDriverEvent};
+use scroll_engine::{EngineDriver, EngineDriverEvent, ForkchoiceState};
 use scroll_network::{
     BlockImportOutcome, NetworkManagerEvent, NewBlockWithPeer, ScrollNetworkManager,
 };
@@ -110,6 +110,8 @@ pub struct RollupNodeManager<
 pub struct RollupManagerStatus {
     /// Whether the rollup manager is syncing.
     pub syncing: bool,
+    /// The current FCS for the manager.
+    pub forkchoice_state: ForkchoiceState,
 }
 
 impl<
@@ -251,12 +253,14 @@ where
                 // update the fcs on new finalized block.
                 self.engine.set_finalized_block_info(finalized_block);
             }
-            IndexerEvent::FinalizedIndexed(l1_block_number, Some(finalized_block)) => {
+            IndexerEvent::FinalizedIndexed(l1_block_number, finalized_block) => {
                 if let Some(sequencer) = self.sequencer.as_mut() {
                     sequencer.set_l1_finalized_block_number(l1_block_number);
                 }
                 // update the fcs on new finalized block.
-                self.engine.set_finalized_block_info(finalized_block);
+                if let Some(finalized_block) = finalized_block {
+                    self.engine.set_finalized_block_info(finalized_block);
+                }
             }
             IndexerEvent::UnwindIndexed {
                 l1_block_number,
@@ -281,6 +285,10 @@ where
 
                 // Handle the reorg in the derivation pipeline.
                 self.derivation_pipeline.handle_reorg(l1_block_number);
+
+                if let Some(event_sender) = self.event_sender.as_ref() {
+                    event_sender.notify(RollupManagerEvent::Reorg(l1_block_number));
+                }
             }
             IndexerEvent::L1MessageIndexed(index) => {
                 if let Some(event_sender) = self.event_sender.as_ref() {
@@ -367,15 +375,23 @@ where
 
     /// Handles an [`L1Notification`] from the L1 watcher.
     fn handle_l1_notification(&mut self, notification: L1Notification) {
-        if let L1Notification::Consensus(ref update) = notification {
-            self.consensus.update_config(update);
+        match notification {
+            L1Notification::Consensus(ref update) => self.consensus.update_config(update),
+            L1Notification::NewBlock(new_block) => {
+                if let Some(sequencer) = self.sequencer.as_mut() {
+                    sequencer.handle_new_l1_block(new_block)
+                }
+            }
+            _ => self.indexer.handle_l1_notification(notification),
         }
-        self.indexer.handle_l1_notification(notification)
     }
 
     /// Returns the current status of the [`RollupNodeManager`].
-    const fn status(&self) -> RollupManagerStatus {
-        RollupManagerStatus { syncing: self.engine.is_syncing() }
+    fn status(&self) -> RollupManagerStatus {
+        RollupManagerStatus {
+            syncing: self.engine.is_syncing(),
+            forkchoice_state: self.engine.forkchoice_state(),
+        }
     }
 
     /// Drives the [`RollupNodeManager`] future until a [`GracefulShutdown`] signal is received.