Skip to content

Commit bfda5c6

Browse files
greged93greged93
authored
feat: integrate derivation pipeline (#49)
* feat: add processed field to batch commits in db * feat: integrate derivation pipeline in RN manager * wip: derivation pipeline integration * feat: integrate derivation pipeline in RNM * fix: don't use Arc for database l1 provider. * test: stream implementation * fix: lints * test: remove single `poll_next_unpin` call * fix: clippy * fix: set the suggested fee recipient to 0x0 * feat: improve `ScrollRollupNodeArgs` * fix: query block timestamp if missing from log * fix: high bytes mask * feat: introduce BeaconProvider trait along with Mock implementation. * fix: answer comments * fix: answer comments * fix: revert constant changes
1 parent 8589790 commit bfda5c6

File tree

33 files changed

+1276
-446
lines changed

33 files changed

+1276
-446
lines changed

‎Cargo.lock‎

Lines changed: 576 additions & 310 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -116,16 +116,17 @@ large_enum_variant = "allow"
116116
[workspace.dependencies]
117117
# alloy
118118
alloy-chains = { version = "0.1.32", default-features = false }
119-
alloy-consensus = { version = "0.12.2", default-features = false }
120-
alloy-eips = { version = "0.12.2", default-features = false }
121-
alloy-json-rpc = { version = "0.12.2", default-features = false }
122-
alloy-network = { version = "0.12.2", default-features = false }
123-
alloy-primitives = { version = "0.8.20", default-features = false }
124-
alloy-provider = { version = "0.12.2", default-features = false }
125-
alloy-rpc-types-engine = { version = "0.12.2", default-features = false }
126-
alloy-rpc-types-eth = { version = "0.12.2", default-features = false }
127-
alloy-sol-types = { version = "0.8.20", default-features = false }
128-
alloy-transport = { version = "0.12.2", default-features = false }
119+
alloy-consensus = { version = "0.13.0", default-features = false }
120+
alloy-eips = { version = "0.13.0", default-features = false }
121+
alloy-json-rpc = { version = "0.13.0", default-features = false }
122+
alloy-network = { version = "0.13.0", default-features = false }
123+
alloy-primitives = { version = "0.8.25", default-features = false }
124+
alloy-provider = { version = "0.13.0", default-features = false }
125+
alloy-rpc-client = { version = "0.13.0", default-features = false }
126+
alloy-rpc-types-engine = { version = "0.13.0", default-features = false }
127+
alloy-rpc-types-eth = { version = "0.13.0", default-features = false }
128+
alloy-sol-types = { version = "0.8.25", default-features = false }
129+
alloy-transport = { version = "0.13.0", default-features = false }
129130

130131
# scroll-alloy
131132
scroll-alloy-consensus = { git = "https://github.com/scroll-tech/reth.git", default-features = false }
@@ -167,10 +168,12 @@ scroll-migration = { path = "crates/database/migration" }
167168
# misc
168169
arbitrary = { version = "1.4", default-features = false }
169170
async-trait = "0.1"
171+
auto_impl = "1.2"
170172
derive_more = { version = "2.0", default-features = false }
171173
eyre = "0.6"
172174
futures = { version = "0.3", default-features = false }
173175
rand = { version = "0.9" }
176+
reqwest = "0.12"
174177
secp256k1 = { version = "0.29", default-features = false }
175178
thiserror = "2.0"
176179
tokio = { version = "1.39", default-features = false }

‎bin/rollup/Cargo.toml‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,13 @@ exclude.workspace = true
1111
alloy-chains.workspace = true
1212
alloy-provider.workspace = true
1313
alloy-rpc-types-engine.workspace = true
14+
alloy-rpc-client.workspace = true
15+
alloy-transport.workspace = true
1416

1517
# scroll-alloy
1618
scroll-alloy-consensus.workspace = true
17-
scroll-alloy-rpc-types-engine.workspace = true
1819
scroll-alloy-provider.workspace = true
20+
scroll-alloy-rpc-types-engine.workspace = true
1921

2022
# reth
2123
reth-cli-util = { git = "https://github.com/scroll-tech/reth.git" }
@@ -39,15 +41,16 @@ reth-scroll-cli = { git = "https://github.com/scroll-tech/reth.git" }
3941
reth-scroll-engine-primitives = { git = "https://github.com/scroll-tech/reth.git" }
4042
reth-scroll-node = { workspace = true, features = ["skip-state-root-validation"] }
4143
reth-scroll-primitives = { workspace = true }
42-
scroll-wire.workspace = true
43-
scroll-network.workspace = true
44-
scroll-engine = { workspace = true, features = ["test-utils"] }
4544
scroll-db = { workspace = true }
45+
scroll-engine = { workspace = true, features = ["test-utils"] }
4646
scroll-migration.workspace = true
47+
scroll-network.workspace = true
48+
scroll-wire.workspace = true
4749

4850
# rollup-node
49-
rollup-node-manager.workspace = true
5051
rollup-node-indexer.workspace = true
52+
rollup-node-manager.workspace = true
53+
rollup-node-providers.workspace = true
5154
rollup-node-watcher.workspace = true
5255

5356
# misc
@@ -72,6 +75,7 @@ reth-tracing = { git = "https://github.com/scroll-tech/reth.git" }
7275

7376
# misc
7477
futures.workspace = true
78+
reqwest.workspace = true
7579
serde_json = { version = "1.0.94", default-features = false, features = ["alloc"] }
7680
tokio = { workspace = true, features = ["full"] }
7781

@@ -85,9 +89,11 @@ test-utils = [
8589
"scroll-engine/test-utils",
8690
"reth-payload-builder/test-utils",
8791
"reth-payload-builder/test-utils",
92+
"reth-scroll-node/test-utils",
8893
"rollup-node-watcher/test-utils",
8994
"scroll-db/test-utils",
9095
"scroll-db/test-utils",
96+
"rollup-node-providers/test-utils",
9197
]
9298
serde = [
9399
"alloy-primitives/serde",

‎bin/rollup/src/args.rs‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,42 @@
1+
use crate::constants;
12
use std::path::PathBuf;
23

34
/// A struct that represents the arguments for the rollup node.
45
#[derive(Debug, clap::Args)]
56
pub struct ScrollRollupNodeArgs {
67
/// A bool to represent if new blocks should be bridged from the eth wire protocol to the
78
/// scroll wire protocol.
9+
#[arg(long, default_value_t = false)]
810
pub enable_eth_scroll_wire_bridge: bool,
911
/// A bool that represents if the scroll wire protocol should be enabled.
12+
#[arg(long, default_value_t = false)]
1013
pub enable_scroll_wire: bool,
1114
/// Database path
15+
#[arg(long)]
1216
pub database_path: Option<PathBuf>,
13-
/// The URL for the L1 RPC URL.
14-
pub l1_rpc_url: Option<reqwest::Url>,
1517
/// The EngineAPI URL.
18+
#[arg(long)]
1619
pub engine_api_url: Option<reqwest::Url>,
20+
/// The provider arguments
21+
#[command(flatten)]
22+
pub l1_provider_args: L1ProviderArgs,
23+
}
24+
25+
#[derive(Debug, clap::Args)]
26+
pub struct L1ProviderArgs {
27+
/// The URL for the L1 RPC URL.
28+
#[arg(long)]
29+
pub l1_rpc_url: Option<reqwest::Url>,
30+
/// The URL for the Beacon RPC URL.
31+
#[arg(long)]
32+
pub beacon_rpc_url: reqwest::Url,
33+
/// The compute units per second for the provider.
34+
#[arg(long)]
35+
pub compute_units_per_second: u64,
36+
/// The max amount of retries for the provider.
37+
#[arg(long, default_value_t = constants::PROVIDER_MAX_RETRIES)]
38+
pub max_retries: u32,
39+
/// The initial backoff for the provider.
40+
#[arg(long, default_value_t = constants::PROVIDER_INITIAL_BACKOFF)]
41+
pub initial_backoff: u64,
1742
}

‎bin/rollup/src/constants.rs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
/// The block number at which to start the L1 watcher.
2+
pub const WATCHER_START_BLOCK_NUMBER: u64 = 18318215;
3+
4+
/// The size of the blob cache for the provider.
5+
pub const PROVIDER_BLOB_CACHE_SIZE: usize = 100;
6+
7+
/// The max retries for the L1 provider.
8+
pub const PROVIDER_MAX_RETRIES: u32 = 10;
9+
10+
/// The initial backoff for the L1 provider.
11+
pub const PROVIDER_INITIAL_BACKOFF: u64 = 100;

‎bin/rollup/src/lib.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
//! Scroll Network Bridge Components.
22
33
mod args;
4-
pub use args::ScrollRollupNodeArgs;
4+
pub use args::{L1ProviderArgs, ScrollRollupNodeArgs};
5+
6+
mod constants;
7+
pub use constants::{PROVIDER_INITIAL_BACKOFF, PROVIDER_MAX_RETRIES, WATCHER_START_BLOCK_NUMBER};
58

69
mod import;
710
pub use import::BridgeBlockImport;

‎bin/rollup/src/network.rs‎

Lines changed: 36 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
use alloy_provider::ProviderBuilder;
2+
use alloy_rpc_client::RpcClient;
3+
use alloy_transport::layers::RetryBackoffLayer;
24
use migration::MigratorTrait;
35
use reth_network::{config::NetworkMode, NetworkManager, PeersInfo};
46
use reth_node_api::TxTy;
@@ -8,18 +10,21 @@ use reth_rpc_builder::config::RethRpcServerConfig;
810
use reth_scroll_chainspec::ScrollChainSpec;
911
use reth_scroll_primitives::ScrollPrimitives;
1012
use reth_transaction_pool::{PoolTransaction, TransactionPool};
11-
use rollup_node_indexer::Indexer;
1213
use rollup_node_manager::{PoAConsensus, RollupNodeManager};
14+
use rollup_node_providers::{beacon_provider, DatabaseL1MessageProvider, OnlineL1Provider};
1315
use rollup_node_watcher::L1Watcher;
1416
use scroll_alloy_provider::ScrollAuthEngineApiProvider;
1517
use scroll_db::{Database, DatabaseConnectionProvider};
1618
use scroll_engine::{test_utils::NoopExecutionPayloadProvider, EngineDriver, ForkchoiceState};
1719
use scroll_network::NetworkManager as ScrollNetworkManager;
1820
use scroll_wire::{ProtocolHandler, ScrollWireConfig};
19-
use std::{path::PathBuf, sync::Arc};
21+
use std::sync::Arc;
2022
use tracing::info;
2123

22-
use crate::ScrollRollupNodeArgs;
24+
use crate::{
25+
constants::PROVIDER_BLOB_CACHE_SIZE, L1ProviderArgs, ScrollRollupNodeArgs,
26+
WATCHER_START_BLOCK_NUMBER,
27+
};
2328

2429
/// The network builder for the eth-wire to scroll-wire bridge.
2530
#[derive(Debug)]
@@ -99,34 +104,53 @@ where
99104

100105
// Instantiate the database
101106
let database_path = if let Some(db_path) = self.config.database_path {
102-
db_path
107+
db_path.to_string_lossy().to_string()
103108
} else {
104-
PathBuf::from("sqlite://").join(ctx.config().datadir().db().join("scroll.db"))
109+
// append the path using strings as using `join(...)` overwrites "sqlite://"
110+
// if the path is absolute.
111+
let path = ctx.config().datadir().db().join("scroll.db");
112+
"sqlite://".to_string() + &*path.to_string_lossy()
105113
};
106-
let db = Database::new(database_path.to_str().unwrap()).await?;
114+
let db = Database::new(&database_path).await?;
107115

108116
// Run the database migrations
109117
migration::Migrator::up(db.get_connection(), None).await?;
110118

111119
// Wrap the database in an Arc
112120
let db = Arc::new(db);
113121

114-
// Spawn the indexer
115-
let indexer = Indexer::new(db.clone());
116-
117122
// Spawn the L1Watcher
118-
let l1_notification_rx = if let Some(l1_rpc_url) = self.config.l1_rpc_url {
119-
Some(L1Watcher::spawn(ProviderBuilder::new().on_http(l1_rpc_url), 20035952).await)
123+
let l1_provider_args = self.config.l1_provider_args;
124+
let l1_notification_rx = if let Some(l1_rpc_url) = l1_provider_args.l1_rpc_url {
125+
let L1ProviderArgs { max_retries, initial_backoff, compute_units_per_second, .. } =
126+
l1_provider_args;
127+
let client = RpcClient::builder()
128+
.layer(RetryBackoffLayer::new(
129+
max_retries,
130+
initial_backoff,
131+
compute_units_per_second,
132+
))
133+
.http(l1_rpc_url);
134+
let provider = ProviderBuilder::new().on_client(client);
135+
Some(L1Watcher::spawn(provider, WATCHER_START_BLOCK_NUMBER).await)
120136
} else {
121137
None
122138
};
123139

140+
// Construct the l1 provider.
141+
let beacon_provider = beacon_provider(l1_provider_args.beacon_rpc_url.to_string());
142+
let l1_messages_provider = DatabaseL1MessageProvider::new(db.clone());
143+
let l1_provider =
144+
OnlineL1Provider::new(beacon_provider, PROVIDER_BLOB_CACHE_SIZE, l1_messages_provider)
145+
.await;
146+
124147
// Spawn the rollup node manager
125148
let rollup_node_manager = RollupNodeManager::new(
126149
scroll_network_manager,
127150
engine,
151+
l1_provider,
152+
db,
128153
l1_notification_rx,
129-
indexer,
130154
ForkchoiceState::genesis(
131155
ctx.config().chain.chain.try_into().expect("must be a named chain"),
132156
),

‎bin/rollup/tests/e2e.rs‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ use reth_scroll_chainspec::ScrollChainSpec;
1414
use reth_scroll_engine_primitives::ScrollPayloadBuilderAttributes;
1515
use reth_scroll_node::{ScrollNetworkPrimitives, ScrollNode};
1616
use reth_tasks::TaskManager;
17-
use rollup_node::ScrollRollupNodeArgs;
17+
use rollup_node::{L1ProviderArgs, ScrollRollupNodeArgs};
1818
use scroll_alloy_rpc_types_engine::ScrollPayloadAttributes;
1919
use scroll_network::{NewBlockWithPeer, SCROLL_MAINNET};
2020
use scroll_wire::ScrollWireConfig;
@@ -136,7 +136,14 @@ pub async fn build_bridge_node(
136136
enable_eth_scroll_wire_bridge: true,
137137
enable_scroll_wire: true,
138138
database_path: Some(PathBuf::from("sqlite::memory:")),
139-
l1_rpc_url: None,
139+
l1_provider_args: L1ProviderArgs {
140+
l1_rpc_url: None,
141+
// <https://docs.arbitrum.io/run-arbitrum-node/l1-ethereum-beacon-chain-rpc-providers>
142+
beacon_rpc_url: reqwest::Url::parse("https://eth-beacon-chain.drpc.org/rest/")?,
143+
compute_units_per_second: 100,
144+
max_retries: 10,
145+
initial_backoff: 100,
146+
},
140147
engine_api_url: None,
141148
};
142149
let node = ScrollNode;

‎crates/codec/src/lib.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ fn get_codec_version(calldata: &[u8]) -> Result<u8, DecodingError> {
8484
const CODEC_VERSION_LEN: usize = 32;
8585
const CODEC_VERSION_OFFSET_END: usize = CODEC_VERSION_OFFSET_START + CODEC_VERSION_LEN;
8686
const HIGH_BYTES_MASK: U256 =
87-
U256::from_limbs([u64::MAX, u64::MAX, u64::MAX, 0xffffffffffffff00]);
87+
U256::from_limbs([0xffffffffffffff00, u64::MAX, u64::MAX, u64::MAX]);
8888

8989
let version = calldata
9090
.get(CODEC_VERSION_OFFSET_START..CODEC_VERSION_OFFSET_END)

‎crates/database/db/Cargo.toml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,9 @@ rollup-node-primitives.workspace = true
2222

2323
# misc
2424
async-trait.workspace = true
25+
auto_impl.workspace = true
2526
futures.workspace = true
2627
sea-orm = { version = "1.1.0", features = ["sqlx-sqlite", "runtime-tokio-native-tls", "macros"] }
27-
serde = { version = "1.0", features = ["derive"] }
28-
serde_json = "1.0"
2928
thiserror.workspace = true
3029
tokio = { workspace = true, features = ["macros", "sync"] }
3130
tracing.workspace = true

0 commit comments

Comments
 (0)