Skip to content

Commit 328b9e9

Browse files
frisitanofrisitano
authored
fix: async header transform (#427)
* header transform await on database insert * batch transform headers * spawn blocking task and use rayon for signature processing * use reth tag scroll-v91.1
1 parent 3d0b03f commit 328b9e9

File tree

8 files changed

+292
-230
lines changed

8 files changed

+292
-230
lines changed

‎Cargo.lock‎

Lines changed: 132 additions & 131 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 50 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -140,59 +140,59 @@ alloy-signer-local = { version = "1.0.37", default-features = false }
140140
alloy-transport = { version = "1.0.37", default-features = false }
141141

142142
# scroll-alloy
143-
scroll-alloy-consensus = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
144-
scroll-alloy-hardforks = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
145-
scroll-alloy-network = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
146-
scroll-alloy-provider = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
147-
scroll-alloy-evm = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
148-
scroll-alloy-rpc-types = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
149-
scroll-alloy-rpc-types-engine = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
143+
scroll-alloy-consensus = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
144+
scroll-alloy-hardforks = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
145+
scroll-alloy-network = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
146+
scroll-alloy-provider = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
147+
scroll-alloy-evm = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
148+
scroll-alloy-rpc-types = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
149+
scroll-alloy-rpc-types-engine = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
150150

151151
# reth
152-
reth-chainspec = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
153-
reth-e2e-test-utils = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91" }
154-
reth-eth-wire = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
155-
reth-eth-wire-types = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
156-
reth-network = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
157-
reth-network-api = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
158-
reth-network-p2p = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
159-
reth-network-peers = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
160-
reth-network-types = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
161-
reth-node-builder = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
162-
reth-node-core = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
163-
reth-node-api = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
164-
reth-node-types = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
165-
reth-payload-primitives = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
166-
reth-primitives = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
167-
reth-primitives-traits = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
168-
reth-provider = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
169-
reth-rpc-api = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
170-
reth-rpc-eth-api = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
171-
reth-rpc-eth-types = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
172-
reth-rpc-layer = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
173-
reth-rpc-server-types = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
174-
reth-storage-api = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
175-
reth-tasks = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
176-
reth-tokio-util = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
177-
reth-tracing = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
178-
reth-transaction-pool = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
179-
reth-trie-db = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
180-
reth-testing-utils = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
181-
reth-revm = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
182-
reth-evm = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
183-
reth-engine-local = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
184-
reth-cli-util = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
152+
reth-chainspec = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
153+
reth-e2e-test-utils = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1" }
154+
reth-eth-wire = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
155+
reth-eth-wire-types = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
156+
reth-network = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
157+
reth-network-api = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
158+
reth-network-p2p = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
159+
reth-network-peers = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
160+
reth-network-types = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
161+
reth-node-builder = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
162+
reth-node-core = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
163+
reth-node-api = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
164+
reth-node-types = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
165+
reth-payload-primitives = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
166+
reth-primitives = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
167+
reth-primitives-traits = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
168+
reth-provider = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
169+
reth-rpc-api = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
170+
reth-rpc-eth-api = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
171+
reth-rpc-eth-types = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
172+
reth-rpc-layer = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
173+
reth-rpc-server-types = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
174+
reth-storage-api = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
175+
reth-tasks = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
176+
reth-tokio-util = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
177+
reth-tracing = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
178+
reth-transaction-pool = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
179+
reth-trie-db = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
180+
reth-testing-utils = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
181+
reth-revm = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
182+
reth-evm = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
183+
reth-engine-local = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
184+
reth-cli-util = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
185185

186186
# reth-scroll
187-
reth-scroll-chainspec = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
188-
reth-scroll-consensus = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
189-
reth-scroll-cli = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
190-
reth-scroll-evm = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
191-
reth-scroll-rpc = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
192-
reth-scroll-engine-primitives = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
193-
reth-scroll-forks = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
194-
reth-scroll-node = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
195-
reth-scroll-primitives = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91", default-features = false }
187+
reth-scroll-chainspec = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
188+
reth-scroll-consensus = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
189+
reth-scroll-cli = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
190+
reth-scroll-evm = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
191+
reth-scroll-rpc = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
192+
reth-scroll-engine-primitives = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
193+
reth-scroll-forks = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
194+
reth-scroll-node = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
195+
reth-scroll-primitives = { git = "https://github.com/scroll-tech/reth.git", tag = "scroll-v91.1", default-features = false }
196196

197197
# rollup node
198198
rollup-node = { path = "crates/node" }
@@ -224,6 +224,7 @@ metrics = "0.24.0"
224224
metrics-derive = "0.1"
225225
parking_lot = "0.12"
226226
rand = { version = "0.9" }
227+
rayon = "1.7"
227228
reqwest = "0.12"
228229
serde = { version = "1.0" }
229230
serde_json = { version = "1.0" }

‎crates/database/db/src/db.rs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -505,6 +505,20 @@ impl DatabaseWriteOperations for Database {
505505
)
506506
}
507507

508+
async fn insert_signatures(
509+
&self,
510+
signatures: Vec<(B256, Signature)>,
511+
) -> Result<(), DatabaseError> {
512+
metered!(
513+
DatabaseOperation::InsertSignatures,
514+
self,
515+
tx_mut(move |tx| {
516+
let signatures = signatures.clone();
517+
async move { tx.insert_signatures(signatures).await }
518+
})
519+
)
520+
}
521+
508522
async fn insert_signature(
509523
&self,
510524
block_hash: B256,

‎crates/database/db/src/metrics.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ pub(crate) enum DatabaseOperation {
5050
PurgeL1MessageToL2BlockMappings,
5151
InsertBatchConsolidationOutcome,
5252
Unwind,
53+
InsertSignatures,
5354
InsertSignature,
5455
// Read operations
5556
GetBatchByIndex,
@@ -118,6 +119,7 @@ impl DatabaseOperation {
118119
Self::PurgeL1MessageToL2BlockMappings => "purge_l1_message_to_l2_block_mappings",
119120
Self::InsertBatchConsolidationOutcome => "insert_batch_consolidation_outcome",
120121
Self::Unwind => "unwind",
122+
Self::InsertSignatures => "insert_signatures",
121123
Self::InsertSignature => "insert_signature",
122124
Self::GetBatchByIndex => "get_batch_by_index",
123125
Self::GetBatchByHash => "get_batch_by_hash",

‎crates/database/db/src/operations.rs‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,12 @@ pub trait DatabaseWriteOperations {
176176
/// block number.
177177
async fn unwind(&self, l1_block_number: u64) -> Result<UnwindResult, DatabaseError>;
178178

179+
/// Store multiple block signatures in the database.
180+
async fn insert_signatures(
181+
&self,
182+
signatures: Vec<(B256, Signature)>,
183+
) -> Result<(), DatabaseError>;
184+
179185
/// Store a block signature in the database.
180186
/// TODO: remove this once we deprecated l2geth.
181187
async fn insert_signature(
@@ -887,6 +893,34 @@ impl<T: WriteConnectionProvider + ?Sized + Sync> DatabaseWriteOperations for T {
887893
Ok(UnwindResult { l1_block_number, queue_index, l2_head_block_number, l2_safe_block_info })
888894
}
889895

896+
async fn insert_signatures(
897+
&self,
898+
signatures: Vec<(B256, Signature)>,
899+
) -> Result<(), DatabaseError> {
900+
tracing::trace!(target: "scroll::db", num_signatures = signatures.len(), "Inserting block signatures into database.");
901+
902+
if signatures.is_empty() {
903+
tracing::debug!(target: "scroll::db", "No signatures to insert.");
904+
return Ok(());
905+
}
906+
907+
models::block_signature::Entity::insert_many(
908+
signatures
909+
.into_iter()
910+
.map(|(block_hash, signature)| (block_hash, signature).into())
911+
.collect::<Vec<models::block_signature::ActiveModel>>(),
912+
)
913+
.on_conflict(
914+
OnConflict::column(models::block_signature::Column::BlockHash)
915+
.update_column(models::block_signature::Column::Signature)
916+
.to_owned(),
917+
)
918+
.exec_without_returning(self.get_connection())
919+
.await?;
920+
921+
Ok(())
922+
}
923+
890924
async fn insert_signature(
891925
&self,
892926
block_hash: B256,

‎crates/node/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ clap = { workspace = true, features = ["derive", "env"] }
100100
eyre.workspace = true
101101
futures.workspace = true
102102
jsonrpsee = { version = "0.26.0", features = ["server", "client", "macros"] }
103+
rayon.workspace = true
103104
reqwest.workspace = true
104105
tokio.workspace = true
105106
tracing.workspace = true

0 commit comments

Comments
 (0)