Indexer Component (#35)

frisitano · greged93 · web-flow · commit 2bba74b5d949 · 2025-03-19T22:42:13.000+07:00
* ci: add dprint * feat: initial work + handle_latest_block * Rollup node manager (#21) * init bridge implementation * extend bridge implementation * refactor and clean up dependencies * add .vscode to .gitignore * lint * spacing * cleanup * wrap inner block import in bridge * add bridge integration test * refactor and clean up * add comments * migrate shared dependencies to workspace * feature propogation * lints and feature fix * remove scroll feature * add scroll feature * integrate reth upstream changes * address PR feedback * stash * add systemd service file * update systemd file * update systemd file * address comments * rollup node manager * merge main * add Cargo.lock * lint * lint * lint * lint * ci remove no-dev-deps * fix serde dep - scroll-wire * skip no_std on scroll-engine and scroll-rollup-node * skip no_std on scroll-engine and scroll-rollup-node * address comments * lints * cleanup * tsets * lint + refactor * refactor engine api call ordering * lint * fix fcu * lint * remove syncing error return on new payload * update forkchoice state after succesfully new payload * update reth following chainspec fork id logic change * PR feedback * lints * lints * feat(watcher): reorg detection * feat(primitives): add batch and l1 transaction * feat(watcher): contracts and constants * feat(watcher): imports * feat(watcher): add error types * feat(watcher): handle l1 messages and commit logs * feat: handle batch commit and finalize * test: handle_latest_block * test: wip events decoding * test: commit and finalize batch logs * test: utils * feat: tracing * test: wip reorg integration * test: reorg detection integration test * test: gaps * fix: lints * feat: point deps to default reth branch * fix: security issue on ring * fix: bump alloy * feat: move abi to separate crate * chore: remove ArbitraryTxBuilder * init database * add database iterator * add test * update streaming implementation * init indexer * fix: answer comments * fix: lints * fix: no std issues * fix: cargo features check * extend indexer * extend and refactor database, migrations and batch primitive * lint * lint * lint * improve database implementation * lint * lint * database transaction refactor * lint: * use GAT for DatabaseConnectionProvider trait * refactor & lint * merge database and refactor * clippy fix * skip rolllup-node-indexer no_std * refactor databse interface for insert_batch_input * refactor * lint * refactor indexer * refactor indexer * lint: * lint --------- Co-authored-by: Gregory Edison <gregory.edison1993@gmail.com>
diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
@@ -127,10 +127,10 @@ jobs:
         include:
           - type: wasm
             target: wasm32-unknown-unknown
-            exclude: scroll-engine,scroll-wire,scroll-bridge,scroll-network,rollup-node-manager,rollup-node-watcher,scroll-db,scroll-migration
+            exclude: scroll-engine,scroll-wire,scroll-bridge,scroll-network,rollup-node-manager,rollup-node-watcher,scroll-db,scroll-migration,rollup-node-indexer
           - type: riscv
             target: riscv32imac-unknown-none-elf
-            exclude: scroll-engine,scroll-wire,scroll-bridge,scroll-network,rollup-node-manager,rollup-node-watcher,scroll-db,scroll-migration
+            exclude: scroll-engine,scroll-wire,scroll-bridge,scroll-network,rollup-node-manager,rollup-node-watcher,scroll-db,scroll-migration,rollup-node-indexer
     steps:
       - uses: actions/checkout@v4
       - uses: rui314/setup-mold@v1
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -11,6 +11,7 @@ members = [
     "crates/database/db",
     "crates/database/migration",
     "crates/engine",
+    "crates/indexer",
     "crates/l1",
     "crates/node",
     "crates/network",
@@ -136,6 +137,7 @@ reth-network-peers = { git = "https://github.com/scroll-tech/reth.git", default-
 reth-primitives = { git = "https://github.com/scroll-tech/reth.git", default-features = false }
 reth-provider = { git = "https://github.com/scroll-tech/reth.git", default-features = false }
 reth-tasks = { git = "https://github.com/scroll-tech/reth.git" }
+reth-tokio-util = { git = "https://github.com/scroll-tech/reth.git", default-features = false }
 
 # reth-scroll
 reth-scroll-chainspec = { git = "https://github.com/scroll-tech/reth.git", default-features = false }
@@ -147,6 +149,7 @@ reth-scroll-primitives = { git = "https://github.com/scroll-tech/reth.git", defa
 rollup-node-manager = { path = "crates/node" }
 rollup-node-primitives = { path = "crates/primitives" }
 rollup-node-watcher = { path = "crates/watcher" }
+scroll-db = { path = "crates/database/db" }
 scroll-engine = { path = "crates/engine" }
 scroll-l1 = { path = "crates/l1" }
 scroll-network = { path = "crates/network" }
diff --git a/crates/indexer/Cargo.toml b/crates/indexer/Cargo.toml
@@ -0,0 +1,38 @@
+[package]
+name = "rollup-node-indexer"
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+exclude.workspace = true
+
+[lints]
+workspace = true
+
+[dependencies]
+# alloy
+alloy-primitives.workspace = true
+
+# rollup-node
+scroll-db.workspace = true
+rollup-node-primitives.workspace = true
+rollup-node-watcher.workspace = true
+
+# misc
+futures.workspace = true
+thiserror.workspace = true
+tokio.workspace = true
+tracing.workspace = true
+
+[dev-dependencies]
+alloy-primitives = { workspace = true, features = ["arbitrary"] }
+
+# rollup-node
+scroll-db = { workspace = true, features = ["test-utils"] }
+rollup-node-primitives = { workspace = true, features = ["arbitrary"] }
+
+# misc
+arbitrary.workspace = true
+futures.workspace = true
+rand.workspace = true
+tokio.workspace = true
diff --git a/crates/indexer/src/action.rs b/crates/indexer/src/action.rs
@@ -0,0 +1,47 @@
+use super::{IndexerError, IndexerEvent};
+use std::{
+    fmt,
+    future::Future,
+    pin::Pin,
+    task::{Context, Poll},
+};
+
+/// A future that resolves to a `Result<IndexerEvent, IndexerError>`.
+type PendingIndexerFuture =
+    Pin<Box<dyn Future<Output = Result<IndexerEvent, IndexerError>> + Send>>;
+
+/// A type that represents a future that is being executed by the indexer.
+pub(super) enum IndexerFuture {
+    HandleReorg(PendingIndexerFuture),
+    HandleBatchCommit(PendingIndexerFuture),
+    HandleBatchFinalization(PendingIndexerFuture),
+    HandleL1Message(PendingIndexerFuture),
+}
+
+impl IndexerFuture {
+    /// Polls the future to completion.
+    pub(super) fn poll(
+        &mut self,
+        cx: &mut Context<'_>,
+    ) -> Poll<Result<IndexerEvent, IndexerError>> {
+        match self {
+            Self::HandleReorg(fut) |
+            Self::HandleBatchCommit(fut) |
+            Self::HandleBatchFinalization(fut) |
+            Self::HandleL1Message(fut) => fut.as_mut().poll(cx),
+        }
+    }
+}
+
+// We implement the Debug trait for IndexerFuture to provide a human-readable representation of the
+// enum variants.
+impl fmt::Debug for IndexerFuture {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::HandleReorg(_) => write!(f, "HandleReorg"),
+            Self::HandleBatchCommit(_) => write!(f, "HandleBatchCommit"),
+            Self::HandleBatchFinalization(_) => write!(f, "HandleBatchFinalization"),
+            Self::HandleL1Message(_) => write!(f, "HandleL1Message"),
+        }
+    }
+}
diff --git a/crates/indexer/src/error.rs b/crates/indexer/src/error.rs
@@ -0,0 +1,9 @@
+use scroll_db::DatabaseError;
+
+/// A type that represents an error that occurred during indexing.
+#[derive(Debug, thiserror::Error)]
+pub enum IndexerError {
+    /// An error occurred while interacting with the database.
+    #[error("indexing failed due to database error: {0}")]
+    DatabaseError(#[from] DatabaseError),
+}
diff --git a/crates/indexer/src/event.rs b/crates/indexer/src/event.rs
@@ -0,0 +1,14 @@
+use alloy_primitives::B256;
+
+/// An event emitted by the indexer.
+#[derive(Debug, Clone, Copy)]
+pub enum IndexerEvent {
+    /// A `BatchCommit` event has been indexed returning the batch index.
+    BatchCommitIndexed(u64),
+    /// A `BatchFinalization` event has been indexed returning the batch index.
+    BatchFinalizationIndexed(B256),
+    /// A `L1Message` event has been indexed returning the message queue index.
+    L1MessageIndexed(u64),
+    /// A `Reorg` event has been indexed returning the reorg block number.
+    ReorgIndexed(u64),
+}
diff --git a/crates/indexer/src/lib.rs b/crates/indexer/src/lib.rs
diff --git a/crates/node/Cargo.toml b/crates/node/Cargo.toml
