feat: update block data hint database models (#180)

Author: greged93

Cargo.lock

@@ -70,6 +70,16 @@ test:
 	--all-features \
 	--no-fail-fast
 
+# Used to update the mainnet-sample.sql data. Provide the path to the sqlite database that should be read from
+# using `DB_PATH`.
+.PHONY: test-data
+export-sample-test-data:
+	sqlite3 "$(DB_PATH)" <<EOF > mainnet-sample.sql
+	.headers on
+	.mode insert block_data
+	SELECT * FROM block_data LIMIT 2000;
+	EOF
+
 .PHONY: docs
 docs:
 	cargo docs --document-private-items

Makefile

@@ -20,7 +20,7 @@ impl<'a> Iterator for BlobSliceIter<'a> {
     type Item = &'a u8;
 
     fn next(&mut self) -> Option<Self::Item> {
-        if self.count % 32 == 0 {
+        if self.count.is_multiple_of(32) {
             let _ = self.iterator.next();
             self.count += 1;
         }

crates/codec/src/decoding/blob.rs

@@ -1,4 +1,4 @@
-use alloy_primitives::U256;
+use alloy_primitives::{Address, B256, U256};
 use scroll_alloy_rpc_types_engine::BlockDataHint;
 use sea_orm::entity::prelude::*;
 
@@ -8,9 +8,11 @@ use sea_orm::entity::prelude::*;
 pub struct Model {
     #[sea_orm(primary_key)]
     number: i64,
-    extra_data: Vec<u8>,
-    state_root: Vec<u8>,
-    difficulty: Vec<u8>,
+    extra_data: Option<Vec<u8>>,
+    state_root: Option<Vec<u8>>,
+    coinbase: Option<Vec<u8>>,
+    nonce: Option<String>,
+    difficulty: Option<i8>,
 }
 
 /// The relation for the extra data model.
@@ -23,8 +25,13 @@ impl ActiveModelBehavior for ActiveModel {}
 impl From<Model> for BlockDataHint {
     fn from(value: Model) -> Self {
         Self {
-            extra_data: value.extra_data.into(),
-            difficulty: U256::from_be_slice(value.difficulty.as_ref()),
+            extra_data: value.extra_data.map(Into::into),
+            state_root: value.state_root.map(|s| B256::from_slice(&s)),
+            coinbase: value.coinbase.as_deref().map(Address::from_slice),
+            nonce: value.nonce.map(|n| {
+                u64::from_str_radix(&n, 16).expect("nonce stored as hex string in database")
+            }),
+            difficulty: value.difficulty.map(U256::from),
         }
     }
 }

crates/database/db/src/models/block_data.rs

@@ -13,9 +13,11 @@ impl MigrationTrait for Migration {
                     .table(BlockData::Table)
                     .if_not_exists()
                     .col(big_unsigned(BlockData::Number).primary_key())
-                    .col(binary(BlockData::ExtraData))
-                    .col(binary_len(BlockData::StateRoot, 32))
-                    .col(binary_len(BlockData::Difficulty, 32))
+                    .col(binary_null(BlockData::ExtraData))
+                    .col(binary_len_null(BlockData::StateRoot, 32))
+                    .col(binary_len_null(BlockData::Coinbase, 20))
+                    .col(text_null(BlockData::Nonce))
+                    .col(tiny_integer_null(BlockData::Difficulty))
                     .to_owned(),
             )
             .await
@@ -32,5 +34,7 @@ enum BlockData {
     Number,
     ExtraData,
     StateRoot,
+    Coinbase,
+    Nonce,
     Difficulty,
 }

crates/database/migration/src/m20250408_132123_add_header_metadata.rs

@@ -1,7 +1,7 @@
 use crate::{migration_info::DataSource, MigrationInfo};
 use std::{collections::HashMap, time::Duration};
 
-use alloy_primitives::{bytes::Buf, B256};
+use alloy_primitives::{bytes::Buf, Address, B256};
 use eyre::{bail, eyre};
 use futures::{stream::FuturesUnordered, StreamExt};
 use indicatif::{ProgressBar, ProgressFinish, ProgressState, ProgressStyle};
@@ -70,19 +70,23 @@ impl<MI: MigrationInfo + Send + Sync> MigrationTrait for Migration<MI> {
 pub struct Model {
     #[sea_orm(primary_key)]
     number: i64,
-    extra_data: Vec<u8>,
-    state_root: Vec<u8>,
-    difficulty: i8,
+    extra_data: Option<Vec<u8>>,
+    state_root: Option<Vec<u8>>,
+    coinbase: Option<Vec<u8>>,
+    nonce: Option<String>,
+    difficulty: Option<i8>,
 }
 impl ActiveModelBehavior for ActiveModel {}
 
 impl From<(i64, HeaderMetadata)> for ActiveModel {
     fn from((bn, header): (i64, HeaderMetadata)) -> Self {
         Self {
             number: ActiveValue::Set(bn),
-            extra_data: ActiveValue::Set(header.extra_data),
-            state_root: ActiveValue::Set(header.state_root.to_vec()),
-            difficulty: ActiveValue::Set(header.difficulty as i8),
+            extra_data: ActiveValue::Set(Some(header.extra_data)),
+            state_root: ActiveValue::Set(Some(header.state_root.to_vec())),
+            coinbase: ActiveValue::Set(header.coinbase.map(|c| c.to_vec())),
+            nonce: ActiveValue::Set(header.nonce.map(|x| format!("{x:x}"))),
+            difficulty: ActiveValue::Set(Some(header.difficulty as i8)),
         }
     }
 }
@@ -107,7 +111,8 @@ async fn download(url: &str) -> eyre::Result<Vec<u8>> {
     if total_size == 0 {
         bail!("empty file");
     }
-    let iterations = total_size / CHUNK_SIZE + if total_size % CHUNK_SIZE != 0 { 1 } else { 0 };
+    let iterations =
+        total_size / CHUNK_SIZE + if !total_size.is_multiple_of(CHUNK_SIZE) { 1 } else { 0 };
 
     // create a progress bar.
     let pb = ProgressBar::new(total_size).
@@ -213,8 +218,8 @@ fn decode_to_headers(data: Vec<u8>) -> eyre::Result<Vec<HeaderMetadata>> {
 
     // decode all available data.
     let mut headers = Vec::with_capacity(data_buf.len() / HEADER_LOWER_SIZE_LIMIT);
-    while let Some(data) = decoder.next(data_buf) {
-        headers.push(data);
+    while !data_buf.is_empty() {
+        headers.push(decoder.next(data_buf).map_err(|err| DbErr::Custom(err.to_string()))?);
     }
 
     Ok(headers)
@@ -225,6 +230,8 @@ fn decode_to_headers(data: Vec<u8>) -> eyre::Result<Vec<HeaderMetadata>> {
 struct HeaderMetadata {
     extra_data: Vec<u8>,
     state_root: Vec<u8>,
+    coinbase: Option<Vec<u8>>,
+    nonce: Option<u64>,
     difficulty: u8,
 }
 
@@ -260,33 +267,56 @@ impl MetadataDecoder {
     }
 
     /// Decodes the next header metadata from the buffer, advancing it.
-    fn next(&self, buf: &mut &[u8]) -> Option<HeaderMetadata> {
+    fn next(&self, buf: &mut &[u8]) -> eyre::Result<HeaderMetadata> {
         // sanity check.
         if buf.len() < 2 {
-            return None
+            bail!("header buffer too small to read seal flag and vanity index");
         }
 
         // get flag and vanity index.
         let flag = buf[0];
         let vanity_index = buf[1];
 
+        let has_coinbase = (flag & 0b00010000) != 0;
+        let has_nonce = (flag & 0b00100000) != 0;
         let difficulty = if flag & 0b01000000 == 0 { 2 } else { 1 };
         let seal_length = if flag & 0b10000000 == 0 { 65 } else { 85 };
-        let vanity = self.vanity.get(&vanity_index)?;
+        let vanity = self.vanity.get(&vanity_index).ok_or(eyre!("vanity not found"))?;
 
-        if buf.len() < B256::len_bytes() + seal_length + 2 {
-            return None
+        // flag + vanity index + state root + coinbase + nonce + seal
+        let total_expected_size = 2 * size_of::<u8>() +
+            B256::len_bytes() +
+            Address::len_bytes() * has_coinbase as usize +
+            size_of::<u64>() * has_nonce as usize +
+            seal_length;
+
+        if buf.len() < total_expected_size {
+            bail!("header buffer too small: got {}, expected {}", buf.len(), total_expected_size);
         }
         buf.advance(2);
 
         let state_root = buf[..B256::len_bytes()].to_vec();
         buf.advance(B256::len_bytes());
 
+        let mut coinbase = None;
+        if has_coinbase {
+            coinbase = Some(buf[..Address::len_bytes()].to_vec());
+            buf.advance(Address::len_bytes());
+        }
+
+        let mut nonce = None;
+        if has_nonce {
+            nonce = Some(u64::from_be_bytes(
+                buf[..size_of::<u64>()].try_into().expect("32 bytes slice"),
+            ));
+            buf.advance(size_of::<u64>());
+        }
+
         let seal = &buf[..seal_length];
         let extra_data = [vanity, seal].concat();
         buf.advance(seal_length);
 
-        Some(HeaderMetadata { extra_data, state_root, difficulty })
+        Ok(HeaderMetadata { extra_data, state_root, coinbase, nonce, difficulty })
     }
 }
 
@@ -316,7 +346,13 @@ mod tests {
         let header_data = bytes!("c00020695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb548c3f81f3d998b6652900e1c3183736c238fe4290000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000");
         let header = decoder.next(&mut header_data.as_ref()).unwrap();
 
-        let expected_header = HeaderMetadata{ extra_data: bytes!("0x000000000000000000000000000000000000000000000000000000000000000048c3f81f3d998b6652900e1c3183736c238fe4290000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000").to_vec(), state_root: b256!("20695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb5").to_vec(), difficulty: 1 };
+        let expected_header = HeaderMetadata {
+            extra_data: bytes!("0x000000000000000000000000000000000000000000000000000000000000000048c3f81f3d998b6652900e1c3183736c238fe4290000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000").to_vec(),
+            state_root: b256!("20695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb5").to_vec(),
+            coinbase: None,
+            nonce: None,
+            difficulty: 1,
+        };
         assert_eq!(header, expected_header)
     }
 }

crates/database/migration/src/m20250408_150338_load_header_metadata.rs

@@ -31,7 +31,7 @@ impl MigrationInfo for ScrollMainnetMigrationInfo {
     }
 
     fn data_hash() -> Option<B256> {
-        Some(b256!("9062e2fa1200dca63bee1d18d429572f134f5f0c98cb4852f62fc394e33cf6e6"))
+        Some(b256!("fa2746026ec9590e37e495cb20046e20a38fd0e7099abd2012640dddf6c88b25"))
     }
 }
 
@@ -58,6 +58,6 @@ impl MigrationInfo for ScrollSepoliaMigrationInfo {
     }
 
     fn data_hash() -> Option<B256> {
-        Some(b256!("3629f5e53250a526ffc46806c4d74b9c52c9209a6d45ecdfebdef5d596bb3f40"))
+        Some(b256!("a02354c12ca0f918bf4768255af9ed13c137db7e56252348f304b17bb4088924"))
     }
 }