hints: Update file format to JIT reads

rustaceanrob · rustaceanrob · commit f60f577b50e5 · 2025-12-08T10:24:07.000Z
Currently breaks the writing side, but allows for parsing from Bitcoin
Core PR #34004
diff --git a/hintfile/src/lib.rs b/hintfile/src/lib.rs
@@ -1,10 +1,11 @@
 use std::{
     collections::BTreeMap,
-    io::{self, Read, Write},
+    fs::File,
+    io::{self, Read, Seek, SeekFrom, Write},
 };
 
-type BlockHash = [u8; 32];
 type BlockHeight = u32;
+type FilePos = u64;
 
 pub fn write_compact_size<W: Write>(value: u64, writer: &mut W) -> Result<(), io::Error> {
     match value {
@@ -45,43 +46,46 @@ pub fn read_compact_size<R: Read>(reader: &mut R) -> Result<u64, io::Error> {
 
 #[derive(Debug)]
 pub struct Hints {
-    map: BTreeMap<BlockHeight, Vec<u64>>,
-    assume_valid: BlockHash,
+    map: BTreeMap<BlockHeight, FilePos>,
+    file: File,
     stop_height: BlockHeight,
 }
 
 impl Hints {
     // # Panics
     //
     // Panics when expected data is not present, or the hintfile overflows the maximum blockheight
-    pub fn from_file<R: Read>(reader: &mut R) -> Self {
+    pub fn from_file(mut file: File) -> Self {
         let mut map = BTreeMap::new();
-        let mut height = 1;
+        let mut magic = [0; 4];
+        file.read_exact(&mut magic).unwrap();
+        assert_eq!(magic, [0x55, 0x54, 0x58, 0x4f]);
+        let mut ver = [0; 1];
+        file.read_exact(&mut ver).unwrap();
+        if u8::from_le_bytes(ver) != 0x00 {
+            panic!("Unsupported file version.");
+        }
         let mut stop_height = [0; 4];
-        reader.read_exact(&mut stop_height).expect("empty file");
-        let mut assume_valid = [0; 32];
-        reader.read_exact(&mut assume_valid).expect("empty file");
-        while let Ok(count) = read_compact_size(reader) {
-            // panics on 32 bit machines
-            let mut offsets = Vec::with_capacity(count as usize);
-            for _ in 0..count {
-                offsets.push(read_compact_size(reader).expect("unexpected end of hintfile"));
-            }
-            map.insert(height, offsets);
-            height += 1;
+        file.read_exact(&mut stop_height).expect("empty file");
+        let stop_height = BlockHeight::from_le_bytes(stop_height);
+        for _ in 1..=stop_height {
+            let mut height = [0; 4];
+            file.read_exact(&mut height)
+                .expect("expected kv pair does not exist.");
+            let height = BlockHeight::from_le_bytes(height);
+            let mut file_pos = [0; 8];
+            file.read_exact(&mut file_pos)
+                .expect("expected kv pair does not exist.");
+            let file_pos = FilePos::from_le_bytes(file_pos);
+            map.insert(height, file_pos);
         }
         Self {
             map,
-            assume_valid,
-            stop_height: BlockHeight::from_le_bytes(stop_height),
+            file,
+            stop_height,
         }
     }
 
-    /// Get the last hash encoded in the hintfile.
-    pub fn stop_hash(&self) -> BlockHash {
-        self.assume_valid
-    }
-
     /// Get the stop height of the hint file.
     pub fn stop_height(&self) -> BlockHeight {
         self.stop_height
@@ -91,15 +95,20 @@ impl Hints {
     ///
     /// If there are no offset present at that height, aka an overflow, or the entry has already
     /// been fetched.
-    pub fn get_indexes(&self, height: BlockHeight) -> Vec<u64> {
-        let offsets = self
+    pub fn get_indexes(&mut self, height: BlockHeight) -> Vec<u64> {
+        let file_pos = self
             .map
             .get(&height)
             .cloned()
             .expect("block height overflow");
-        let mut indexes = Vec::with_capacity(offsets.len());
+        self.file
+            .seek(SeekFrom::Start(file_pos))
+            .expect("missing file position.");
+        let num_unspents = read_compact_size(&mut self.file).expect("unexpected missing hints.");
+        let mut indexes = Vec::new();
         let mut prev = 0;
-        for offset in offsets {
+        for _ in 0..num_unspents {
+            let offset = read_compact_size(&mut self.file).expect("unexpected missing hints.");
             let next = prev + offset;
             indexes.push(next);
             prev = next;
diff --git a/node/src/bin/ibd.rs b/node/src/bin/ibd.rs
@@ -5,7 +5,7 @@ use std::{
     time::{Duration, Instant},
 };
 
-use bitcoin::{consensus, BlockHash, Network};
+use bitcoin::Network;
 use hintfile::Hints;
 use kernel::{ChainstateManager, ChainstateManagerOptions, ContextBuilder};
 
@@ -41,17 +41,16 @@ fn main() {
     tracing::subscriber::set_global_default(subscriber).unwrap();
     let hintfile_start_time = Instant::now();
     tracing::info!("Reading in {hint_path}");
-    let mut hintfile = File::open(hint_path).expect("invalid hintfile path");
-    let hints = Arc::new(Hints::from_file(&mut hintfile));
+    let hintfile = File::open(hint_path).expect("invalid hintfile path");
+    let hints = Hints::from_file(hintfile);
+    let stop_height = hints.stop_height();
     elapsed_time(hintfile_start_time);
     tracing::info!("Syncing to height {}", hints.stop_height());
     let block_file_path = blocks_dir.map(PathBuf::from);
     if let Some(block_file_path) = block_file_path.as_ref() {
         std::fs::create_dir(block_file_path).expect("could not create block file directory");
     }
-    let stop_hash =
-        consensus::deserialize::<BlockHash>(&hints.stop_hash()).expect("stop hash is not valid");
-    tracing::info!("Assume valid hash: {stop_hash}");
+    tracing::info!("Assume valid height: {stop_height}");
     tracing::info!("Finding peers with DNS");
     let dns_start_time = Instant::now();
     let peers = bootstrap_dns(network);
@@ -68,7 +67,13 @@ fn main() {
     let tip = chainman.best_header().height();
     tracing::info!("Kernel best header: {tip}");
     let chain = Arc::new(chainman);
-    sync_block_headers(stop_hash, &peers, Arc::clone(&chain), network, timeout_conf);
+    sync_block_headers(
+        stop_height,
+        &peers,
+        Arc::clone(&chain),
+        network,
+        timeout_conf,
+    );
     tracing::info!("Assume valid height: {}", chain.best_header().height());
     let (tx, rx) = channel();
     let main_routine_time = Instant::now();
@@ -78,6 +83,7 @@ fn main() {
     let mut tasks = Vec::new();
     let hashes = hashes_from_chain(Arc::clone(&chain));
     let arc_hashes = Arc::new(Mutex::new(hashes));
+    let hints = Arc::new(Mutex::new(hints));
     for task_id in 0..task_num {
         let chain = Arc::clone(&chain);
         let tx = tx.clone();
@@ -93,7 +99,7 @@ fn main() {
                 network,
                 block_file_path,
                 chain,
-                &hints,
+                hints,
                 peers,
                 tx,
                 hashes,
diff --git a/node/src/lib.rs b/node/src/lib.rs
@@ -74,7 +74,7 @@ pub fn bootstrap_dns(network: Network) -> Vec<SocketAddr> {
 }
 
 pub fn sync_block_headers(
-    stop_hash: BlockHash,
+    stop_height: u32,
     hosts: &[SocketAddr],
     chainman: Arc<ChainstateManager>,
     network: Network,
@@ -99,9 +99,10 @@ pub fn sync_block_headers(
             Err(_) => continue,
         };
         tracing::info!("Connection established");
-        let curr = chainman.best_header().block_hash().hash;
-        let locator = BlockHash::from_byte_array(curr);
-        if locator.eq(&stop_hash) {
+        let curr = chainman.best_header();
+        let locator = BlockHash::from_byte_array(curr.block_hash().hash);
+        let curr_height = curr.height() as u32;
+        if curr_height.eq(&stop_height) {
             tracing::info!("Using existing header state");
             return;
         }
@@ -124,7 +125,8 @@ pub fn sync_block_headers(
                         chainman
                             .process_new_block_headers(&consensus::serialize(&header), true)
                             .expect("process headers failed");
-                        if header.block_hash().eq(&stop_hash) {
+                        let curr_height = chainman.best_header().height() as u32;
+                        if curr_height.eq(&stop_height) {
                             tracing::info!("Done syncing block headers");
                             if let Some(message_rate) =
                                 metrics.message_rate(p2p::TimedMessage::BlockHeaders)
@@ -175,7 +177,7 @@ pub fn get_blocks_for_range(
     network: Network,
     block_dir: Option<PathBuf>,
     chain: Arc<ChainstateManager>,
-    hints: &Hints,
+    hints: Arc<Mutex<Hints>>,
     peers: Arc<Mutex<Vec<SocketAddr>>>,
     updater: Sender<AccumulatorUpdate>,
     hashes: Arc<Mutex<Vec<Vec<BlockHash>>>>,
@@ -231,8 +233,10 @@ pub fn get_blocks_for_range(
                         .block_index_by_hash(kernal_hash)
                         .expect("header is in best chain.");
                     let block_height = block_index.height().unsigned_abs();
-                    let unspent_indexes: HashSet<u64> =
-                        hints.get_indexes(block_height).into_iter().collect();
+                    let unspent_indexes: HashSet<u64> = {
+                        let mut hint_ref = hints.lock().unwrap();
+                        hint_ref.get_indexes(block_height).into_iter().collect()
+                    };
                     if let Some(block_dir) = block_dir.as_ref() {
                         let file_path = block_dir.join(format!("{hash}.block"));
                         let file = File::create_new(file_path);
