feat: stacks-inspect reads mock mined blocks from files. No replay yet

Author: jbencin

stacks-common/src/util/macros.rs

@@ -617,6 +617,28 @@ macro_rules! impl_byte_array_serde {
     };
 }
 
+#[allow(unused_macros)]
+#[macro_export]
+macro_rules! impl_file_io_serde_json {
+    ($thing:ident) => {
+        impl $thing {
+            pub fn serialize_to_file<P>(&self, path: P) -> Result<(), std::io::Error>
+            where
+                P: AsRef<std::path::Path>,
+            {
+                $crate::util::serialize_json_to_file(self, path)
+            }
+
+            pub fn deserialize_from_file<P>(path: P) -> Result<Self, std::io::Error>
+            where
+                P: AsRef<std::path::Path>,
+            {
+                $crate::util::deserialize_json_from_file(path)
+            }
+        }
+    };
+}
+
 // print debug statements while testing
 #[allow(unused_macros)]
 #[macro_export]

stacks-common/src/util/mod.rs

@@ -28,6 +28,9 @@ pub mod uint;
 pub mod vrf;
 
 use std::collections::HashMap;
+use std::fs::File;
+use std::io::{BufReader, BufWriter, Write};
+use std::path::Path;
 use std::time::{SystemTime, UNIX_EPOCH};
 use std::{error, fmt, thread, time};
 
@@ -120,3 +123,26 @@ pub mod db_common {
         true
     }
 }
+
+/// Write any `serde_json` object directly to a file
+pub fn serialize_json_to_file<J, P>(json: &J, path: P) -> Result<(), std::io::Error>
+where
+    J: ?Sized + serde::Serialize,
+    P: AsRef<Path>,
+{
+    let file = File::create(path)?;
+    let mut writer = BufWriter::new(file);
+    serde_json::to_writer(&mut writer, json)?;
+    writer.flush()
+}
+
+/// Read any `serde_json` object directly from a file
+pub fn deserialize_json_from_file<J, P>(path: P) -> Result<J, std::io::Error>
+where
+    J: serde::de::DeserializeOwned,
+    P: AsRef<Path>,
+{
+    let file = File::open(path)?;
+    let reader = BufReader::new(file);
+    serde_json::from_reader::<_, J>(reader).map_err(std::io::Error::from)
+}

stackslib/src/main.rs

@@ -26,6 +26,7 @@ extern crate stacks_common;
 #[macro_use(o, slog_log, slog_trace, slog_debug, slog_info, slog_warn, slog_error)]
 extern crate slog;
 
+use regex::Regex;
 use stacks_common::types::MempoolCollectionBehavior;
 #[cfg(not(any(target_os = "macos", target_os = "windows", target_arch = "arm")))]
 use tikv_jemallocator::Jemalloc;
@@ -38,6 +39,7 @@ use std::collections::{BTreeMap, HashMap, HashSet};
 use std::fs::{File, OpenOptions};
 use std::io::prelude::*;
 use std::io::BufReader;
+use std::path::PathBuf;
 use std::time::Instant;
 use std::{env, fs, io, process, thread};
 
@@ -92,7 +94,7 @@ use stacks_common::util::hash::{hex_bytes, to_hex, Hash160};
 use stacks_common::util::retry::LogReader;
 use stacks_common::util::secp256k1::{Secp256k1PrivateKey, Secp256k1PublicKey};
 use stacks_common::util::vrf::VRFProof;
-use stacks_common::util::{get_epoch_time_ms, log, sleep_ms};
+use stacks_common::util::{deserialize_json_from_file, get_epoch_time_ms, log, sleep_ms};
 
 #[cfg_attr(test, mutants::skip)]
 fn main() {
@@ -1339,6 +1341,88 @@ simulating a miner.
         );
         return;
     }
+    if argv[1] == "replay-mock-mining" {
+        let print_help_and_exit = || {
+            let n = &argv[0];
+            eprintln!("Usage:");
+            eprintln!("  {n} <mock-miner-output-dir>");
+            process::exit(1);
+        };
+
+        // Process CLI args
+        let dir = argv
+            .get(2)
+            .map(PathBuf::from)
+            .map(fs::canonicalize)
+            .transpose()
+            .unwrap_or_else(|e| panic!("Not a valid path: {e}"))
+            .unwrap_or_else(print_help_and_exit);
+
+        if !dir.is_dir() {
+            panic!("Not a valid directory: {dir:?}");
+        }
+
+        // Read entries in directory
+        let dir_entries = dir
+            .read_dir()
+            .unwrap_or_else(|e| panic!("Failed to read directory: {e}"))
+            .filter_map(|e| e.ok());
+
+        // Get filenames, filtering out anything that isn't a regular file
+        let filenames = dir_entries.filter_map(|e| match e.file_type() {
+            Ok(t) if t.is_file() => e.file_name().into_string().ok(),
+            _ => None,
+        });
+
+        // Get vec of (block_height, filename), to prepare for sorting
+        //
+        // NOTE: Trusting the filename is not ideal. We could sort on data read from the file,
+        // but that requires reading all files
+        let re = Regex::new(r"^([0-9]+\.json)$").unwrap();
+        let mut indexed_files = filenames
+            .filter_map(|filename| {
+                // Use regex to extract block number from filename
+                let Some(cap) = re.captures(&filename) else {
+                    return None;
+                };
+                let Some(m) = cap.get(0) else {
+                    return None;
+                };
+                let Ok(bh) = m.as_str().parse::<u64>() else {
+                    return None;
+                };
+                Some((bh, filename))
+            })
+            .collect::<Vec<_>>();
+
+        // Sort by block height
+        indexed_files.sort_by_key(|(bh, _)| *bh);
+
+        if indexed_files.is_empty() {
+            panic!("No block files found");
+        }
+
+        info!(
+            "Replaying {} blocks starting at {}",
+            indexed_files.len(),
+            indexed_files[0].0
+        );
+
+        for (bh, filename) in indexed_files {
+            let filepath = dir.join(filename);
+            info!("Replaying block from file";
+                "block_height" => bh,
+                "filepath" => ?filepath
+            );
+            // let block = AssembledAnchorBlock::deserialize_json_from_file(filepath)
+            //     .unwrap_or_else(|e| panic!("Error reading block {block} from file: {e}"));
+            // debug!("Replaying block from {filepath:?}";
+            //     "block_height" => bh,
+            //     "block" => %block
+            // );
+            // TODO: Actually replay block
+        }
+    }
 
     if argv.len() < 4 {
         eprintln!("Usage: {} blockchain network working_dir", argv[0]);

testnet/stacks-node/src/neon_node.rs

@@ -140,14 +140,13 @@
 use std::cmp;
 use std::cmp::Ordering as CmpOrdering;
 use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
-use std::fs::{self, File};
-use std::io::{BufWriter, Read, Write};
+use std::io::{Read, Write};
 use std::net::SocketAddr;
 use std::path::Path;
 use std::sync::mpsc::{Receiver, TrySendError};
 use std::thread::JoinHandle;
 use std::time::Duration;
-use std::{mem, thread};
+use std::{fs, mem, thread};
 
 use clarity::vm::ast::ASTRules;
 use clarity::vm::costs::ExecutionCost;
@@ -191,6 +190,7 @@ use stacks::net::stackerdb::{StackerDBConfig, StackerDBSync, StackerDBs};
 use stacks::net::{
     Error as NetError, NetworkResult, PeerNetworkComms, RPCHandlerArgs, ServiceFlags,
 };
+use stacks::util::{deserialize_json_from_file, serialize_json_to_file};
 use stacks::util_lib::strings::{UrlString, VecDisplay};
 use stacks_common::codec::StacksMessageCodec;
 use stacks_common::types::chainstate::{
@@ -239,7 +239,7 @@ pub(crate) enum MinerThreadResult {
 /// Fully-assembled Stacks anchored, block as well as some extra metadata pertaining to how it was
 /// linked to the burnchain and what view(s) the miner had of the burnchain before and after
 /// completing the block.
-#[derive(Clone, Serialize)]
+#[derive(Clone, Serialize, Deserialize)]
 pub struct AssembledAnchorBlock {
     /// Consensus hash of the parent Stacks block
     parent_consensus_hash: ConsensusHash,
@@ -256,28 +256,7 @@ pub struct AssembledAnchorBlock {
     /// Epoch timestamp in milliseconds when we started producing the block.
     tenure_begin: u128,
 }
-
-/// Write any `serde_json` object to a file
-/// TODO: Move this somewhere else
-pub fn serialize_json_to_file<J, P>(json: &J, filepath: P) -> Result<(), std::io::Error>
-where
-    J: ?Sized + serde::Serialize,
-    P: AsRef<Path>,
-{
-    let file = File::create(filepath)?;
-    let mut writer = BufWriter::new(file);
-    serde_json::to_writer(&mut writer, json)?;
-    writer.flush()
-}
-
-impl AssembledAnchorBlock {
-    pub fn serialize_to_file<P>(&self, filepath: P) -> Result<(), std::io::Error>
-    where
-        P: AsRef<Path>,
-    {
-        serialize_json_to_file(self, filepath)
-    }
-}
+impl_file_io_serde_json!(AssembledAnchorBlock);
 
 /// Miner chain tip, on top of which to build microblocks
 #[derive(Debug, Clone, PartialEq)]