Merge pull request #5060 from jbencin/feat/neon-miner-replay

feat: Neon mock miner replay

Author: jcnelson

.github/workflows/bitcoin-tests.yml

@@ -72,6 +72,7 @@ jobs:
           - tests::neon_integrations::confirm_unparsed_ongoing_ops
           - tests::neon_integrations::min_txs
           - tests::neon_integrations::vote_for_aggregate_key_burn_op_test
+          - tests::neon_integrations::mock_miner_replay
           - tests::epoch_25::microblocks_disabled
           - tests::should_succeed_handling_malformed_and_valid_txs
           - tests::nakamoto_integrations::simple_neon_integration

stacks-common/src/types/chainstate.rs

@@ -30,11 +30,11 @@ impl_byte_array_serde!(TrieHash);
 
 pub const TRIEHASH_ENCODED_SIZE: usize = 32;
 
-#[derive(Serialize, Deserialize)]
 pub struct BurnchainHeaderHash(pub [u8; 32]);
 impl_array_newtype!(BurnchainHeaderHash, u8, 32);
 impl_array_hexstring_fmt!(BurnchainHeaderHash);
 impl_byte_array_newtype!(BurnchainHeaderHash, u8, 32);
+impl_byte_array_serde!(BurnchainHeaderHash);
 
 pub struct BlockHeaderHash(pub [u8; 32]);
 impl_array_newtype!(BlockHeaderHash, u8, 32);

stacks-common/src/util/macros.rs

@@ -617,6 +617,28 @@ macro_rules! impl_byte_array_serde {
     };
 }
 
+#[allow(unused_macros)]
+#[macro_export]
+macro_rules! impl_file_io_serde_json {
+    ($thing:ident) => {
+        impl $thing {
+            pub fn serialize_to_file<P>(&self, path: P) -> Result<(), std::io::Error>
+            where
+                P: AsRef<std::path::Path>,
+            {
+                $crate::util::serialize_json_to_file(self, path)
+            }
+
+            pub fn deserialize_from_file<P>(path: P) -> Result<Self, std::io::Error>
+            where
+                P: AsRef<std::path::Path>,
+            {
+                $crate::util::deserialize_json_from_file(path)
+            }
+        }
+    };
+}
+
 // print debug statements while testing
 #[allow(unused_macros)]
 #[macro_export]

stacks-common/src/util/mod.rs

@@ -28,6 +28,9 @@ pub mod uint;
 pub mod vrf;
 
 use std::collections::HashMap;
+use std::fs::File;
+use std::io::{BufReader, BufWriter, Write};
+use std::path::Path;
 use std::time::{SystemTime, UNIX_EPOCH};
 use std::{error, fmt, thread, time};
 
@@ -120,3 +123,26 @@ pub mod db_common {
         true
     }
 }
+
+/// Write any `serde_json` object directly to a file
+pub fn serialize_json_to_file<J, P>(json: &J, path: P) -> Result<(), std::io::Error>
+where
+    J: ?Sized + serde::Serialize,
+    P: AsRef<Path>,
+{
+    let file = File::create(path)?;
+    let mut writer = BufWriter::new(file);
+    serde_json::to_writer(&mut writer, json)?;
+    writer.flush()
+}
+
+/// Read any `serde_json` object directly from a file
+pub fn deserialize_json_from_file<J, P>(path: P) -> Result<J, std::io::Error>
+where
+    J: serde::de::DeserializeOwned,
+    P: AsRef<Path>,
+{
+    let file = File::open(path)?;
+    let reader = BufReader::new(file);
+    serde_json::from_reader::<_, J>(reader).map_err(std::io::Error::from)
+}

stackslib/src/chainstate/stacks/block.rs

@@ -651,6 +651,14 @@ impl StacksBlock {
     pub fn has_microblock_parent(&self) -> bool {
         self.header.has_microblock_parent()
     }
+
+    /// Returns size in bytes of `StacksMessageCodec` representation
+    /// Note that this will serialize the block, so don't call if there is a better way to get block size
+    pub fn block_size(&self) -> Result<usize, codec_error> {
+        let mut buf = vec![];
+        self.consensus_serialize(&mut buf)?;
+        Ok(buf.len())
+    }
 }
 
 impl StacksMessageCodec for StacksMicroblockHeader {

stackslib/src/chainstate/stacks/db/blocks.rs

@@ -3716,30 +3716,44 @@ impl StacksChainState {
         blocks_conn: &DBConn,
         staging_block: &StagingBlock,
     ) -> Result<Option<Vec<StacksMicroblock>>, Error> {
-        if staging_block.parent_microblock_hash == EMPTY_MICROBLOCK_PARENT_HASH
-            && staging_block.parent_microblock_seq == 0
-        {
+        Self::inner_find_parent_microblock_stream(
+            blocks_conn,
+            &staging_block.anchored_block_hash,
+            &staging_block.parent_anchored_block_hash,
+            &staging_block.parent_consensus_hash,
+            &staging_block.parent_microblock_hash,
+            staging_block.parent_microblock_seq,
+        )
+    }
+
+    /// Allow `find_parent_microblock_stream()` to be called without `StagingBlock`
+    pub fn inner_find_parent_microblock_stream(
+        blocks_conn: &DBConn,
+        anchored_block_hash: &BlockHeaderHash,
+        parent_anchored_block_hash: &BlockHeaderHash,
+        parent_consensus_hash: &ConsensusHash,
+        parent_microblock_hash: &BlockHeaderHash,
+        parent_microblock_seq: u16,
+    ) -> Result<Option<Vec<StacksMicroblock>>, Error> {
+        if *parent_microblock_hash == EMPTY_MICROBLOCK_PARENT_HASH && parent_microblock_seq == 0 {
             // no parent microblocks, ever
             return Ok(Some(vec![]));
         }
 
         // find the microblock stream fork that this block confirms
         match StacksChainState::load_microblock_stream_fork(
             blocks_conn,
-            &staging_block.parent_consensus_hash,
-            &staging_block.parent_anchored_block_hash,
-            &staging_block.parent_microblock_hash,
+            parent_consensus_hash,
+            parent_anchored_block_hash,
+            parent_microblock_hash,
         )? {
             Some(microblocks) => {
                 return Ok(Some(microblocks));
             }
             None => {
                 // parent microblocks haven't arrived yet, or there are none
                 debug!(
-                    "No parent microblock stream for {}: expected a stream with tail {},{}",
-                    staging_block.anchored_block_hash,
-                    staging_block.parent_microblock_hash,
-                    staging_block.parent_microblock_seq
+                    "No parent microblock stream for {anchored_block_hash}: expected a stream with tail {parent_microblock_hash},{parent_microblock_seq}",
                 );
                 return Ok(None);
             }
@@ -5997,29 +6011,26 @@ impl StacksChainState {
     /// the given block.
     pub fn extract_connecting_microblocks(
         parent_block_header_info: &StacksHeaderInfo,
-        next_staging_block: &StagingBlock,
+        next_block_consensus_hash: &ConsensusHash,
+        next_block_hash: &BlockHeaderHash,
         block: &StacksBlock,
         mut next_microblocks: Vec<StacksMicroblock>,
     ) -> Result<Vec<StacksMicroblock>, Error> {
         // NOTE: since we got the microblocks from staging, where their signatures were already
         // validated, we don't need to validate them again.
-        let microblock_terminus = match StacksChainState::validate_parent_microblock_stream(
+        let Some((microblock_terminus, _)) = StacksChainState::validate_parent_microblock_stream(
             parent_block_header_info
                 .anchored_header
                 .as_stacks_epoch2()
                 .ok_or_else(|| Error::InvalidChildOfNakomotoBlock)?,
             &block.header,
             &next_microblocks,
             false,
-        ) {
-            Some((terminus, _)) => terminus,
-            None => {
-                debug!(
-                    "Stopping at block {}/{} -- discontiguous header stream",
-                    next_staging_block.consensus_hash, next_staging_block.anchored_block_hash,
-                );
-                return Ok(vec![]);
-            }
+        ) else {
+            debug!(
+                "Stopping at block {next_block_consensus_hash}/{next_block_hash} -- discontiguous header stream"
+            );
+            return Ok(vec![]);
         };
 
         // do not consider trailing microblocks that this anchored block does _not_ confirm
@@ -6214,7 +6225,8 @@ impl StacksChainState {
         // block's parent to this block.
         let next_microblocks = StacksChainState::extract_connecting_microblocks(
             &parent_header_info,
-            &next_staging_block,
+            &parent_header_info.consensus_hash,
+            &next_staging_block.anchored_block_hash,
             &block,
             next_microblocks,
         )?;

stackslib/src/chainstate/stacks/miner.rs

@@ -66,6 +66,30 @@ use crate::monitoring::{
 use crate::net::relay::Relayer;
 use crate::net::Error as net_error;
 
+/// Fully-assembled Stacks anchored, block as well as some extra metadata pertaining to how it was
+/// linked to the burnchain and what view(s) the miner had of the burnchain before and after
+/// completing the block.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct AssembledAnchorBlock {
+    /// Consensus hash of the parent Stacks block
+    pub parent_consensus_hash: ConsensusHash,
+    /// Consensus hash this Stacks block
+    pub consensus_hash: ConsensusHash,
+    /// Burnchain tip's block hash when we finished mining
+    pub burn_hash: BurnchainHeaderHash,
+    /// Burnchain tip's block height when we finished mining
+    pub burn_block_height: u64,
+    /// Burnchain tip's block hash when we started mining (could be different)
+    pub orig_burn_hash: BurnchainHeaderHash,
+    /// The block we produced
+    pub anchored_block: StacksBlock,
+    /// The attempt count of this block (multiple blocks will be attempted per burnchain block)
+    pub attempt: u64,
+    /// Epoch timestamp in milliseconds when we started producing the block.
+    pub tenure_begin: u128,
+}
+impl_file_io_serde_json!(AssembledAnchorBlock);
+
 /// System status for mining.
 /// The miner can be Ready, in which case a miner is allowed to run
 /// The miner can be Blocked, in which case the miner *should not start* and/or *should terminate*