Merge branch 'develop' into fix/5193-stackerdb-decoherence

Author: saralab

.github/workflows/bitcoin-tests.yml

@@ -81,6 +81,7 @@ jobs:
           - tests::epoch_25::microblocks_disabled
           - tests::should_succeed_handling_malformed_and_valid_txs
           - tests::nakamoto_integrations::simple_neon_integration
+          - tests::nakamoto_integrations::simple_neon_integration_with_flash_blocks_on_epoch_3
           - tests::nakamoto_integrations::mine_multiple_per_tenure_integration
           - tests::nakamoto_integrations::block_proposal_api_endpoint
           - tests::nakamoto_integrations::miner_writes_proposed_block_to_stackerdb

clarity/src/vm/docs/mod.rs

@@ -1831,7 +1831,7 @@ and block times are accurate only to within two hours. See [BIP113](https://gith
 For a block mined after epoch 3.0, this timestamp comes from the Stacks block header. **Note**: this is the time, according to the miner, when
 the mining of this block started, but is not guaranteed to be accurate. This time will be validated by the signers to be:
   - Greater than the timestamp of the previous block
-  - Less than 15 seconds into the future (according to their own local clocks)
+  - At most 15 seconds into the future (according to their own local clocks)
 ",
     example: "(get-stacks-block-info? time u0) ;; Returns (some u1557860301)
 (get-stacks-block-info? header-hash u0) ;; Returns (some 0x374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb)

contrib/tools/relay-server/src/http.rs

@@ -8,7 +8,9 @@ use crate::to_io_result::ToIoResult;
 pub struct Request {
     pub method: String,
     pub url: String,
+    #[allow(dead_code)]
     pub protocol: String,
+    #[allow(dead_code)]
     pub headers: HashMap<String, String>,
     pub content: Vec<u8>,
 }

stacks-common/Cargo.toml

@@ -78,6 +78,7 @@ testing = ["canonical"]
 serde = []
 bech32_std = []
 bech32_strict = []
+strason = []
 
 [target.'cfg(all(any(target_arch = "x86_64", target_arch = "x86", target_arch = "aarch64"), not(any(target_os="windows"))))'.dependencies]
 sha2 = { version = "0.10", features = ["asm"] }

stacks-signer/src/v0/signer.rs

@@ -185,6 +185,14 @@ impl SignerTrait<SignerMessage> for Signer {
                             );
                         }
                         SignerMessage::BlockPushed(b) => {
+                            let block_push_result = stacks_client.post_block(b);
+                            if let Err(ref e) = &block_push_result {
+                                warn!(
+                                    "{self}: Failed to post block {} (id {}): {e:?}",
+                                    &b.header.signer_signature_hash(),
+                                    &b.block_id()
+                                );
+                            };
                             // This will infinitely loop until the block is acknowledged by the node
                             info!(
                                 "{self}: Got block pushed message";
@@ -235,17 +243,17 @@ impl SignerTrait<SignerMessage> for Signer {
                 received_time,
             } => {
                 info!("{self}: Received a new burn block event for block height {burn_height}");
-                if let Err(e) =
-                    self.signer_db
-                        .insert_burn_block(burn_header_hash, *burn_height, received_time)
-                {
-                    warn!(
-                        "Failed to write burn block event to signerdb";
-                        "err" => ?e,
-                        "burn_header_hash" => %burn_header_hash,
-                        "burn_height" => burn_height
-                    );
-                }
+                self.signer_db
+                    .insert_burn_block(burn_header_hash, *burn_height, received_time)
+                    .unwrap_or_else(|e| {
+                        error!(
+                            "Failed to write burn block event to signerdb";
+                            "err" => ?e,
+                            "burn_header_hash" => %burn_header_hash,
+                            "burn_height" => burn_height
+                        );
+                        panic!("{self} Failed to write burn block event to signerdb: {e}");
+                    });
                 *sortition_state = None;
             }
         }
@@ -774,7 +782,8 @@ impl Signer {
             warn!("{self}: Failed to mark block as globally rejected: {e:?}",);
         }
         if let Err(e) = self.signer_db.insert_block(&block_info) {
-            warn!("{self}: Failed to update block state: {e:?}",);
+            error!("{self}: Failed to update block state: {e:?}",);
+            panic!("{self} Failed to update block state: {e}");
         }
     }
 
@@ -895,7 +904,7 @@ impl Signer {
                 "Failed to set group threshold signature timestamp for {}: {:?}",
                 block_hash, &e
             );
-            e
+            panic!("{self} Failed to write block to signerdb: {e}");
         });
         #[cfg(any(test, feature = "testing"))]
         {
@@ -935,24 +944,8 @@ impl Signer {
         block.header.signer_signature = signatures;
 
         #[cfg(any(test, feature = "testing"))]
-        {
-            if *TEST_SKIP_BLOCK_BROADCAST.lock().unwrap() == Some(true) {
-                warn!(
-                    "{self}: Skipping block broadcast due to testing directive";
-                    "block_id" => %block.block_id(),
-                    "height" => block.header.chain_length,
-                    "consensus_hash" => %block.header.consensus_hash
-                );
-
-                if let Err(e) = self.signer_db.set_block_broadcasted(
-                    self.reward_cycle,
-                    &block_hash,
-                    get_epoch_time_secs(),
-                ) {
-                    warn!("{self}: Failed to set block broadcasted for {block_hash}: {e:?}");
-                }
-                return;
-            }
+        if self.test_skip_block_broadcast(&block) {
+            return;
         }
         debug!(
             "{self}: Broadcasting Stacks block {} to node",
@@ -977,6 +970,29 @@ impl Signer {
         }
     }
 
+    #[cfg(any(test, feature = "testing"))]
+    fn test_skip_block_broadcast(&self, block: &NakamotoBlock) -> bool {
+        if *TEST_SKIP_BLOCK_BROADCAST.lock().unwrap() == Some(true) {
+            let block_hash = block.header.signer_signature_hash();
+            warn!(
+                "{self}: Skipping block broadcast due to testing directive";
+                "block_id" => %block.block_id(),
+                "height" => block.header.chain_length,
+                "consensus_hash" => %block.header.consensus_hash
+            );
+
+            if let Err(e) = self.signer_db.set_block_broadcasted(
+                self.reward_cycle,
+                &block_hash,
+                get_epoch_time_secs(),
+            ) {
+                warn!("{self}: Failed to set block broadcasted for {block_hash}: {e:?}");
+            }
+            return true;
+        }
+        false
+    }
+
     /// Send a mock signature to stackerdb to prove we are still alive
     fn mock_sign(&mut self, mock_proposal: MockProposal) {
         info!("{self}: Mock signing mock proposal: {mock_proposal:?}");

stacks-signer/src/v1/signer.rs

@@ -239,12 +239,13 @@ impl SignerTrait<SignerMessage> for Signer {
                     self.signer_db
                         .insert_burn_block(burn_header_hash, *burn_height, received_time)
                 {
-                    warn!(
+                    error!(
                         "Failed to write burn block event to signerdb";
                         "err" => ?e,
                         "burn_header_hash" => %burn_header_hash,
                         "burn_height" => burn_height
                     );
+                    panic!("Failed to write burn block event to signerdb");
                 }
             }
         }

stackslib/src/chainstate/burn/db/sortdb.rs

@@ -574,7 +574,7 @@ const SORTITION_DB_INITIAL_SCHEMA: &'static [&'static str] = &[
         block_height INTEGER NOT NULL,
         burn_header_hash TEXT NOT NULL,
         sortition_id TEXT NOT NULL,
-        
+
         consensus_hash TEXT NOT NULL,
         public_key TEXT NOT NULL,
         memo TEXT,
@@ -619,7 +619,7 @@ const SORTITION_DB_INITIAL_SCHEMA: &'static [&'static str] = &[
         stacked_ustx TEXT NOT NULL,
         num_cycles INTEGER NOT NULL,
 
-        -- The primary key here is (txid, burn_header_hash) because 
+        -- The primary key here is (txid, burn_header_hash) because
         -- this transaction will be accepted regardless of which sortition
         -- history it is in.
         PRIMARY KEY(txid,burn_header_hash)
@@ -636,7 +636,7 @@ const SORTITION_DB_INITIAL_SCHEMA: &'static [&'static str] = &[
         transfered_ustx TEXT NOT NULL,
         memo TEXT NOT NULL,
 
-        -- The primary key here is (txid, burn_header_hash) because 
+        -- The primary key here is (txid, burn_header_hash) because
         -- this transaction will be accepted regardless of which sortition
         -- history it is in.
         PRIMARY KEY(txid,burn_header_hash)
@@ -687,11 +687,11 @@ const SORTITION_DB_SCHEMA_4: &'static [&'static str] = &[
         delegated_ustx TEXT NOT NULL,
         until_burn_height INTEGER,
 
-        PRIMARY KEY(txid,burn_header_Hash)
+        PRIMARY KEY(txid,burn_header_hash)
     );"#,
     r#"
     CREATE TABLE ast_rule_heights (
-        ast_rule_id INTEGER PRIMAR KEY NOT NULL,
+        ast_rule_id INTEGER PRIMARY KEY NOT NULL,
         block_height INTEGER NOT NULL
     );"#,
 ];
@@ -2261,7 +2261,7 @@ impl<'a> SortitionHandleConn<'a> {
 
     /// Get a block commit by txid. In the event of a burnchain fork, this may not be unique.
     ///   this function simply returns one of those block commits: only use data that is
-    ///   immutable across burnchain/pox forks, e.g., parent block ptr,  
+    ///   immutable across burnchain/pox forks, e.g., parent block ptr,
     pub fn get_block_commit_by_txid(
         &self,
         sort_id: &SortitionId,
@@ -2909,7 +2909,6 @@ impl SortitionDB {
         SortitionDB::apply_schema_6(&db_tx, epochs_ref)?;
         SortitionDB::apply_schema_7(&db_tx, epochs_ref)?;
         SortitionDB::apply_schema_8_tables(&db_tx, epochs_ref)?;
-        SortitionDB::apply_schema_9(&db_tx, epochs_ref)?;
 
         db_tx.instantiate_index()?;
 
@@ -2930,6 +2929,11 @@ impl SortitionDB {
         // NOTE: we don't need to provide a migrator here because we're not migrating
         self.apply_schema_8_migration(None)?;
 
+        let db_tx = SortitionHandleTx::begin(self, &SortitionId::sentinel())?;
+        SortitionDB::apply_schema_9(&db_tx, epochs_ref)?;
+
+        db_tx.commit()?;
+
         self.add_indexes()?;
 
         debug!("Instantiated SortDB");
@@ -5046,7 +5050,7 @@ impl SortitionDB {
         conn: &Connection,
         sortition: &SortitionId,
     ) -> Result<Option<u16>, db_error> {
-        let qry = "SELECT vtxindex FROM block_commits WHERE sortition_id = ?1 
+        let qry = "SELECT vtxindex FROM block_commits WHERE sortition_id = ?1
                     AND txid = (
                       SELECT winning_block_txid FROM snapshots WHERE sortition_id = ?2 LIMIT 1) LIMIT 1";
         let args = params![sortition, sortition];

stackslib/src/chainstate/nakamoto/miner.rs

@@ -108,7 +108,7 @@ impl NakamotoTenureInfo {
 }
 
 pub struct NakamotoBlockBuilder {
-    /// If there's a parent (i.e., not a genesis), this is Some(parent_header)    
+    /// If there's a parent (i.e., not a genesis), this is Some(parent_header)
     parent_header: Option<StacksHeaderInfo>,
     /// Signed coinbase tx, if starting a new tenure
     coinbase_tx: Option<StacksTransaction>,
@@ -280,8 +280,11 @@ impl NakamotoBlockBuilder {
             &self.header.parent_block_id,
         ).map_err(|e| {
             warn!(
-                "Cannot process Nakamoto block: could not load reward set that elected the block";
+                "Cannot process Nakamoto block: could not find height at which the PoX reward set was calculated";
                 "err" => ?e,
+                "stacks_tip" => %self.header.parent_block_id,
+                "elected_height" => elected_height,
+                "elected_cycle" => elected_in_cycle
             );
             Error::NoSuchBlockError
         })?;

stackslib/src/chainstate/nakamoto/mod.rs

@@ -612,7 +612,7 @@ pub struct NakamotoBlockHeader {
     /// A Unix time timestamp of when this block was mined, according to the miner.
     /// For the signers to consider a block valid, this timestamp must be:
     ///  * Greater than the timestamp of its parent block
-    ///  * Less than 15 seconds into the future
+    ///  * At most 15 seconds into the future
     pub timestamp: u64,
     /// Recoverable ECDSA signature from the tenure's miner.
     pub miner_signature: MessageSignature,
@@ -734,6 +734,7 @@ impl NakamotoBlockHeader {
         write_next(fd, &self.tx_merkle_root)?;
         write_next(fd, &self.state_index_root)?;
         write_next(fd, &self.timestamp)?;
+        write_next(fd, &self.pox_treatment)?;
         Ok(Sha512Trunc256Sum::from_hasher(hasher))
     }
 
@@ -1876,7 +1877,7 @@ impl NakamotoChainState {
                         "stacks_block_id" => %next_ready_block.header.block_id(),
                         "parent_block_id" => %next_ready_block.header.parent_block_id
                     );
-                    ChainstateError::InvalidStacksBlock("Failed to load burn view of parent block ID".into())                    
+                    ChainstateError::InvalidStacksBlock("Failed to load burn view of parent block ID".into())
                 })?;
                 let handle = sort_db.index_handle_at_ch(&tenure_change.burn_view_consensus_hash)?;
                 let connected_sort_id = get_ancestor_sort_id(&handle, parent_burn_view_sn.block_height, &handle.context.chain_tip)?
@@ -1888,7 +1889,7 @@ impl NakamotoChainState {
                             "stacks_block_id" => %next_ready_block.header.block_id(),
                             "parent_block_id" => %next_ready_block.header.parent_block_id
                         );
-                        ChainstateError::InvalidStacksBlock("Failed to load burn view of parent block ID".into())                    
+                        ChainstateError::InvalidStacksBlock("Failed to load burn view of parent block ID".into())
                     })?;
                 if connected_sort_id != parent_burn_view_sn.sortition_id {
                     warn!(
@@ -3608,7 +3609,7 @@ impl NakamotoChainState {
                                 .map_err(|_| ChainstateError::InvalidStacksBlock("Reward set index outside of u16".into()))?;
                             let bitvec_value = block_bitvec.get(ix)
                                 .unwrap_or_else(|| {
-                                    info!("Block header's bitvec is smaller than the reward set, defaulting higher indexes to 1");
+                                    warn!("Block header's bitvec is smaller than the reward set, defaulting higher indexes to 1");
                                     true
                                 });
                             Ok(bitvec_value)