Merge branch 'develop' into feat/rpc-endpoints-to-fetch-data-from-key

Author: jcnelson

.github/actions/dockerfiles/Dockerfile.debian-source

@@ -24,5 +24,5 @@ RUN --mount=type=tmpfs,target=${BUILD_DIR} cp -R /src/. ${BUILD_DIR}/ \
     && cp -R ${BUILD_DIR}/target/${TARGET}/release/. /out
 
 FROM --platform=${TARGETPLATFORM} debian:bookworm
-COPY --from=build /out/stacks-node /out/stacks-signer /bin/
+COPY --from=build /out/stacks-node /out/stacks-signer /out/stacks-inspect /bin/
 CMD ["stacks-node", "mainnet"]

.github/workflows/bitcoin-tests.yml

@@ -126,6 +126,7 @@ jobs:
           - tests::signer::v0::block_commit_delay
           - tests::signer::v0::continue_after_fast_block_no_sortition
           - tests::signer::v0::block_validation_response_timeout
+          - tests::signer::v0::tenure_extend_after_bad_commit
           - tests::nakamoto_integrations::burn_ops_integration_test
           - tests::nakamoto_integrations::check_block_heights
           - tests::nakamoto_integrations::clarity_burn_state
@@ -139,6 +140,7 @@ jobs:
           - tests::nakamoto_integrations::utxo_check_on_startup_panic
           - tests::nakamoto_integrations::utxo_check_on_startup_recover
           - tests::nakamoto_integrations::v3_signer_api_endpoint
+          - tests::nakamoto_integrations::test_shadow_recovery
           - tests::nakamoto_integrations::signer_chainstate
           - tests::nakamoto_integrations::clarity_cost_spend_down
           - tests::nakamoto_integrations::v3_blockbyheight_api_endpoint

.github/workflows/p2p-tests.yml

@@ -43,6 +43,10 @@ jobs:
           - net::tests::convergence::test_walk_star_15_org_biased
           - net::tests::convergence::test_walk_inbound_line_15
           - net::api::tests::postblock_proposal::test_try_make_response
+          - net::server::tests::test_http_10_threads_getinfo
+          - net::server::tests::test_http_10_threads_getblock
+          - net::server::tests::test_http_too_many_clients
+          - net::server::tests::test_http_slow_client
     steps:
       ## Setup test environment
       - name: Setup Test Environment

CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE
 - Add `block_commit_delay_ms` to the config file to control the time to wait after seeing a new burn block, before submitting a block commit, to allow time for the first Nakamoto block of the new tenure to be mined, allowing this miner to avoid the need to RBF the block commit.
 - Add `tenure_cost_limit_per_block_percentage` to the miner config file to control the percentage remaining tenure cost limit to consume per nakamoto block.
 - Add `/v3/blocks/height/:block_height` rpc endpoint
+- If the winning miner of a sortition is committed to the wrong parent tenure, the previous miner can immediately tenure extend and continue mining since the winning miner would never be able to propose a valid block. (#5361)
 
 ## [3.0.0.0.1]
 

stacks-common/src/types/mod.rs

@@ -170,6 +170,21 @@ impl StacksEpochId {
         }
     }
 
+    /// Whether or not this epoch supports shadow blocks
+    pub fn supports_shadow_blocks(&self) -> bool {
+        match self {
+            StacksEpochId::Epoch10
+            | StacksEpochId::Epoch20
+            | StacksEpochId::Epoch2_05
+            | StacksEpochId::Epoch21
+            | StacksEpochId::Epoch22
+            | StacksEpochId::Epoch23
+            | StacksEpochId::Epoch24
+            | StacksEpochId::Epoch25 => false,
+            StacksEpochId::Epoch30 => true,
+        }
+    }
+
     /// Does this epoch support unlocking PoX contributors that miss a slot?
     ///
     /// Epoch 2.0 - 2.05 didn't support this feature, but they weren't epoch-guarded on it. Instead,

stacks-signer/CHANGELOG.md

@@ -11,6 +11,8 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE
 
 ### Changed
 
+- Allow a miner to extend their tenure immediately if the winner of the next tenure has committed to the wrong parent tenure (#5361)
+
 ## [3.0.0.0.1.0]
 
 ### Changed

stacks-signer/src/chainstate.rs

@@ -203,7 +203,40 @@ impl SortitionsView {
                 "current_sortition_consensus_hash" => ?self.cur_sortition.consensus_hash,
             );
             self.cur_sortition.miner_status = SortitionMinerStatus::InvalidatedBeforeFirstBlock;
+        } else if let Some(tip) = signer_db.get_canonical_tip()? {
+            // Check if the current sortition is aligned with the expected tenure:
+            // - If the tip is in the current tenure, we are in the process of mining this tenure.
+            // - If the tip is not in the current tenure, then we’re starting a new tenure,
+            //   and the current sortition's parent tenure must match the tenure of the tip.
+            // - If the tip is not building off of the current sortition's parent tenure, then
+            //   check to see if the tip's parent is within the first proposal burn block timeout,
+            //   which allows for forks when a burn block arrives quickly.
+            // - Else the miner of the current sortition has committed to an incorrect parent tenure.
+            let consensus_hash_match =
+                self.cur_sortition.consensus_hash == tip.block.header.consensus_hash;
+            let parent_tenure_id_match =
+                self.cur_sortition.parent_tenure_id == tip.block.header.consensus_hash;
+            if !consensus_hash_match && !parent_tenure_id_match {
+                // More expensive check, so do it only if we need to.
+                let is_valid_parent_tenure = Self::check_parent_tenure_choice(
+                    &self.cur_sortition,
+                    block,
+                    signer_db,
+                    client,
+                    &self.config.first_proposal_burn_block_timing,
+                )?;
+                if !is_valid_parent_tenure {
+                    warn!(
+                        "Current sortition does not build off of canonical tip tenure, marking as invalid";
+                        "current_sortition_parent" => ?self.cur_sortition.parent_tenure_id,
+                        "tip_consensus_hash" => ?tip.block.header.consensus_hash,
+                    );
+                    self.cur_sortition.miner_status =
+                        SortitionMinerStatus::InvalidatedBeforeFirstBlock;
+                }
+            }
         }
+
         if let Some(last_sortition) = self.last_sortition.as_mut() {
             if last_sortition.is_timed_out(self.config.block_proposal_timeout, signer_db)? {
                 info!(
@@ -304,6 +337,7 @@ impl SortitionsView {
                         "Miner block proposal is from last sortition winner, when the new sortition winner is still valid. Considering proposal invalid.";
                         "proposed_block_consensus_hash" => %block.header.consensus_hash,
                         "proposed_block_signer_sighash" => %block.header.signer_signature_hash(),
+                        "current_sortition_miner_status" => ?self.cur_sortition.miner_status,
                     );
                     return Ok(false);
                 }
@@ -439,6 +473,8 @@ impl SortitionsView {
                             "violating_tenure_proposed_time" => local_block_info.proposed_time,
                             "new_tenure_received_time" => sortition_state_received_time,
                             "new_tenure_burn_timestamp" => sortition_state.burn_header_timestamp,
+                            "first_proposal_burn_block_timing_secs" => first_proposal_burn_block_timing.as_secs(),
+                            "proposal_to_sortition" => proposal_to_sortition,
                         );
                         continue;
                     }

stacks-signer/src/client/stacks_client.rs

@@ -173,6 +173,9 @@ impl StacksClient {
         &self,
         consensus_hash: &ConsensusHash,
     ) -> Result<StacksBlockHeaderTypes, ClientError> {
+        debug!("StacksClient: Getting tenure tip";
+            "consensus_hash" => %consensus_hash,
+        );
         let send_request = || {
             self.stacks_node_client
                 .get(self.tenure_tip_path(consensus_hash))
@@ -192,6 +195,7 @@ impl StacksClient {
 
     /// Get the last set reward cycle stored within the stackerdb contract
     pub fn get_last_set_cycle(&self) -> Result<u128, ClientError> {
+        debug!("StacksClient: Getting last set cycle");
         let signer_stackerdb_contract_id = boot_code_id(SIGNERS_NAME, self.mainnet);
         let function_name_str = "get-last-set-cycle";
         let function_name = ClarityName::from(function_name_str);
@@ -210,6 +214,10 @@ impl StacksClient {
         stackerdb_contract: &QualifiedContractIdentifier,
         page: u32,
     ) -> Result<Vec<(StacksAddress, u128)>, ClientError> {
+        debug!("StacksClient: Getting signer slots";
+            "stackerdb_contract" => %stackerdb_contract,
+            "page" => page,
+        );
         let function_name_str = "stackerdb-get-signer-slots-page";
         let function_name = ClarityName::from(function_name_str);
         let function_args = &[ClarityValue::UInt(page.into())];
@@ -250,6 +258,9 @@ impl StacksClient {
         &self,
         reward_cycle: u64,
     ) -> Result<HashMap<StacksAddress, SignerSlotID>, ClientError> {
+        debug!("StacksClient: Getting parsed signer slots";
+            "reward_cycle" => reward_cycle,
+        );
         let signer_set =
             u32::try_from(reward_cycle % 2).expect("FATAL: reward_cycle % 2 exceeds u32::MAX");
         let signer_stackerdb_contract_id = boot_code_id(SIGNERS_NAME, self.mainnet);
@@ -272,6 +283,7 @@ impl StacksClient {
 
     /// Determine the stacks node current epoch
     pub fn get_node_epoch(&self) -> Result<StacksEpochId, ClientError> {
+        debug!("StacksClient: Getting node epoch");
         let pox_info = self.get_pox_data()?;
         let burn_block_height = self.get_burn_block_height()?;
 
@@ -302,7 +314,7 @@ impl StacksClient {
 
     /// Submit the block proposal to the stacks node. The block will be validated and returned via the HTTP endpoint for Block events.
     pub fn submit_block_for_validation(&self, block: NakamotoBlock) -> Result<(), ClientError> {
-        debug!("stacks_node_client: Submitting block for validation...";
+        debug!("StacksClient: Submitting block for validation";
             "signer_sighash" => %block.header.signer_signature_hash(),
             "block_id" => %block.header.block_id(),
             "block_height" => %block.header.chain_length,
@@ -337,6 +349,10 @@ impl StacksClient {
         chosen_parent: &ConsensusHash,
         last_sortition: &ConsensusHash,
     ) -> Result<Vec<TenureForkingInfo>, ClientError> {
+        debug!("StacksClient: Getting tenure forking info";
+            "chosen_parent" => %chosen_parent,
+            "last_sortition" => %last_sortition,
+        );
         let mut tenures: VecDeque<TenureForkingInfo> =
             self.get_tenure_forking_info_step(chosen_parent, last_sortition)?;
         if tenures.is_empty() {
@@ -373,7 +389,7 @@ impl StacksClient {
         chosen_parent: &ConsensusHash,
         last_sortition: &ConsensusHash,
     ) -> Result<VecDeque<TenureForkingInfo>, ClientError> {
-        debug!("stacks_node_client: Getting tenure forking info...";
+        debug!("StacksClient: Getting tenure forking info";
             "chosen_parent" => %chosen_parent,
             "last_sortition" => %last_sortition,
         );
@@ -402,7 +418,7 @@ impl StacksClient {
 
     /// Get the current winning sortition and the last winning sortition
     pub fn get_current_and_last_sortition(&self) -> Result<CurrentAndLastSortition, ClientError> {
-        debug!("stacks_node_client: Getting current and prior sortition...");
+        debug!("StacksClient: Getting current and prior sortition");
         let path = format!("{}/latest_and_last", self.sortition_info_path());
         let timer = crate::monitoring::new_rpc_call_timer(&path, &self.http_origin);
         let send_request = || {
@@ -443,7 +459,7 @@ impl StacksClient {
 
     /// Get the current peer info data from the stacks node
     pub fn get_peer_info(&self) -> Result<PeerInfo, ClientError> {
-        debug!("stacks_node_client: Getting peer info...");
+        debug!("StacksClient: Getting peer info");
         let timer =
             crate::monitoring::new_rpc_call_timer(&self.core_info_path(), &self.http_origin);
         let send_request = || {
@@ -466,7 +482,9 @@ impl StacksClient {
         &self,
         reward_cycle: u64,
     ) -> Result<Option<Vec<NakamotoSignerEntry>>, ClientError> {
-        debug!("stacks_node_client: Getting reward set signers for reward cycle {reward_cycle}...");
+        debug!("StacksClient: Getting reward set signers";
+            "reward_cycle" => reward_cycle,
+        );
         let timer = crate::monitoring::new_rpc_call_timer(
             &format!("{}/v3/stacker_set/:reward_cycle", self.http_origin),
             &self.http_origin,
@@ -502,7 +520,7 @@ impl StacksClient {
 
     /// Retrieve the current pox data from the stacks node
     pub fn get_pox_data(&self) -> Result<RPCPoxInfoData, ClientError> {
-        debug!("stacks_node_client: Getting pox data...");
+        debug!("StacksClient: Getting pox data");
         let timer = crate::monitoring::new_rpc_call_timer(&self.pox_path(), &self.http_origin);
         let send_request = || {
             self.stacks_node_client
@@ -521,11 +539,13 @@ impl StacksClient {
 
     /// Helper function to retrieve the burn tip height from the stacks node
     fn get_burn_block_height(&self) -> Result<u64, ClientError> {
+        debug!("StacksClient: Getting burn block height");
         self.get_peer_info().map(|info| info.burn_block_height)
     }
 
     /// Get the current reward cycle info from the stacks node
     pub fn get_current_reward_cycle_info(&self) -> Result<RewardCycleInfo, ClientError> {
+        debug!("StacksClient: Getting current reward cycle info");
         let pox_data = self.get_pox_data()?;
         let blocks_mined = pox_data
             .current_burnchain_block_height
@@ -548,7 +568,9 @@ impl StacksClient {
         &self,
         address: &StacksAddress,
     ) -> Result<AccountEntryResponse, ClientError> {
-        debug!("stacks_node_client: Getting account info...");
+        debug!("StacksClient: Getting account info";
+            "address" => %address,
+        );
         let timer_label = format!("{}/v2/accounts/:principal", self.http_origin);
         let timer = crate::monitoring::new_rpc_call_timer(&timer_label, &self.http_origin);
         let send_request = || {
@@ -570,6 +592,11 @@ impl StacksClient {
     ///
     /// In tests, this panics if the retry takes longer than 30 seconds.
     pub fn post_block_until_ok<F: Display>(&self, log_fmt: &F, block: &NakamotoBlock) -> bool {
+        debug!("StacksClient: Posting block to stacks node";
+            "signer_sighash" => %block.header.signer_signature_hash(),
+            "block_id" => %block.header.block_id(),
+            "block_height" => %block.header.chain_length,
+        );
         let start_time = Instant::now();
         loop {
             match self.post_block(block) {
@@ -595,7 +622,8 @@ impl StacksClient {
     /// Returns `true` if the block was accepted or `false` if the block
     ///   was rejected.
     pub fn post_block(&self, block: &NakamotoBlock) -> Result<bool, ClientError> {
-        debug!("stacks_node_client: Posting block to the stacks node...";
+        debug!("StacksClient: Posting block to the stacks node";
+            "signer_sighash" => %block.header.signer_signature_hash(),
             "block_id" => %block.header.block_id(),
             "block_height" => %block.header.chain_length,
         );
@@ -630,7 +658,9 @@ impl StacksClient {
         function_name: &ClarityName,
         function_args: &[ClarityValue],
     ) -> Result<ClarityValue, ClientError> {
-        debug!("stacks_node_client: Calling read-only function {function_name} with args {function_args:?}...");
+        debug!(
+            "StacksClient: Calling read-only function {function_name} with args {function_args:?}"
+        );
         let args = function_args
             .iter()
             .filter_map(|arg| arg.serialize_to_hex().ok())