Merge branch 'develop' into logs/info-neighbor-add-and-remove

Author: ASuciuX

.github/workflows/bitcoin-tests.yml

@@ -140,6 +140,8 @@ jobs:
           - tests::nakamoto_integrations::utxo_check_on_startup_recover
           - tests::nakamoto_integrations::v3_signer_api_endpoint
           - tests::nakamoto_integrations::signer_chainstate
+          - tests::nakamoto_integrations::clarity_cost_spend_down
+          - tests::nakamoto_integrations::v3_blockbyheight_api_endpoint
           # TODO: enable these once v1 signer is supported by a new nakamoto epoch
           # - tests::signer::v1::dkg
           # - tests::signer::v1::sign_request_rejected

.github/workflows/p2p-tests.yml

@@ -42,6 +42,7 @@ jobs:
           - net::tests::convergence::test_walk_star_15_pingback
           - net::tests::convergence::test_walk_star_15_org_biased
           - net::tests::convergence::test_walk_inbound_line_15
+          - net::api::tests::postblock_proposal::test_try_make_response
     steps:
       ## Setup test environment
       - name: Setup Test Environment

CHANGELOG.md

@@ -12,6 +12,8 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE
 - Remove the panic for reporting DB deadlocks (just error and continue waiting)
 - Add index to `metadata_table` in Clarity DB on `blockhash`
 - Add `block_commit_delay_ms` to the config file to control the time to wait after seeing a new burn block, before submitting a block commit, to allow time for the first Nakamoto block of the new tenure to be mined, allowing this miner to avoid the need to RBF the block commit.
+- Add `tenure_cost_limit_per_block_percentage` to the miner config file to control the percentage remaining tenure cost limit to consume per nakamoto block.
+- Add `/v3/blocks/height/:block_height` rpc endpoint
 
 ## [3.0.0.0.1]
 

clarity/src/vm/costs/mod.rs

@@ -896,6 +896,7 @@ impl LimitedCostTracker {
             Self::Free => ExecutionCost::max_value(),
         }
     }
+
     pub fn get_memory(&self) -> u64 {
         match self {
             Self::Limited(TrackerData { memory, .. }) => *memory,
@@ -1170,6 +1171,7 @@ pub trait CostOverflowingMath<T> {
     fn cost_overflow_mul(self, other: T) -> Result<T>;
     fn cost_overflow_add(self, other: T) -> Result<T>;
     fn cost_overflow_sub(self, other: T) -> Result<T>;
+    fn cost_overflow_div(self, other: T) -> Result<T>;
 }
 
 impl CostOverflowingMath<u64> for u64 {
@@ -1185,6 +1187,10 @@ impl CostOverflowingMath<u64> for u64 {
         self.checked_sub(other)
             .ok_or_else(|| CostErrors::CostOverflow)
     }
+    fn cost_overflow_div(self, other: u64) -> Result<u64> {
+        self.checked_div(other)
+            .ok_or_else(|| CostErrors::CostOverflow)
+    }
 }
 
 impl ExecutionCost {
@@ -1293,6 +1299,15 @@ impl ExecutionCost {
         Ok(())
     }
 
+    pub fn divide(&mut self, divisor: u64) -> Result<()> {
+        self.runtime = self.runtime.cost_overflow_div(divisor)?;
+        self.read_count = self.read_count.cost_overflow_div(divisor)?;
+        self.read_length = self.read_length.cost_overflow_div(divisor)?;
+        self.write_length = self.write_length.cost_overflow_div(divisor)?;
+        self.write_count = self.write_count.cost_overflow_div(divisor)?;
+        Ok(())
+    }
+
     /// Returns whether or not this cost exceeds any dimension of the
     ///  other cost.
     pub fn exceeds(&self, other: &ExecutionCost) -> bool {

docs/rpc-endpoints.md

@@ -503,6 +503,17 @@ data.
 
 This will return 404 if the block does not exist.
 
+### GET /v3/blocks/height/[Block Height]
+
+Fetch a Nakamoto block given its block height.  This returns the raw block
+data.
+
+This will return 404 if the block does not exist.
+
+This endpoint also accepts a querystring parameter `?tip=` which when supplied 
+will return the block relative to the specified tip allowing the querying of 
+sibling blocks (same height, different tip) too.
+
 ### GET /v3/tenures/[Block ID]
 
 Fetch a Nakamoto block and all of its ancestors in the same tenure, given its

docs/rpc/openapi.yaml

@@ -627,6 +627,40 @@ paths:
           content:
             application/text-plain: {}
 
+  /v3/blocks/height/{block_height}:
+    get:
+      summary: Fetch a Nakamoto block by its height and optional tip
+      tags:
+        - Blocks
+      operationId: get_block_v3_by_height
+      description:
+        Fetch a Nakamoto block by its height and optional tip.
+      parameters:
+        - name: block_height
+          in: path
+          description: The block's height
+          required: true
+          schema:
+            type: integer
+        - name: tip
+          in: query
+          schema:
+            type: string
+          description: The Stacks chain tip to query from. If tip == latest or empty, the query will be run
+            from the latest known tip.
+      responses:
+        "200":
+          description: The raw SIP-003-encoded block will be returned.
+          content:
+            application/octet-stream:
+              schema:
+                type: string
+                format: binary
+        "404":
+          description: The block could not be found
+          content:
+            application/text-plain: {}
+
   /v3/tenures/info:
     get:
       summary: Fetch metadata about the ongoing Nakamoto tenure

stacks-common/src/types/mod.rs

@@ -1,5 +1,6 @@
 use std::cmp::Ordering;
 use std::fmt;
+use std::ops::{Deref, DerefMut, Index, IndexMut};
 
 #[cfg(feature = "canonical")]
 pub mod sqlite;
@@ -460,3 +461,83 @@ impl<L: PartialEq + Eq> Ord for StacksEpoch<L> {
         self.epoch_id.cmp(&other.epoch_id)
     }
 }
+
+/// A wrapper for holding a list of Epochs, indexable by StacksEpochId
+#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)]
+pub struct EpochList<L: Clone>(Vec<StacksEpoch<L>>);
+
+impl<L: Clone> EpochList<L> {
+    pub fn new(epochs: &[StacksEpoch<L>]) -> EpochList<L> {
+        EpochList(epochs.to_vec())
+    }
+
+    pub fn get(&self, index: StacksEpochId) -> Option<&StacksEpoch<L>> {
+        self.0.get(StacksEpoch::find_epoch_by_id(&self.0, index)?)
+    }
+
+    pub fn get_mut(&mut self, index: StacksEpochId) -> Option<&mut StacksEpoch<L>> {
+        let index = StacksEpoch::find_epoch_by_id(&self.0, index)?;
+        self.0.get_mut(index)
+    }
+
+    /// Truncates the list after the given epoch id
+    pub fn truncate_after(&mut self, epoch_id: StacksEpochId) {
+        if let Some(index) = StacksEpoch::find_epoch_by_id(&self.0, epoch_id) {
+            self.0.truncate(index + 1);
+        }
+    }
+
+    /// Determine which epoch, if any, a given burnchain height falls into.
+    pub fn epoch_id_at_height(&self, height: u64) -> Option<StacksEpochId> {
+        StacksEpoch::find_epoch(self, height).map(|idx| self.0[idx].epoch_id)
+    }
+
+    /// Determine which epoch, if any, a given burnchain height falls into.
+    pub fn epoch_at_height(&self, height: u64) -> Option<StacksEpoch<L>> {
+        StacksEpoch::find_epoch(self, height).map(|idx| self.0[idx].clone())
+    }
+
+    /// Pushes a new `StacksEpoch` to the end of the list
+    pub fn push(&mut self, epoch: StacksEpoch<L>) {
+        if let Some(last) = self.0.last() {
+            assert!(
+                epoch.start_height == last.end_height && epoch.epoch_id > last.epoch_id,
+                "Epochs must be pushed in order"
+            );
+        }
+        self.0.push(epoch);
+    }
+
+    pub fn to_vec(&self) -> Vec<StacksEpoch<L>> {
+        self.0.clone()
+    }
+}
+
+impl<L: Clone> Index<StacksEpochId> for EpochList<L> {
+    type Output = StacksEpoch<L>;
+    fn index(&self, index: StacksEpochId) -> &StacksEpoch<L> {
+        self.get(index)
+            .expect("Invalid StacksEpochId: could not find corresponding epoch")
+    }
+}
+
+impl<L: Clone> IndexMut<StacksEpochId> for EpochList<L> {
+    fn index_mut(&mut self, index: StacksEpochId) -> &mut StacksEpoch<L> {
+        self.get_mut(index)
+            .expect("Invalid StacksEpochId: could not find corresponding epoch")
+    }
+}
+
+impl<L: Clone> Deref for EpochList<L> {
+    type Target = [StacksEpoch<L>];
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl<L: Clone> DerefMut for EpochList<L> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}

stacks-signer/src/signerdb.rs

@@ -258,6 +258,22 @@ impl BlockInfo {
         self.state = state;
         Ok(())
     }
+
+    /// Check if the block is globally accepted or rejected
+    pub fn has_reached_consensus(&self) -> bool {
+        matches!(
+            self.state,
+            BlockState::GloballyAccepted | BlockState::GloballyRejected
+        )
+    }
+
+    /// Check if the block is locally accepted or rejected
+    pub fn is_locally_finalized(&self) -> bool {
+        matches!(
+            self.state,
+            BlockState::LocallyAccepted | BlockState::LocallyRejected
+        )
+    }
 }
 
 /// This struct manages a SQLite database connection

stacks-signer/src/v0/signer.rs

@@ -21,6 +21,7 @@ use blockstack_lib::chainstate::nakamoto::{NakamotoBlock, NakamotoBlockHeader};
 use blockstack_lib::net::api::postblock_proposal::{
     BlockValidateOk, BlockValidateReject, BlockValidateResponse,
 };
+use blockstack_lib::util_lib::db::Error as DBError;
 use clarity::types::chainstate::StacksPrivateKey;
 use clarity::types::{PrivateKey, StacksEpochId};
 use clarity::util::hash::MerkleHashFunc;
@@ -496,7 +497,7 @@ impl Signer {
             // Do not store KNOWN invalid blocks as this could DOS the signer. We only store blocks that are valid or unknown.
             self.signer_db
                 .insert_block(&block_info)
-                .unwrap_or_else(|_| panic!("{self}: Failed to insert block in DB"));
+                .unwrap_or_else(|e| self.handle_insert_block_error(e));
         }
     }
 
@@ -539,9 +540,7 @@ impl Signer {
             .block_lookup(self.reward_cycle, &signer_signature_hash)
         {
             Ok(Some(block_info)) => {
-                if block_info.state == BlockState::GloballyRejected
-                    || block_info.state == BlockState::GloballyAccepted
-                {
+                if block_info.is_locally_finalized() {
                     debug!("{self}: Received block validation for a block that is already marked as {}. Ignoring...", block_info.state);
                     return None;
                 }
@@ -558,8 +557,11 @@ impl Signer {
             }
         };
         if let Err(e) = block_info.mark_locally_accepted(false) {
-            warn!("{self}: Failed to mark block as locally accepted: {e:?}",);
-            return None;
+            if !block_info.has_reached_consensus() {
+                warn!("{self}: Failed to mark block as locally accepted: {e:?}",);
+                return None;
+            }
+            block_info.signed_self.get_or_insert(get_epoch_time_secs());
         }
         let signature = self
             .private_key
@@ -568,7 +570,7 @@ impl Signer {
 
         self.signer_db
             .insert_block(&block_info)
-            .unwrap_or_else(|_| panic!("{self}: Failed to insert block in DB"));
+            .unwrap_or_else(|e| self.handle_insert_block_error(e));
         let accepted = BlockAccepted::new(block_info.signer_signature_hash(), signature);
         // have to save the signature _after_ the block info
         self.handle_block_signature(stacks_client, &accepted);
@@ -597,9 +599,7 @@ impl Signer {
             .block_lookup(self.reward_cycle, &signer_signature_hash)
         {
             Ok(Some(block_info)) => {
-                if block_info.state == BlockState::GloballyRejected
-                    || block_info.state == BlockState::GloballyAccepted
-                {
+                if block_info.is_locally_finalized() {
                     debug!("{self}: Received block validation for a block that is already marked as {}. Ignoring...", block_info.state);
                     return None;
                 }
@@ -616,8 +616,10 @@ impl Signer {
             }
         };
         if let Err(e) = block_info.mark_locally_rejected() {
-            warn!("{self}: Failed to mark block as locally rejected: {e:?}",);
-            return None;
+            if !block_info.has_reached_consensus() {
+                warn!("{self}: Failed to mark block as locally rejected: {e:?}",);
+                return None;
+            }
         }
         let block_rejection = BlockRejection::from_validate_rejection(
             block_validate_reject.clone(),
@@ -626,7 +628,7 @@ impl Signer {
         );
         self.signer_db
             .insert_block(&block_info)
-            .unwrap_or_else(|_| panic!("{self}: Failed to insert block in DB"));
+            .unwrap_or_else(|e| self.handle_insert_block_error(e));
         self.handle_block_rejection(&block_rejection);
         Some(BlockResponse::Rejected(block_rejection))
     }
@@ -739,7 +741,7 @@ impl Signer {
         }
         self.signer_db
             .insert_block(&block_info)
-            .unwrap_or_else(|_| panic!("{self}: Failed to insert block in DB"));
+            .unwrap_or_else(|e| self.handle_insert_block_error(e));
     }
 
     /// Compute the signing weight, given a list of signatures
@@ -1095,7 +1097,7 @@ impl Signer {
             // in case this is the first time we saw this block. Safe to do since this is testing case only.
             self.signer_db
                 .insert_block(block_info)
-                .unwrap_or_else(|_| panic!("{self}: Failed to insert block in DB"));
+                .unwrap_or_else(|e| self.handle_insert_block_error(e));
             Some(BlockResponse::rejected(
                 block_proposal.block.header.signer_signature_hash(),
                 RejectCode::TestingDirective,
@@ -1119,4 +1121,10 @@ impl Signer {
             warn!("{self}: Failed to send mock signature to stacker-db: {e:?}",);
         }
     }
+
+    /// Helper for logging insert_block error
+    fn handle_insert_block_error(&self, e: DBError) {
+        error!("{self}: Failed to insert block into signer-db: {e:?}");
+        panic!("{self} Failed to write block to signerdb: {e}");
+    }
 }

stackslib/src/burnchains/bitcoin/indexer.rs

@@ -45,7 +45,7 @@ use crate::burnchains::{
     Burnchain, BurnchainBlockHeader, Error as burnchain_error, MagicBytes, BLOCKSTACK_MAGIC_MAINNET,
 };
 use crate::core::{
-    StacksEpoch, StacksEpochExtension, STACKS_EPOCHS_MAINNET, STACKS_EPOCHS_REGTEST,
+    EpochList, StacksEpoch, StacksEpochExtension, STACKS_EPOCHS_MAINNET, STACKS_EPOCHS_REGTEST,
     STACKS_EPOCHS_TESTNET,
 };
 use crate::util_lib::db::Error as DBError;
@@ -91,11 +91,11 @@ impl TryFrom<u32> for BitcoinNetworkType {
 /// Get the default epochs definitions for the given BitcoinNetworkType.
 /// Should *not* be used except by the BitcoinIndexer when no epochs vector
 /// was specified.
-pub fn get_bitcoin_stacks_epochs(network_id: BitcoinNetworkType) -> Vec<StacksEpoch> {
+pub fn get_bitcoin_stacks_epochs(network_id: BitcoinNetworkType) -> EpochList {
     match network_id {
-        BitcoinNetworkType::Mainnet => STACKS_EPOCHS_MAINNET.to_vec(),
-        BitcoinNetworkType::Testnet => STACKS_EPOCHS_TESTNET.to_vec(),
-        BitcoinNetworkType::Regtest => STACKS_EPOCHS_REGTEST.to_vec(),
+        BitcoinNetworkType::Mainnet => (*STACKS_EPOCHS_MAINNET).clone(),
+        BitcoinNetworkType::Testnet => (*STACKS_EPOCHS_TESTNET).clone(),
+        BitcoinNetworkType::Regtest => (*STACKS_EPOCHS_REGTEST).clone(),
     }
 }
 
@@ -112,7 +112,7 @@ pub struct BitcoinIndexerConfig {
     pub spv_headers_path: String,
     pub first_block: u64,
     pub magic_bytes: MagicBytes,
-    pub epochs: Option<Vec<StacksEpoch>>,
+    pub epochs: Option<EpochList>,
 }
 
 #[derive(Debug)]
@@ -1041,7 +1041,7 @@ impl BurnchainIndexer for BitcoinIndexer {
     /// 2) Use hard-coded static values, otherwise.
     ///
     /// It is an error (panic) to set custom epochs if running on `Mainnet`.
-    fn get_stacks_epochs(&self) -> Vec<StacksEpoch> {
+    fn get_stacks_epochs(&self) -> EpochList {
         StacksEpoch::get_epochs(self.runtime.network_id, self.config.epochs.as_ref())
     }