Merge pull request #5509 from stacks-network/clarity-wasm-merge-develop

Merge develop onto clarity wasm

Author: hugoclrd

.github/actions/dockerfiles/Dockerfile.debian-source

@@ -24,5 +24,5 @@ RUN --mount=type=tmpfs,target=${BUILD_DIR} cp -R /src/. ${BUILD_DIR}/ \
     && cp -R ${BUILD_DIR}/target/${TARGET}/release/. /out
 
 FROM --platform=${TARGETPLATFORM} debian:bookworm
-COPY --from=build /out/stacks-node /out/stacks-signer /bin/
+COPY --from=build /out/stacks-node /out/stacks-signer /out/stacks-inspect /bin/
 CMD ["stacks-node", "mainnet"]

.github/workflows/bitcoin-tests.yml

@@ -99,6 +99,8 @@ jobs:
           - tests::signer::v0::forked_tenure_okay
           - tests::signer::v0::forked_tenure_invalid
           - tests::signer::v0::empty_sortition
+          - tests::signer::v0::empty_sortition_before_approval
+          - tests::signer::v0::empty_sortition_before_proposal
           - tests::signer::v0::bitcoind_forking_test
           - tests::signer::v0::multiple_miners
           - tests::signer::v0::mock_sign_epoch_25
@@ -112,6 +114,7 @@ jobs:
           - tests::signer::v0::locally_accepted_blocks_overriden_by_global_rejection
           - tests::signer::v0::locally_rejected_blocks_overriden_by_global_acceptance
           - tests::signer::v0::reorg_locally_accepted_blocks_across_tenures_succeeds
+          - tests::signer::v0::reorg_locally_accepted_blocks_across_tenures_fails
           - tests::signer::v0::miner_recovers_when_broadcast_block_delay_across_tenures_occurs
           - tests::signer::v0::multiple_miners_with_nakamoto_blocks
           - tests::signer::v0::partial_tenure_fork
@@ -120,6 +123,10 @@ jobs:
           - tests::signer::v0::signing_in_0th_tenure_of_reward_cycle
           - tests::signer::v0::continue_after_tenure_extend
           - tests::signer::v0::multiple_miners_with_custom_chain_id
+          - tests::signer::v0::block_commit_delay
+          - tests::signer::v0::continue_after_fast_block_no_sortition
+          - tests::signer::v0::block_validation_response_timeout
+          - tests::signer::v0::tenure_extend_after_bad_commit
           - tests::nakamoto_integrations::burn_ops_integration_test
           - tests::nakamoto_integrations::check_block_heights
           - tests::nakamoto_integrations::clarity_burn_state
@@ -133,6 +140,10 @@ jobs:
           - tests::nakamoto_integrations::utxo_check_on_startup_panic
           - tests::nakamoto_integrations::utxo_check_on_startup_recover
           - tests::nakamoto_integrations::v3_signer_api_endpoint
+          - tests::nakamoto_integrations::test_shadow_recovery
+          - tests::nakamoto_integrations::signer_chainstate
+          - tests::nakamoto_integrations::clarity_cost_spend_down
+          - tests::nakamoto_integrations::v3_blockbyheight_api_endpoint
           # TODO: enable these once v1 signer is supported by a new nakamoto epoch
           # - tests::signer::v1::dkg
           # - tests::signer::v1::sign_request_rejected

.github/workflows/p2p-tests.yml

@@ -42,6 +42,11 @@ jobs:
           - net::tests::convergence::test_walk_star_15_pingback
           - net::tests::convergence::test_walk_star_15_org_biased
           - net::tests::convergence::test_walk_inbound_line_15
+          - net::api::tests::postblock_proposal::test_try_make_response
+          - net::server::tests::test_http_10_threads_getinfo
+          - net::server::tests::test_http_10_threads_getblock
+          - net::server::tests::test_http_too_many_clients
+          - net::server::tests::test_http_slow_client
     steps:
       ## Setup test environment
       - name: Setup Test Environment

CHANGELOG.md

@@ -7,6 +7,48 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE
 
 ## [Unreleased]
 
+### Added
+
+### Changed
+
+
+## [3.0.0.0.3]
+
+### Added
+
+### Changed
+- Add index for StacksBlockId to nakamoto block headers table (improves node performance)
+- Remove the panic for reporting DB deadlocks (just error and continue waiting)
+- Add index to `metadata_table` in Clarity DB on `blockhash`
+- Add `block_commit_delay_ms` to the config file to control the time to wait after seeing a new burn block, before submitting a block commit, to allow time for the first Nakamoto block of the new tenure to be mined, allowing this miner to avoid the need to RBF the block commit.
+- Add `tenure_cost_limit_per_block_percentage` to the miner config file to control the percentage remaining tenure cost limit to consume per nakamoto block.
+- Add `/v3/blocks/height/:block_height` rpc endpoint
+- If the winning miner of a sortition is committed to the wrong parent tenure, the previous miner can immediately tenure extend and continue mining since the winning miner would never be able to propose a valid block. (#5361)
+
+## [3.0.0.0.2]
+
+### Added
+
+### Changed
+- Fixes  a few bugs in the relayer and networking stack
+  - detects and deprioritizes unhealthy replicas
+  - fixes an issue in the p2p stack which was preventing it from caching the reward set.
+
+## [3.0.0.0.1]
+
+### Changed
+- Add index for StacksBlockId to nakamoto block headers table (improves node performance)
+- Remove the panic for reporting DB deadlocks (just error and continue waiting)
+- Various test fixes for CI (5353, 5368, 5372, 5371, 5380, 5378, 5387, 5396, 5390, 5394)
+- Various log fixes:
+    - don't say proceeding to mine blocks if not a miner
+    - misc. warns downgraded to debugs
+- 5391: Update default block proposal timeout to 10 minutes
+- 5406: After block rejection, miner pauses
+- Docs fixes
+    - Fix signer docs link
+    - Specify burn block in clarity docs
+
 ## [3.0.0.0.0]
 
 ### Added

Cargo.lock

@@ -22,6 +22,7 @@ rand = "0.8"
 rand_chacha = "0.3.1"
 tikv-jemallocator = "0.5.4"
 rusqlite = { version = "0.31.0", features = ["blob", "serde_json", "i128_blob", "bundled", "trace"] }
+thiserror = { version = "1.0.65" }
 
 # Use a bit more than default optimization for
 #  dev builds to speed up test execution

Cargo.toml

@@ -904,6 +904,7 @@ impl LimitedCostTracker {
             Self::Free => ExecutionCost::max_value(),
         }
     }
+
     pub fn get_memory(&self) -> u64 {
         match self {
             Self::Limited(TrackerData { memory, .. }) => *memory,
@@ -1178,6 +1179,7 @@ pub trait CostOverflowingMath<T> {
     fn cost_overflow_mul(self, other: T) -> Result<T>;
     fn cost_overflow_add(self, other: T) -> Result<T>;
     fn cost_overflow_sub(self, other: T) -> Result<T>;
+    fn cost_overflow_div(self, other: T) -> Result<T>;
 }
 
 impl CostOverflowingMath<u64> for u64 {
@@ -1193,6 +1195,10 @@ impl CostOverflowingMath<u64> for u64 {
         self.checked_sub(other)
             .ok_or_else(|| CostErrors::CostOverflow)
     }
+    fn cost_overflow_div(self, other: u64) -> Result<u64> {
+        self.checked_div(other)
+            .ok_or_else(|| CostErrors::CostOverflow)
+    }
 }
 
 impl ExecutionCost {
@@ -1301,6 +1307,15 @@ impl ExecutionCost {
         Ok(())
     }
 
+    pub fn divide(&mut self, divisor: u64) -> Result<()> {
+        self.runtime = self.runtime.cost_overflow_div(divisor)?;
+        self.read_count = self.read_count.cost_overflow_div(divisor)?;
+        self.read_length = self.read_length.cost_overflow_div(divisor)?;
+        self.write_length = self.write_length.cost_overflow_div(divisor)?;
+        self.write_count = self.write_count.cost_overflow_div(divisor)?;
+        Ok(())
+    }
+
     /// Returns whether or not this cost exceeds any dimension of the
     ///  other cost.
     pub fn exceeds(&self, other: &ExecutionCost) -> bool {

clarity/src/vm/costs/mod.rs

@@ -23,7 +23,7 @@ use stacks_common::consts::{
 };
 use stacks_common::types::chainstate::{
     BlockHeaderHash, BurnchainHeaderHash, ConsensusHash, SortitionId, StacksAddress, StacksBlockId,
-    VRFSeed,
+    TrieHash, VRFSeed,
 };
 use stacks_common::types::{Address, StacksEpoch as GenericStacksEpoch, StacksEpochId};
 use stacks_common::util::hash::{to_hex, Hash160, Sha256Sum, Sha512Trunc256Sum};
@@ -76,6 +76,68 @@ pub enum StoreType {
     PoxUnlockHeight = 0x15,
 }
 
+impl TryFrom<&str> for StoreType {
+    type Error = String;
+
+    fn try_from(value: &str) -> core::result::Result<Self, Self::Error> {
+        use self::StoreType::*;
+
+        let hex_value = u8::from_str_radix(value, 10).map_err(|e| e.to_string())?;
+        match hex_value {
+            0x00 => Ok(DataMap),
+            0x01 => Ok(Variable),
+            0x02 => Ok(FungibleToken),
+            0x03 => Ok(CirculatingSupply),
+            0x04 => Ok(NonFungibleToken),
+            0x05 => Ok(DataMapMeta),
+            0x06 => Ok(VariableMeta),
+            0x07 => Ok(FungibleTokenMeta),
+            0x08 => Ok(NonFungibleTokenMeta),
+            0x09 => Ok(Contract),
+            0x10 => Ok(SimmedBlock),
+            0x11 => Ok(SimmedBlockHeight),
+            0x12 => Ok(Nonce),
+            0x13 => Ok(STXBalance),
+            0x14 => Ok(PoxSTXLockup),
+            0x15 => Ok(PoxUnlockHeight),
+            _ => Err("Invalid StoreType".into()),
+        }
+    }
+}
+
+pub enum ContractDataVarName {
+    Contract,
+    ContractSize,
+    ContractSrc,
+    ContractDataSize,
+}
+
+impl ContractDataVarName {
+    pub fn as_str(&self) -> &str {
+        match self {
+            Self::Contract => "contract",
+            Self::ContractSize => "contract-size",
+            Self::ContractSrc => "contract-src",
+            Self::ContractDataSize => "contract-data-size",
+        }
+    }
+}
+
+impl TryFrom<&str> for ContractDataVarName {
+    type Error = String;
+
+    fn try_from(value: &str) -> core::result::Result<Self, Self::Error> {
+        use self::ContractDataVarName::*;
+        match value {
+            "contract" => Ok(Contract),
+            "contract-size" => Ok(ContractSize),
+            "contract-src" => Ok(ContractSrc),
+            "contract-data-size" => Ok(ContractDataSize),
+            _ => Err("Invalid ContractDataVarName".into()),
+        }
+    }
+}
+
 pub struct ClarityDatabase<'a> {
     pub store: RollbackWrapper<'a>,
     headers_db: &'a dyn HeadersDB,
@@ -465,6 +527,13 @@ impl<'a> ClarityDatabase<'a> {
         self.store.get_data::<T>(key)
     }
 
+    pub fn get_data_by_hash<T>(&mut self, hash: &TrieHash) -> Result<Option<T>>
+    where
+        T: ClarityDeserializable<T>,
+    {
+        self.store.get_data_by_hash::<T>(hash)
+    }
+
     pub fn put_value(&mut self, key: &str, value: Value, epoch: &StacksEpochId) -> Result<()> {
         self.put_value_with_size(key, value, epoch)?;
         Ok(())
@@ -522,6 +591,16 @@ impl<'a> ClarityDatabase<'a> {
         self.store.get_data_with_proof(key)
     }
 
+    pub fn get_data_with_proof_by_hash<T>(
+        &mut self,
+        hash: &TrieHash,
+    ) -> Result<Option<(T, Vec<u8>)>>
+    where
+        T: ClarityDeserializable<T>,
+    {
+        self.store.get_data_with_proof_by_hash(hash)
+    }
+
     pub fn make_key_for_trip(
         contract_identifier: &QualifiedContractIdentifier,
         data: StoreType,
@@ -559,12 +638,18 @@ impl<'a> ClarityDatabase<'a> {
         self.store
             .prepare_for_contract_metadata(contract_identifier, hash)?;
         // insert contract-size
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-size");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::ContractSize.as_str(),
+        );
         self.insert_metadata(contract_identifier, &key, &(contract_content.len() as u64))?;
 
         // insert contract-src
         if STORE_CONTRACT_SRC_INTERFACE {
-            let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-src");
+            let key = ClarityDatabase::make_metadata_key(
+                StoreType::Contract,
+                ContractDataVarName::ContractSrc.as_str(),
+            );
             self.insert_metadata(contract_identifier, &key, &contract_content.to_string())?;
         }
         Ok(())
@@ -574,7 +659,10 @@ impl<'a> ClarityDatabase<'a> {
         &mut self,
         contract_identifier: &QualifiedContractIdentifier,
     ) -> Option<String> {
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-src");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::ContractSrc.as_str(),
+        );
         self.fetch_metadata(contract_identifier, &key)
             .ok()
             .flatten()
@@ -683,15 +771,21 @@ impl<'a> ClarityDatabase<'a> {
         &mut self,
         contract_identifier: &QualifiedContractIdentifier,
     ) -> Result<u64> {
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-size");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::ContractSize.as_str(),
+        );
         let contract_size: u64 =
             self.fetch_metadata(contract_identifier, &key)?
                 .ok_or_else(|| {
                     InterpreterError::Expect(
             "Failed to read non-consensus contract metadata, even though contract exists in MARF."
         .into())
                 })?;
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-data-size");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::ContractDataSize.as_str(),
+        );
         let data_size: u64 = self
             .fetch_metadata(contract_identifier, &key)?
             .ok_or_else(|| {
@@ -710,7 +804,10 @@ impl<'a> ClarityDatabase<'a> {
         contract_identifier: &QualifiedContractIdentifier,
         data_size: u64,
     ) -> Result<()> {
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-size");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::ContractSize.as_str(),
+        );
         let contract_size: u64 =
             self.fetch_metadata(contract_identifier, &key)?
                 .ok_or_else(|| {
@@ -720,7 +817,10 @@ impl<'a> ClarityDatabase<'a> {
                 })?;
         contract_size.cost_overflow_add(data_size)?;
 
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-data-size");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::ContractDataSize.as_str(),
+        );
         self.insert_metadata(contract_identifier, &key, &data_size)?;
         Ok(())
     }
@@ -730,21 +830,30 @@ impl<'a> ClarityDatabase<'a> {
         contract_identifier: &QualifiedContractIdentifier,
         contract: Contract,
     ) -> Result<()> {
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::Contract.as_str(),
+        );
         self.insert_metadata(contract_identifier, &key, &contract)?;
         Ok(())
     }
 
     pub fn has_contract(&mut self, contract_identifier: &QualifiedContractIdentifier) -> bool {
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::Contract.as_str(),
+        );
         self.store.has_metadata_entry(contract_identifier, &key)
     }
 
     pub fn get_contract(
         &mut self,
         contract_identifier: &QualifiedContractIdentifier,
     ) -> Result<Contract> {
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::Contract.as_str(),
+        );
         let mut data: Contract = self.fetch_metadata(contract_identifier, &key)?
             .ok_or_else(|| InterpreterError::Expect(
                 "Failed to read non-consensus contract metadata, even though contract exists in MARF."