Merge branch 'develop' of https://github.com/stacks-network/stacks-core into bug/validation-queue-race-condition

Author: jferrant

CHANGELOG.md

@@ -7,6 +7,15 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE
 
 ## [Unreleased]
 
+### Added
+
+### Changed
+
+
+## [3.0.0.0.3]
+
+### Added
+
 ### Changed
 - Add index for StacksBlockId to nakamoto block headers table (improves node performance)
 - Remove the panic for reporting DB deadlocks (just error and continue waiting)
@@ -16,6 +25,15 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE
 - Add `/v3/blocks/height/:block_height` rpc endpoint
 - If the winning miner of a sortition is committed to the wrong parent tenure, the previous miner can immediately tenure extend and continue mining since the winning miner would never be able to propose a valid block. (#5361)
 
+## [3.0.0.0.2]
+
+### Added
+
+### Changed
+- Fixes  a few bugs in the relayer and networking stack
+  - detects and deprioritizes unhealthy replicas
+  - fixes an issue in the p2p stack which was preventing it from caching the reward set.
+
 ## [3.0.0.0.1]
 
 ### Changed

clarity/src/vm/database/clarity_db.rs

@@ -23,7 +23,7 @@ use stacks_common::consts::{
 };
 use stacks_common::types::chainstate::{
     BlockHeaderHash, BurnchainHeaderHash, ConsensusHash, SortitionId, StacksAddress, StacksBlockId,
-    VRFSeed,
+    TrieHash, VRFSeed,
 };
 use stacks_common::types::{Address, StacksEpoch as GenericStacksEpoch, StacksEpochId};
 use stacks_common::util::hash::{to_hex, Hash160, Sha256Sum, Sha512Trunc256Sum};
@@ -76,6 +76,68 @@ pub enum StoreType {
     PoxUnlockHeight = 0x15,
 }
 
+impl TryFrom<&str> for StoreType {
+    type Error = String;
+
+    fn try_from(value: &str) -> core::result::Result<Self, Self::Error> {
+        use self::StoreType::*;
+
+        let hex_value = u8::from_str_radix(value, 10).map_err(|e| e.to_string())?;
+        match hex_value {
+            0x00 => Ok(DataMap),
+            0x01 => Ok(Variable),
+            0x02 => Ok(FungibleToken),
+            0x03 => Ok(CirculatingSupply),
+            0x04 => Ok(NonFungibleToken),
+            0x05 => Ok(DataMapMeta),
+            0x06 => Ok(VariableMeta),
+            0x07 => Ok(FungibleTokenMeta),
+            0x08 => Ok(NonFungibleTokenMeta),
+            0x09 => Ok(Contract),
+            0x10 => Ok(SimmedBlock),
+            0x11 => Ok(SimmedBlockHeight),
+            0x12 => Ok(Nonce),
+            0x13 => Ok(STXBalance),
+            0x14 => Ok(PoxSTXLockup),
+            0x15 => Ok(PoxUnlockHeight),
+            _ => Err("Invalid StoreType".into()),
+        }
+    }
+}
+
+pub enum ContractDataVarName {
+    Contract,
+    ContractSize,
+    ContractSrc,
+    ContractDataSize,
+}
+
+impl ContractDataVarName {
+    pub fn as_str(&self) -> &str {
+        match self {
+            Self::Contract => "contract",
+            Self::ContractSize => "contract-size",
+            Self::ContractSrc => "contract-src",
+            Self::ContractDataSize => "contract-data-size",
+        }
+    }
+}
+
+impl TryFrom<&str> for ContractDataVarName {
+    type Error = String;
+
+    fn try_from(value: &str) -> core::result::Result<Self, Self::Error> {
+        use self::ContractDataVarName::*;
+        match value {
+            "contract" => Ok(Contract),
+            "contract-size" => Ok(ContractSize),
+            "contract-src" => Ok(ContractSrc),
+            "contract-data-size" => Ok(ContractDataSize),
+            _ => Err("Invalid ContractDataVarName".into()),
+        }
+    }
+}
+
 pub struct ClarityDatabase<'a> {
     pub store: RollbackWrapper<'a>,
     headers_db: &'a dyn HeadersDB,
@@ -465,6 +527,13 @@ impl<'a> ClarityDatabase<'a> {
         self.store.get_data::<T>(key)
     }
 
+    pub fn get_data_by_hash<T>(&mut self, hash: &TrieHash) -> Result<Option<T>>
+    where
+        T: ClarityDeserializable<T>,
+    {
+        self.store.get_data_by_hash::<T>(hash)
+    }
+
     pub fn put_value(&mut self, key: &str, value: Value, epoch: &StacksEpochId) -> Result<()> {
         self.put_value_with_size(key, value, epoch)?;
         Ok(())
@@ -522,6 +591,16 @@ impl<'a> ClarityDatabase<'a> {
         self.store.get_data_with_proof(key)
     }
 
+    pub fn get_data_with_proof_by_hash<T>(
+        &mut self,
+        hash: &TrieHash,
+    ) -> Result<Option<(T, Vec<u8>)>>
+    where
+        T: ClarityDeserializable<T>,
+    {
+        self.store.get_data_with_proof_by_hash(hash)
+    }
+
     pub fn make_key_for_trip(
         contract_identifier: &QualifiedContractIdentifier,
         data: StoreType,
@@ -559,12 +638,18 @@ impl<'a> ClarityDatabase<'a> {
         self.store
             .prepare_for_contract_metadata(contract_identifier, hash)?;
         // insert contract-size
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-size");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::ContractSize.as_str(),
+        );
         self.insert_metadata(contract_identifier, &key, &(contract_content.len() as u64))?;
 
         // insert contract-src
         if STORE_CONTRACT_SRC_INTERFACE {
-            let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-src");
+            let key = ClarityDatabase::make_metadata_key(
+                StoreType::Contract,
+                ContractDataVarName::ContractSrc.as_str(),
+            );
             self.insert_metadata(contract_identifier, &key, &contract_content.to_string())?;
         }
         Ok(())
@@ -574,7 +659,10 @@ impl<'a> ClarityDatabase<'a> {
         &mut self,
         contract_identifier: &QualifiedContractIdentifier,
     ) -> Option<String> {
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-src");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::ContractSrc.as_str(),
+        );
         self.fetch_metadata(contract_identifier, &key)
             .ok()
             .flatten()
@@ -683,15 +771,21 @@ impl<'a> ClarityDatabase<'a> {
         &mut self,
         contract_identifier: &QualifiedContractIdentifier,
     ) -> Result<u64> {
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-size");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::ContractSize.as_str(),
+        );
         let contract_size: u64 =
             self.fetch_metadata(contract_identifier, &key)?
                 .ok_or_else(|| {
                     InterpreterError::Expect(
             "Failed to read non-consensus contract metadata, even though contract exists in MARF."
         .into())
                 })?;
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-data-size");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::ContractDataSize.as_str(),
+        );
         let data_size: u64 = self
             .fetch_metadata(contract_identifier, &key)?
             .ok_or_else(|| {
@@ -710,7 +804,10 @@ impl<'a> ClarityDatabase<'a> {
         contract_identifier: &QualifiedContractIdentifier,
         data_size: u64,
     ) -> Result<()> {
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-size");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::ContractSize.as_str(),
+        );
         let contract_size: u64 =
             self.fetch_metadata(contract_identifier, &key)?
                 .ok_or_else(|| {
@@ -720,7 +817,10 @@ impl<'a> ClarityDatabase<'a> {
                 })?;
         contract_size.cost_overflow_add(data_size)?;
 
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-data-size");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::ContractDataSize.as_str(),
+        );
         self.insert_metadata(contract_identifier, &key, &data_size)?;
         Ok(())
     }
@@ -730,21 +830,30 @@ impl<'a> ClarityDatabase<'a> {
         contract_identifier: &QualifiedContractIdentifier,
         contract: Contract,
     ) -> Result<()> {
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::Contract.as_str(),
+        );
         self.insert_metadata(contract_identifier, &key, &contract)?;
         Ok(())
     }
 
     pub fn has_contract(&mut self, contract_identifier: &QualifiedContractIdentifier) -> bool {
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::Contract.as_str(),
+        );
         self.store.has_metadata_entry(contract_identifier, &key)
     }
 
     pub fn get_contract(
         &mut self,
         contract_identifier: &QualifiedContractIdentifier,
     ) -> Result<Contract> {
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::Contract.as_str(),
+        );
         let mut data: Contract = self.fetch_metadata(contract_identifier, &key)?
             .ok_or_else(|| InterpreterError::Expect(
                 "Failed to read non-consensus contract metadata, even though contract exists in MARF."

clarity/src/vm/database/clarity_store.rs

@@ -18,7 +18,7 @@ use std::path::PathBuf;
 
 #[cfg(feature = "canonical")]
 use rusqlite::Connection;
-use stacks_common::types::chainstate::{BlockHeaderHash, StacksBlockId, VRFSeed};
+use stacks_common::types::chainstate::{BlockHeaderHash, StacksBlockId, TrieHash, VRFSeed};
 use stacks_common::util::hash::{hex_bytes, to_hex, Hash160, Sha512Trunc256Sum};
 
 use crate::vm::analysis::AnalysisDatabase;
@@ -64,9 +64,15 @@ pub trait ClarityBackingStore {
     fn put_all_data(&mut self, items: Vec<(String, String)>) -> Result<()>;
     /// fetch K-V out of the committed datastore
     fn get_data(&mut self, key: &str) -> Result<Option<String>>;
+    /// fetch Hash(K)-V out of the commmitted datastore
+    fn get_data_from_path(&mut self, hash: &TrieHash) -> Result<Option<String>>;
     /// fetch K-V out of the committed datastore, along with the byte representation
     ///  of the Merkle proof for that key-value pair
     fn get_data_with_proof(&mut self, key: &str) -> Result<Option<(String, Vec<u8>)>>;
+    fn get_data_with_proof_from_path(
+        &mut self,
+        hash: &TrieHash,
+    ) -> Result<Option<(String, Vec<u8>)>>;
     fn has_entry(&mut self, key: &str) -> Result<bool> {
         Ok(self.get_data(key)?.is_some())
     }
@@ -209,10 +215,21 @@ impl ClarityBackingStore for NullBackingStore {
         panic!("NullBackingStore can't retrieve data")
     }
 
+    fn get_data_from_path(&mut self, _hash: &TrieHash) -> Result<Option<String>> {
+        panic!("NullBackingStore can't retrieve data")
+    }
+
     fn get_data_with_proof(&mut self, _key: &str) -> Result<Option<(String, Vec<u8>)>> {
         panic!("NullBackingStore can't retrieve data")
     }
 
+    fn get_data_with_proof_from_path(
+        &mut self,
+        _hash: &TrieHash,
+    ) -> Result<Option<(String, Vec<u8>)>> {
+        panic!("NullBackingStore can't retrieve data")
+    }
+
     #[cfg(feature = "canonical")]
     fn get_side_store(&mut self) -> &Connection {
         panic!("NullBackingStore has no side store")

clarity/src/vm/database/key_value_wrapper.rs

@@ -17,7 +17,7 @@
 use std::hash::Hash;
 
 use hashbrown::HashMap;
-use stacks_common::types::chainstate::StacksBlockId;
+use stacks_common::types::chainstate::{StacksBlockId, TrieHash};
 use stacks_common::types::StacksEpochId;
 use stacks_common::util::hash::Sha512Trunc256Sum;
 
@@ -369,6 +369,21 @@ impl<'a> RollbackWrapper<'a> {
             .transpose()
     }
 
+    /// this function will only return commitment proofs for values _already_ materialized
+    ///  in the underlying store. otherwise it returns None.
+    pub fn get_data_with_proof_by_hash<T>(
+        &mut self,
+        hash: &TrieHash,
+    ) -> InterpreterResult<Option<(T, Vec<u8>)>>
+    where
+        T: ClarityDeserializable<T>,
+    {
+        self.store
+            .get_data_with_proof_from_path(hash)?
+            .map(|(value, proof)| Ok((T::deserialize(&value)?, proof)))
+            .transpose()
+    }
+
     pub fn get_data<T>(&mut self, key: &str) -> InterpreterResult<Option<T>>
     where
         T: ClarityDeserializable<T>,
@@ -392,6 +407,23 @@ impl<'a> RollbackWrapper<'a> {
             .transpose()
     }
 
+    /// DO NOT USE IN CONSENSUS CODE.
+    ///
+    /// Load data directly from the underlying store, given its trie hash.  The lookup map will not
+    /// be used.
+    ///
+    /// This should never be called from within the Clarity VM, or via block-processing.  It's only
+    /// meant to be used by the RPC system.
+    pub fn get_data_by_hash<T>(&mut self, hash: &TrieHash) -> InterpreterResult<Option<T>>
+    where
+        T: ClarityDeserializable<T>,
+    {
+        self.store
+            .get_data_from_path(hash)?
+            .map(|x| T::deserialize(&x))
+            .transpose()
+    }
+
     pub fn deserialize_value(
         value_hex: &str,
         expected: &TypeSignature,

clarity/src/vm/database/sqlite.rs

@@ -19,7 +19,7 @@ use rusqlite::{
     params, Connection, Error as SqliteError, ErrorCode as SqliteErrorCode, OptionalExtension, Row,
     Savepoint,
 };
-use stacks_common::types::chainstate::{BlockHeaderHash, StacksBlockId};
+use stacks_common::types::chainstate::{BlockHeaderHash, StacksBlockId, TrieHash};
 use stacks_common::types::sqlite::NO_PARAMS;
 use stacks_common::util::db::tx_busy_handler;
 use stacks_common::util::hash::Sha512Trunc256Sum;
@@ -330,10 +330,21 @@ impl ClarityBackingStore for MemoryBackingStore {
         SqliteConnection::get(self.get_side_store(), key)
     }
 
+    fn get_data_from_path(&mut self, hash: &TrieHash) -> Result<Option<String>> {
+        SqliteConnection::get(self.get_side_store(), hash.to_string().as_str())
+    }
+
     fn get_data_with_proof(&mut self, key: &str) -> Result<Option<(String, Vec<u8>)>> {
         Ok(SqliteConnection::get(self.get_side_store(), key)?.map(|x| (x, vec![])))
     }
 
+    fn get_data_with_proof_from_path(
+        &mut self,
+        hash: &TrieHash,
+    ) -> Result<Option<(String, Vec<u8>)>> {
+        self.get_data_with_proof(&hash.to_string())
+    }
+
     fn get_side_store(&mut self) -> &Connection {
         &self.side_store
     }