Merge pull request #4997 from stacks-network/feat/rpc-endpoints-to-fetch-data-from-key

Feat/rpc endpoints to fetch data from key

Author: obycode

clarity/src/vm/database/clarity_db.rs

@@ -23,7 +23,7 @@ use stacks_common::consts::{
 };
 use stacks_common::types::chainstate::{
     BlockHeaderHash, BurnchainHeaderHash, ConsensusHash, SortitionId, StacksAddress, StacksBlockId,
-    VRFSeed,
+    TrieHash, VRFSeed,
 };
 use stacks_common::types::{Address, StacksEpoch as GenericStacksEpoch, StacksEpochId};
 use stacks_common::util::hash::{to_hex, Hash160, Sha256Sum, Sha512Trunc256Sum};
@@ -76,6 +76,68 @@ pub enum StoreType {
     PoxUnlockHeight = 0x15,
 }
 
+impl TryFrom<&str> for StoreType {
+    type Error = String;
+
+    fn try_from(value: &str) -> core::result::Result<Self, Self::Error> {
+        use self::StoreType::*;
+
+        let hex_value = u8::from_str_radix(value, 10).map_err(|e| e.to_string())?;
+        match hex_value {
+            0x00 => Ok(DataMap),
+            0x01 => Ok(Variable),
+            0x02 => Ok(FungibleToken),
+            0x03 => Ok(CirculatingSupply),
+            0x04 => Ok(NonFungibleToken),
+            0x05 => Ok(DataMapMeta),
+            0x06 => Ok(VariableMeta),
+            0x07 => Ok(FungibleTokenMeta),
+            0x08 => Ok(NonFungibleTokenMeta),
+            0x09 => Ok(Contract),
+            0x10 => Ok(SimmedBlock),
+            0x11 => Ok(SimmedBlockHeight),
+            0x12 => Ok(Nonce),
+            0x13 => Ok(STXBalance),
+            0x14 => Ok(PoxSTXLockup),
+            0x15 => Ok(PoxUnlockHeight),
+            _ => Err("Invalid StoreType".into()),
+        }
+    }
+}
+
+pub enum ContractDataVarName {
+    Contract,
+    ContractSize,
+    ContractSrc,
+    ContractDataSize,
+}
+
+impl ContractDataVarName {
+    pub fn as_str(&self) -> &str {
+        match self {
+            Self::Contract => "contract",
+            Self::ContractSize => "contract-size",
+            Self::ContractSrc => "contract-src",
+            Self::ContractDataSize => "contract-data-size",
+        }
+    }
+}
+
+impl TryFrom<&str> for ContractDataVarName {
+    type Error = String;
+
+    fn try_from(value: &str) -> core::result::Result<Self, Self::Error> {
+        use self::ContractDataVarName::*;
+        match value {
+            "contract" => Ok(Contract),
+            "contract-size" => Ok(ContractSize),
+            "contract-src" => Ok(ContractSrc),
+            "contract-data-size" => Ok(ContractDataSize),
+            _ => Err("Invalid ContractDataVarName".into()),
+        }
+    }
+}
+
 pub struct ClarityDatabase<'a> {
     pub store: RollbackWrapper<'a>,
     headers_db: &'a dyn HeadersDB,
@@ -465,6 +527,13 @@ impl<'a> ClarityDatabase<'a> {
         self.store.get_data::<T>(key)
     }
 
+    pub fn get_data_by_hash<T>(&mut self, hash: &TrieHash) -> Result<Option<T>>
+    where
+        T: ClarityDeserializable<T>,
+    {
+        self.store.get_data_by_hash::<T>(hash)
+    }
+
     pub fn put_value(&mut self, key: &str, value: Value, epoch: &StacksEpochId) -> Result<()> {
         self.put_value_with_size(key, value, epoch)?;
         Ok(())
@@ -522,6 +591,16 @@ impl<'a> ClarityDatabase<'a> {
         self.store.get_data_with_proof(key)
     }
 
+    pub fn get_data_with_proof_by_hash<T>(
+        &mut self,
+        hash: &TrieHash,
+    ) -> Result<Option<(T, Vec<u8>)>>
+    where
+        T: ClarityDeserializable<T>,
+    {
+        self.store.get_data_with_proof_by_hash(hash)
+    }
+
     pub fn make_key_for_trip(
         contract_identifier: &QualifiedContractIdentifier,
         data: StoreType,
@@ -559,12 +638,18 @@ impl<'a> ClarityDatabase<'a> {
         self.store
             .prepare_for_contract_metadata(contract_identifier, hash)?;
         // insert contract-size
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-size");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::ContractSize.as_str(),
+        );
         self.insert_metadata(contract_identifier, &key, &(contract_content.len() as u64))?;
 
         // insert contract-src
         if STORE_CONTRACT_SRC_INTERFACE {
-            let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-src");
+            let key = ClarityDatabase::make_metadata_key(
+                StoreType::Contract,
+                ContractDataVarName::ContractSrc.as_str(),
+            );
             self.insert_metadata(contract_identifier, &key, &contract_content.to_string())?;
         }
         Ok(())
@@ -574,7 +659,10 @@ impl<'a> ClarityDatabase<'a> {
         &mut self,
         contract_identifier: &QualifiedContractIdentifier,
     ) -> Option<String> {
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-src");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::ContractSrc.as_str(),
+        );
         self.fetch_metadata(contract_identifier, &key)
             .ok()
             .flatten()
@@ -683,15 +771,21 @@ impl<'a> ClarityDatabase<'a> {
         &mut self,
         contract_identifier: &QualifiedContractIdentifier,
     ) -> Result<u64> {
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-size");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::ContractSize.as_str(),
+        );
         let contract_size: u64 =
             self.fetch_metadata(contract_identifier, &key)?
                 .ok_or_else(|| {
                     InterpreterError::Expect(
             "Failed to read non-consensus contract metadata, even though contract exists in MARF."
         .into())
                 })?;
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-data-size");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::ContractDataSize.as_str(),
+        );
         let data_size: u64 = self
             .fetch_metadata(contract_identifier, &key)?
             .ok_or_else(|| {
@@ -710,7 +804,10 @@ impl<'a> ClarityDatabase<'a> {
         contract_identifier: &QualifiedContractIdentifier,
         data_size: u64,
     ) -> Result<()> {
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-size");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::ContractSize.as_str(),
+        );
         let contract_size: u64 =
             self.fetch_metadata(contract_identifier, &key)?
                 .ok_or_else(|| {
@@ -720,7 +817,10 @@ impl<'a> ClarityDatabase<'a> {
                 })?;
         contract_size.cost_overflow_add(data_size)?;
 
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract-data-size");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::ContractDataSize.as_str(),
+        );
         self.insert_metadata(contract_identifier, &key, &data_size)?;
         Ok(())
     }
@@ -730,21 +830,30 @@ impl<'a> ClarityDatabase<'a> {
         contract_identifier: &QualifiedContractIdentifier,
         contract: Contract,
     ) -> Result<()> {
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::Contract.as_str(),
+        );
         self.insert_metadata(contract_identifier, &key, &contract)?;
         Ok(())
     }
 
     pub fn has_contract(&mut self, contract_identifier: &QualifiedContractIdentifier) -> bool {
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::Contract.as_str(),
+        );
         self.store.has_metadata_entry(contract_identifier, &key)
     }
 
     pub fn get_contract(
         &mut self,
         contract_identifier: &QualifiedContractIdentifier,
     ) -> Result<Contract> {
-        let key = ClarityDatabase::make_metadata_key(StoreType::Contract, "contract");
+        let key = ClarityDatabase::make_metadata_key(
+            StoreType::Contract,
+            ContractDataVarName::Contract.as_str(),
+        );
         let mut data: Contract = self.fetch_metadata(contract_identifier, &key)?
             .ok_or_else(|| InterpreterError::Expect(
                 "Failed to read non-consensus contract metadata, even though contract exists in MARF."

clarity/src/vm/database/clarity_store.rs

@@ -18,7 +18,7 @@ use std::path::PathBuf;
 
 #[cfg(feature = "canonical")]
 use rusqlite::Connection;
-use stacks_common::types::chainstate::{BlockHeaderHash, StacksBlockId, VRFSeed};
+use stacks_common::types::chainstate::{BlockHeaderHash, StacksBlockId, TrieHash, VRFSeed};
 use stacks_common::util::hash::{hex_bytes, to_hex, Hash160, Sha512Trunc256Sum};
 
 use crate::vm::analysis::AnalysisDatabase;
@@ -64,9 +64,15 @@ pub trait ClarityBackingStore {
     fn put_all_data(&mut self, items: Vec<(String, String)>) -> Result<()>;
     /// fetch K-V out of the committed datastore
     fn get_data(&mut self, key: &str) -> Result<Option<String>>;
+    /// fetch Hash(K)-V out of the commmitted datastore
+    fn get_data_from_path(&mut self, hash: &TrieHash) -> Result<Option<String>>;
     /// fetch K-V out of the committed datastore, along with the byte representation
     ///  of the Merkle proof for that key-value pair
     fn get_data_with_proof(&mut self, key: &str) -> Result<Option<(String, Vec<u8>)>>;
+    fn get_data_with_proof_from_path(
+        &mut self,
+        hash: &TrieHash,
+    ) -> Result<Option<(String, Vec<u8>)>>;
     fn has_entry(&mut self, key: &str) -> Result<bool> {
         Ok(self.get_data(key)?.is_some())
     }
@@ -209,10 +215,21 @@ impl ClarityBackingStore for NullBackingStore {
         panic!("NullBackingStore can't retrieve data")
     }
 
+    fn get_data_from_path(&mut self, _hash: &TrieHash) -> Result<Option<String>> {
+        panic!("NullBackingStore can't retrieve data")
+    }
+
     fn get_data_with_proof(&mut self, _key: &str) -> Result<Option<(String, Vec<u8>)>> {
         panic!("NullBackingStore can't retrieve data")
     }
 
+    fn get_data_with_proof_from_path(
+        &mut self,
+        _hash: &TrieHash,
+    ) -> Result<Option<(String, Vec<u8>)>> {
+        panic!("NullBackingStore can't retrieve data")
+    }
+
     #[cfg(feature = "canonical")]
     fn get_side_store(&mut self) -> &Connection {
         panic!("NullBackingStore has no side store")

clarity/src/vm/database/key_value_wrapper.rs

@@ -17,7 +17,7 @@
 use std::hash::Hash;
 
 use hashbrown::HashMap;
-use stacks_common::types::chainstate::StacksBlockId;
+use stacks_common::types::chainstate::{StacksBlockId, TrieHash};
 use stacks_common::types::StacksEpochId;
 use stacks_common::util::hash::Sha512Trunc256Sum;
 
@@ -369,6 +369,21 @@ impl<'a> RollbackWrapper<'a> {
             .transpose()
     }
 
+    /// this function will only return commitment proofs for values _already_ materialized
+    ///  in the underlying store. otherwise it returns None.
+    pub fn get_data_with_proof_by_hash<T>(
+        &mut self,
+        hash: &TrieHash,
+    ) -> InterpreterResult<Option<(T, Vec<u8>)>>
+    where
+        T: ClarityDeserializable<T>,
+    {
+        self.store
+            .get_data_with_proof_from_path(hash)?
+            .map(|(value, proof)| Ok((T::deserialize(&value)?, proof)))
+            .transpose()
+    }
+
     pub fn get_data<T>(&mut self, key: &str) -> InterpreterResult<Option<T>>
     where
         T: ClarityDeserializable<T>,
@@ -392,6 +407,23 @@ impl<'a> RollbackWrapper<'a> {
             .transpose()
     }
 
+    /// DO NOT USE IN CONSENSUS CODE.
+    ///
+    /// Load data directly from the underlying store, given its trie hash.  The lookup map will not
+    /// be used.
+    ///
+    /// This should never be called from within the Clarity VM, or via block-processing.  It's only
+    /// meant to be used by the RPC system.
+    pub fn get_data_by_hash<T>(&mut self, hash: &TrieHash) -> InterpreterResult<Option<T>>
+    where
+        T: ClarityDeserializable<T>,
+    {
+        self.store
+            .get_data_from_path(hash)?
+            .map(|x| T::deserialize(&x))
+            .transpose()
+    }
+
     pub fn deserialize_value(
         value_hex: &str,
         expected: &TypeSignature,

clarity/src/vm/database/sqlite.rs

@@ -19,7 +19,7 @@ use rusqlite::{
     params, Connection, Error as SqliteError, ErrorCode as SqliteErrorCode, OptionalExtension, Row,
     Savepoint,
 };
-use stacks_common::types::chainstate::{BlockHeaderHash, StacksBlockId};
+use stacks_common::types::chainstate::{BlockHeaderHash, StacksBlockId, TrieHash};
 use stacks_common::types::sqlite::NO_PARAMS;
 use stacks_common::util::db::tx_busy_handler;
 use stacks_common::util::hash::Sha512Trunc256Sum;
@@ -330,10 +330,21 @@ impl ClarityBackingStore for MemoryBackingStore {
         SqliteConnection::get(self.get_side_store(), key)
     }
 
+    fn get_data_from_path(&mut self, hash: &TrieHash) -> Result<Option<String>> {
+        SqliteConnection::get(self.get_side_store(), hash.to_string().as_str())
+    }
+
     fn get_data_with_proof(&mut self, key: &str) -> Result<Option<(String, Vec<u8>)>> {
         Ok(SqliteConnection::get(self.get_side_store(), key)?.map(|x| (x, vec![])))
     }
 
+    fn get_data_with_proof_from_path(
+        &mut self,
+        hash: &TrieHash,
+    ) -> Result<Option<(String, Vec<u8>)>> {
+        self.get_data_with_proof(&hash.to_string())
+    }
+
     fn get_side_store(&mut self) -> &Connection {
         &self.side_store
     }

docs/rpc-endpoints.md

@@ -172,6 +172,35 @@ Where data is the hex serialization of the variable value.
 This endpoint also accepts a querystring parameter `?proof=` which when supplied `0`, will return the
 JSON object _without_ the `proof` field.
 
+### GET /v2/clarity/marf/[Clarity MARF Key]
+Attempt to fetch the value of a MARF key. The key is identified with [Clarity MARF Key].
+
+Returns JSON data in the form:
+
+```json
+{
+  "data": "0x01ce...",
+  "proof": "0x01ab...",
+}
+```
+
+Where data is the hex serialization of the value.
+
+### GET /v2/clarity/metadata/[Stacks Address]/[Contract Name]/[Clarity Metadata Key]
+Attempt to fetch the metadata of a contract.
+ The contract is identified with [Stacks Address] and [Contract Name] in the URL path.
+ The metadata key is identified with [Clarity Metadata Key].
+
+Returns JSON data in the form:
+
+```json
+{
+  "data": "'{\"contract_identifier\":{...}'",
+}
+```
+
+Where data is the metadata formatted as a JSON string.
+
 ### GET /v2/constant_val/[Stacks Address]/[Contract Name]/[Constant Name]
 Attempt to fetch a constant from a contract. The contract is identified with [Stacks Address] and 
  [Contract Name] in the URL path. The constant is identified with [Constant Name].

docs/rpc/api/core-node/get-clarity-marf-value.example.json

@@ -0,0 +1,4 @@
+{
+  "data": "0x0a0c000000010a6d6f6e737465722d69640100000000000000000000000000000001",
+  "proof": "0x123..."
+}