Merge pull request #5942 from rdeioris/feat/txindex

Feat/txindex

Author: wileyj

CHANGELOG.md

@@ -13,6 +13,7 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE
 - Add `max_execution_time_secs` to miner config for limiting duration of contract calls
 - When a miner's config file is updated (ie with a new fee rate), a new block commit is issued using
   the new values ([#5924](https://github.com/stacks-network/stacks-core/pull/5924))
+- Add `txindex` configuration option enabling the storage (and querying via api) of transactions. Note: the old STACKS_TRANSACTION_LOG environment var configuration is no longer available.
 
 ### Changed
 

docs/rpc-endpoints.md

@@ -578,3 +578,20 @@ tenure, `tip_block_id` identifies the highest-known block in this tenure, and
 Get number of blocks signed by signer during a given reward cycle
 
 Returns a non-negative integer
+
+### GET /v3/transaction/[Transaction ID]
+
+Returns the index_block_hash, the transaction body (as hex) and the result given the TXID.
+
+```json
+{
+  "index_block_hash": "e0b6c25b1dac0c0e1c75e41ab46bd6d70d9a2d02ffed8f2c0733b6e686289c38",
+  "tx": "(ok true)",
+  "result": "808000000004008bc5147525b8f477f0bc4522a88c8339b2494db5000000000000001a0000000000000000010123eab800bc9f639c5aa05d154148a981c89fd21064a6f8cafd8649800a56c9ea77e2e46bed8bd9ef0f173b19b20d6c43e2a9f37b078df5c74b9e6f9be75b650e01020000000007588687edeb02248d402c316ed33e22ea0e73af8703ce5011f3e25f5ce12f00f903ca504742117ee0588687edeb02248d402c316ed33e22ea0e73af87c54e0d94e4dd298cf19778352906a2fcf0af74582b07dfb57c710288874f71ca00000001006bc51b33e9f3626944eb879147e18111581f8f9b"
+}
+```
+
+This feature requires enabling of transaction indexing by setting the `txindex` node option.
+
+This will return 404 if the transaction does not exist and 501 (Not Implemented) if
+transaction indexing is not enabled.

docs/rpc/api/core-node/get_transaction.json

@@ -0,0 +1,5 @@
+{
+  "index_block_hash": "e0b6c25b1dac0c0e1c75e41ab46bd6d70d9a2d02ffed8f2c0733b6e686289c38",
+  "result": "(ok true)",
+  "tx": "808000000004008bc5147525b8f477f0bc4522a88c8339b2494db5000000000000001a0000000000000000010123eab800bc9f639c5aa05d154148a981c89fd21064a6f8cafd8649800a56c9ea77e2e46bed8bd9ef0f173b19b20d6c43e2a9f37b078df5c74b9e6f9be75b650e01020000000007588687edeb02248d402c316ed33e22ea0e73af8703ce5011f3e25f5ce12f00f903ca504742117ee0588687edeb02248d402c316ed33e22ea0e73af87c54e0d94e4dd298cf19778352906a2fcf0af74582b07dfb57c710288874f71ca00000001006bc51b33e9f3626944eb879147e18111581f8f9b"
+}

docs/rpc/openapi.yaml

@@ -866,3 +866,32 @@ paths:
               schema:
                 type: integer
                 example: 7
+  /v3/transaction/{txid}:
+    post:
+      summary: Retrieve transaction details by TXID
+      tags:
+        - Transactions
+      description: Get a JSON with the transaction details including the `index_block_hash`, the hex-encoded transaction body, and the `result`.
+      operationId: get_transaction
+      parameters:
+      - name: txid
+        in: path
+        required: true
+        description: Transaction ID
+        schema:
+          type: string
+      responses:
+        "200":
+          description: Transaction JSON with index_block_hash, transaction body and result
+          content:
+            application/json:
+              example:
+                $ref: ./api/core-node/get_transaction.json
+        "404":
+          description: Transaction not found
+          content:
+            application/text-plain: {}
+        "501":
+          description: Transaction indexing not enabled
+          content:
+            application/text-plain: {}

stackslib/src/burnchains/affirmation.rs

@@ -896,7 +896,7 @@ fn inner_find_heaviest_block_commit_ptr(
     // consider ancestor candidates in _highest_-first order
     for ((height, vtxindex), (block_set, burnt)) in ancestor_confirmations.iter().rev() {
         let confs = block_set.len() as u64;
-        if confs < anchor_threshold.into() {
+        if confs < u64::from(anchor_threshold) {
             continue;
         }
 

stackslib/src/chainstate/coordinator/mod.rs

@@ -208,6 +208,9 @@ pub struct ChainsCoordinatorConfig {
     /// true: always wait for canonical anchor blocks, even if it stalls the chain
     /// false: proceed to process new chain history even if we're missing an anchor block.
     pub require_affirmed_anchor_blocks: bool,
+    /// true: enable transactions indexing
+    /// false: no transactions indexing
+    pub txindex: bool,
 }
 
 impl ChainsCoordinatorConfig {
@@ -216,14 +219,16 @@ impl ChainsCoordinatorConfig {
             always_use_affirmation_maps: true,
             require_affirmed_anchor_blocks: true,
             assume_present_anchor_blocks: true,
+            txindex: false,
         }
     }
 
-    pub fn test_new() -> ChainsCoordinatorConfig {
+    pub fn test_new(txindex: bool) -> ChainsCoordinatorConfig {
         ChainsCoordinatorConfig {
             always_use_affirmation_maps: false,
             require_affirmed_anchor_blocks: false,
             assume_present_anchor_blocks: false,
+            txindex,
         }
     }
 }
@@ -249,7 +254,7 @@ pub struct ChainsCoordinator<
     pub reward_set_provider: R,
     pub notifier: N,
     pub atlas_config: AtlasConfig,
-    config: ChainsCoordinatorConfig,
+    pub config: ChainsCoordinatorConfig,
     burnchain_indexer: B,
     /// Used to tell the P2P thread that the stackerdb
     ///  needs to be refreshed.
@@ -646,6 +651,7 @@ impl<T: BlockEventDispatcher, U: RewardSetProvider, B: BurnchainHeaderReader>
         path: &str,
         reward_set_provider: U,
         indexer: B,
+        txindex: bool,
     ) -> ChainsCoordinator<'a, T, (), U, (), (), B> {
         ChainsCoordinator::test_new_full(
             burnchain,
@@ -655,6 +661,7 @@ impl<T: BlockEventDispatcher, U: RewardSetProvider, B: BurnchainHeaderReader>
             None,
             indexer,
             None,
+            txindex,
         )
     }
 
@@ -668,6 +675,7 @@ impl<T: BlockEventDispatcher, U: RewardSetProvider, B: BurnchainHeaderReader>
         dispatcher: Option<&'a T>,
         burnchain_indexer: B,
         atlas_config: Option<AtlasConfig>,
+        txindex: bool,
     ) -> ChainsCoordinator<'a, T, (), U, (), (), B> {
         let burnchain = burnchain.clone();
 
@@ -713,7 +721,7 @@ impl<T: BlockEventDispatcher, U: RewardSetProvider, B: BurnchainHeaderReader>
             notifier: (),
             atlas_config,
             atlas_db: Some(atlas_db),
-            config: ChainsCoordinatorConfig::test_new(),
+            config: ChainsCoordinatorConfig::test_new(txindex),
             burnchain_indexer,
             refresh_stacker_db: Arc::new(AtomicBool::new(false)),
             in_nakamoto_epoch: false,

stackslib/src/chainstate/coordinator/tests.rs

@@ -470,13 +470,15 @@ pub fn make_coordinator<'a>(
         path,
         OnChainRewardSetProvider(None),
         indexer,
+        false,
     )
 }
 
 pub fn make_coordinator_atlas<'a>(
     path: &str,
     burnchain: Option<Burnchain>,
     atlas_config: Option<AtlasConfig>,
+    txindex: bool,
 ) -> ChainsCoordinator<
     'a,
     NullEventDispatcher,
@@ -496,6 +498,7 @@ pub fn make_coordinator_atlas<'a>(
         None,
         indexer,
         atlas_config,
+        txindex,
     )
 }
 
@@ -545,6 +548,7 @@ fn make_reward_set_coordinator<'a>(
         path,
         StubbedRewardSetProvider(addrs),
         indexer,
+        false,
     )
 }
 
@@ -1552,7 +1556,7 @@ fn missed_block_commits_2_1() {
             // did we have a bad missed commit in this window?
             // bad missed commits land in the prepare phase.
             let have_bad_missed_commit = b.is_in_prepare_phase(last_bad_op_height)
-                && ix >= MINING_COMMITMENT_WINDOW.into()
+                && ix >= usize::from(MINING_COMMITMENT_WINDOW)
                 && last_bad_op_height + (MINING_COMMITMENT_WINDOW as u64) > tip.block_height;
             if have_bad_missed_commit {
                 // bad commit breaks the chain if its PoX outputs are invalid
@@ -4659,6 +4663,7 @@ fn atlas_stop_start() {
         path,
         Some(burnchain_conf.clone()),
         Some(atlas_config.clone()),
+        false,
     );
 
     coord.handle_new_burnchain_block().unwrap();
@@ -4853,7 +4858,7 @@ fn atlas_stop_start() {
     // now, we'll shut down all the coordinator connections and reopen them
     //  to ensure that the queue remains in place
     let coord = (); // dispose of the coordinator, closing all its connections
-    let coord = make_coordinator_atlas(path, Some(burnchain_conf), Some(atlas_config));
+    let coord = make_coordinator_atlas(path, Some(burnchain_conf), Some(atlas_config), false);
 
     let atlas_queue = coord
         .atlas_db
@@ -5162,7 +5167,7 @@ fn test_epoch_verify_active_pox_contract() {
 
         let active_pox_contract = b.pox_constants.active_pox_contract(burn_block_height);
 
-        if burn_block_height <= pox_v1_unlock_ht.into() {
+        if burn_block_height <= u64::from(pox_v1_unlock_ht) {
             assert_eq!(active_pox_contract, POX_1_NAME);
             if curr_reward_cycle == 1 {
                 // This is a result of the first stack stx sent.

stackslib/src/chainstate/nakamoto/coordinator/mod.rs

@@ -801,6 +801,7 @@ impl<
                 &mut self.sortition_db,
                 &canonical_sortition_tip,
                 self.dispatcher,
+                self.config.txindex,
             ) {
                 Ok(receipt_opt) => receipt_opt,
                 Err(ChainstateError::InvalidStacksBlock(msg)) => {

stackslib/src/chainstate/nakamoto/coordinator/tests.rs

@@ -34,7 +34,7 @@ use stacks_common::types::chainstate::{
     BurnchainHeaderHash, StacksAddress, StacksBlockId, StacksPrivateKey, StacksPublicKey,
 };
 use stacks_common::types::{Address, StacksEpoch, StacksEpochId, StacksPublicKeyBuffer};
-use stacks_common::util::hash::Hash160;
+use stacks_common::util::hash::{to_hex, Hash160};
 use stacks_common::util::secp256k1::Secp256k1PrivateKey;
 use stacks_common::util::vrf::VRFProof;
 
@@ -1568,6 +1568,135 @@ fn pox_treatment() {
     );
 }
 
+#[test]
+// Test Transactions indexing system
+fn transactions_indexing() {
+    let private_key = StacksPrivateKey::from_seed(&[2]);
+    let addr = StacksAddress::p2pkh(false, &StacksPublicKey::from_private(&private_key));
+
+    let num_stackers: u32 = 4;
+    let mut signing_key_seed = num_stackers.to_be_bytes().to_vec();
+    signing_key_seed.extend_from_slice(&[1, 1, 1, 1]);
+    let signing_key = StacksPrivateKey::from_seed(signing_key_seed.as_slice());
+    let test_stackers = (0..num_stackers)
+        .map(|index| TestStacker {
+            signer_private_key: signing_key.clone(),
+            stacker_private_key: StacksPrivateKey::from_seed(&index.to_be_bytes()),
+            amount: u64::MAX as u128 - 10000,
+            pox_addr: Some(PoxAddress::Standard(
+                StacksAddress::new(
+                    C32_ADDRESS_VERSION_TESTNET_SINGLESIG,
+                    Hash160::from_data(&index.to_be_bytes()),
+                )
+                .unwrap(),
+                Some(AddressHashMode::SerializeP2PKH),
+            )),
+            max_amount: None,
+        })
+        .collect::<Vec<_>>();
+    let test_signers = TestSigners::new(vec![signing_key]);
+    let mut pox_constants = TestPeerConfig::default().burnchain.pox_constants;
+    pox_constants.reward_cycle_length = 10;
+    pox_constants.v2_unlock_height = 21;
+    pox_constants.pox_3_activation_height = 26;
+    pox_constants.v3_unlock_height = 27;
+    pox_constants.pox_4_activation_height = 28;
+
+    let mut boot_plan = NakamotoBootPlan::new(function_name!())
+        .with_test_stackers(test_stackers.clone())
+        .with_test_signers(test_signers.clone())
+        .with_private_key(private_key)
+        .with_txindex(true);
+    boot_plan.pox_constants = pox_constants;
+
+    let mut peer = boot_plan.boot_into_nakamoto_peer(vec![], None);
+
+    // generate a new block with txindex
+    let (tracked_block, burn_height, ..) =
+        peer.single_block_tenure(&private_key, |_| {}, |_| {}, |_| false);
+
+    assert_eq!(peer.try_process_block(&tracked_block).unwrap(), true);
+
+    let tracked_block_id = tracked_block.block_id();
+
+    let chainstate = &peer.stacks_node.unwrap().chainstate;
+
+    // compare transactions to what has been tracked
+    for tx in tracked_block.txs {
+        let current_tx_hex = to_hex(&tx.serialize_to_vec());
+        let (index_block_hash, tx_hex, _) =
+            NakamotoChainState::get_tx_info_from_txid(&chainstate.index_conn(), tx.txid())
+                .unwrap()
+                .unwrap();
+        assert_eq!(index_block_hash, tracked_block_id);
+        assert_eq!(tx_hex, current_tx_hex);
+    }
+}
+
+#[test]
+// Test Transactions indexing system (not indexing)
+fn transactions_not_indexing() {
+    let private_key = StacksPrivateKey::from_seed(&[2]);
+    let addr = StacksAddress::p2pkh(false, &StacksPublicKey::from_private(&private_key));
+
+    let num_stackers: u32 = 4;
+    let mut signing_key_seed = num_stackers.to_be_bytes().to_vec();
+    signing_key_seed.extend_from_slice(&[1, 1, 1, 1]);
+    let signing_key = StacksPrivateKey::from_seed(signing_key_seed.as_slice());
+    let test_stackers = (0..num_stackers)
+        .map(|index| TestStacker {
+            signer_private_key: signing_key.clone(),
+            stacker_private_key: StacksPrivateKey::from_seed(&index.to_be_bytes()),
+            amount: u64::MAX as u128 - 10000,
+            pox_addr: Some(PoxAddress::Standard(
+                StacksAddress::new(
+                    C32_ADDRESS_VERSION_TESTNET_SINGLESIG,
+                    Hash160::from_data(&index.to_be_bytes()),
+                )
+                .unwrap(),
+                Some(AddressHashMode::SerializeP2PKH),
+            )),
+            max_amount: None,
+        })
+        .collect::<Vec<_>>();
+    let test_signers = TestSigners::new(vec![signing_key]);
+    let mut pox_constants = TestPeerConfig::default().burnchain.pox_constants;
+    pox_constants.reward_cycle_length = 10;
+    pox_constants.v2_unlock_height = 21;
+    pox_constants.pox_3_activation_height = 26;
+    pox_constants.v3_unlock_height = 27;
+    pox_constants.pox_4_activation_height = 28;
+
+    let mut boot_plan = NakamotoBootPlan::new(function_name!())
+        .with_test_stackers(test_stackers.clone())
+        .with_test_signers(test_signers.clone())
+        .with_private_key(private_key)
+        .with_txindex(false);
+    boot_plan.pox_constants = pox_constants;
+
+    let mut peer = boot_plan.boot_into_nakamoto_peer(vec![], None);
+
+    // generate a new block but without txindex
+    let (untracked_block, burn_height, ..) =
+        peer.single_block_tenure(&private_key, |_| {}, |_| {}, |_| false);
+
+    assert_eq!(peer.try_process_block(&untracked_block).unwrap(), true);
+
+    let untracked_block_id = untracked_block.block_id();
+
+    let chainstate = &peer.stacks_node.unwrap().chainstate;
+
+    // ensure untracked transactions are not recorded
+    for tx in untracked_block.txs {
+        assert_eq!(
+            NakamotoChainState::get_tx_info_from_txid(&chainstate.index_conn(), tx.txid(),)
+                .unwrap()
+                .is_none(),
+            true
+        );
+    }
+}
+
 /// Test chainstate getters against an instantiated epoch2/Nakamoto chain.
 /// There are 11 epoch2 blocks and 2 nakamto tenure with 10 nakamoto blocks each
 /// Tests:

stackslib/src/chainstate/nakamoto/miner.rs

@@ -693,7 +693,7 @@ impl BlockBuilder for NakamotoBlockBuilder {
         ast_rules: ASTRules,
         max_execution_time: Option<std::time::Duration>,
     ) -> TransactionResult {
-        if self.bytes_so_far + tx_len >= MAX_EPOCH_SIZE.into() {
+        if self.bytes_so_far + tx_len >= u64::from(MAX_EPOCH_SIZE) {
             return TransactionResult::skipped_due_to_error(tx, Error::BlockTooBigError);
         }