feat: add 2 heuristics to miner for nakamoto

* for the first block in a tenure, just mine an empty block
* estimate the time it takes to eval a tx, and see if it will interfere with block deadline

Author: kantai

stackslib/src/chainstate/stacks/miner.rs

@@ -18,6 +18,7 @@ use std::collections::{HashMap, HashSet};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex};
 use std::thread::ThreadId;
+use std::time::Instant;
 use std::{cmp, fs, mem};
 
 use clarity::vm::analysis::{CheckError, CheckErrors};
@@ -2211,6 +2212,15 @@ impl StacksBlockBuilder {
             );
         }
 
+        // nakamoto miner tenure start heuristic:
+        //  mine an empty block so you can start your tenure quickly!
+        if let Some(tx) = initial_txs.first() {
+            if matches!(&tx.payload, TransactionPayload::TenureChange(_)) {
+                debug!("Nakamoto miner heuristic: during tenure change blocks, produce a fast short block to begin tenure");
+                return Ok((false, tx_events));
+            }
+        }
+
         mempool.reset_nonce_cache()?;
         mempool.estimate_tx_rates(100, &block_limit, &stacks_epoch_id)?;
 
@@ -2221,6 +2231,7 @@ impl StacksBlockBuilder {
 
         let mut invalidated_txs = vec![];
         let mut to_drop_and_blacklist = vec![];
+        let mut update_timings = vec![];
 
         let deadline = ts_start + u128::from(max_miner_time_ms);
         let mut num_txs = 0;
@@ -2250,10 +2261,27 @@ impl StacksBlockBuilder {
                         if block_limit_hit == BlockLimitFunction::LIMIT_REACHED {
                             return Ok(None);
                         }
-                        if get_epoch_time_ms() >= deadline {
+                        let time_now = get_epoch_time_ms();
+                        if time_now >= deadline {
                             debug!("Miner mining time exceeded ({} ms)", max_miner_time_ms);
                             return Ok(None);
                         }
+                        if let Some(time_estimate) = txinfo.metadata.time_estimate_ms {
+                            if time_now.saturating_add(time_estimate.into()) > deadline {
+                                debug!("Mining tx would cause us to exceed our deadline, skipping";
+                                       "txid" => %txinfo.tx.txid(),
+                                       "deadline" => deadline,
+                                       "now" => time_now,
+                                       "estimate" => time_estimate);
+                                return Ok(Some(
+                                    TransactionResult::skipped(
+                                        &txinfo.tx,
+                                        "Transaction would exceed deadline.".into(),
+                                    )
+                                    .convert_to_event(),
+                                ));
+                            }
+                        }
 
                         // skip transactions early if we can
                         if considered.contains(&txinfo.tx.txid()) {
@@ -2303,6 +2331,7 @@ impl StacksBlockBuilder {
                         considered.insert(txinfo.tx.txid());
                         num_considered += 1;
 
+                        let tx_start = Instant::now();
                         let tx_result = builder.try_mine_tx_with_len(
                             epoch_tx,
                             &txinfo.tx,
@@ -2314,6 +2343,21 @@ impl StacksBlockBuilder {
                         let result_event = tx_result.convert_to_event();
                         match tx_result {
                             TransactionResult::Success(TransactionSuccess { receipt, .. }) => {
+                                if txinfo.metadata.time_estimate_ms.is_none() {
+                                    // use i64 to avoid running into issues when storing in
+                                    //  rusqlite.
+                                    let time_estimate_ms: i64 = tx_start
+                                        .elapsed()
+                                        .as_millis()
+                                        .try_into()
+                                        .unwrap_or_else(|_| i64::MAX);
+                                    let time_estimate_ms: u64 = time_estimate_ms
+                                        .try_into()
+                                        // should be unreachable
+                                        .unwrap_or_else(|_| 0);
+                                    update_timings.push((txinfo.tx.txid(), time_estimate_ms));
+                                }
+
                                 num_txs += 1;
                                 if update_estimator {
                                     if let Err(e) = estimator.notify_event(
@@ -2386,6 +2430,12 @@ impl StacksBlockBuilder {
                     },
                 );
 
+                if !update_timings.is_empty() {
+                    if let Err(e) = mempool.update_tx_time_estimates(&update_timings) {
+                        warn!("Error while updating time estimates for mempool"; "err" => ?e);
+                    }
+                }
+
                 if to_drop_and_blacklist.len() > 0 {
                     let _ = mempool.drop_and_blacklist_txs(&to_drop_and_blacklist);
                 }

stackslib/src/core/mempool.rs

@@ -460,6 +460,7 @@ pub struct MemPoolTxMetadata {
     pub last_known_origin_nonce: Option<u64>,
     pub last_known_sponsor_nonce: Option<u64>,
     pub accept_time: u64,
+    pub time_estimate_ms: Option<u64>,
 }
 
 impl MemPoolTxMetadata {
@@ -594,6 +595,7 @@ impl FromRow<MemPoolTxMetadata> for MemPoolTxMetadata {
         let sponsor_nonce = u64::from_column(row, "sponsor_nonce")?;
         let last_known_sponsor_nonce = u64::from_column(row, "last_known_sponsor_nonce")?;
         let last_known_origin_nonce = u64::from_column(row, "last_known_origin_nonce")?;
+        let time_estimate_ms: Option<u64> = row.get("time_estimate_ms")?;
 
         Ok(MemPoolTxMetadata {
             txid,
@@ -609,6 +611,7 @@ impl FromRow<MemPoolTxMetadata> for MemPoolTxMetadata {
             last_known_origin_nonce,
             last_known_sponsor_nonce,
             accept_time,
+            time_estimate_ms,
         })
     }
 }
@@ -624,10 +627,7 @@ impl FromRow<MemPoolTxInfo> for MemPoolTxInfo {
             return Err(db_error::ParseError);
         }
 
-        Ok(MemPoolTxInfo {
-            tx: tx,
-            metadata: md,
-        })
+        Ok(MemPoolTxInfo { tx, metadata: md })
     }
 }
 
@@ -803,6 +803,16 @@ const MEMPOOL_SCHEMA_6_NONCES: &'static [&'static str] = &[
     "#,
 ];
 
+const MEMPOOL_SCHEMA_7_TIME_ESTIMATES: &'static [&'static str] = &[
+    r#"
+    -- ALLOW NULL
+    ALTER TABLE mempool ADD COLUMN time_estimate_ms NUMBER;
+    "#,
+    r#"
+    INSERT INTO schema_version (version) VALUES (7)
+    "#,
+];
+
 const MEMPOOL_INDEXES: &'static [&'static str] = &[
     "CREATE INDEX IF NOT EXISTS by_txid ON mempool(txid);",
     "CREATE INDEX IF NOT EXISTS by_height ON mempool(height);",
@@ -1287,6 +1297,9 @@ impl MemPoolDB {
                     MemPoolDB::instantiate_nonces(tx)?;
                 }
                 6 => {
+                    MemPoolDB::instantiate_schema_7(tx)?;
+                }
+                7 => {
                     break;
                 }
                 _ => {
@@ -1363,6 +1376,16 @@ impl MemPoolDB {
         Ok(())
     }
 
+    /// Add the nonce table
+    #[cfg_attr(test, mutants::skip)]
+    fn instantiate_schema_7(tx: &DBTx) -> Result<(), db_error> {
+        for sql_exec in MEMPOOL_SCHEMA_7_TIME_ESTIMATES {
+            tx.execute_batch(sql_exec)?;
+        }
+
+        Ok(())
+    }
+
     #[cfg_attr(test, mutants::skip)]
     pub fn db_path(chainstate_root_path: &str) -> Result<String, db_error> {
         let mut path = PathBuf::from(chainstate_root_path);
@@ -2650,6 +2673,24 @@ impl MemPoolDB {
         Ok(())
     }
 
+    /// Drop and blacklist transactions, so we don't re-broadcast them or re-fetch them.
+    /// Do *NOT* remove them from the bloom filter.  This will cause them to continue to be
+    /// reported as present, which is exactly what we want because we don't want these transactions
+    /// to be seen again (so we don't want anyone accidentally "helpfully" pushing them to us, nor
+    /// do we want the mempool sync logic to "helpfully" re-discover and re-download them).
+    pub fn update_tx_time_estimates(&mut self, txs: &[(Txid, u64)]) -> Result<(), db_error> {
+        let sql = "UPDATE mempool SET time_estimate_ms = ? WHERE txid = ?";
+        let mempool_tx = self.tx_begin()?;
+        for (txid, time_estimate_ms) in txs.iter() {
+            mempool_tx
+                .tx
+                .execute(sql, params![time_estimate_ms, txid])?;
+        }
+        mempool_tx.commit()?;
+
+        Ok(())
+    }
+
     /// Drop and blacklist transactions, so we don't re-broadcast them or re-fetch them.
     /// Do *NOT* remove them from the bloom filter.  This will cause them to continue to be
     /// reported as present, which is exactly what we want because we don't want these transactions