Merge pull request #5176 from stacks-network/release/2.5.0.0.7

merge 2.5.0.0.7 release branch to develop

Author: jferrant

CHANGELOG.md

@@ -17,6 +17,28 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE
     - `get-tenure-info?` added
     - `get-block-info?` removed
 
+## [2.5.0.0.7]
+
+### Added
+
+- Add warn logs for block validate rejections (#5079)
+- Neon mock miner replay (#5060)
+
+### Changed
+
+- Revert BurnchainHeaderHash serialization change (#5094)
+- boot_to_epoch_3 in SignerTest should wait for a new commit (#5087)
+- Fix block proposal rejection test (#5084)
+- Mock signing revamp (#5070)
+- Multi miner fixes jude (#5040)
+- Remove spurious deadlock condition whenever the sortition DB is opened
+
+## [2.5.0.0.6]
+
+### Changed
+
+- If there is a getchunk/putchunk that fails due to a stale (or future) version NACK, the StackerDB sync state machine should immediately retry sync (#5066)
+
 ## [2.5.0.0.5]
 
 ### Added

stackslib/src/net/stackerdb/db.rs

@@ -293,6 +293,15 @@ impl<'a> StackerDBTx<'a> {
         Ok(())
     }
 
+    /// Shrink a StackerDB.  Remove all slots at and beyond a particular slot ID.
+    fn shrink_stackerdb(&self, stackerdb_id: i64, first_slot_id: u32) -> Result<(), net_error> {
+        let qry = "DELETE FROM chunks WHERE stackerdb_id = ?1 AND slot_id >= ?2";
+        let args = params![&stackerdb_id, &first_slot_id];
+        let mut stmt = self.sql_tx.prepare(&qry)?;
+        stmt.execute(args)?;
+        Ok(())
+    }
+
     /// Update a database's storage slots, e.g. from new configuration state in its smart contract.
     /// Chunk data for slots that no longer exist will be dropped.
     /// Newly-created slots will be instantiated with empty data.
@@ -343,6 +352,8 @@ impl<'a> StackerDBTx<'a> {
                 stmt.execute(args)?;
             }
         }
+        debug!("Shrink {} to {} slots", smart_contract, total_slots_read);
+        self.shrink_stackerdb(stackerdb_id, total_slots_read)?;
         Ok(())
     }
 

stackslib/src/net/stackerdb/mod.rs

@@ -341,8 +341,14 @@ impl StackerDBs {
                         &e
                     );
                 }
-            } else if new_config != stackerdb_config && new_config.signers.len() > 0 {
+            } else if (new_config != stackerdb_config && new_config.signers.len() > 0)
+                || (new_config == stackerdb_config
+                    && new_config.signers.len()
+                        != self.get_slot_versions(&stackerdb_contract_id)?.len())
+            {
                 // only reconfigure if the config has changed
+                // (that second check on the length is needed in case the node is a victim of
+                // #5142, which was a bug whereby a stackerdb could never shrink)
                 if let Err(e) = self.reconfigure_stackerdb(&stackerdb_contract_id, &new_config) {
                     warn!(
                         "Failed to create or reconfigure StackerDB {stackerdb_contract_id}: DB error {:?}",

stackslib/src/net/stackerdb/tests/db.rs

@@ -20,6 +20,7 @@ use std::path::Path;
 use clarity::vm::types::QualifiedContractIdentifier;
 use clarity::vm::ContractName;
 use libstackerdb::SlotMetadata;
+use rusqlite::params;
 use stacks_common::address::{
     AddressHashMode, C32_ADDRESS_VERSION_MAINNET_MULTISIG, C32_ADDRESS_VERSION_MAINNET_SINGLESIG,
 };
@@ -649,6 +650,16 @@ fn test_reconfigure_stackerdb() {
         initial_metadata.push((slot_metadata, chunk_data));
     }
 
+    tx.commit().unwrap();
+
+    let db_slot_metadata = db.get_db_slot_metadata(&sc).unwrap();
+    assert_eq!(db_slot_metadata.len(), pks.len());
+    for (i, slot_md) in db_slot_metadata.iter().enumerate() {
+        let slot_metadata = db.get_slot_metadata(&sc, i as u32).unwrap().unwrap();
+        assert_eq!(slot_metadata, *slot_md);
+    }
+
+    let tx = db.tx_begin(StackerDBConfig::noop()).unwrap();
     let new_pks: Vec<_> = (0..10).map(|_| StacksPrivateKey::new()).collect();
     let reconfigured_pks = vec![
         // first five slots are unchanged
@@ -722,6 +733,91 @@ fn test_reconfigure_stackerdb() {
             assert_eq!(chunk.len(), 0);
         }
     }
+
+    let db_slot_metadata = db.get_db_slot_metadata(&sc).unwrap();
+    assert_eq!(db_slot_metadata.len(), reconfigured_pks.len());
+    for (i, slot_md) in db_slot_metadata.iter().enumerate() {
+        let slot_metadata = db.get_slot_metadata(&sc, i as u32).unwrap().unwrap();
+        assert_eq!(slot_metadata, *slot_md);
+    }
+
+    // reconfigure with fewer slots
+    let new_pks: Vec<_> = (0..10).map(|_| StacksPrivateKey::new()).collect();
+    let reconfigured_pks = vec![
+        // first five slots are unchanged
+        pks[0], pks[1], pks[2], pks[3], pks[4],
+        // next five slots are different, so their contents will be dropped and versions and write
+        // timestamps reset
+        new_pks[0], new_pks[1], new_pks[2], new_pks[3],
+        new_pks[4],
+        // slots 10-15 will disappear
+    ];
+    let reconfigured_addrs: Vec<_> = reconfigured_pks
+        .iter()
+        .map(|pk| {
+            StacksAddress::from_public_keys(
+                C32_ADDRESS_VERSION_MAINNET_SINGLESIG,
+                &AddressHashMode::SerializeP2PKH,
+                1,
+                &vec![StacksPublicKey::from_private(&pk)],
+            )
+            .unwrap()
+        })
+        .collect();
+
+    let tx = db.tx_begin(StackerDBConfig::noop()).unwrap();
+
+    // reconfigure
+    tx.reconfigure_stackerdb(
+        &sc,
+        &reconfigured_addrs
+            .clone()
+            .into_iter()
+            .map(|addr| (addr, 1))
+            .collect::<Vec<_>>(),
+    )
+    .unwrap();
+
+    tx.commit().unwrap();
+
+    for (i, pk) in new_pks.iter().enumerate() {
+        if i < 5 {
+            // first five are unchanged
+            let chunk_data = StackerDBChunkData {
+                slot_id: i as u32,
+                slot_version: 1,
+                sig: MessageSignature::empty(),
+                data: vec![i as u8; 128],
+            };
+
+            let slot_metadata = db.get_slot_metadata(&sc, i as u32).unwrap().unwrap();
+            let chunk = db.get_latest_chunk(&sc, i as u32).unwrap().unwrap();
+
+            assert_eq!(initial_metadata[i].0, slot_metadata);
+            assert_eq!(initial_metadata[i].1.data, chunk);
+        } else if i < 10 {
+            // next five are wiped
+            let slot_metadata = db.get_slot_metadata(&sc, i as u32).unwrap().unwrap();
+            assert_eq!(slot_metadata.slot_id, i as u32);
+            assert_eq!(slot_metadata.slot_version, 0);
+            assert_eq!(slot_metadata.data_hash, Sha512Trunc256Sum([0x00; 32]));
+            assert_eq!(slot_metadata.signature, MessageSignature::empty());
+
+            let chunk = db.get_latest_chunk(&sc, i as u32).unwrap().unwrap();
+            assert_eq!(chunk.len(), 0);
+        } else {
+            // final five are gone
+            let slot_metadata_opt = db.get_slot_metadata(&sc, i as u32).unwrap();
+            assert!(slot_metadata_opt.is_none());
+        }
+    }
+
+    let db_slot_metadata = db.get_db_slot_metadata(&sc).unwrap();
+    assert_eq!(db_slot_metadata.len(), reconfigured_pks.len());
+    for (i, slot_md) in db_slot_metadata.iter().enumerate() {
+        let slot_metadata = db.get_slot_metadata(&sc, i as u32).unwrap().unwrap();
+        assert_eq!(slot_metadata, *slot_md);
+    }
 }
 
 // TODO: max chunk size