Merge pull request #3552 from stacks-network/feat/stackerdb-discovery

Feat/stackerdb discovery

Author: jcnelson

stackslib/src/burnchains/bitcoin/indexer.rs

@@ -245,7 +245,10 @@ impl BitcoinIndexer {
             true,
             false,
         )
-        .unwrap();
+        .expect(&format!(
+            "Failed to open {:?}",
+            &working_dir_path.to_str().unwrap().to_string()
+        ));
 
         BitcoinIndexer {
             config: BitcoinIndexerConfig::default_regtest(

stackslib/src/chainstate/coordinator/mod.rs

@@ -503,8 +503,12 @@ impl<'a, T: BlockEventDispatcher, U: RewardSetProvider, B: BurnchainHeaderReader
             SortitionDB::get_canonical_sortition_tip(sortition_db.conn()).unwrap();
 
         let atlas_config = atlas_config.unwrap_or(AtlasConfig::new(false));
-        let atlas_db =
-            AtlasDB::connect(atlas_config.clone(), &format!("{}/atlas", path), true).unwrap();
+        let atlas_db = AtlasDB::connect(
+            atlas_config.clone(),
+            &format!("{}/atlas.sqlite", path),
+            true,
+        )
+        .unwrap();
 
         ChainsCoordinator {
             canonical_sortition_tip: Some(canonical_sortition_tip),

stackslib/src/main.rs

@@ -580,6 +580,7 @@ check if the associated microblocks can be downloaded
             None,
             0,
             UrlString::try_from("abc").unwrap(),
+            vec![],
         );
 
         let header_hashes = {

stackslib/src/net/chat.rs

@@ -467,12 +467,19 @@ impl Neighbor {
         Ok(())
     }
 
-    pub fn from_handshake(
+    /// Instantiate a Neighbor from HandshakeData, merging the information we have on-disk in the
+    /// PeerDB with information in the handshake.
+    /// * If we already know about this neighbor, then all previously-calculated state and local
+    /// configuration state will be loaded as well.  This includes things like the calculated
+    /// in/out-degree and last-contact time, as well as the allow/deny time limits.
+    /// * If we do not know about this neighbor, then the above state will not be loaded.
+    /// Returns (the neighbor, whether or not the neighbor was known)
+    pub fn load_and_update(
         conn: &DBConn,
         peer_version: u32,
         network_id: u32,
         handshake_data: &HandshakeData,
-    ) -> Result<Neighbor, net_error> {
+    ) -> Result<(Neighbor, bool), net_error> {
         let addr = NeighborKey::from_handshake(peer_version, network_id, handshake_data);
         let pubk = handshake_data
             .node_public_key
@@ -482,15 +489,15 @@ impl Neighbor {
         let peer_opt = PeerDB::get_peer(conn, network_id, &addr.addrbytes, addr.port)
             .map_err(net_error::DBError)?;
 
-        let mut neighbor = match peer_opt {
+        let (mut neighbor, present) = match peer_opt {
             Some(neighbor) => {
                 let mut ret = neighbor;
                 ret.addr = addr.clone();
-                ret
+                (ret, true)
             }
             None => {
                 let ret = Neighbor::empty(&addr, &pubk, handshake_data.expire_block_height);
-                ret
+                (ret, false)
             }
         };
 
@@ -510,7 +517,7 @@ impl Neighbor {
         }
 
         neighbor.handshake_update(conn, &handshake_data)?;
-        Ok(neighbor)
+        Ok((neighbor, present))
     }
 
     pub fn from_conversation(
@@ -1250,13 +1257,13 @@ impl ConversationP2P {
         if updated {
             // save the new key
             let mut tx = peerdb.tx_begin().map_err(net_error::DBError)?;
-            let mut neighbor = Neighbor::from_handshake(
+            let (mut neighbor, _) = Neighbor::load_and_update(
                 &mut tx,
                 message.preamble.peer_version,
                 message.preamble.network_id,
                 &handshake_data,
             )?;
-            neighbor.save_update(&mut tx)?;
+            neighbor.save_update(&mut tx, None)?;
             tx.commit()
                 .map_err(|e| net_error::DBError(db_error::SqliteError(e)))?;
 
@@ -1277,10 +1284,7 @@ impl ConversationP2P {
                 accept_data,
                 StackerDBHandshakeData {
                     rc_consensus_hash: chain_view.rc_consensus_hash.clone(),
-                    // placeholder sbtc address for now
-                    smart_contracts: vec![
-                        ContractId::parse("SP000000000000000000002Q6VF78.sbtc").unwrap()
-                    ],
+                    smart_contracts: local_peer.stacker_dbs.clone(),
                 },
             )
         } else {
@@ -1401,11 +1405,12 @@ impl ConversationP2P {
         let epoch = self.get_current_epoch(chain_view.burn_block_height);
 
         // get neighbors at random as long as they're fresh, and as long as they're compatible with
-        // the current system epoch
-        let mut neighbors = PeerDB::get_random_neighbors(
+        // the current system epoch.
+        let mut neighbors = PeerDB::get_fresh_random_neighbors(
             peer_dbconn,
             self.network_id,
             epoch.network_epoch,
+            (get_epoch_time_secs() as u64).saturating_sub(self.connection.options.max_neighbor_age),
             MAX_NEIGHBORS_DATA_LEN,
             chain_view.burn_block_height,
             false,
@@ -2664,6 +2669,7 @@ mod test {
             data_url.clone(),
             &asn4_entries,
             Some(&initial_neighbors),
+            &vec![ContractId::parse("SP000000000000000000002Q6VF78.sbtc").unwrap()],
         )
         .unwrap();
         let sortdb = SortitionDB::connect(
@@ -3188,15 +3194,16 @@ mod test {
                         assert_eq!(data.handshake.data_url, "http://peer2.com".into());
                         assert_eq!(data.heartbeat_interval, conn_opts.heartbeat);
 
-                        // remote peer always replies with its supported smart contracts
-                        assert_eq!(
-                            db_data.smart_contracts,
-                            vec![ContractId::parse("SP000000000000000000002Q6VF78.sbtc").unwrap()]
-                        );
-
                         if peer_1_rc_consensus_hash == peer_2_rc_consensus_hash {
                             assert_eq!(db_data.rc_consensus_hash, chain_view_1.rc_consensus_hash);
 
+                            // remote peer always replies with its supported smart contracts
+                            assert_eq!(
+                                db_data.smart_contracts,
+                                vec![ContractId::parse("SP000000000000000000002Q6VF78.sbtc")
+                                    .unwrap()]
+                            );
+
                             // peers learn each others' smart contract DBs
                             eprintln!(
                                 "{:?}, {:?}",
@@ -5519,6 +5526,7 @@ mod test {
             None,
             get_epoch_time_secs() + 123456,
             UrlString::try_from("http://foo.com").unwrap(),
+            vec![],
         );
         let mut convo = ConversationP2P::new(
             123,

stackslib/src/net/connection.rs

@@ -52,8 +52,9 @@ use crate::net::StacksP2P;
 use crate::net::download::BLOCK_DOWNLOAD_INTERVAL;
 use crate::net::inv::{INV_REWARD_CYCLES, INV_SYNC_INTERVAL};
 use crate::net::neighbors::{
-    NEIGHBOR_REQUEST_TIMEOUT, NEIGHBOR_WALK_INTERVAL, NUM_INITIAL_WALKS, WALK_MAX_DURATION,
-    WALK_MIN_DURATION, WALK_RESET_INTERVAL, WALK_RESET_PROB, WALK_RETRY_COUNT, WALK_STATE_TIMEOUT,
+    MAX_NEIGHBOR_AGE, NEIGHBOR_REQUEST_TIMEOUT, NEIGHBOR_WALK_INTERVAL, NUM_INITIAL_WALKS,
+    WALK_MAX_DURATION, WALK_MIN_DURATION, WALK_RESET_INTERVAL, WALK_RESET_PROB, WALK_RETRY_COUNT,
+    WALK_STATE_TIMEOUT,
 };
 
 use clarity::vm::{costs::ExecutionCost, types::BOUND_VALUE_SERIALIZATION_HEX};
@@ -339,6 +340,7 @@ pub struct ConnectionOptions {
     pub max_neighbors_of_neighbor: u64,
     pub max_http_clients: u64,
     pub neighbor_request_timeout: u64,
+    pub max_neighbor_age: u64,
     pub num_initial_walks: u64,
     pub walk_retry_count: u64,
     pub walk_interval: u64,
@@ -423,6 +425,7 @@ impl std::default::Default for ConnectionOptions {
             max_neighbors_of_neighbor: 10,
             max_http_clients: 10,
             neighbor_request_timeout: NEIGHBOR_REQUEST_TIMEOUT, // how long to wait for a neighbor request
+            max_neighbor_age: MAX_NEIGHBOR_AGE,
             num_initial_walks: NUM_INITIAL_WALKS,
             walk_retry_count: WALK_RETRY_COUNT,
             walk_interval: NEIGHBOR_WALK_INTERVAL, // how often to do a neighbor walk.