Do not panic on stale messages from connections (#3764)

Fixes #3759

Author: andrewjstone

bootstore/src/schemes/v0/peer.rs

@@ -752,14 +752,17 @@ impl Node {
                 let Some(accepted_handle) =
                    self.accepted_connections.remove(&accepted_addr) else
                 {
-                    error!(
+                    warn!(
                         self.log,
-                        "Missing AcceptedConnHandle";
+                        concat!(
+                            "Missing AcceptedConnHandle: ",
+                            "Stale ConnectedAcceptor msg"
+                        );
                         "accepted_addr" => accepted_addr.to_string(),
                         "addr" => addr.to_string(),
                         "remote_peer_id" => peer_id.to_string()
                     );
-                    panic!("Missing AcceptedConnHandle");
+                    return;
                 };
 
                 // Ignore the stale message if the unique_id doesn't match what
@@ -818,13 +821,13 @@ impl Node {
                     self.established_connections
                         .insert(peer_id.clone(), handle);
                 } else {
-                    error!(
+                    warn!(
                         self.log,
-                        "Missing PeerConnHandle";
+                        "Missing PeerConnHandle; Stale ConnectedInitiator msg";
                         "addr" => addr.to_string(),
                         "remote_peer_id" => peer_id.to_string()
                     );
-                    panic!("Missing PeerConnHandle");
+                    return;
                 }
 
                 if let Err(e) =
@@ -846,12 +849,12 @@ impl Node {
                         return;
                     }
                 } else {
-                    error!(
+                    warn!(
                         self.log,
-                        "Missing PeerConnHandle";
+                        "Missing PeerConnHandle: Stale Disconnected msg";
                         "remote_peer_id" => peer_id.to_string()
                     );
-                    panic!("Missing PeerConnHandle");
+                    return;
                 }
                 warn!(self.log, "peer disconnected {peer_id}");
                 let handle =
@@ -932,6 +935,7 @@ impl Node {
                     self.conn_tx.clone(),
                 )
                 .await;
+                info!(self.log, "Initiating connection to new peer: {addr}");
                 self.initiating_connections.insert(addr, handle);
             }
         }
@@ -952,6 +956,11 @@ impl Node {
     async fn remove_peer(&mut self, addr: SocketAddrV6) {
         if let Some(handle) = self.initiating_connections.remove(&addr) {
             // The connection has not yet completed its handshake
+            info!(
+                self.log,
+                "Peer removed: deleting initiating connection";
+                "remote_addr" => addr.to_string()
+            );
             let _ = handle.tx.send(MainToConnMsg::Close).await;
         } else {
             // Do we have an established connection?
@@ -960,6 +969,12 @@ impl Node {
                 .iter()
                 .find(|(_, handle)| handle.addr == addr)
             {
+                info!(
+                    self.log,
+                    "Peer removed: deleting established connection";
+                    "remote_addr" => addr.to_string(),
+                    "remote_peer_id" => id.to_string(),
+                );
                 let _ = handle.tx.send(MainToConnMsg::Close).await;
                 // probably a better way to avoid borrowck issues
                 let id = id.clone();