TQ: Add protocol support for LRTQ upgrade (#9065)

This PR completes the first version of the sans-io trust quorum protocol
implementation.

LRTQ upgrade can now be started via
`Node::coordinate_upgrade_from_lrtq`.
This triggers the coordinating node to start collecting the LRTQ key
shares so that they can be used to construct the LRTQ rack secret via
the bootstore code. After this occurs, a Prepare message is sent out
with this old rack secret encrypted in a manner identical to a normal
reconfiguration. The prepare and commit paths remain the same.

The cluster proptest was updated to sometimes start out with an
existing LRTQ configuration and then to upgrade from there. Like normal
reconfigurations it allows aborting and pre-empting of the LRTQ upgrade
with a new attempt at a higher epoch. In production this is how we
"retry"
if the coordinating node crashes prior to commit, or more accurately, if
nexus can't talk to the coordinating node for some period of time and
just moves on. After the LRTQ upgrade commits, normal reconfigurations
are run.

We also remove unnecessary config related messages in this commit.
Since a `Configuration` does not contain sensitive information it can be
retrieved when Nexus polls the coordinator before it commits. Then Nexus
can save this info and send it in `PrepareAndCommit` messages rather
than having the receiving node try to find a live peer with the config
prior to collecting shares. This is a nice optimization that reduces
protocol complexity a bit. This removal allowed removing the TODO in the
message `match` statement in `Node::handle` and completing the protocol.

Author: andrewjstone

Cargo.lock

@@ -11,3 +11,9 @@ use serde::{Deserialize, Serialize};
     Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize,
 )]
 pub struct Sha3_256Digest([u8; 32]);
+
+impl Sha3_256Digest {
+    pub fn new(bytes: [u8; 32]) -> Self {
+        Sha3_256Digest(bytes)
+    }
+}

bootstore/src/lib.rs

@@ -90,7 +90,6 @@ impl RackSecret {
     }
 
     /// Combine a set of shares and return a RackSecret
-    #[allow(unused)]
     pub fn combine_shares(
         shares: &[Vec<u8>],
     ) -> Result<RackSecret, vsss_rs::Error> {

bootstore/src/trust_quorum/rack_secret.rs

@@ -5,7 +5,6 @@
 //! A configuration of a trust quroum at a given epoch
 
 use crate::crypto::{EncryptedRackSecrets, RackSecret, Sha3_256Digest};
-use crate::validators::ValidatedReconfigureMsg;
 use crate::{Epoch, PlatformId, Threshold};
 use daft::Diffable;
 use gfss::shamir::{Share, SplitError};
@@ -15,7 +14,7 @@ use secrecy::ExposeSecret;
 use serde::{Deserialize, Serialize};
 use serde_with::serde_as;
 use slog_error_chain::SlogInlineError;
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
 
 #[derive(Debug, Clone, thiserror::Error, PartialEq, Eq, SlogInlineError)]
 pub enum ConfigurationError {
@@ -75,22 +74,30 @@ impl IdOrdItem for Configuration {
     id_upcast!();
 }
 
+pub struct NewConfigParams<'a> {
+    pub rack_id: RackUuid,
+    pub epoch: Epoch,
+    pub members: &'a BTreeSet<PlatformId>,
+    pub threshold: Threshold,
+    pub coordinator_id: &'a PlatformId,
+}
+
 impl Configuration {
     /// Create a new configuration for the trust quorum
     ///
     /// `previous_configuration` is never filled in upon construction. A
     /// coordinator will fill this in as necessary after retrieving shares for
     /// the last committed epoch.
     pub fn new(
-        reconfigure_msg: &ValidatedReconfigureMsg,
+        params: NewConfigParams<'_>,
     ) -> Result<(Configuration, BTreeMap<PlatformId, Share>), ConfigurationError>
     {
-        let coordinator = reconfigure_msg.coordinator_id().clone();
+        let coordinator = params.coordinator_id.clone();
         let rack_secret = RackSecret::new();
         let shares = rack_secret.split(
-            reconfigure_msg.threshold(),
-            reconfigure_msg
-                .members()
+            params.threshold,
+            params
+                .members
                 .len()
                 .try_into()
                 .map_err(|_| ConfigurationError::TooManyMembers)?,
@@ -106,19 +113,19 @@ impl Configuration {
         let mut members: BTreeMap<PlatformId, Sha3_256Digest> = BTreeMap::new();
         let mut shares: BTreeMap<PlatformId, Share> = BTreeMap::new();
         for (platform_id, (share, digest)) in
-            reconfigure_msg.members().iter().cloned().zip(shares_and_digests)
+            params.members.iter().cloned().zip(shares_and_digests)
         {
             members.insert(platform_id.clone(), digest);
             shares.insert(platform_id, share);
         }
 
         Ok((
             Configuration {
-                rack_id: reconfigure_msg.rack_id(),
-                epoch: reconfigure_msg.epoch(),
+                rack_id: params.rack_id,
+                epoch: params.epoch,
                 coordinator,
                 members,
-                threshold: reconfigure_msg.threshold(),
+                threshold: params.threshold,
                 encrypted_rack_secrets: None,
             },
             shares,