diff --git a/trust-quorum/gfss/src/shamir.rs b/trust-quorum/gfss/src/shamir.rs index 41f09a3b7cd..c315b8dfd1a 100644 --- a/trust-quorum/gfss/src/shamir.rs +++ b/trust-quorum/gfss/src/shamir.rs @@ -23,7 +23,17 @@ pub enum SplitError { TooFewTotalShares { n: u8, k: u8 }, } -#[derive(Debug, Clone, thiserror::Error, PartialEq, Eq)] +#[derive( + Debug, + Clone, + thiserror::Error, + PartialEq, + Eq, + PartialOrd, + Ord, + Serialize, + Deserialize, +)] pub enum CombineError { #[error("must be at least 2 shares to combine")] TooFewShares, diff --git a/trust-quorum/src/alarm.rs b/trust-quorum/src/alarm.rs new file mode 100644 index 00000000000..0580c024d34 --- /dev/null +++ b/trust-quorum/src/alarm.rs @@ -0,0 +1,36 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Mechanism for reporting protocol invariant violations + +use serde::{Deserialize, Serialize}; + +use crate::{Configuration, Epoch, PlatformId}; + +#[allow(clippy::large_enum_variant)] +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, +)] +pub enum Alarm { + /// Different configurations found for the same epoch + /// + /// Reason: Nexus creates configurations and stores them in CRDB before + /// sending them to a coordinator of its choosing. Nexus will not send the + /// same reconfiguration request to different coordinators. If it does those + /// coordinators will generate different key shares. However, since Nexus + /// will not tell different nodes to coordinate the same configuration, this + /// state should be impossible to reach. + MismatchedConfigurations { + config1: Configuration, + config2: Configuration, + from: PlatformId, + }, + + /// The `keyShareComputer` could not compute this node's share + /// + /// Reason: A threshold of valid key shares were received based on the the + /// share digests in the Configuration. However, computation of the share + /// still failed. This should be impossible. + ShareComputationFailed { epoch: Epoch, err: gfss::shamir::CombineError }, +} diff --git a/trust-quorum/src/compute_key_share.rs b/trust-quorum/src/compute_key_share.rs index 2bee03abbea..8cc780f752e 100644 --- a/trust-quorum/src/compute_key_share.rs +++ b/trust-quorum/src/compute_key_share.rs @@ -9,7 +9,9 @@ //! other nodes so that it can compute its own key share. use crate::crypto::Sha3_256Digest; -use crate::{Configuration, Epoch, NodeHandlerCtx, PeerMsgKind, PlatformId}; +use crate::{ + Alarm, Configuration, Epoch, NodeHandlerCtx, PeerMsgKind, PlatformId, +}; use gfss::gf256::Gf256; use gfss::shamir::{self, Share}; use slog::{Logger, error, o, warn}; @@ -101,6 +103,7 @@ impl KeyShareComputer { "epoch" => %epoch, "from" => %from ); + return false; } // A valid share was received. Is it new? @@ -116,12 +119,23 @@ impl KeyShareComputer { // What index are we in the configuration? This is our "x-coordinate" // for our key share calculation. We always start indexing from 1, since // 0 is the rack secret. - let index = self - .config - .members - .keys() - .position(|id| id == ctx.platform_id()) - .expect("node exists"); + let index = + self.config.members.keys().position(|id| id == ctx.platform_id()); + + let Some(index) = index else { + let msg = concat!( + "Failed to get index for ourselves in current configuration. ", + "We are not a member, and must have been expunged." + ); + error!( + self.log, + "{msg}"; + "platform_id" => %ctx.platform_id(), + "config" => ?self.config + ); + return false; + }; + let x_coordinate = Gf256::new(u8::try_from(index + 1).expect("index fits in u8")); @@ -137,11 +151,9 @@ impl KeyShareComputer { }); true } - Err(e) => { - // TODO: put the node into into an `Alarm` state similar to - // https://github.com/oxidecomputer/omicron/pull/8062 once we - // have alarms? - error!(self.log, "Failed to compute share: {}", e); + Err(err) => { + error!(self.log, "Failed to compute share: {}", err); + ctx.raise_alarm(Alarm::ShareComputationFailed { epoch, err }); false } } diff --git a/trust-quorum/src/configuration.rs b/trust-quorum/src/configuration.rs index 23b4f6f3c8d..a6057c62ed1 100644 --- a/trust-quorum/src/configuration.rs +++ b/trust-quorum/src/configuration.rs @@ -30,7 +30,9 @@ pub enum ConfigurationError { /// The configuration for a given epoch. /// /// Only valid for non-lrtq configurations -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive( + Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, +)] pub struct Configuration { /// Unique Id of the rack pub rack_id: RackUuid, diff --git a/trust-quorum/src/lib.rs b/trust-quorum/src/lib.rs index 39418714296..8bb8d8de5d3 100644 --- a/trust-quorum/src/lib.rs +++ b/trust-quorum/src/lib.rs @@ -23,7 +23,9 @@ mod persistent_state; mod validators; pub use configuration::Configuration; pub use coordinator_state::{CoordinatorOperation, CoordinatorState}; +mod alarm; +pub use alarm::Alarm; pub use crypto::RackSecret; pub use messages::*; pub use node::Node; diff --git a/trust-quorum/src/node.rs b/trust-quorum/src/node.rs index 72d38a5b674..a6613f9062f 100644 --- a/trust-quorum/src/node.rs +++ b/trust-quorum/src/node.rs @@ -20,7 +20,7 @@ use crate::validators::{ MismatchedRackIdError, ReconfigurationError, ValidatedReconfigureMsg, }; use crate::{ - Configuration, CoordinatorState, Epoch, NodeHandlerCtx, PlatformId, + Alarm, Configuration, CoordinatorState, Epoch, NodeHandlerCtx, PlatformId, messages::*, }; use gfss::shamir::Share; @@ -308,28 +308,33 @@ impl Node { "epoch" => %config.epoch ); ctx.update_persistent_state(|ps| ps.commits.insert(config.epoch)); + return; } // Do we have the configuration in our persistent state? If not save it. - ctx.update_persistent_state(|ps| { - if let Err(e) = ps.configs.insert_unique(config.clone()) { - let existing = - e.duplicates().first().expect("duplicate exists"); - if *existing != &config { - error!( - self.log, - "Received a configuration mismatch"; - "from" => %from, - "existing_config" => #?existing, - "received_config" => #?config - ); - // TODO: Alarm - } - false - } else { - true + if let Some(existing) = + ctx.persistent_state().configuration(config.epoch) + { + if existing != &config { + error!( + self.log, + "Received a configuration mismatch"; + "from" => %from, + "existing_config" => #?existing, + "received_config" => #?config + ); + ctx.raise_alarm(Alarm::MismatchedConfigurations { + config1: (*existing).clone(), + config2: config.clone(), + from: from.clone(), + }); } - }); + } else { + ctx.update_persistent_state(|ps| { + ps.configs.insert_unique(config.clone()).expect("new config"); + true + }); + } // Are we coordinating for an older epoch? If so, cancel. if let Some(cs) = &self.coordinator_state { @@ -343,14 +348,14 @@ impl Node { "received_epoch" => %config.epoch ); self.coordinator_state = None; + // Intentionally fall through } else if coordinating_epoch == config.epoch { - error!( + info!( self.log, "Received CommitAdvance while coordinating for same epoch!"; "from" => %from, "epoch" => %config.epoch ); - // TODO: Alarm return; } else { info!( @@ -399,7 +404,8 @@ impl Node { } } - // We either were collectiong shares for an old epoch or haven't started yet. + // We either were collectiong shares for an old epoch or haven't started + // yet. self.key_share_computer = Some(KeyShareComputer::new(&self.log, ctx, config)); } @@ -414,6 +420,18 @@ impl Node { ctx.persistent_state().latest_committed_configuration() { if latest_committed_config.epoch > epoch { + if !latest_committed_config.members.contains_key(&from) { + info!( + self.log, + "Received a GetShare message from expunged node"; + "from" => %from, + "latest_committed_epoch" => + %latest_committed_config.epoch, + "requested_epoch" => %epoch + ); + // TODO: Send an expunged message + return; + } info!( self.log, concat!( @@ -432,6 +450,20 @@ impl Node { } } + // Do we have the configuration? Is the requesting peer a member? + if let Some(config) = ctx.persistent_state().configuration(epoch) { + if !config.members.contains_key(&from) { + info!( + self.log, + "Received a GetShare message from expunged node"; + "from" => %from, + "epoch" => %epoch + ); + // TODO: Send an expunged message + return; + } + } + // If we have the share for the requested epoch, we always return it. We // know that it is at least as new as the last committed epoch. We might // not have learned about the configuration being committed yet, but diff --git a/trust-quorum/src/node_ctx.rs b/trust-quorum/src/node_ctx.rs index 30cfe15b174..e3a4f7fed32 100644 --- a/trust-quorum/src/node_ctx.rs +++ b/trust-quorum/src/node_ctx.rs @@ -4,7 +4,9 @@ //! Parameter to Node API calls that allows interaction with the system at large -use crate::{Envelope, PeerMsg, PeerMsgKind, PersistentState, PlatformId}; +use crate::{ + Alarm, Envelope, PeerMsg, PeerMsgKind, PersistentState, PlatformId, +}; use std::collections::BTreeSet; /// An API shared by [`NodeCallerCtx`] and [`NodeHandlerCtx`] @@ -12,6 +14,7 @@ pub trait NodeCommonCtx { fn platform_id(&self) -> &PlatformId; fn persistent_state(&self) -> &PersistentState; fn connected(&self) -> &BTreeSet; + fn alarms(&self) -> &BTreeSet; } /// An API for an [`NodeCtx`] usable from a [`crate::Node`] @@ -54,6 +57,9 @@ pub trait NodeHandlerCtx: NodeCommonCtx { /// Remove a peer from the connected set fn remove_connection(&mut self, id: &PlatformId); + + /// Record (in-memory) that an alarm has occurred + fn raise_alarm(&mut self, alarm: Alarm); } /// Common parameter to [`crate::Node`] methods @@ -79,6 +85,9 @@ pub struct NodeCtx { /// Connected peer nodes connected: BTreeSet, + + /// Any alarms that have occurred + alarms: BTreeSet, } impl NodeCtx { @@ -89,6 +98,7 @@ impl NodeCtx { persistent_state_changed: false, outgoing: Vec::new(), connected: BTreeSet::new(), + alarms: BTreeSet::new(), } } } @@ -105,6 +115,10 @@ impl NodeCommonCtx for NodeCtx { fn connected(&self) -> &BTreeSet { &self.connected } + + fn alarms(&self) -> &BTreeSet { + &self.alarms + } } impl NodeHandlerCtx for NodeCtx { @@ -138,6 +152,10 @@ impl NodeHandlerCtx for NodeCtx { fn remove_connection(&mut self, id: &PlatformId) { self.connected.remove(id); } + + fn raise_alarm(&mut self, alarm: Alarm) { + self.alarms.insert(alarm); + } } impl NodeCallerCtx for NodeCtx { diff --git a/trust-quorum/tests/cluster.proptest-regressions b/trust-quorum/tests/cluster.proptest-regressions new file mode 100644 index 00000000000..bb7577eca65 --- /dev/null +++ b/trust-quorum/tests/cluster.proptest-regressions @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc aa229dbfa63db6ed21d624e6c474f3591be97204fc051fe2b7b6d1af78409532 # shrinks to input = _TestTrustQuorumProtocolArgs { input: TestInput { initial_config: GeneratedConfiguration { members: {2, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 21}, threshold: Index(0) }, initial_down_nodes: {}, actions: [DeliverEnvelopes([Index(14493870343628933413), Index(0), Index(2635249153387079100), Index(7659274142974149207), Index(10538101999560721780), Index(9052169730812637246), Index(4315275857016173272), Index(1740015673248333392), Index(13987162340212539423)]), DeliverEnvelopes([Index(9041798398182823156), Index(13868876044032355650)]), DeliverEnvelopes([Index(3011348768397116341), Index(8660180482978775225), Index(9182847551331361735), Index(16121357191531813301), Index(3788564878545978377), Index(17393195961508650460), Index(3346167418243659948), Index(3573581286569374339), Index(13250013466448051467), Index(4820013139542767201), Index(8039736122411843390), Index(17013067520142601277), Index(16122004323767661676), Index(17569103901503114628), Index(1932949711347229750), Index(4319513761709523737), Index(4584565250890182062), Index(1279075436995347802), Index(17062300948886540969)]), PollPrepareAcks, Commit([Index(17389235045933586108), Index(8560515657089253499), Index(10855801671237462872), Index(13083943140650726535), Index(6596360928255073579), Index(12912936435579810120), Index(13555346217578137976), Index(1669092046242131079)]), DeliverNexusReplies(6), Reconfigure { num_added_nodes: 3, removed_nodes: [Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }], threshold: Index(1703742760083928913), coordinator: Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 } }, Reconfigure { num_added_nodes: 0, removed_nodes: [], threshold: Index(5259489828371974487), coordinator: Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 } }, Reconfigure { num_added_nodes: 0, removed_nodes: [Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }, Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }], threshold: Index(14446131236589207074), coordinator: Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 } }, DeliverEnvelopes([Index(2073236698708178332)]), DeliverEnvelopes([Index(3453196791926166099), Index(568387599084033335), Index(3409840665184802609), Index(12286030226091071672), Index(7286708483759199339), Index(2055658592347030945), Index(1173213978658236675), Index(8649978366863725550)]), Reconfigure { num_added_nodes: 2, removed_nodes: [Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }], threshold: Index(9996579049375553061), coordinator: Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 } }, Reconfigure { num_added_nodes: 2, removed_nodes: [Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }, Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }], threshold: Index(17841859598948908161), coordinator: Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 } }, Reconfigure { num_added_nodes: 1, removed_nodes: [], threshold: Index(8689652795643676030), coordinator: Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 } }, DeliverEnvelopes([Index(13933299526419321128), Index(5225873483252317482), Index(5244472839316863196), Index(15065206300363858421), Index(1331991639281083654), Index(9969299029824142306), Index(10418763353833126365)]), Reconfigure { num_added_nodes: 2, removed_nodes: [Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }, Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }], threshold: Index(17441105865747581233), coordinator: Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 } }, DeliverEnvelopes([Index(9629417235951452701), Index(17699594351241716570), Index(6446358179225870872), Index(10477494232845674894), Index(226357282448375442), Index(17511469032238010586), Index(5868458999694820988), Index(9008597573607186863), Index(14889469521169823547), Index(16216562580497901897), Index(10976472819558126038), Index(8297592251652111615)]), DeliverEnvelopes([Index(6225179597717370182), Index(3456377812674314165), Index(4870110218414913904), Index(12933525926916895630), Index(5251263308628043457), Index(14960131063828769882), Index(954293712144926568), Index(15613441814815191176), Index(7681056590698508795), Index(3497512649074481205)]), DeliverEnvelopes([Index(5458890086829541581), Index(4300690563898456475), Index(469725690344733484), Index(3627582562296635119), Index(13485721932604488035), Index(838877244787605597), Index(17616669406954758185), Index(10912008557925580957), Index(2064843088888151834), Index(13592137105530054033), Index(464925174084213053), Index(12477077518224200179), Index(7062021569203679845)]), DeliverEnvelopes([Index(15629509651933908728), Index(7593448817552421286), Index(6069697258859901799), Index(7447724974852314797), Index(6976391078682036797), Index(15954111707567866812), Index(9739269646592130920), Index(7840844524173616953), Index(17251031726274733868), Index(3236370306236870045), Index(10198745931145551826), Index(11526777721718675946), Index(7855592439162647423), Index(17024415010144483949), Index(4229155328323504098), Index(5472111026499192469)]), DeliverEnvelopes([Index(4411786472941543389), Index(2352452285251361371), Index(7822068502251264399), Index(10423488690252370246), Index(6800625973591759468), Index(14891503574606637490), Index(4929072166198014740), Index(15521754316309705239), Index(8420972048384534331), Index(14271222805435196129), Index(15731413232976718648), Index(7620446100066998584), Index(16071224481319847817), Index(3144698519065113997), Index(2809650099793229888), Index(5232005717265660268), Index(7349929989771864370), Index(5385953262323237136), Index(5995048832340625664)]), DeliverEnvelopes([Index(17995955460911334433), Index(4708641667603374303), Index(7732664601632819026), Index(17423877575181254121), Index(13237365779440508178), Index(16219013419076737978)]), DeliverEnvelopes([Index(8304946145816664660), Index(862057291908146042), Index(15725440490586181656), Index(16995826471578986413), Index(2215118704959889028), Index(5868186664753940774), Index(8206076788005089132), Index(5701603538374868810), Index(7882633833049924931), Index(14920895499854492495), Index(9744657067126826847), Index(17217199825680194648)]), DeliverEnvelopes([Index(8037762189481305143), Index(7161454571813702078), Index(16624697899890776024), Index(6776753215496123550), Index(3552663706596513637), Index(13168137737732814449), Index(17773039112791415335), Index(5357529117839942678), Index(11360548594432633510), Index(17152936725964058392), Index(3735622760167456433), Index(3729434446292991499), Index(8239146570926600189), Index(17213753850553118615), Index(3586505624037292033), Index(5676170410763085337), Index(7854336347370863376), Index(3296193835517592123)]), DeliverEnvelopes([Index(7450515981053560543), Index(7281049720524507073), Index(11805621530410984780), Index(14212481354680952709), Index(11677201894475474096), Index(15579047847775346789), Index(8933612206363673270), Index(14589715969816048496), Index(9146028479816005688), Index(15678520830606285653), Index(13646715682213052692), Index(16542748546042997802), Index(3018629145641526806), Index(3279754668523137850), Index(9190080933222987746)]), DeliverEnvelopes([Index(4442754352407199000), Index(6726639451905550261), Index(276325053358083205), Index(8309639208446119622), Index(17071310980996705540), Index(13681920258353201127), Index(6824753331681591316), Index(9564449636752909493), Index(11781421339321328484), Index(17060651525797945236), Index(6256326521108731728), Index(882364767508101600), Index(18155289916698303405), Index(3604801106996280475), Index(18030768360505361161), Index(10828708533993573006), Index(10140811484372325170)]), DeliverEnvelopes([Index(14675779565613765883), Index(7875237876508366946), Index(1080598541644522468), Index(9795208880339871790), Index(16788173384975729322), Index(7862287353069942075), Index(18287036425108952119), Index(438554228774108801), Index(16936278258143324901), Index(17297858047970493225), Index(16280794788037943206)]), DeliverEnvelopes([Index(14318968306187897375), Index(15260180391589155604), Index(15065168303490293135), Index(16283608916485249870), Index(13275033248234956364), Index(2830029696975378334), Index(10826126602511348625), Index(3503489386636778134), Index(4143173347019462961), Index(14194382779242307605), Index(6222922790114912969), Index(4040121466287304927), Index(9113628229529655615), Index(5415084019849945730), Index(1626553538397648371), Index(15501687776809538824)]), DeliverEnvelopes([Index(5723406985380791968), Index(1155733174009260305), Index(7032746625126254072), Index(4743835759260085268), Index(7266303314157497289), Index(9385508013014315790), Index(17143596343958976419), Index(12019102422355865994), Index(17164688997776033771), Index(5601248314369099348), Index(8909509441657159134), Index(434646333244277371), Index(2059402500670808177), Index(10728320671205025035), Index(10839709495944012941), Index(13074991387519599432), Index(16684428541064080728), Index(1516391557911492208)]), PollPrepareAcks, Commit([Index(10338703738028056944), Index(18153730223737876722), Index(7878439710336920970), Index(16934827903768804991)]), DeliverNexusReplies(7), DeliverEnvelopes([Index(12043925503899430601), Index(13756482947028862137), Index(16512953710007648247), Index(17789811749467591337), Index(10320232143017481771), Index(1459556606276719356), Index(845781064116242142), Index(1956519909910251783), Index(3164835970372175816), Index(14120105518532170713), Index(10524629999594340195), Index(2646948465709084646), Index(15782024841931074195), Index(4857968548620380892), Index(10321553606781845802), Index(2234980294207553151), Index(14471054211349921145)]), DeliverEnvelopes([Index(3767742994858916851), Index(5436520998904493278), Index(10088257053026246598), Index(1498263247328214760), Index(14106058185302211861), Index(6061739775064231445), Index(2023011050214169455), Index(7250794870201811289), Index(17600053360168057888), Index(1740642731188709568), Index(2663072436983654497), Index(809252200146375679), Index(1869788575371162759), Index(4781549348184833023), Index(3523664822500907563)]), Commit([Index(14652573856526224379), Index(5781738428227077912), Index(12854715630867365273), Index(1911123227362573689)]), DeliverEnvelopes([Index(5483103026978400083), Index(14279025710451447544), Index(4937116848351103172), Index(16927987341169693780), Index(9896521330207311715)]), DeliverEnvelopes([Index(16187414459998690745), Index(65365848735262627), Index(16029309594081455593), Index(8738644556875519057), Index(8210105897167225670), Index(15476129597523288716), Index(5183379879833502204), Index(11717624468492776210), Index(15229030068062309892), Index(11851728872298706575), Index(10702820350636537827), Index(2687836824532707096), Index(2996236953690420395)]), Commit([Index(10707814609213298068), Index(12699776013897419563)]), Commit([Index(10157815118387593249), Index(5123540499752415756), Index(15808738534837068446)]), DeliverEnvelopes([Index(1541961047234018357), Index(1525243190754021203), Index(12707227425511543885), Index(1363732177719220534), Index(6262127490926569596), Index(15087875812043424072), Index(7757884893278869289), Index(11575998173089334269), Index(7651957407489583015), Index(384152082221813754), Index(13402749590934510602), Index(9025488436292943022)]), Commit([Index(2522165368668848218), Index(13296047016915538553), Index(15622931071258404786), Index(3114595942384622948), Index(12412617090084599671), Index(11664985786937734966)]), PollPrepareAcks, Commit([Index(13180524526118109937), Index(17193218842005402908), Index(17790813705747893301), Index(739262928517866423)]), PollPrepareAcks, PollPrepareAcks, DeliverNexusReplies(3), DeliverEnvelopes([Index(652695315058809065), Index(727847806737003567), Index(5404780742271511029), Index(16694185347801040155), Index(4098481543012673009), Index(7791187420263002075)]), Reconfigure { num_added_nodes: 1, removed_nodes: [Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }], threshold: Index(199832759506365010), coordinator: Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 } }, DeliverEnvelopes([Index(9168374719730985799), Index(10794102811332111465), Index(17028470677432611953)]), DeliverNexusReplies(3), Reconfigure { num_added_nodes: 1, removed_nodes: [Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }], threshold: Index(18233548140847603442), coordinator: Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 } }, PollPrepareAcks, DeliverNexusReplies(7), DeliverNexusReplies(6), PollPrepareAcks, Reconfigure { num_added_nodes: 4, removed_nodes: [Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }, Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }], threshold: Index(6273134768375120567), coordinator: Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 } }, DeliverNexusReplies(1), DeliverNexusReplies(8), Reconfigure { num_added_nodes: 4, removed_nodes: [Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 }], threshold: Index(15266709888734590000), coordinator: Selector { rng: TestRng { rng: ChaCha(ChaCha20Rng { rng: BlockRng { core: ChaChaXCore {}, result_len: 64, index: 64 } }) }, bias_increment: 0 } }, DeliverEnvelopes([Index(9098454520105484266), Index(5401214547661529702), Index(729613142155688864), Index(18165312717926349510), Index(13596594098232187840), Index(17864609912832060176), Index(6576389962946327472), Index(6748861532334916574), Index(6270337184347778455), Index(4621189873065888654), Index(1329701404467837166), Index(15253391592315247523), Index(7548879285352433967), Index(1519898158722201861), Index(9221399647181265582), Index(14775495632976400799)]), DeliverNexusReplies(7), DeliverNexusReplies(7), DeliverEnvelopes([Index(6844824367736706288), Index(15644179546212865649), Index(4684448243389276339), Index(7196508453975880576)]), PollPrepareAcks, PollPrepareAcks, DeliverNexusReplies(8), DeliverEnvelopes([Index(5940056868932490196), Index(7905305070747045242), Index(18010824367591471345)]), PollPrepareAcks, DeliverEnvelopes([Index(11313344465270829281), Index(14770881005951866888), Index(14791521919982503413), Index(13986963003913738122), Index(15146722185811966483), Index(5670599236968951456), Index(17258904022024285117), Index(9970983405711455637), Index(1037749372107185535), Index(14590287308375860226), Index(14401361711946635290), Index(11544751816992154044)]), DeliverNexusReplies(2), DeliverNexusReplies(5), PollPrepareAcks, Commit([Index(4768387033456628201), Index(4474917877175302218), Index(16571851457975840247), Index(15255821624041417534), Index(16757765128366937769), Index(2630722755438967257), Index(9285796362852384853), Index(14015858312178579018)]), DeliverEnvelopes([Index(15871463961649635696), Index(18163198373939775702), Index(2600226515950816111)]), DeliverEnvelopes([Index(3137454651926879610), Index(4284065310722833297), Index(16055853410816331071), Index(15166506491941155629), Index(12590336857066018677)]), Commit([Index(5641127885216290645), Index(11167110485586573589), Index(1014364951816464696), Index(12696711847665095026), Index(17978099696654320596)]), DeliverNexusReplies(8), Commit([Index(4029921677534804644), Index(15185718783597090882), Index(11314695466132516141), Index(5987253657617228904), Index(12519437426641924797), Index(6263639313484088578), Index(1682708770096528814), Index(17514314903812047682), Index(11457587421741508124)]), Commit([Index(9774639841375717942), Index(10038533130474007843), Index(17060633271273876314), Index(7000629506295319285)]), DeliverEnvelopes([Index(15061341259133073864), Index(5027422415066345386), Index(9379255978562524739), Index(6150067533282387651), Index(213696060201653414), Index(16059755076226721578), Index(16824493282550752097), Index(15722153504680899440), Index(5759912639700558277), Index(4924300060842152804), Index(9327426990216358558), Index(17734827167724690660), Index(1357167295908663567), Index(13405304061122455105)]), Commit([Index(17426666830188603729), Index(12810851899054823276), Index(5888927766144618223), Index(18098665668806060067), Index(8508378395275761614), Index(1650853648920734436), Index(5663558801920224835)]), DeliverEnvelopes([Index(15295993199711016882), Index(3859998607405058373), Index(16246966108341368602), Index(2383124365549873480), Index(16408321263194864709), Index(14444156841149526789), Index(6495747943129340003), Index(17772779185757377240), Index(14604635448037405545), Index(4847143012976409235), Index(1145991671159544728), Index(6507514046328470924), Index(13464350535917834973), Index(14483501505596509725), Index(4511687336665279669), Index(9620101944261959773), Index(484390403697139876), Index(9857456322741422009), Index(3341222595136110266)]), PollPrepareAcks, DeliverNexusReplies(5), PollPrepareAcks, PollPrepareAcks, DeliverEnvelopes([Index(213757553854772440), Index(3794994835931277050), Index(1230995856688641619), Index(615222509626393634), Index(11235657579233697266), Index(380444266263585305), Index(17306335717947557462), Index(8806641371157136569), Index(544411057042609898), Index(14217584839742171129), Index(17179942378194078060), Index(18111746867895443277), Index(4705170374898011720), Index(3617427108480506863), Index(4184498722027448066), Index(17681194391468303007), Index(12378580499825910547), Index(1587437098921461308), Index(5646864724861626575)]), Commit([Index(2895265222191031484), Index(17950221019931177667), Index(9263853737651325899), Index(3842185010035964272), Index(12827291054333377474)]), Commit([Index(15759430573635913179), Index(9798702836106401259), Index(5201468168437920159), Index(2657315818353075169), Index(8828036642617900104), Index(16884723312357945697), Index(14437294211944068336)]), DeliverNexusReplies(5), DeliverEnvelopes([Index(7768905154491618039), Index(3438302915553662917), Index(14091186660758608634)]), Commit([Index(17866948588368032403), Index(11604953794048654451)]), DeliverNexusReplies(2), DeliverNexusReplies(7), Commit([Index(13712612038145219747), Index(12085627743433809088), Index(8699014604701166458), Index(2994093657804205227), Index(16225639718416622872), Index(6900536451203328895), Index(11227963432608776486), Index(13049675711087060279), Index(5214683319822193770)]), DeliverNexusReplies(4), PollPrepareAcks, DeliverNexusReplies(2), DeliverEnvelopes([Index(6044528710470823771), Index(7296331177112754699), Index(9457525726669633929), Index(5079497433348808932), Index(17348870535556670336), Index(512738896472721452), Index(12256764748176290216), Index(7604437520746076905), Index(6769996048460626115), Index(16864410288957109112), Index(3088590075318437038), Index(2095012960284709716)]), Commit([Index(15776963438323035828), Index(16592172527267608874), Index(4574510404731010758), Index(9801172050530399304), Index(12591565060149844827), Index(17758031771693043328), Index(2356648783237251688), Index(2403981669904249702), Index(4639941478079293934)])] } } diff --git a/trust-quorum/tests/cluster.rs b/trust-quorum/tests/cluster.rs index 9dec6000640..9bc7da94c65 100644 --- a/trust-quorum/tests/cluster.rs +++ b/trust-quorum/tests/cluster.rs @@ -834,6 +834,7 @@ impl TestState { self.invariant_nodes_have_prepared_if_coordinator_has_acks()?; self.invariant_nodes_have_committed_if_nexus_has_acks()?; self.invariant_nodes_not_coordinating_and_computing_key_share_simultaneously()?; + self.invariant_no_alarms()?; Ok(()) } @@ -953,6 +954,20 @@ impl TestState { Ok(()) } + + // Ensure there has been no alarm at any node + fn invariant_no_alarms(&self) -> Result<(), TestCaseError> { + for (id, (_, ctx)) in &self.sut.nodes { + let alarms = ctx.alarms(); + prop_assert!( + alarms.is_empty(), + "Alarms found for {}: {:#?}", + id, + alarms + ); + } + Ok(()) + } } /// Broken out of `TestState` to alleviate borrow checker woes