Skip to content

Commit 2d8fc67

Browse files
andrewjstoneandrewjstone
committed
WIP: TQ: Support adding sleds via trust quorum
This PR introduces two new external APIs to allow adding multiple sleds to a rack at once and to query status about the ongoing operation. Both are currently experimental and live under `/v1/trust-quorum`. They need to be moved under `/system/hardware` like the original `sled-add` command. They also need to be reworked to not report trust quorum specific details if it can be avoided. Most of that should be in omdb for debugging. I may add some of that support to this PR. This PR also introduces a background task for driving the trust quorum reconfiguration to completion. Reconfiguration is driven by two steps. Synchronously updating the DB in the new external endpoint handler and then asynchronously trying to commit the operation via the background task. I tested this on a4x2 and it works as expected. See the trace from the original external API test below: ``` ➜ oxide.rs git:(main) ✗ echo '{"rack_id": "0dbef452-a6dd-4831-bbdc-769ea3353f28", "sled_ids": [{"part": "PPP-PPPPPPP","serial": "00000000002"}]}' | target/debug/oxide --profile recovery api /v1/trust-quorum/new-members --method POST --input - ➜ oxide.rs git:(main) ✗ target/debug/oxide --profile recovery api /v1/trust-quorum/config/latest/0dbef452-a6dd-4831-bbdc-769ea3353f28 { "abort_reason": null, "commit_crash_tolerance": 1, "coordinator": { "part_number": "PPP-PPPPPPP", "serial_number": "00000000003" }, "encrypted_rack_secrets": null, "epoch": 2, "last_committed_epoch": 1, "members": { "PPP-PPPPPPP:00000000000": { "share_digest": null, "state": "unacked", "time_committed": null, "time_prepared": null }, "PPP-PPPPPPP:00000000001": { "share_digest": null, "state": "unacked", "time_committed": null, "time_prepared": null }, "PPP-PPPPPPP:00000000002": { "share_digest": null, "state": "unacked", "time_committed": null, "time_prepared": null }, "PPP-PPPPPPP:00000000003": { "share_digest": null, "state": "unacked", "time_committed": null, "time_prepared": null } }, "rack_id": "0dbef452-a6dd-4831-bbdc-769ea3353f28", "state": "preparing", "threshold": 3, "time_aborted": null, "time_committed": null, "time_committing": null, "time_created": "2026-01-14T21:32:18.780136Z" } ➜ oxide.rs git:(main) ✗ target/debug/oxide --profile recovery api /v1/trust-quorum/config/latest/0dbef452-a6dd-4831-bbdc-769ea3353f28 { "abort_reason": null, "commit_crash_tolerance": 1, "coordinator": { "part_number": "PPP-PPPPPPP", "serial_number": "00000000003" }, "encrypted_rack_secrets": null, "epoch": 2, "last_committed_epoch": 1, "members": { "PPP-PPPPPPP:00000000000": { "share_digest": "fcfb09128c84d82cc81b200c6c682510f63160a4417856f4041b1886445e8b14", "state": "prepared", "time_committed": null, "time_prepared": "2026-01-14T21:32:55.826622Z" }, "PPP-PPPPPPP:00000000001": { "share_digest": "d8cad02bd3bccd08109a79e3bf6d8dab0d460a0ba879bf42887dc0fc8d855786", "state": "prepared", "time_committed": null, "time_prepared": "2026-01-14T21:32:55.848235Z" }, "PPP-PPPPPPP:00000000002": { "share_digest": "dd57ad8e271734fabfe97d6180d6da3e5c3805e17dacf58e0f2a6d5ed7f1242b", "state": "prepared", "time_committed": null, "time_prepared": "2026-01-14T21:32:55.806644Z" }, "PPP-PPPPPPP:00000000003": { "share_digest": "6b27327ca49976ccca83972e6578ef195c99489e62811e8d0a0cb061fca9c0c4", "state": "prepared", "time_committed": null, "time_prepared": "2026-01-14T21:32:55.837154Z" } }, "rack_id": "0dbef452-a6dd-4831-bbdc-769ea3353f28", "state": "preparing", "threshold": 3, "time_aborted": null, "time_committed": null, "time_committing": null, "time_created": "2026-01-14T21:32:18.780136Z" } ➜ oxide.rs git:(main) ✗ target/debug/oxide --profile recovery api /v1/trust-quorum/config/latest/0dbef452-a6dd-4831-bbdc-769ea3353f28 { "abort_reason": null, "commit_crash_tolerance": 1, "coordinator": { "part_number": "PPP-PPPPPPP", "serial_number": "00000000003" }, "encrypted_rack_secrets": { "data": "53de7731deec3f298a7f5067e256a63bb2869a91c9710d9b23dbf3d261d1b730039d9cb11b543c14906ff77cd409d32953959e9ff8933858", "salt": "ec609ed5ff7aee94e2e88ad94af56e0cbb8a66a683294005c7888f60a627956a" }, "epoch": 2, "last_committed_epoch": 1, "members": { "PPP-PPPPPPP:00000000000": { "share_digest": "fcfb09128c84d82cc81b200c6c682510f63160a4417856f4041b1886445e8b14", "state": "committed", "time_committed": "2026-01-14T21:33:03.864617Z", "time_prepared": "2026-01-14T21:32:55.826622Z" }, "PPP-PPPPPPP:00000000001": { "share_digest": "d8cad02bd3bccd08109a79e3bf6d8dab0d460a0ba879bf42887dc0fc8d855786", "state": "committed", "time_committed": "2026-01-14T21:33:03.864617Z", "time_prepared": "2026-01-14T21:32:55.848235Z" }, "PPP-PPPPPPP:00000000002": { "share_digest": "dd57ad8e271734fabfe97d6180d6da3e5c3805e17dacf58e0f2a6d5ed7f1242b", "state": "committed", "time_committed": "2026-01-14T21:33:03.864617Z", "time_prepared": "2026-01-14T21:32:55.806644Z" }, "PPP-PPPPPPP:00000000003": { "share_digest": "6b27327ca49976ccca83972e6578ef195c99489e62811e8d0a0cb061fca9c0c4", "state": "committed", "time_committed": "2026-01-14T21:33:03.864617Z", "time_prepared": "2026-01-14T21:32:55.837154Z" } }, "rack_id": "0dbef452-a6dd-4831-bbdc-769ea3353f28", "state": "committed", "threshold": 3, "time_aborted": null, "time_committed": "2026-01-14T21:33:04.652543Z", "time_committing": "2026-01-14T21:32:55.861158Z", "time_created": "2026-01-14T21:32:18.780136Z" } ➜ oxide.rs git:(main) ✗ ```
1 parent c197cca commit 2d8fc67

File tree

26 files changed

+31048
-14
lines changed

26 files changed

+31048
-14
lines changed

‎Cargo.lock‎

Lines changed: 4 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎clients/sled-agent-client/Cargo.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,5 +24,7 @@ schemars.workspace = true
2424
serde.workspace = true
2525
serde_json.workspace = true
2626
sled-agent-types.workspace = true
27+
sled-hardware-types.workspace = true
28+
trust-quorum-types.workspace = true
2729
slog.workspace = true
2830
uuid.workspace = true

‎clients/sled-agent-client/src/lib.rs‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ use std::convert::TryFrom;
1313
use uuid::Uuid;
1414

1515
pub use propolis_client::{CrucibleOpts, VolumeConstructionRequest};
16-
1716
progenitor::generate_api!(
1817
spec = "../../openapi/sled-agent/sled-agent-latest.json",
1918
interface = Positional,
@@ -47,14 +46,19 @@ progenitor::generate_api!(
4746
},
4847
replace = {
4948
Baseboard = sled_agent_types_versions::latest::inventory::Baseboard,
49+
BaseboardId = sled_hardware_types::BaseboardId,
5050
ByteCount = omicron_common::api::external::ByteCount,
51+
CommitRequest = trust_quorum_types::messages::CommitRequest,
52+
CommitStatus = trust_quorum_types::status::CommitStatus,
53+
CoordinatorStatus = trust_quorum_types::status::CoordinatorStatus,
5154
DatasetsConfig = omicron_common::disk::DatasetsConfig,
5255
DatasetManagementStatus = omicron_common::disk::DatasetManagementStatus,
5356
DatasetKind = omicron_common::api::internal::shared::DatasetKind,
5457
DiskIdentity = omicron_common::disk::DiskIdentity,
5558
DiskManagementStatus = omicron_common::disk::DiskManagementStatus,
5659
DiskManagementError = omicron_common::disk::DiskManagementError,
5760
DiskVariant = omicron_common::disk::DiskVariant,
61+
Epoch = trust_quorum_types::types::Epoch,
5862
ExternalIpGatewayMap = omicron_common::api::internal::shared::ExternalIpGatewayMap,
5963
ExternalIpConfig = omicron_common::api::internal::shared::ExternalIpConfig,
6064
ExternalIpv4Config = omicron_common::api::internal::shared::ExternalIpv4Config,
@@ -79,15 +83,18 @@ progenitor::generate_api!(
7983
OmicronZonesConfig = sled_agent_types_versions::latest::inventory::OmicronZonesConfig,
8084
PortFec = omicron_common::api::internal::shared::PortFec,
8185
PortSpeed = omicron_common::api::internal::shared::PortSpeed,
82-
RouterId = omicron_common::api::internal::shared::RouterId,
86+
PrepareAndCommitRequest = trust_quorum_types::messages::PrepareAndCommitRequest,
87+
ReconfigureMsg = trust_quorum_types::messages::ReconfigureMsg,
8388
ResolvedVpcFirewallRule = omicron_common::api::internal::shared::ResolvedVpcFirewallRule,
8489
ResolvedVpcRoute = omicron_common::api::internal::shared::ResolvedVpcRoute,
8590
ResolvedVpcRouteSet = omicron_common::api::internal::shared::ResolvedVpcRouteSet,
91+
RouterId = omicron_common::api::internal::shared::RouterId,
8692
RouterTarget = omicron_common::api::internal::shared::RouterTarget,
8793
RouterVersion = omicron_common::api::internal::shared::RouterVersion,
8894
SledRole = sled_agent_types_versions::latest::inventory::SledRole,
8995
SourceNatConfigGeneric = omicron_common::api::internal::shared::SourceNatConfigGeneric,
9096
SwitchLocation = omicron_common::api::external::SwitchLocation,
97+
Threshold = trust_quorum_types::types::Threshold,
9198
Vni = omicron_common::api::external::Vni,
9299
VpcFirewallIcmpFilter = omicron_common::api::external::VpcFirewallIcmpFilter,
93100
ZpoolKind = omicron_common::zpool_name::ZpoolKind,

‎nexus-config/src/nexus_config.rs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -434,6 +434,8 @@ pub struct BackgroundTaskConfig {
434434
pub probe_distributor: ProbeDistributorConfig,
435435
/// configuration for multicast reconciler (group+members) task
436436
pub multicast_reconciler: MulticastGroupReconcilerConfig,
437+
/// configuration for trust quorum manager task
438+
pub trust_quorum: TrustQuorumConfig,
437439
}
438440

439441
#[serde_as]
@@ -965,6 +967,15 @@ pub struct ProbeDistributorConfig {
965967
pub period_secs: Duration,
966968
}
967969

970+
#[serde_as]
971+
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
972+
pub struct TrustQuorumConfig {
973+
/// period (in seconds) for periodic activations of the background task that
974+
/// completes trust quorum reconfigurations.
975+
#[serde_as(as = "DurationSeconds<u64>")]
976+
pub period_secs: Duration,
977+
}
978+
968979
/// Configuration for a nexus server
969980
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
970981
pub struct PackageConfig {
@@ -1269,6 +1280,7 @@ mod test {
12691280
fm.sitrep_gc_period_secs = 49
12701281
probe_distributor.period_secs = 50
12711282
multicast_reconciler.period_secs = 60
1283+
trust_quorum.period_secs = 60
12721284
[default_region_allocation_strategy]
12731285
type = "random"
12741286
seed = 0
@@ -1526,6 +1538,9 @@ mod test {
15261538
sled_cache_ttl_secs: MulticastGroupReconcilerConfig::default_sled_cache_ttl_secs(),
15271539
backplane_cache_ttl_secs: MulticastGroupReconcilerConfig::default_backplane_cache_ttl_secs(),
15281540
},
1541+
trust_quorum: TrustQuorumConfig {
1542+
period_secs: Duration::from_secs(50),
1543+
},
15291544
},
15301545
multicast: MulticastConfig { enabled: false },
15311546
default_region_allocation_strategy:
@@ -1629,6 +1644,7 @@ mod test {
16291644
fm.sitrep_gc_period_secs = 46
16301645
probe_distributor.period_secs = 47
16311646
multicast_reconciler.period_secs = 60
1647+
trust_quorum.period_secs = 60
16321648
16331649
[default_region_allocation_strategy]
16341650
type = "random"

‎nexus/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ tokio = { workspace = true, features = ["full"] }
119119
tokio-postgres = { workspace = true, features = ["with-serde_json-1"] }
120120
tokio-util = { workspace = true, features = ["codec", "rt"] }
121121
tough.workspace = true
122+
trust-quorum-types.workspace = true
122123
tufaceous-artifact.workspace = true
123124
usdt.workspace = true
124125
uuid.workspace = true

‎nexus/background-task-interface/src/init.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ pub struct BackgroundTasks {
5555
pub task_fm_sitrep_gc: Activator,
5656
pub task_probe_distributor: Activator,
5757
pub task_multicast_reconciler: Activator,
58+
pub task_trust_quorum_manager: Activator,
5859

5960
// Handles to activate background tasks that do not get used by Nexus
6061
// at-large. These background tasks are implementation details as far as

‎nexus/db-queries/src/db/datastore/sled.rs‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -737,6 +737,32 @@ impl DataStore {
737737
Ok(rack_id.map(RackUuid::from))
738738
}
739739

740+
// Return the commissioned sled if it exists in the given rack, given its
741+
// `BaseboardId`.
742+
pub async fn sled_get_commissioned_by_baseboard_and_rack_id(
743+
&self,
744+
opctx: &OpContext,
745+
rack_id: RackUuid,
746+
baseboard_id: &BaseboardId,
747+
) -> Result<Option<Sled>, Error> {
748+
opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?;
749+
let conn = &*self.pool_connection_authorized(opctx).await?;
750+
use nexus_db_schema::schema::sled::dsl;
751+
let sled = dsl::sled
752+
.filter(dsl::time_deleted.is_null())
753+
.filter(dsl::part_number.eq(baseboard_id.part_number.clone()))
754+
.filter(dsl::serial_number.eq(baseboard_id.serial_number.clone()))
755+
.filter(dsl::rack_id.eq(rack_id.into_untyped_uuid()))
756+
.sled_filter(SledFilter::Commissioned)
757+
.select(Sled::as_select())
758+
.get_result_async::<Sled>(conn)
759+
.await
760+
.optional()
761+
.map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?;
762+
763+
Ok(sled)
764+
}
765+
740766
pub async fn sled_list(
741767
&self,
742768
opctx: &OpContext,

‎nexus/db-queries/src/db/datastore/trust_quorum.rs‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ use nexus_db_model::TrustQuorumConfiguration as DbTrustQuorumConfiguration;
2323
use nexus_db_model::TrustQuorumMember as DbTrustQuorumMember;
2424
use nexus_types::trust_quorum::ProposedTrustQuorumConfig;
2525
use nexus_types::trust_quorum::{
26-
TrustQuorumConfig, TrustQuorumMemberData, TrustQuorumMemberState,
26+
TrustQuorumConfig, TrustQuorumConfigState, TrustQuorumMemberData,
27+
TrustQuorumMemberState,
2728
};
2829
use omicron_common::api::external::Error;
2930
use omicron_common::api::external::OptionalLookupResult;
@@ -163,6 +164,23 @@ impl DataStore {
163164
.map_err(|err| err.into_public_ignore_retries())
164165
}
165166

167+
/// Get the trust quorum configuration from the database for the given Epoch
168+
pub async fn tq_get_config(
169+
&self,
170+
opctx: &OpContext,
171+
rack_id: RackUuid,
172+
epoch: Epoch,
173+
) -> OptionalLookupResult<TrustQuorumConfig> {
174+
opctx.authorize(authz::Action::Read, &authz::FLEET).await?;
175+
let conn = &*self.pool_connection_authorized(opctx).await?;
176+
177+
Self::tq_get_config_with_members_from_epoch_conn(
178+
opctx, conn, rack_id, epoch,
179+
)
180+
.await
181+
.map_err(|err| err.into_public_ignore_retries())
182+
}
183+
166184
async fn tq_get_latest_config_with_members_conn(
167185
opctx: &OpContext,
168186
conn: &async_bb8_diesel::Connection<DbConnection>,
@@ -591,7 +609,7 @@ impl DataStore {
591609
opctx: &OpContext,
592610
config: trust_quorum_types::configuration::Configuration,
593611
acked_prepares: BTreeSet<BaseboardId>,
594-
) -> Result<(), Error> {
612+
) -> Result<TrustQuorumConfigState, Error> {
595613
opctx.authorize(authz::Action::Modify, &authz::FLEET).await?;
596614
let conn = &*self.pool_connection_authorized(opctx).await?;
597615

@@ -739,9 +757,11 @@ impl DataStore {
739757
)
740758
.await
741759
.map_err(|txn_error| txn_error.into_diesel(&err))?;
760+
761+
return Ok(TrustQuorumConfigState::Committing);
742762
}
743763

744-
Ok(())
764+
Ok(db_config.state.into())
745765
}
746766
})
747767
.await

‎nexus/external-api/output/nexus_tags.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,8 @@ support_bundle_list GET /experimental/v1/system/suppor
7070
support_bundle_update PUT /experimental/v1/system/support-bundles/{bundle_id}
7171
support_bundle_view GET /experimental/v1/system/support-bundles/{bundle_id}
7272
timeseries_query POST /v1/timeseries/query
73+
trust_quorum_add_sleds POST /v1/trust-quorum/new-members
74+
trust_quorum_get_latest_config GET /v1/trust-quorum/config/latest/{rack_id}
7375

7476
API operations found with tag "floating-ips"
7577
OPERATION ID METHOD URL PATH

‎nexus/external-api/src/lib.rs‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ use nexus_types::{
2323
headers, params, shared,
2424
views::{self, MulticastGroupMember},
2525
},
26+
trust_quorum::TrustQuorumConfig,
2627
};
2728
use omicron_common::api::external::{
2829
http_pagination::{
@@ -70,6 +71,7 @@ api_versions!([
7071
// | date-based version should be at the top of the list.
7172
// v
7273
// (next_yyyymmddnn, IDENT),
74+
(2026011400, TRUST_QUORUM_ADD_SLEDS_AND_GET_LATEST_CONFIG),
7375
(2026011300, DOC_LINT_SUMMARY_TRAILING_PERIOD),
7476
(2026011100, MULTICAST_JOIN_LEAVE_DOCS),
7577
(2026010800, MULTICAST_IMPLICIT_LIFECYCLE_UPDATES),
@@ -3993,6 +3995,37 @@ pub trait NexusExternalApi {
39933995
query_params: Query<params::DeleteInternetGatewayElementSelector>,
39943996
) -> Result<HttpResponseDeleted, HttpError>;
39953997

3998+
//
3999+
// Trust Quorum
4000+
//
4001+
4002+
/// Add new sleds to the trust quorum membership
4003+
///
4004+
/// This will write a new configuration to the database and then issue a
4005+
/// reconfiguration request to a trust quorum coordinator.
4006+
#[endpoint {
4007+
method = POST,
4008+
path = "/v1/trust-quorum/new-members",
4009+
tags = ["experimental"],
4010+
versions = VERSION_TRUST_QUORUM_ADD_SLEDS_AND_GET_LATEST_CONFIG..
4011+
}]
4012+
async fn trust_quorum_add_sleds(
4013+
rqctx: RequestContext<Self::Context>,
4014+
req: TypedBody<params::TrustQuorumAddSledsRequest>,
4015+
) -> Result<HttpResponseUpdatedNoContent, HttpError>;
4016+
4017+
/// Retrieve the latest trust quorum configuration, including member status
4018+
#[endpoint {
4019+
method = GET,
4020+
path = "/v1/trust-quorum/config/latest/{rack_id}",
4021+
tags = ["experimental"],
4022+
versions = VERSION_TRUST_QUORUM_ADD_SLEDS_AND_GET_LATEST_CONFIG..
4023+
}]
4024+
async fn trust_quorum_get_latest_config(
4025+
rqctx: RequestContext<Self::Context>,
4026+
path_params: Path<params::RackPath>,
4027+
) -> Result<HttpResponseOk<Option<TrustQuorumConfig>>, HttpError>;
4028+
39964029
// Racks
39974030

39984031
/// List racks

0 commit comments

Comments
 (0)