update omdb

Author: davepacheco

dev-tools/omdb/src/bin/omdb/nexus/quiesce.rs

@@ -12,6 +12,8 @@ use chrono::Utc;
 use clap::Args;
 use clap::Subcommand;
 use nexus_client::types::QuiesceState;
+use nexus_client::types::QuiesceStatus;
+use nexus_client::types::SagaQuiesceStatus;
 use std::time::Duration;
 
 #[derive(Debug, Args)]
@@ -31,9 +33,9 @@ pub enum QuiesceCommands {
 
 #[derive(Debug, Args)]
 pub struct QuiesceShowArgs {
-    /// Show details about held database connections
+    /// Show stack traces for held database connections
     #[clap(short, long, default_value_t = false)]
-    verbose: bool,
+    stacks: bool,
 }
 
 pub async fn cmd_nexus_quiesce(
@@ -60,7 +62,10 @@ async fn quiesce_show(
         .await
         .context("fetching quiesce state")?
         .into_inner();
-    match quiesce.state {
+
+    let QuiesceStatus { db_claims, sagas, state } = quiesce;
+
+    match state {
         QuiesceState::Undetermined => {
             println!("has not yet determined if it is quiescing");
         }
@@ -145,25 +150,70 @@ async fn quiesce_show(
         }
     }
 
-    println!("sagas running: {}", quiesce.sagas_pending.len());
-    for saga in &quiesce.sagas_pending {
+    let SagaQuiesceStatus {
+        sagas_pending,
+        drained_blueprint_id,
+        first_recovery_complete,
+        new_sagas_allowed,
+        reassignment_blueprint_id,
+        reassignment_generation,
+        reassignment_pending,
+        recovered_blueprint_id,
+        recovered_reassignment_generation,
+    } = sagas;
+
+    println!("saga quiesce: {:?}", new_sagas_allowed);
+    println!(
+        "drained as of blueprint: {}",
+        drained_blueprint_id
+            .map(|s| s.to_string())
+            .as_deref()
+            .unwrap_or("none")
+    );
+    println!(
+        "    blueprint for last recovery pass: {}",
+        recovered_blueprint_id
+            .map(|s| s.to_string())
+            .as_deref()
+            .unwrap_or("none")
+    );
+    println!(
+        "    blueprint for last reassignment pass: {}",
+        reassignment_blueprint_id
+            .map(|s| s.to_string())
+            .as_deref()
+            .unwrap_or("none")
+    );
+    println!(
+        "    reassignment generation: {} (pass running: {})",
+        reassignment_generation,
+        if reassignment_pending { "yes" } else { "no " }
+    );
+    println!("    recovered generation: {}", recovered_reassignment_generation);
+    println!(
+        "    recovered at least once successfully: {}",
+        if first_recovery_complete { "yes" } else { "no" },
+    );
+
+    println!("    sagas running: {}", sagas_pending.len());
+    for saga in &sagas_pending {
         println!(
-            "    saga {} pending since {} ({})",
+            "        saga {} pending since {} ({})",
             saga.saga_id,
             humantime::format_rfc3339_millis(saga.time_pending.into()),
             saga.saga_name
         );
     }
 
-    println!("database connections held: {}", quiesce.db_claims.len());
-    for claim in &quiesce.db_claims {
+    println!("database connections held: {}", db_claims.len());
+    for claim in &db_claims {
         println!(
             "    claim {} held since {} ({} ago)",
             claim.id,
             claim.held_since,
             format_time_delta(Utc::now() - claim.held_since),
         );
-        if args.verbose {
+        if args.stacks {
             println!("    acquired by:");
             println!("{}", textwrap::indent(&claim.debug, "        "));
         }
@@ -177,7 +227,7 @@ async fn quiesce_start(
     _token: DestructiveOperationToken,
 ) -> Result<(), anyhow::Error> {
     client.quiesce_start().await.context("quiescing Nexus")?;
-    quiesce_show(client, &QuiesceShowArgs { verbose: false }).await
+    quiesce_show(client, &QuiesceShowArgs { stacks: false }).await
 }
 
 fn format_duration_ms(duration: Duration) -> String {

nexus/src/app/quiesce.rs

@@ -32,9 +32,9 @@ impl super::Nexus {
     ) -> LookupResult<QuiesceStatus> {
         opctx.authorize(authz::Action::Read, &authz::QUIESCE_STATE).await?;
         let state = self.quiesce.state();
-        let sagas_pending = self.quiesce.sagas().sagas_pending();
+        let sagas = self.quiesce.sagas().status();
         let db_claims = self.datastore().claims_held();
-        Ok(QuiesceStatus { state, sagas_pending, db_claims })
+        Ok(QuiesceStatus { state, sagas, db_claims })
     }
 }
 
@@ -281,7 +281,7 @@ mod test {
         assert!(duration_total >= duration_draining_db);
         assert!(duration_total >= duration_recording_quiesce);
         assert!(duration_total <= (after - before).to_std().unwrap());
-        assert!(status.sagas_pending.is_empty());
+        assert!(status.sagas.sagas_pending.is_empty());
         assert!(status.db_claims.is_empty());
     }
 
@@ -355,7 +355,9 @@ mod test {
             quiesce_status.state,
             QuiesceState::DrainingSagas { .. }
         );
-        assert!(quiesce_status.sagas_pending.contains_key(&demo_saga.saga_id));
+        assert!(
+            quiesce_status.sagas.sagas_pending.contains_key(&demo_saga.saga_id)
+        );
         // We should see at least one held database claim from the one we took
         // above.
         assert!(!quiesce_status.db_claims.is_empty());
@@ -419,7 +421,7 @@ mod test {
                 if !matches!(rv.state, QuiesceState::DrainingDb { .. }) {
                     return Err(CondCheckError::<NexusClientError>::NotYet);
                 }
-                assert!(rv.sagas_pending.is_empty());
+                assert!(rv.sagas.sagas_pending.is_empty());
                 // The database claim we took is still held.
                 assert!(!rv.db_claims.is_empty());
                 Ok(())

nexus/types/src/internal_api/views.rs

@@ -7,6 +7,7 @@ use crate::deployment::TargetReleaseDescription;
 use crate::inventory::BaseboardId;
 use crate::inventory::CabooseWhich;
 use crate::inventory::Collection;
+use crate::quiesce::SagaQuiesceStatus;
 use chrono::DateTime;
 use chrono::SecondsFormat;
 use chrono::Utc;
@@ -978,12 +979,8 @@ pub struct QuiesceStatus {
     /// what stage of quiescing is Nexus at
     pub state: QuiesceState,
 
-    /// what sagas are currently running or known needing to be recovered
-    ///
-    /// This should only be non-empty when state is `Running` or
-    /// `WaitingForSagas`.  Entries here prevent transitioning from
-    /// `WaitingForSagas` to `WaitingForDb`.
-    pub sagas_pending: IdOrdMap<PendingSagaInfo>,
+    /// information about saga quiescing
+    pub sagas: SagaQuiesceStatus,
 
     /// what database claims are currently held (by any part of Nexus)
     ///

nexus/types/src/quiesce.rs

@@ -12,6 +12,8 @@ use iddqd::IdOrdMap;
 use omicron_common::api::external::Error;
 use omicron_common::api::external::Generation;
 use omicron_uuid_kinds::BlueprintUuid;
+use schemars::JsonSchema;
+use serde::Serialize;
 use slog::Logger;
 use slog::error;
 use slog::info;
@@ -25,7 +27,8 @@ use tokio::sync::watch;
 ///
 /// This is used by Nexus quiesce to disallow creation of new sagas when we're
 /// trying to quiesce Nexus.
-#[derive(Debug, Clone, Copy, Eq, PartialEq)]
+#[derive(Debug, Clone, Copy, Eq, PartialEq, JsonSchema, Serialize)]
+#[serde(rename_all = "snake_case")]
 enum SagasAllowed {
     /// New sagas may be started (normal condition)
     Allowed,
@@ -101,11 +104,11 @@ pub struct SagaQuiesceHandle {
     //     cancellation behavior is abysmal), but we don't want to block on a
     //     std `Condvar` in an async thread.  There are options here (e.g.,
     //     `block_on`), but they're not pleasant.
-    inner: watch::Sender<SagaQuiesceInner>,
+    inner: watch::Sender<SagaQuiesceStatus>,
 }
 
-#[derive(Debug, Clone)]
-struct SagaQuiesceInner {
+#[derive(Debug, Clone, Serialize, JsonSchema)]
+pub struct SagaQuiesceStatus {
     /// current policy: are we allowed to *create* new sagas?
     ///
     /// This also affects re-assigning sagas from expunged Nexus instances to
@@ -183,12 +186,13 @@ struct SagaQuiesceInner {
     /// whether a saga recovery operation is ongoing, and if one is:
     /// - what `reassignment_generation` was when it started
     /// - which blueprint id we'll be fully caught up to upon completion
+    #[serde(skip)] // XXX-dap
     recovery_pending: Option<(Generation, Option<BlueprintUuid>)>,
 }
 
 impl SagaQuiesceHandle {
     pub fn new(log: Logger) -> SagaQuiesceHandle {
-        let (inner, _) = watch::channel(SagaQuiesceInner {
+        let (inner, _) = watch::channel(SagaQuiesceStatus {
             new_sagas_allowed: SagasAllowed::DisallowedUnknown,
             sagas_pending: IdOrdMap::new(),
             first_recovery_complete: false,
@@ -253,6 +257,10 @@ impl SagaQuiesceHandle {
     }
 
     /// Returns the blueprint id as of which sagas are fully drained
+    ///
+    /// We may become un-drained if another re-assignment pass starts for a
+    /// subsequent blueprint, but this fact will still be true that we *were*
+    /// fully drained as of expungements included up through this blueprint.
     pub fn fully_drained_blueprint(&self) -> Option<BlueprintUuid> {
         self.inner.borrow().drained_blueprint_id
     }
@@ -286,8 +294,14 @@ impl SagaQuiesceHandle {
             .await;
     }
 
+    /// Returns a summary of internal state for debugging (involves a clone)
+    pub fn status(&self) -> SagaQuiesceStatus {
+        self.inner.borrow().clone()
+    }
+
     /// Returns information about running sagas (involves a clone)
-    pub fn sagas_pending(&self) -> IdOrdMap<PendingSagaInfo> {
+    #[cfg(test)]
+    fn sagas_pending(&self) -> IdOrdMap<PendingSagaInfo> {
         self.inner.borrow().sagas_pending.clone()
     }
 
@@ -593,7 +607,7 @@ impl SagaQuiesceHandle {
     }
 }
 
-impl SagaQuiesceInner {
+impl SagaQuiesceStatus {
     /// Returns whether sagas are fully drained
     ///
     /// This condition is not permanent.  New sagas can be re-assigned to this
@@ -647,7 +661,7 @@ impl SagaQuiesceInner {
 #[must_use = "must record the saga completion future once the saga is running"]
 pub struct NewlyPendingSagaRef {
     log: Logger,
-    quiesce: watch::Sender<SagaQuiesceInner>,
+    quiesce: watch::Sender<SagaQuiesceStatus>,
     saga_id: steno::SagaId,
     init_finished: bool,
 }