self-review + regenerate API spec

Author: davepacheco

dev-tools/omdb/src/bin/omdb/nexus/quiesce.rs

@@ -61,20 +61,23 @@ async fn quiesce_show(
         .context("fetching quiesce state")?
         .into_inner();
     match quiesce.state {
+        QuiesceState::Undetermined => {
+            println!("has not yet determined if it is quiescing");
+        }
         QuiesceState::Running => {
             println!("running normally (not quiesced, not quiescing)");
         }
-        QuiesceState::WaitingForSagas { time_requested } => {
+        QuiesceState::DrainingSagas { time_requested } => {
             println!(
                 "quiescing since {} ({} ago)",
                 humantime::format_rfc3339_millis(time_requested.into()),
                 format_time_delta(now - time_requested),
             );
             println!("details: waiting for running sagas to finish");
         }
-        QuiesceState::WaitingForDb {
+        QuiesceState::DrainingDb {
             time_requested,
-            duration_waiting_for_sagas,
+            duration_draining_sagas,
             ..
         } => {
             println!(
@@ -87,13 +90,34 @@ async fn quiesce_show(
             );
             println!(
                 "    previously: waiting for sagas took {}",
-                format_duration_ms(duration_waiting_for_sagas.into()),
+                format_duration_ms(duration_draining_sagas.into()),
+            );
+        }
+        QuiesceState::RecordingQuiesce {
+            time_requested,
+            duration_draining_sagas,
+            duration_draining_db,
+            ..
+        } => {
+            println!(
+                "quiescing since {} ({} ago)",
+                humantime::format_rfc3339_millis(time_requested.into()),
+                format_time_delta(now - time_requested),
+            );
+            println!(
+                "    waiting for sagas took {}",
+                format_duration_ms(duration_draining_sagas.into()),
+            );
+            println!(
+                "    waiting for db quiesce took {}",
+                format_duration_ms(duration_draining_db.into()),
             );
         }
         QuiesceState::Quiesced {
             time_quiesced,
-            duration_waiting_for_sagas,
-            duration_waiting_for_db,
+            duration_draining_sagas,
+            duration_draining_db,
+            duration_recording_quiesce,
             duration_total,
             ..
         } => {
@@ -104,11 +128,15 @@ async fn quiesce_show(
             );
             println!(
                 "    waiting for sagas took {}",
-                format_duration_ms(duration_waiting_for_sagas.into()),
+                format_duration_ms(duration_draining_sagas.into()),
             );
             println!(
                 "    waiting for db quiesce took {}",
-                format_duration_ms(duration_waiting_for_db.into()),
+                format_duration_ms(duration_draining_db.into()),
+            );
+            println!(
+                "    recording quiesce took {}",
+                format_duration_ms(duration_recording_quiesce.into()),
             );
             println!(
                 "    total quiesce time: {}",

nexus/src/app/background/tasks/blueprint_execution.rs

@@ -108,7 +108,7 @@ impl BlueprintExecutor {
         // blueprint before enabling sagas, since we already know if we're
         // quiescing or not); and (2) because we want to do it even if blueprint
         // execution is disabled.
-        match blueprint.nexus_quiescing(self.nexus_id) {
+        match blueprint.is_nexus_quiescing(self.nexus_id) {
             Ok(quiescing) => {
                 debug!(
                     &opctx.log,

nexus/src/app/quiesce.rs

@@ -250,24 +250,27 @@ mod test {
         let QuiesceState::Quiesced {
             time_requested,
             time_quiesced,
-            duration_waiting_for_sagas,
-            duration_waiting_for_db,
+            duration_draining_sagas,
+            duration_draining_db,
+            duration_recording_quiesce,
             duration_total,
         } = status.state
         else {
             panic!("not quiesced");
         };
         let duration_total = Duration::from(duration_total);
-        let duration_waiting_for_sagas =
-            Duration::from(duration_waiting_for_sagas);
-        let duration_waiting_for_db = Duration::from(duration_waiting_for_db);
+        let duration_draining_sagas = Duration::from(duration_draining_sagas);
+        let duration_draining_db = Duration::from(duration_draining_db);
+        let duration_recording_quiesce =
+            Duration::from(duration_recording_quiesce);
         assert!(time_requested >= before);
         assert!(time_requested <= after);
         assert!(time_quiesced >= before);
         assert!(time_quiesced <= after);
         assert!(time_quiesced >= time_requested);
-        assert!(duration_total >= duration_waiting_for_sagas);
-        assert!(duration_total >= duration_waiting_for_db);
+        assert!(duration_total >= duration_draining_sagas);
+        assert!(duration_total >= duration_draining_db);
+        assert!(duration_total >= duration_recording_quiesce);
         assert!(duration_total <= (after - before).to_std().unwrap());
         assert!(status.sagas_pending.is_empty());
         assert!(status.db_claims.is_empty());
@@ -341,7 +344,7 @@ mod test {
         debug!(log, "found quiesce status"; "status" => ?quiesce_status);
         assert_matches!(
             quiesce_status.state,
-            QuiesceState::WaitingForSagas { .. }
+            QuiesceState::DrainingSagas { .. }
         );
         assert!(quiesce_status.sagas_pending.contains_key(&demo_saga.saga_id));
         // We should see at least one held database claim from the one we took
@@ -404,7 +407,7 @@ mod test {
                     .map_err(|e| CondCheckError::Failed(e))?
                     .into_inner();
                 debug!(log, "found quiesce state"; "state" => ?rv);
-                if !matches!(rv.state, QuiesceState::WaitingForDb { .. }) {
+                if !matches!(rv.state, QuiesceState::DrainingDb { .. }) {
                     return Err(CondCheckError::<NexusClientError>::NotYet);
                 }
                 assert!(rv.sagas_pending.is_empty());

nexus/types/src/deployment.rs

@@ -388,7 +388,7 @@ impl Blueprint {
 
     /// Returns whether the given Nexus instance should be quiescing or quiesced
     /// in preparation for handoff to the next generation
-    pub fn nexus_quiescing(
+    pub fn is_nexus_quiescing(
         &self,
         nexus_id: OmicronZoneUuid,
     ) -> Result<bool, anyhow::Error> {

openapi/nexus-internal.json

@@ -7477,8 +7477,23 @@
         ]
       },
       "QuiesceState": {
-        "description": "See [`QuiesceStatus`] for more on Nexus quiescing.\n\nAt any given time, Nexus is always in one of these states:\n\n```text Running             (normal operation) | | quiesce starts v WaitingForSagas     (no new sagas are allowed, but some are still running) | | no more sagas running v WaitingForDb        (no sagas running; no new db connections may be acquired by Nexus at-large, but some are still held) | | no more database connections held v Quiesced            (no sagas running, no database connections in use) ```\n\nQuiescing is (currently) a one-way trip: once a Nexus process starts quiescing, it will never go back to normal operation.  It will never go back to an earlier stage, either.",
+        "description": "See [`QuiesceStatus`] for more on Nexus quiescing.\n\nAt any given time, Nexus is always in one of these states:\n\n```text Undetermined        (have not loaded persistent state; don't know yet) | | load persistent state and find we're not quiescing v Running             (normal operation) | | quiesce starts v DrainingSagas       (no new sagas are allowed, but some are still running) | | no more sagas running v DrainingDb          (no sagas running; no new db connections may be |                  acquired by Nexus at-large, but some are still held) | | no more database connections held v RecordingQuiesce    (everything is quiesced aside from one connection being |                  used to record our final quiesced state) | | finish recording quiesce state in database v Quiesced            (no sagas running, no database connections in use) ```\n\nQuiescing is (currently) a one-way trip: once a Nexus process starts quiescing, it will never go back to normal operation.  It will never go back to an earlier stage, either.",
         "oneOf": [
+          {
+            "description": "We have not yet determined based on persistent state if we're supposed to be quiesced or not",
+            "type": "object",
+            "properties": {
+              "state": {
+                "type": "string",
+                "enum": [
+                  "undetermined"
+                ]
+              }
+            },
+            "required": [
+              "state"
+            ]
+          },
           {
             "description": "Normal operation",
             "type": "object",
@@ -7495,7 +7510,7 @@
             ]
           },
           {
-            "description": "New sagas disallowed, but some are still running.",
+            "description": "New sagas disallowed, but some are still running on some Nexus instances",
             "type": "object",
             "properties": {
               "quiesce_details": {
@@ -7513,7 +7528,7 @@
               "state": {
                 "type": "string",
                 "enum": [
-                  "waiting_for_sagas"
+                  "draining_sagas"
                 ]
               }
             },
@@ -7523,13 +7538,13 @@
             ]
           },
           {
-            "description": "No sagas running, no new database connections may be claimed, but some database connections are still held.",
+            "description": "No sagas running on any Nexus instances\n\nNo new database connections may be claimed, but some database connections are still held.",
             "type": "object",
             "properties": {
               "quiesce_details": {
                 "type": "object",
                 "properties": {
-                  "duration_waiting_for_sagas": {
+                  "duration_draining_sagas": {
                     "$ref": "#/components/schemas/Duration"
                   },
                   "time_requested": {
@@ -7538,14 +7553,50 @@
                   }
                 },
                 "required": [
-                  "duration_waiting_for_sagas",
+                  "duration_draining_sagas",
                   "time_requested"
                 ]
               },
               "state": {
                 "type": "string",
                 "enum": [
-                  "waiting_for_db"
+                  "draining_db"
+                ]
+              }
+            },
+            "required": [
+              "quiesce_details",
+              "state"
+            ]
+          },
+          {
+            "description": "No database connections in use except to record the final \"quiesced\" state",
+            "type": "object",
+            "properties": {
+              "quiesce_details": {
+                "type": "object",
+                "properties": {
+                  "duration_draining_db": {
+                    "$ref": "#/components/schemas/Duration"
+                  },
+                  "duration_draining_sagas": {
+                    "$ref": "#/components/schemas/Duration"
+                  },
+                  "time_requested": {
+                    "type": "string",
+                    "format": "date-time"
+                  }
+                },
+                "required": [
+                  "duration_draining_db",
+                  "duration_draining_sagas",
+                  "time_requested"
+                ]
+              },
+              "state": {
+                "type": "string",
+                "enum": [
+                  "recording_quiesce"
                 ]
               }
             },
@@ -7561,13 +7612,16 @@
               "quiesce_details": {
                 "type": "object",
                 "properties": {
-                  "duration_total": {
+                  "duration_draining_db": {
                     "$ref": "#/components/schemas/Duration"
                   },
-                  "duration_waiting_for_db": {
+                  "duration_draining_sagas": {
                     "$ref": "#/components/schemas/Duration"
                   },
-                  "duration_waiting_for_sagas": {
+                  "duration_recording_quiesce": {
+                    "$ref": "#/components/schemas/Duration"
+                  },
+                  "duration_total": {
                     "$ref": "#/components/schemas/Duration"
                   },
                   "time_quiesced": {
@@ -7580,9 +7634,10 @@
                   }
                 },
                 "required": [
+                  "duration_draining_db",
+                  "duration_draining_sagas",
+                  "duration_recording_quiesce",
                   "duration_total",
-                  "duration_waiting_for_db",
-                  "duration_waiting_for_sagas",
                   "time_quiesced",
                   "time_requested"
                 ]