tests need to wait for sagas to be enabled

Author: davepacheco

nexus/src/app/mod.rs

@@ -621,6 +621,14 @@ impl Nexus {
         }
     }
 
+    // Waits for Nexus to determine whether sagas are supposed to be quiesced
+    //
+    // This is used by the test suite because most tests assume that sagas are
+    // operational as soon as they start.
+    pub(crate) async fn wait_for_saga_determination(&self) {
+        self.quiesce.sagas().wait_for_determination().await;
+    }
+
     pub(crate) async fn external_tls_config(
         &self,
         tls_enabled: bool,

nexus/src/app/quiesce.rs

@@ -92,13 +92,13 @@ impl NexusQuiesceHandle {
             }
         });
 
+        // Immediately (synchronously) update the saga quiesce status.  It's
+        // okay to do this even if there wasn't a change.
+        self.sagas.set_quiescing(quiescing);
+
         if changed && quiescing {
-            // Immediately quiesce sagas.
-            self.sagas.set_quiescing(quiescing);
             // Asynchronously complete the rest of the quiesce process.
-            if quiescing {
-                tokio::spawn(do_quiesce(self.clone()));
-            }
+            tokio::spawn(do_quiesce(self.clone()));
         }
     }
 }

nexus/src/lib.rs

@@ -138,6 +138,15 @@ impl Server {
         // the external server we're about to start.
         apictx.context.nexus.await_ip_allowlist_plumbing().await;
 
+        // Wait until Nexus has determined if sagas are supposed to be quiesced.
+        // This is not strictly necessary.  The goal here is to prevent 503
+        // errors to clients that reach this Nexus while it's starting up and
+        // before it's figured out that it doesn't need to quiesce.  The risk of
+        // doing this is that Nexus gets stuck here, but that should only happen
+        // if it's unable to load the current blueprint, in which case
+        // something's pretty wrong and it's likely pretty stuck anyway.
+        apictx.context.nexus.wait_for_saga_determination().await;
+
         // Launch the external server.
         let tls_config = apictx
             .context
@@ -332,6 +341,16 @@ impl nexus_test_interface::NexusServer for Server {
             .await
             .expect("Could not initialize rack");
 
+        // Now that we have a blueprint, determination of whether sagas are
+        // quiesced can complete.  Wait for that so that tests can assume they
+        // can immediately kick off sagas.
+        internal_server
+            .apictx
+            .context
+            .nexus
+            .wait_for_saga_determination()
+            .await;
+
         // Start the Nexus external API.
         Server::start(internal_server).await.unwrap()
     }

nexus/types/src/quiesce.rs

@@ -218,6 +218,18 @@ impl SagaQuiesceHandle {
         let _ = self.inner.subscribe().wait_for(|q| q.is_fully_drained()).await;
     }
 
+    /// Wait for the initial determination to be made about whether sagas are
+    /// allowed or not.
+    pub async fn wait_for_determination(&self) {
+        let _ = self
+            .inner
+            .subscribe()
+            .wait_for(|q| {
+                q.new_sagas_allowed != SagasAllowed::DisallowedUnknown
+            })
+            .await;
+    }
+
     /// Returns information about running sagas (involves a clone)
     pub fn sagas_pending(&self) -> IdOrdMap<PendingSagaInfo> {
         self.inner.borrow().sagas_pending.clone()