Add measurements to sled-agent (#9626)

Author: labbott

Cargo.lock

@@ -136,6 +136,7 @@ members = [
     "sled-agent/bootstrap-agent-api",
     "sled-agent/config-reconciler",
     "sled-agent/health-monitor",
+    "sled-agent/measurements",
     "sled-agent/repo-depot-api",
     "sled-agent/types",
     "sled-agent/types/versions",
@@ -308,6 +309,7 @@ default-members = [
     "sled-agent/bootstrap-agent-api",
     "sled-agent/config-reconciler",
     "sled-agent/health-monitor",
+    "sled-agent/measurements",
     "sled-agent/repo-depot-api",
     "sled-agent/types",
     "sled-agent/types/versions",
@@ -745,6 +747,7 @@ sled-agent-api = { path = "sled-agent/api" }
 sled-agent-client = { path = "clients/sled-agent-client" }
 sled-agent-config-reconciler = { path = "sled-agent/config-reconciler" }
 sled-agent-health-monitor = { path = "sled-agent/health-monitor" }
+sled-agent-measurements = { path = "sled-agent/measurements" }
 sled-agent-types = { path = "sled-agent/types" }
 sled-agent-types-versions = { path = "sled-agent/types/versions" }
 sled-agent-resolvable-files = { path = "sled-agent/resolvable-files" }

Cargo.toml

@@ -89,6 +89,7 @@ sled-agent-api.workspace = true
 sled-agent-client.workspace = true
 sled-agent-config-reconciler.workspace = true
 sled-agent-health-monitor.workspace = true
+sled-agent-measurements.workspace = true
 sled-agent-types.workspace = true
 sled-agent-types-versions.workspace = true
 sled-agent-resolvable-files.workspace = true

sled-agent/Cargo.toml

@@ -22,6 +22,7 @@ use slog::Logger;
 use std::collections::HashSet;
 use std::sync::Arc;
 use std::sync::OnceLock;
+use tokio::sync::oneshot;
 use tokio::sync::watch;
 
 #[cfg(feature = "testing")]
@@ -82,18 +83,29 @@ pub enum TimeSyncConfig {
 }
 
 #[derive(Debug)]
-pub struct ConfigReconcilerSpawnToken {
+struct SpawnTokenCommon {
     key_requester: StorageKeyRequester,
     time_sync_config: TimeSyncConfig,
     reconciler_result_tx: watch::Sender<ReconcilerResult>,
     currently_managed_zpools_tx: watch::Sender<Arc<CurrentlyManagedZpools>>,
     external_disks_tx: watch::Sender<HashSet<Disk>>,
     former_zone_root_archiver: FormerZoneRootArchiver,
     raw_disks_rx: RawDisksReceiver,
-    ledger_task_log: Logger,
     reconciler_task_log: Logger,
 }
 
+#[derive(Debug)]
+pub struct LedgerTaskSpawnToken {
+    common: SpawnTokenCommon,
+    ledger_task_log: Logger,
+}
+
+#[derive(Debug)]
+pub struct ConfigReconcilerSpawnToken {
+    common: SpawnTokenCommon,
+    ledger_rx: watch::Receiver<CurrentSledConfig>,
+}
+
 #[derive(Debug)]
 pub struct ConfigReconcilerHandle {
     raw_disks_tx: RawDisksSender,
@@ -110,18 +122,25 @@ impl ConfigReconcilerHandle {
     /// Create a `ConfigReconcilerHandle` and spawn many of the early-sled-agent
     /// background tasks (e.g., managing internal disks).
     ///
-    /// The config reconciler subsystem splits initialization into two phases:
-    /// the main reconcilation task will not be spawned until
-    /// `spawn_reconciliation_task()` is called on the returned handle.
+    /// The config reconciler subsystem splits initialization into three phases:
+    /// - This function returns a `LedgerTaskSpawnToken`
+    /// - `spawn_ledger_task` takes a `LedgerTaskSpawnToken` and is responsible
+    /// for spawning the ledger task and returning a `ConfigReconcilerSpawnToken`.
+    /// The ledger task needs to be spawned early for access to reference measurements.
+    ///  - The main reconciliation task will not be spawned until
+    /// `spawn_reconciliation_task()` is called with a `ConfigReconcilerSpawnToken`.
     /// `spawn_reconciliation_task()` cannot be called by sled-agent proper
     /// until rack setup has occurred (or sled-agent has found its config from a
     /// prior rack setup, during a cold boot).
+    ///
+    /// This is designed to be difficult (if not impossible!) to mess up the
+    /// ordering of functions to call
     pub fn new(
         mount_config: MountConfig,
         key_requester: StorageKeyRequester,
         time_sync_config: TimeSyncConfig,
         base_log: &Logger,
-    ) -> (Self, ConfigReconcilerSpawnToken) {
+    ) -> (Self, LedgerTaskSpawnToken) {
         let mount_config = Arc::new(mount_config);
 
         // Spawn the task that monitors our internal disks (M.2s).
@@ -168,54 +187,49 @@ impl ConfigReconcilerHandle {
             // Stash the dependencies the reconciler task will need in
             // `spawn_reconciliation_task()` inside this token that the caller
             // has to hold until it has the other outside dependencies ready.
-            ConfigReconcilerSpawnToken {
-                key_requester,
-                time_sync_config,
-                reconciler_result_tx,
-                currently_managed_zpools_tx,
-                external_disks_tx,
-                former_zone_root_archiver,
-                raw_disks_rx,
+            LedgerTaskSpawnToken {
+                common: SpawnTokenCommon {
+                    key_requester,
+                    time_sync_config,
+                    reconciler_result_tx,
+                    currently_managed_zpools_tx,
+                    external_disks_tx,
+                    former_zone_root_archiver,
+                    raw_disks_rx,
+                    reconciler_task_log: base_log
+                        .new(slog::o!("component" => "ConfigReconcilerTask")),
+                },
                 ledger_task_log: base_log
                     .new(slog::o!("component" => "SledConfigLedgerTask")),
-                reconciler_task_log: base_log
-                    .new(slog::o!("component" => "ConfigReconcilerTask")),
             },
         )
     }
 
-    /// Spawn the primary config reconciliation task.
+    /// Spawn the ledger task
+    ///
+    /// This is the first half of spawning the reconciliation task. We need to
+    /// spawn the ledger task early to allow for access to the ledger for
+    /// early measurement reconciliation.
     ///
     /// This method can effectively only be called once, because the caller must
     /// supply the `token` returned by `new()` when this handle was created.
     ///
+    /// This returns a watch channel for use by the measurement handler to
+    /// know when the ledger task has started running and has found M.2 disks.
+    ///
     /// # Panics
     ///
     /// Panics if called multiple times, which is statically impossible outside
-    /// shenanigans to get a second [`ConfigReconcilerSpawnToken`].
-    pub fn spawn_reconciliation_task<
-        T: SledAgentFacilities,
-        U: SledAgentArtifactStore + Clone,
-    >(
+    /// shenanigans to get a second `LedgerTaskSpawnToken`.
+    pub fn spawn_ledger_task<U: SledAgentArtifactStore + Clone>(
         &self,
-        sled_agent_facilities: T,
         sled_agent_artifact_store: U,
-        token: ConfigReconcilerSpawnToken,
-    ) {
-        let ConfigReconcilerSpawnToken {
-            key_requester,
-            time_sync_config,
-            reconciler_result_tx,
-            currently_managed_zpools_tx,
-            external_disks_tx,
-            former_zone_root_archiver,
-            raw_disks_rx,
-            ledger_task_log,
-            reconciler_task_log,
-        } = token;
+        token: LedgerTaskSpawnToken,
+    ) -> (ConfigReconcilerSpawnToken, oneshot::Receiver<()>) {
+        let LedgerTaskSpawnToken { common, ledger_task_log } = token;
 
         // Spawn the task that manages our config ledger.
-        let (ledger_task, current_config_rx) =
+        let (ledger_task, current_config_rx, ledger_task_run_rx) =
             LedgerTaskHandle::spawn_ledger_task(
                 self.internal_disks_rx.clone(),
                 sled_agent_artifact_store.clone(),
@@ -227,18 +241,51 @@ impl ConfigReconcilerHandle {
             // we document that we panic if called multiple times via some
             // multi-token shenanigans.
             Err(_) => {
-                panic!(
-                    "spawn_reconciliation_task() called with multiple tokens"
-                )
+                panic!("spawn_ledger_task() called with multiple tokens")
             }
         }
 
+        (
+            ConfigReconcilerSpawnToken { common, ledger_rx: current_config_rx },
+            ledger_task_run_rx,
+        )
+    }
+
+    /// Spawn the primary config reconciliation task.
+    ///
+    /// This method can effectively only be called once, because the caller must
+    /// supply the `token` returned by `spawn_ledger_task()` when this handle was created.
+    ///
+    pub fn spawn_reconciliation_task<
+        T: SledAgentFacilities,
+        U: SledAgentArtifactStore + Clone,
+    >(
+        &self,
+        sled_agent_facilities: T,
+        sled_agent_artifact_store: U,
+        token: ConfigReconcilerSpawnToken,
+    ) {
+        let ConfigReconcilerSpawnToken {
+            common:
+                SpawnTokenCommon {
+                    key_requester,
+                    time_sync_config,
+                    reconciler_result_tx,
+                    currently_managed_zpools_tx,
+                    external_disks_tx,
+                    former_zone_root_archiver,
+                    raw_disks_rx,
+                    reconciler_task_log,
+                },
+            ledger_rx,
+        } = token;
+
         reconciler_task::spawn(
             Arc::clone(self.internal_disks_rx.mount_config()),
             self.dataset_task.clone(),
             key_requester,
             time_sync_config,
-            current_config_rx,
+            ledger_rx,
             reconciler_result_tx,
             currently_managed_zpools_tx,
             self.internal_disks_rx.clone(),

sled-agent/config-reconciler/src/handle.rs

@@ -127,7 +127,7 @@ impl LedgerTaskHandle {
         internal_disks_rx: InternalDisksReceiver,
         artifact_store: T,
         log: Logger,
-    ) -> (Self, watch::Receiver<CurrentSledConfig>) {
+    ) -> (Self, watch::Receiver<CurrentSledConfig>, oneshot::Receiver<()>) {
         // We only accept two kinds of requests on this channel, both of which
         // come from HTTP requests to sled agent:
         //
@@ -152,6 +152,11 @@ impl LedgerTaskHandle {
         let (current_config_tx, current_config_rx) =
             watch::channel(CurrentSledConfig::WaitingForInternalDisks);
 
+        // The measurement handler relies on the ledger task running.
+        // Give a channel to wait for that to happen instead of relying
+        // on polling.
+        let (ledger_run_tx, ledger_run_rx) = oneshot::channel();
+
         tokio::spawn(
             LedgerTask {
                 artifact_store,
@@ -160,12 +165,13 @@ impl LedgerTaskHandle {
                 current_config_tx,
                 log,
             }
-            .run(),
+            .run(ledger_run_tx),
         );
 
         (
             Self { request_tx, current_config_rx: current_config_rx.clone() },
             current_config_rx,
+            ledger_run_rx,
         )
     }
 
@@ -255,10 +261,10 @@ struct LedgerTask<T> {
 }
 
 impl<T: SledAgentArtifactStore> LedgerTask<T> {
-    async fn run(self) {
+    async fn run(self, ledger_run_tx: oneshot::Sender<()>) {
         // This pattern match looks strange, but `run_impl()` cannot return
         // `Ok(_)`; it must run forever (or until failure).
-        let Err((log, err)) = self.run_impl().await;
+        let Err((log, err)) = self.run_impl(ledger_run_tx).await;
         error!(
             log,
             "LedgerTask::run() unexpectedly exited; this should only be \
@@ -269,6 +275,7 @@ impl<T: SledAgentArtifactStore> LedgerTask<T> {
 
     async fn run_impl(
         mut self,
+        ledger_run_tx: oneshot::Sender<()>,
     ) -> Result<Infallible, (Logger, LedgerTaskExit)> {
         // We created `self.current_config_tx` in `spawn_ledger_task()` and own
         // the only sender, so it should start out in the `WaitingForM2Disks`
@@ -283,6 +290,9 @@ impl<T: SledAgentArtifactStore> LedgerTask<T> {
             CurrentSledConfig::WaitingForInternalDisks
         );
 
+        // We've gotten far enough that our disks should be ready!
+        let _ = ledger_run_tx.send(());
+
         loop {
             let Some(request) = self.request_rx.recv().await else {
                 return Err((
@@ -800,13 +810,16 @@ mod tests {
             }
 
             // Spawn the ledger task.
-            let (task_handle, mut current_config_rx) =
+            let (task_handle, mut current_config_rx, ledger_task_run_rx) =
                 LedgerTaskHandle::spawn_ledger_task(
                     internal_disks_rx.clone(),
                     fake_artifact_store,
                     log,
                 );
 
+            // This better run!
+            let _ = ledger_task_run_rx.await.unwrap();
+
             // Wait for the task to check our fake disk and progress to either
             // `Ledgered` (if we copied in a config) or
             // `WaitingForInitialConfig` (if we didn't).
@@ -896,7 +909,7 @@ mod tests {
 
         // Spawn the ledger task. It should sit in the `WaitingForInternalDisks`
         // state.
-        let (_task_handle, mut current_config_rx) =
+        let (_task_handle, mut current_config_rx, ledger_task_run_rx) =
             LedgerTaskHandle::spawn_ledger_task(
                 internal_disks_rx,
                 FakeArtifactStore::default(),
@@ -911,6 +924,8 @@ mod tests {
         // Populate a fake disk.
         disks_tx.send(vec![make_fake_disk()]).expect("receiver still exists");
 
+        let _ = ledger_task_run_rx.await.unwrap();
+
         // Confirm the ledger task notices and progresses to
         // `WaitingForInitialConfig`.
         wait_for_watch_channel_condition(

sled-agent/config-reconciler/src/ledger.rs

@@ -236,7 +236,6 @@ impl LatestReconciliationResult {
             zones: self.zones_inventory.clone(),
             boot_partitions: self.boot_partitions.clone(),
             remove_mupdate_override: self.remove_mupdate_override.clone(),
-            // TODO: this will come in another PR
             measurements: IdOrdMap::new(),
         }
     }