Use measurements with sprockets

Author: labbott

Cargo.lock

@@ -41,6 +41,7 @@ use sled_agent_config_reconciler::ConfigReconcilerHandle;
 use sled_agent_config_reconciler::InternalDisksReceiver;
 use sled_agent_types::artifact::ArtifactConfig;
 use sled_agent_types::artifact::{ArtifactListResponse, ArtifactPutResponse};
+use sled_agent_config_reconciler::SledAgentArtifactStore;
 use slog::{Logger, error, info};
 use slog_error_chain::{InlineErrorChain, SlogInlineError};
 use tokio::fs::File;
@@ -54,6 +55,22 @@ use tufaceous_artifact::ArtifactHash;
 const LEDGER_PATH: &str = "artifact-config.json";
 const TEMP_SUBDIR: &str = "tmp";
 
+// Workaround wrapper for orphan rules.
+#[derive(Clone)]
+pub(crate) struct SledAgentArtifactStoreWrapper(
+    pub Arc<ArtifactStore<InternalDisksReceiver>>,
+);
+
+impl SledAgentArtifactStore for SledAgentArtifactStoreWrapper {
+    async fn get_artifact(
+        &self,
+        artifact: ArtifactHash,
+    ) -> anyhow::Result<tokio::fs::File> {
+        let file = self.0.get(artifact).await?;
+        Ok(file)
+    }
+}
+
 /// Content-addressable local storage for software artifacts.
 ///
 /// If you need to read a file managed by the artifact store from somewhere else

sled-agent/src/artifact_store.rs

@@ -10,6 +10,7 @@ use super::params::version;
 use super::views::SledAgentResponse;
 use crate::bootstrap::views::Response;
 use crate::bootstrap::views::ResponseEnvelope;
+use sled_agent_config_reconciler::MeasurementsReceiver;
 use sled_agent_types::sled::StartSledAgentRequest;
 use slog::Logger;
 use slog_error_chain::SlogInlineError;
@@ -77,15 +78,17 @@ pub(crate) struct Client {
     addr: SocketAddrV6,
     log: Logger,
     sprockets_conf: SprocketsConfig,
+    measurements_rx: MeasurementsReceiver,
 }
 
 impl Client {
     pub(crate) fn new(
         addr: SocketAddrV6,
         sprockets_conf: SprocketsConfig,
+        measurements_rx: MeasurementsReceiver,
         log: Logger,
     ) -> Self {
-        Self { addr, sprockets_conf, log }
+        Self { addr, sprockets_conf, log, measurements_rx }
     }
 
     /// Start sled agent by sending an initialization request determined from
@@ -117,8 +120,14 @@ impl Client {
         // Establish connection and sprockets connection (if possible).
         // The sprockets client loads the associated root certificates at this point.
         //
-        // TODO: Use a real corpus
-        let corpus = vec![];
+        let corpus = self.measurements_rx.latest_measurements();
+        if corpus.is_empty() {
+            error!(self.log, "The measurement log shouldn't be empty");
+        }
+        for c in &corpus {
+            info!(self.log, "Using file {c} in corpus");
+        }
+
         let stream = SprocketsClient::connect(
             self.sprockets_conf.clone(),
             self.addr,

sled-agent/src/bootstrap/client.rs

@@ -24,6 +24,7 @@ use omicron_common::api::external::Error;
 use omicron_uuid_kinds::RackInitUuid;
 use omicron_uuid_kinds::RackResetUuid;
 use sled_agent_config_reconciler::InternalDisksReceiver;
+use sled_agent_config_reconciler::MeasurementsReceiver;
 use sled_agent_types::rack_init::{
     RackInitializeRequest, RackInitializeRequestParams,
 };
@@ -48,6 +49,7 @@ pub(crate) struct BootstrapServerContext {
         mpsc::Sender<oneshot::Sender<Result<(), BootstrapError>>>,
     pub(crate) sprockets: SprocketsConfig,
     pub(crate) trust_quorum_handle: trust_quorum::NodeTaskHandle,
+    pub(crate) measurements_rx: MeasurementsReceiver,
 }
 
 impl BootstrapServerContext {
@@ -60,6 +62,7 @@ impl BootstrapServerContext {
             self.sprockets.clone(),
             self.global_zone_bootstrap_ip,
             &self.internal_disks_rx,
+            &self.measurements_rx,
             &self.bootstore_node_handle,
             &self.trust_quorum_handle,
             request,
@@ -132,6 +135,7 @@ impl BootstrapAgentApi for BootstrapAgentImpl {
             .start_reset(
                 &ctx.base_log,
                 ctx.sprockets.clone(),
+                ctx.measurements_rx.clone(),
                 ctx.global_zone_bootstrap_ip,
             )
             .map_err(|err| HttpError::for_bad_request(None, err.to_string()))?;

sled-agent/src/bootstrap/http_entrypoints.rs

@@ -10,6 +10,7 @@ use bootstore::schemes::v0 as bootstore;
 use omicron_uuid_kinds::RackInitUuid;
 use omicron_uuid_kinds::RackResetUuid;
 use sled_agent_config_reconciler::InternalDisksReceiver;
+use sled_agent_config_reconciler::MeasurementsReceiver;
 use sled_agent_types::rack_init::RackInitializeRequestParams;
 use sled_agent_types::rack_ops::{RackOperationStatus, RssStep};
 use slog::Logger;
@@ -149,6 +150,7 @@ impl RssAccess {
         sprockets: SprocketsConfig,
         global_zone_bootstrap_ip: Ipv6Addr,
         internal_disks_rx: &InternalDisksReceiver,
+        measurements_rx: &MeasurementsReceiver,
         bootstore_node_handle: &bootstore::NodeHandle,
         trust_quorum_handle: &trust_quorum::NodeTaskHandle,
         request: RackInitializeRequestParams,
@@ -188,6 +190,7 @@ impl RssAccess {
                 mem::drop(status);
                 let parent_log = parent_log.clone();
                 let internal_disks_rx = internal_disks_rx.clone();
+                let measurements_rx = measurements_rx.clone();
                 let bootstore_node_handle = bootstore_node_handle.clone();
                 let status = Arc::clone(&self.status);
                 let trust_quorum_handle = trust_quorum_handle.clone();
@@ -197,6 +200,7 @@ impl RssAccess {
                         sprockets,
                         global_zone_bootstrap_ip,
                         internal_disks_rx,
+                        measurements_rx,
                         bootstore_node_handle,
                         trust_quorum_handle,
                         request,
@@ -224,6 +228,7 @@ impl RssAccess {
         &self,
         parent_log: &Logger,
         sprockets: SprocketsConfig,
+        measurements_rx: MeasurementsReceiver,
         global_zone_bootstrap_ip: Ipv6Addr,
     ) -> Result<RackResetUuid, RssAccessError> {
         let mut status = self.status.lock().unwrap();
@@ -264,6 +269,7 @@ impl RssAccess {
                     let result = rack_reset(
                         &parent_log,
                         sprockets,
+                        measurements_rx,
                         global_zone_bootstrap_ip,
                     )
                     .await;
@@ -339,6 +345,7 @@ async fn rack_initialize(
     sprockets: SprocketsConfig,
     global_zone_bootstrap_ip: Ipv6Addr,
     internal_disks_rx: InternalDisksReceiver,
+    measurements_rx: MeasurementsReceiver,
     bootstore_node_handle: bootstore::NodeHandle,
     trust_quorum_handle: trust_quorum::NodeTaskHandle,
     request: RackInitializeRequestParams,
@@ -350,6 +357,7 @@ async fn rack_initialize(
         request,
         global_zone_bootstrap_ip,
         internal_disks_rx,
+        measurements_rx,
         bootstore_node_handle,
         trust_quorum_handle,
         step_tx,
@@ -360,8 +368,14 @@ async fn rack_initialize(
 async fn rack_reset(
     parent_log: &Logger,
     sprockets: SprocketsConfig,
+    measurements_rx: MeasurementsReceiver,
     global_zone_bootstrap_ip: Ipv6Addr,
 ) -> Result<(), SetupServiceError> {
-    RssHandle::run_rss_reset(parent_log, global_zone_bootstrap_ip, sprockets)
-        .await
+    RssHandle::run_rss_reset(
+        parent_log,
+        global_zone_bootstrap_ip,
+        sprockets,
+        measurements_rx,
+    )
+    .await
 }

sled-agent/src/bootstrap/rack_ops.rs

@@ -15,6 +15,7 @@ use omicron_common::backoff::BackoffError;
 use omicron_common::backoff::retry_notify;
 use omicron_common::backoff::retry_policy_local;
 use sled_agent_config_reconciler::InternalDisksReceiver;
+use sled_agent_config_reconciler::MeasurementsReceiver;
 use sled_agent_types::rack_init::RackInitializeRequestParams;
 use sled_agent_types::rack_ops::RssStep;
 use sled_agent_types::sled::StartSledAgentRequest;
@@ -52,11 +53,16 @@ impl RssHandle {
         config: RackInitializeRequestParams,
         our_bootstrap_address: Ipv6Addr,
         internal_disks_rx: InternalDisksReceiver,
+        measurements_rx: MeasurementsReceiver,
         bootstore: bootstore::NodeHandle,
         trust_quorum: trust_quorum::NodeTaskHandle,
         step_tx: watch::Sender<RssStep>,
     ) -> Result<(), SetupServiceError> {
-        let (tx, rx) = rss_channel(our_bootstrap_address, sprockets);
+        let (tx, rx) = rss_channel(
+            our_bootstrap_address,
+            sprockets,
+            measurements_rx.clone(),
+        );
 
         let rss = RackSetupService::new(
             log.new(o!("component" => "RSS")),
@@ -77,8 +83,13 @@ impl RssHandle {
         log: &Logger,
         our_bootstrap_address: Ipv6Addr,
         sprockets: SprocketsConfig,
+        measurements_rx: MeasurementsReceiver,
     ) -> Result<(), SetupServiceError> {
-        let (tx, rx) = rss_channel(our_bootstrap_address, sprockets);
+        let (tx, rx) = rss_channel(
+            our_bootstrap_address,
+            sprockets,
+            measurements_rx.clone(),
+        );
 
         let rss = RackSetupService::new_reset_rack(
             log.new(o!("component" => "RSS")),
@@ -95,11 +106,13 @@ async fn initialize_sled_agent(
     log: &Logger,
     bootstrap_addr: SocketAddrV6,
     sprockets: SprocketsConfig,
+    measurements_rx: MeasurementsReceiver,
     request: &StartSledAgentRequest,
 ) -> Result<(), bootstrap_agent_client::Error> {
     let client = bootstrap_agent_client::Client::new(
         bootstrap_addr,
         sprockets,
+        measurements_rx,
         log.new(o!("BootstrapAgentClient" => bootstrap_addr.to_string())),
     );
 
@@ -131,11 +144,12 @@ async fn initialize_sled_agent(
 fn rss_channel(
     our_bootstrap_address: Ipv6Addr,
     sprockets: SprocketsConfig,
+    measurements_rx: MeasurementsReceiver,
 ) -> (BootstrapAgentHandle, BootstrapAgentHandleReceiver) {
     let (tx, rx) = mpsc::channel(32);
     (
         BootstrapAgentHandle { inner: tx, our_bootstrap_address },
-        BootstrapAgentHandleReceiver { inner: rx, sprockets },
+        BootstrapAgentHandleReceiver { inner: rx, sprockets, measurements_rx },
     )
 }
 
@@ -207,6 +221,7 @@ impl BootstrapAgentHandle {
 struct BootstrapAgentHandleReceiver {
     inner: mpsc::Receiver<Request>,
     sprockets: SprocketsConfig,
+    measurements_rx: MeasurementsReceiver,
 }
 
 impl BootstrapAgentHandleReceiver {
@@ -227,10 +242,12 @@ impl BootstrapAgentHandleReceiver {
                 // of the initialization requests, allowing them to run concurrently.
 
                 let s = self.sprockets.clone();
+                let mx = self.measurements_rx.clone();
                 let mut futs = requests
                     .into_iter()
                     .map(|(bootstrap_addr, request)| {
                         let value = s.clone();
+                        let measurements_rx = mx.clone();
                         async move {
                             info!(
                                 log, "Received initialization request from RSS";
@@ -242,6 +259,7 @@ impl BootstrapAgentHandleReceiver {
                                 log,
                                 bootstrap_addr,
                                 value,
+                                measurements_rx,
                                 &request,
                             )
                             .await

sled-agent/src/bootstrap/rss_handle.rs

@@ -204,6 +204,60 @@ impl Server {
         // enqueue another, and we can send back an HTTP busy.
         let (sled_reset_tx, sled_reset_rx) = mpsc::channel(1);
 
+        while let Err(
+            sled_agent_config_reconciler::InventoryError::WaitingOnLedger,
+        ) =
+            long_running_task_handles.config_reconciler.ledgered_sled_config()
+        {
+            // waiting for our ledger to run...
+        }
+
+        // Get our pre-boot measurements, first we check the ledger
+        let pre_boot = match long_running_task_handles
+            .config_reconciler
+            .ledgered_sled_config()
+        {
+            // Lie down. Try not to cry. Cry a lot.
+            Err(_) => {
+                vec![]
+            }
+            // We haven't run RSS, we'll take what we get from the measurement manifest
+            Ok(None) => match long_running_task_handles
+                .zone_image_resolver
+                .status()
+                .to_inventory()
+                .measurement_manifest
+                .boot_inventory
+            {
+                Err(_) => {
+                    vec![]
+                }
+                Ok(s) => s
+                    .artifacts
+                    .iter()
+                    .filter_map(|entry| match entry.status {
+                        Ok(_) => Some(entry.path.clone()),
+                        Err(_) => None,
+                    })
+                    .collect(),
+            },
+            // Do an early resolution
+            Ok(Some(s)) => {
+                long_running_task_handles
+                    .config_reconciler
+                    .bootstrap_measurement_reconciler(
+                        &long_running_task_handles.zone_image_resolver.status(),
+                        &long_running_task_handles
+                            .config_reconciler
+                            .internal_disks_rx()
+                            .current(),
+                        &s.measurements,
+                        &base_log,
+                    )
+                    .await
+            }
+        };
+
         // Start the bootstrap dropshot server.
         let bootstrap_context = BootstrapServerContext {
             base_log: base_log.clone(),
@@ -216,13 +270,24 @@ impl Server {
             sled_reset_tx,
             sprockets: config.sprockets.clone(),
             trust_quorum_handle: long_running_task_handles.trust_quorum.clone(),
+            measurements_rx: long_running_task_handles
+                .config_reconciler
+                .measurement_corpus_rx(pre_boot.clone())
+                .await
+                .clone(),
         };
         let bootstrap_http_server = start_dropshot_server(bootstrap_context)?;
 
         // Create a channel for proxying sled-initialization requests that land
         // in the sprockets server to our bootstrap agent `Inner` task.
         let (sled_init_tx, sled_init_rx) = mpsc::channel(1);
 
+        let measurements = long_running_task_handles
+            .config_reconciler
+            .measurement_corpus_rx(pre_boot)
+            .await
+            .clone();
+
         // We don't bother to wrap this bind in a
         // `wait_while_handling_hardware_updates()` because (a) binding should
         // be fast and (b) can succeed regardless of any pending hardware
@@ -235,6 +300,7 @@ impl Server {
                 0,
             ),
             sled_init_tx,
+            measurements,
             config.sprockets.clone(),
             &base_log,
         )