[sp-sim] Enhancements to support host OS testing (#8731)

This is pulled out of the upcoming Reconfigurator host OS update work to
make that PR slightly smaller. This has a few small enhancements to our
SP simulator:

* Simulated sleds now always report something for both phase 1 slots.
The initial values for the two slots are different. (This isn't really
necessary but makes debugging some tests easier, since without this both
slots report the same hash value for their contents.)
* We expose a count of the number of times we've performed a power state
transition. (Will be used to test whether an update triggers a reboot.)
* Implement `component_get_active_slot()` for host boot flash.
* Simulated sled only: We expose a watch channel with the current power
state, and if the state is A0, it also includes which host boot flash
slot was active at the time we transitioned to A0. (We'll hook this up
to a fake sled-agent so it can report a different boot disk after an
sp-sim "reboot".)

Author: jgallagher

common/src/disk.rs

@@ -626,6 +626,34 @@ pub enum M2Slot {
     B,
 }
 
+impl M2Slot {
+    /// Flip from `A` to `B` or vice versa.
+    pub fn toggled(self) -> Self {
+        match self {
+            Self::A => Self::B,
+            Self::B => Self::A,
+        }
+    }
+
+    /// Convert this slot to an MGS "firmware slot" index.
+    pub fn to_mgs_firmware_slot(self) -> u16 {
+        match self {
+            Self::A => 0,
+            Self::B => 1,
+        }
+    }
+
+    /// Convert a putative MGS "firmware slot" index to an `M2Slot`, returning
+    /// `None` if `slot` is invalid.
+    pub fn from_mgs_firmware_slot(slot: u16) -> Option<Self> {
+        match slot {
+            0 => Some(Self::A),
+            1 => Some(Self::B),
+            _ => None,
+        }
+    }
+}
+
 impl fmt::Display for M2Slot {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {

dev-tools/omdb/tests/successes.out

@@ -110,8 +110,8 @@ SPs FOUND THROUGH IGNITION
 SERVICE PROCESSOR STATES
 
     TYPE   SLOT MODEL            SERIAL      REV HUBRIS           PWR ROT_ACTIVE 
-    Sled   0    i86pc            SimGimlet00 0   0000000000000000 A2  slot A     
-    Sled   1    i86pc            SimGimlet01 0   0000000000000000 A2  slot A     
+    Sled   0    i86pc            SimGimlet00 0   0000000000000000 A0  slot A     
+    Sled   1    i86pc            SimGimlet01 0   0000000000000000 A0  slot A     
     Switch 0    FAKE_SIM_SIDECAR SimSidecar0 0   0000000000000000 A2  slot A     
     Switch 1    FAKE_SIM_SIDECAR SimSidecar1 0   0000000000000000 A2  slot A     
 

nexus/mgs-updates/tests/host_phase1_hash.rs

@@ -128,9 +128,21 @@ async fn test_host_phase1_hashing() {
         .await
         .expect("starting hashing while hashing should be okay");
 
-    // Calculate the hash we expect to see.
+    // Calculate the hashes we expect to see.
     let expected_sha256_0 = Sha256::digest(
-        sp_sim.last_host_phase1_update_data(0).await.as_deref().unwrap_or(&[]),
+        sp_sim
+            .host_phase1_data(0)
+            .await
+            .as_deref()
+            .expect("sled should have data in slot 0"),
+    )
+    .into();
+    let expected_sha256_1 = Sha256::digest(
+        sp_sim
+            .host_phase1_data(1)
+            .await
+            .as_deref()
+            .expect("sled should have data in slot 1"),
     )
     .into();
 
@@ -155,7 +167,7 @@ async fn test_host_phase1_hashing() {
     phase1_checker
         .assert_status(&[
             (0, ComponentFirmwareHashStatus::Hashed(expected_sha256_0)),
-            (1, ComponentFirmwareHashStatus::Hashed(expected_sha256_0)),
+            (1, ComponentFirmwareHashStatus::Hashed(expected_sha256_1)),
         ])
         .await;
 
@@ -222,7 +234,7 @@ async fn test_host_phase1_hashing() {
     }
 
     // Confirm the simulator wrote the expected data in slot 1.
-    let slot_1_data = sp_sim.last_host_phase1_update_data(1).await.unwrap();
+    let slot_1_data = sp_sim.host_phase1_data(1).await.unwrap();
     assert_eq!(*slot_1_data, *fake_phase1);
 
     // Writing an update should have put slot 1 back into the "needs hashing"

nexus/mgs-updates/tests/host_phase1_updater.rs

@@ -64,7 +64,7 @@ async fn test_host_phase1_updater_updates_sled() {
 
         // Ensure the SP received the complete update.
         let last_update_image = mgstestctx.simrack.gimlets[sp_slot as usize]
-            .last_host_phase1_update_data(target_host_slot)
+            .host_phase1_data(target_host_slot)
             .await
             .expect("simulated host phase1 did not receive an update");
 
@@ -150,7 +150,7 @@ async fn test_host_phase1_updater_remembers_successful_mgs_instance() {
     host_phase1_updater.update(&mut mgs_clients).await.expect("update failed");
 
     let last_update_image = mgstestctx.simrack.gimlets[sp_slot as usize]
-        .last_host_phase1_update_data(target_host_slot)
+        .host_phase1_data(target_host_slot)
         .await
         .expect("simulated host phase1 did not receive an update");
 
@@ -361,7 +361,7 @@ async fn test_host_phase1_updater_switches_mgs_instances_on_failure() {
     );
 
     let last_update_image = mgstestctx.simrack.gimlets[sp_slot as usize]
-        .last_host_phase1_update_data(target_host_slot)
+        .host_phase1_data(target_host_slot)
         .await
         .expect("simulated host phase1 did not receive an update");
 
@@ -460,7 +460,7 @@ async fn test_host_phase1_updater_delivers_progress() {
     do_update_task.await.expect("update task panicked").expect("update failed");
 
     let last_update_image = target_sp
-        .last_host_phase1_update_data(target_host_slot)
+        .host_phase1_data(target_host_slot)
         .await
         .expect("simulated host phase1 did not receive an update");
 

sp-sim/src/gimlet.rs

@@ -48,12 +48,15 @@ use gateway_messages::{ComponentDetails, Message, MgsError, StartupOptions};
 use gateway_messages::{DiscoverResponse, IgnitionState, PowerState};
 use gateway_messages::{MessageKind, version};
 use gateway_types::component::SpState;
+use omicron_common::disk::M2Slot;
 use slog::{Logger, debug, error, info, warn};
 use std::cell::Cell;
 use std::collections::HashMap;
 use std::iter;
 use std::net::{SocketAddr, SocketAddrV6};
 use std::pin::Pin;
+use std::sync::atomic::AtomicUsize;
+use std::sync::atomic::Ordering;
 use std::sync::{Arc, Mutex};
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::net::{TcpListener, TcpStream, UdpSocket};
@@ -86,6 +89,24 @@ pub enum SimSpHandledRequest {
     NotImplemented,
 }
 
+/// Current power state and, if in A0, which M2 slot was active at the time
+/// we transitioned to A0. (This represents what disk the OS would attempt to
+/// boot from, if we were a real SP connected to a real sled.)
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum GimletPowerState {
+    A2,
+    A0(M2Slot),
+}
+
+impl From<GimletPowerState> for PowerState {
+    fn from(value: GimletPowerState) -> Self {
+        match value {
+            GimletPowerState::A2 => Self::A2,
+            GimletPowerState::A0(_) => Self::A0,
+        }
+    }
+}
+
 pub struct Gimlet {
     local_addrs: Option<[SocketAddrV6; 2]>,
     ereport_addrs: Option<[SocketAddrV6; 2]>,
@@ -94,7 +115,9 @@ pub struct Gimlet {
     commands: mpsc::UnboundedSender<Command>,
     inner_tasks: Vec<JoinHandle<()>>,
     responses_sent_count: Option<watch::Receiver<usize>>,
+    power_state_changes: Arc<AtomicUsize>,
     last_request_handled: Arc<Mutex<Option<SimSpHandledRequest>>>,
+    power_state_rx: Option<watch::Receiver<GimletPowerState>>,
 }
 
 impl Drop for Gimlet {
@@ -149,13 +172,10 @@ impl SimulatedSp for Gimlet {
         handler.update_state.last_rot_update_data()
     }
 
-    async fn last_host_phase1_update_data(
-        &self,
-        slot: u16,
-    ) -> Option<Box<[u8]>> {
+    async fn host_phase1_data(&self, slot: u16) -> Option<Vec<u8>> {
         let handler = self.handler.as_ref()?;
         let handler = handler.lock().await;
-        handler.update_state.last_host_phase1_update_data(slot)
+        handler.update_state.host_phase1_data(slot)
     }
 
     async fn current_update_status(&self) -> gateway_messages::UpdateStatus {
@@ -166,6 +186,10 @@ impl SimulatedSp for Gimlet {
         handler.lock().await.update_state.status()
     }
 
+    fn power_state_changes(&self) -> usize {
+        self.power_state_changes.load(Ordering::Relaxed)
+    }
+
     fn responses_sent_count(&self) -> Option<watch::Receiver<usize>> {
         self.responses_sent_count.clone()
     }
@@ -235,6 +259,8 @@ impl Gimlet {
                 inner_tasks,
                 responses_sent_count: None,
                 last_request_handled,
+                power_state_rx: None,
+                power_state_changes: Arc::new(AtomicUsize::new(0)),
             });
         };
 
@@ -374,6 +400,9 @@ impl Gimlet {
             }
         }
         let local_addrs = [servers[0].local_addr(), servers[1].local_addr()];
+        let (power_state, power_state_rx) =
+            watch::channel(GimletPowerState::A0(M2Slot::A));
+        let power_state_changes = Arc::new(AtomicUsize::new(0));
         let (inner, handler, responses_sent_count) = UdpTask::new(
             servers,
             ereport_servers,
@@ -382,11 +411,13 @@ impl Gimlet {
             attached_mgs,
             gimlet.common.serial_number.clone(),
             incoming_console_tx,
+            power_state,
             commands_rx,
             Arc::clone(&last_request_handled),
             log,
             gimlet.common.old_rot_state,
             update_state,
+            Arc::clone(&power_state_changes),
         );
         inner_tasks
             .push(task::spawn(async move { inner.run().await.unwrap() }));
@@ -400,9 +431,15 @@ impl Gimlet {
             inner_tasks,
             responses_sent_count: Some(responses_sent_count),
             last_request_handled,
+            power_state_rx: Some(power_state_rx),
+            power_state_changes,
         })
     }
 
+    pub fn power_state_rx(&self) -> Option<watch::Receiver<GimletPowerState>> {
+        self.power_state_rx.clone()
+    }
+
     pub fn serial_console_addr(&self, component: &str) -> Option<SocketAddrV6> {
         self.serial_console_addrs.get(component).copied()
     }
@@ -625,21 +662,25 @@ impl UdpTask {
         attached_mgs: AttachedMgsSerialConsole,
         serial_number: String,
         incoming_serial_console: HashMap<SpComponent, UnboundedSender<Vec<u8>>>,
+        power_state: watch::Sender<GimletPowerState>,
         commands: mpsc::UnboundedReceiver<Command>,
         last_request_handled: Arc<Mutex<Option<SimSpHandledRequest>>>,
         log: Logger,
         old_rot_state: bool,
         update_state: SimSpUpdate,
+        power_state_changes: Arc<AtomicUsize>,
     ) -> (Self, Arc<TokioMutex<Handler>>, watch::Receiver<usize>) {
         let [udp0, udp1] = servers;
         let handler = Arc::new(TokioMutex::new(Handler::new(
             serial_number,
             components,
             attached_mgs,
             incoming_serial_console,
+            power_state,
             log.clone(),
             old_rot_state,
             update_state,
+            power_state_changes,
         )));
         let responses_sent_count = watch::Sender::new(0);
         let responses_sent_count_rx = responses_sent_count.subscribe();
@@ -782,7 +823,8 @@ struct Handler {
 
     attached_mgs: AttachedMgsSerialConsole,
     incoming_serial_console: HashMap<SpComponent, UnboundedSender<Vec<u8>>>,
-    power_state: PowerState,
+    power_state: watch::Sender<GimletPowerState>,
+    power_state_changes: Arc<AtomicUsize>,
     startup_options: StartupOptions,
     update_state: SimSpUpdate,
     reset_pending: Option<SpComponent>,
@@ -801,14 +843,17 @@ struct Handler {
 }
 
 impl Handler {
+    #[allow(clippy::too_many_arguments)]
     fn new(
         serial_number: String,
         components: Vec<SpComponentConfig>,
         attached_mgs: AttachedMgsSerialConsole,
         incoming_serial_console: HashMap<SpComponent, UnboundedSender<Vec<u8>>>,
+        power_state: watch::Sender<GimletPowerState>,
         log: Logger,
         old_rot_state: bool,
         update_state: SimSpUpdate,
+        power_state_changes: Arc<AtomicUsize>,
     ) -> Self {
         let mut leaked_component_device_strings =
             Vec::with_capacity(components.len());
@@ -835,14 +880,15 @@ impl Handler {
             serial_number,
             attached_mgs,
             incoming_serial_console,
-            power_state: PowerState::A2,
             startup_options: StartupOptions::empty(),
             update_state,
             reset_pending: None,
+            power_state,
             last_request_handled: None,
             should_fail_to_respond_signal: None,
             old_rot_state,
             sp_dumps,
+            power_state_changes,
         }
     }
 
@@ -859,7 +905,7 @@ impl Handler {
             model,
             revision: 0,
             base_mac_address: [0; 6],
-            power_state: self.power_state,
+            power_state: (*self.power_state.borrow()).into(),
             rot: Ok(rot_state_v2(self.update_state.rot_state())),
         }
     }
@@ -1195,19 +1241,21 @@ impl SpHandler for Handler {
     }
 
     fn power_state(&mut self) -> Result<PowerState, SpError> {
+        let power_state = *self.power_state.borrow();
         debug!(
             &self.log, "received power state";
-            "power_state" => ?self.power_state,
+            "power_state" => ?power_state,
         );
-        Ok(self.power_state)
+        Ok(power_state.into())
     }
 
     fn set_power_state(
         &mut self,
         sender: Sender<Self::VLanId>,
         power_state: PowerState,
     ) -> Result<PowerStateTransition, SpError> {
-        let transition = if power_state != self.power_state {
+        let prev_power_state = *self.power_state.borrow();
+        let transition = if power_state != prev_power_state.into() {
             PowerStateTransition::Changed
         } else {
             PowerStateTransition::Unchanged
@@ -1216,11 +1264,38 @@ impl SpHandler for Handler {
         debug!(
             &self.log, "received set power state";
             "sender" => ?sender,
-            "prev_power_state" => ?self.power_state,
+            "prev_power_state" => ?power_state,
             "power_state" => ?power_state,
             "transition" => ?transition,
         );
-        self.power_state = power_state;
+
+        let new_power_state = match power_state {
+            PowerState::A0 => {
+                let slot = self
+                    .update_state
+                    .component_get_active_slot(SpComponent::HOST_CPU_BOOT_FLASH)
+                    .expect("can always get active slot for valid component");
+                let slot = M2Slot::from_mgs_firmware_slot(slot)
+                    .expect("sp-sim ensures host slot is always valid");
+                GimletPowerState::A0(slot)
+            }
+            // `A1` is a transitory state that we can't even observe on real
+            // devices as of https://github.com/oxidecomputer/hubris/pull/2107.
+            // Our tests really care about "host powered on" (A0) or "host
+            // powered off" (A2), so just squish the transitory state down to
+            // "host powered off".
+            PowerState::A1 | PowerState::A2 => GimletPowerState::A2,
+        };
+        self.power_state.send_modify(|s| {
+            *s = new_power_state;
+        });
+        match transition {
+            PowerStateTransition::Changed => {
+                self.power_state_changes.fetch_add(1, Ordering::Relaxed);
+            }
+            PowerStateTransition::Unchanged => (),
+        }
+
         Ok(transition)
     }