feat: Add command to manually swap the GuestOS alternative (#9154)

We already have an automatic mechanism that rolls back the GuestOS to
the other boot alternative if the GuestOS boot fails. This PR adds a
manual mechanism that node operators can use to swap the boot
alternative manually.

The new commands are available through the limited host console:

```
guestos-alternative show
guestos-alternative swap
guestos-alternative swap A
```

---------

Co-authored-by: IDX GitHub Automation <infra+github-automation@dfinity.org>

Author: frankdavid

Cargo.lock

@@ -22,6 +22,7 @@ show_menu() {
     echo "  ip-addresses            - Show all IP addresses"
     echo "  ping-gateway            - Test connectivity to IPv6 gateway"
     echo "  routes                  - Show routing table"
+    echo "  guestos-alternative     - Show or swap the GuestOS boot alternative (A/B)"
     echo "  manual-recovery         - Manual node recovery (ONLY FOR EMERGENCY USE)"
     echo "  rbash-console           - Drop into restricted bash console"
     echo "  clear                   - Clear the console"
@@ -119,6 +120,21 @@ execute_command() {
             echo "IPv6 routes:"
             /bin/ip -6 route show
             ;;
+        "guestos-alternative")
+            case "${2:-}" in
+                "show")
+                    /opt/ic/bin/hostos_tool guestos-alternative show
+                    ;;
+                "swap")
+                    sudo /opt/ic/bin/hostos_tool guestos-alternative swap ${3:+"$3"}
+                    ;;
+                *)
+                    echo "Usage: guestos-alternative show|swap [A|B]"
+                    echo "  show      - Show the current GuestOS boot alternative"
+                    echo "  swap [A|B]- Swap GuestOS boot alternative (defaults to opposite of current)"
+                    ;;
+            esac
+            ;;
         "manual-recovery")
             echo "=== Manual Node Recovery ==="
             echo "This will perform a manual node recovery. Do not attempt this unless you are certain it is appropriate."

ic-os/components/hostos/misc/limited-console

@@ -32,4 +32,7 @@ root	ALL=(ALL:ALL) NOPASSWD:ALL
 # Allow limited-console to run recovery via the secure launcher script
 limited-console ALL=(ALL:ALL) NOPASSWD: /opt/ic/bin/guestos-recovery-launcher.sh mode=* version=* recovery-hash-prefix=*
 
+# Allow limited-console to swap the GuestOS boot alternative
+limited-console ALL=(ALL:ALL) NOPASSWD: /opt/ic/bin/hostos_tool guestos-alternative swap *
+
 # See sudoers(5) for more information on "#include" directives:

ic-os/components/hostos/misc/sudoers

@@ -5,6 +5,7 @@ package(default_visibility = ["//rs:ic-os-pkg"])
 DEPENDENCIES = [
     # Keep sorted.
     "//rs/sys",
+    "@crate_index//:clap",
     "@crate_index//:regex",
     "@crate_index//:strum",
     "@crate_index//:thiserror",

rs/ic_os/boot/grub/BUILD.bazel

@@ -4,6 +4,7 @@ version = "1.0.0"
 edition.workspace = true
 
 [dependencies]
+clap = { workspace = true }
 ic-sys = { path = "../../../sys" }
 regex = { workspace = true }
 strum = { workspace = true }

rs/ic_os/boot/grub/Cargo.toml

@@ -9,12 +9,14 @@ use thiserror::Error;
 
 const GRUB_ENV_SIZE: usize = 1024;
 
-#[derive(Debug, Eq, PartialEq, Clone, Copy, EnumString, Display)]
+#[derive(Debug, Eq, PartialEq, Clone, Copy, EnumString, Display, clap::ValueEnum)]
 pub enum BootAlternative {
     // Bash scripts depend on the string representations, be very careful if you want to change them
     #[strum(serialize = "A")]
+    #[clap(name = "A")]
     A,
     #[strum(serialize = "B")]
+    #[clap(name = "B")]
     B,
 }
 

rs/ic_os/boot/grub/src/lib.rs

@@ -6,7 +6,7 @@ use gpt::GptDisk;
 use std::fs::File;
 use std::path::{Path, PathBuf};
 #[cfg(target_os = "linux")]
-use sys_mount::{FilesystemType, Mount, Unmount, UnmountFlags};
+use sys_mount::{FilesystemType, Mount, MountFlags, Unmount, UnmountFlags};
 use tempfile::TempDir;
 use uuid::Uuid;
 
@@ -88,6 +88,7 @@ impl FileSystem {
 #[derive(Copy, Clone)]
 pub struct MountOptions {
     pub file_system: FileSystem,
+    pub read_only: bool,
 }
 
 /// Represents a mounted partition with access to its filesystem.
@@ -187,10 +188,13 @@ impl PartitionProvider for UdevPartitionProvider {
         );
 
         let tempdir = TempDir::new()?;
+        let mut builder =
+            Mount::builder().fstype(FilesystemType::Manual(options.file_system.as_str()));
+        if options.read_only {
+            builder = builder.flags(MountFlags::RDONLY);
+        }
         Ok(Box::new(TempDeviceMount {
-            mount: Mount::builder()
-                .fstype(FilesystemType::Manual(options.file_system.as_str()))
-                .mount(device_path, &tempdir)?,
+            mount: builder.mount(device_path, &tempdir)?,
             _loop_device: None,
             _tempdir: tempdir,
         }))
@@ -252,14 +256,17 @@ impl Mounter for LoopDeviceMounter {
 
         // Sometimes the mount can fail with EIO when udev is not ready yet
         let mount = retry_if_io_error(nix::Error::EIO, || {
-            Mount::builder()
-                .fstype(FilesystemType::Manual(options.file_system.as_str()))
-                .mount(
-                    loop_device
-                        .path()
-                        .ok_or_else(|| std::io::Error::other("Loop device has no path"))?,
-                    mount_point,
-                )
+            let mut builder =
+                Mount::builder().fstype(FilesystemType::Manual(options.file_system.as_str()));
+            if options.read_only {
+                builder = builder.flags(MountFlags::RDONLY);
+            }
+            builder.mount(
+                loop_device
+                    .path()
+                    .ok_or_else(|| std::io::Error::other("Loop device has no path"))?,
+                mount_point,
+            )
         })
         .context("Failed to create mount")?;
 
@@ -451,6 +458,7 @@ mod tests {
                 0,
                 MountOptions {
                     file_system: FileSystem::Ext4,
+                    read_only: true,
                 },
             )
             .unwrap();

rs/ic_os/device/src/mount.rs

@@ -11,7 +11,7 @@ use sev_guest::attestation_package::generate_attestation_package;
 use sev_guest::firmware::SevGuestFirmware;
 use std::path::Path;
 
-pub const CONFIG_PARTITION_LABEL: &str = "CONFIG";
+pub const CONFIG_DEVICE_LABEL: &str = "CONFIG";
 pub const RECOVERY_PROPOSAL_FILE_NAME: &str = "alternative_guestos_proposal.cbor";
 
 /// Reads and verifies an alternative GuestOS proposal, returning the rootfs hash from it.
@@ -24,9 +24,10 @@ pub fn extract_and_verify_recovery_rootfs_hash(
 ) -> Result<String> {
     let config_mount = partition_provider
         .mount_partition(
-            PartitionSelector::ByLabel(CONFIG_PARTITION_LABEL.to_string()),
+            PartitionSelector::ByLabel(CONFIG_DEVICE_LABEL.to_string()),
             MountOptions {
                 file_system: FileSystem::Vfat,
+                read_only: true,
             },
         )
         .context("Failed to mount CONFIG partition")?;
@@ -46,6 +47,7 @@ pub fn extract_and_verify_recovery_rootfs_hash(
         PartitionSelector::ByUuid(get_boot_partition_uuid(root_device, command_runner)?),
         MountOptions {
             file_system: FileSystem::Ext4,
+            read_only: false, // Partition may need repair
         },
     )?;
 

rs/ic_os/open_rootfs/src/recovery.rs

@@ -1,5 +1,5 @@
 use crate::partitions::{A_BOOT_UUID, A_ROOT_UUID, B_BOOT_UUID, B_ROOT_UUID};
-use crate::recovery::{CONFIG_PARTITION_LABEL, RECOVERY_PROPOSAL_FILE_NAME};
+use crate::recovery::{CONFIG_DEVICE_LABEL, RECOVERY_PROPOSAL_FILE_NAME};
 use anyhow::{Context, Result};
 use candid::Encode;
 use command_runner::MockCommandRunner;
@@ -72,7 +72,7 @@ impl TestFixture {
             Arc::new(TempDir::with_prefix("b_boot").unwrap()),
         );
         partitions.insert(
-            PartitionSelector::ByLabel(CONFIG_PARTITION_LABEL.to_string()),
+            PartitionSelector::ByLabel(CONFIG_DEVICE_LABEL.to_string()),
             config_media,
         );
 
@@ -222,9 +222,7 @@ impl TestFixture {
         // Write NNS public key override to CONFIG media
         fs::write(
             self.partition_provider
-                .get_partition(PartitionSelector::ByLabel(
-                    CONFIG_PARTITION_LABEL.to_string(),
-                ))
+                .get_partition(PartitionSelector::ByLabel(CONFIG_DEVICE_LABEL.to_string()))
                 .unwrap()
                 .join("nns_public_key_override.pem"),
             &nns_public_key,
@@ -440,9 +438,7 @@ fn test_nns_root_key_mismatch() {
     fs::remove_file(
         fixture
             .partition_provider
-            .get_partition(PartitionSelector::ByLabel(
-                CONFIG_PARTITION_LABEL.to_string(),
-            ))
+            .get_partition(PartitionSelector::ByLabel(CONFIG_DEVICE_LABEL.to_string()))
             .unwrap()
             .join("nns_public_key_override.pem"),
     )

rs/ic_os/open_rootfs/src/tests.rs

@@ -76,6 +76,7 @@ pub async fn prepare_direct_boot(
             PartitionSelector::ByUuid(GRUB_PARTITION_UUID),
             MountOptions {
                 file_system: GRUB_PARTITION_FS,
+                read_only: false, // Partition may need repair
             },
         )
         .context("Could not mount grub partition")?;
@@ -97,6 +98,8 @@ pub async fn prepare_direct_boot(
         boot_alternative = boot_alternative.get_opposite();
     }
 
+    println!("Will boot into {boot_alternative} from {guest_vm_type:?} GuestVM");
+
     // The variable name inside 'boot_args' that contains the kernel command line parameters.
     // Note that this depends on the boot alternative since they contain the root partition and
     // other boot alternative-specific parameters.
@@ -110,6 +113,7 @@ pub async fn prepare_direct_boot(
             PartitionSelector::ByUuid(boot_partition_uuid),
             MountOptions {
                 file_system: BOOT_PARTITION_FS,
+                read_only: false, // Partition may need repair
             },
         )
         .with_context(|| format!("Could not mount boot partition {boot_alternative}"))?;