diff --git a/Cargo.lock b/Cargo.lock index dbb4a14..4bcd7b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,13 +19,25 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.3" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "const-random", + "getrandom 0.3.1", "once_cell", "version_check", + "zerocopy 0.8.25", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", ] [[package]] @@ -119,6 +131,17 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +[[package]] +name = "async-recursion" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "async-trait" version = "0.1.88" @@ -338,10 +361,33 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", + "pure-rust-locales", + "serde", "wasm-bindgen", "windows-targets 0.52.6", ] +[[package]] +name = "chrono-tz" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efdce149c370f133a071ca8ef6ea340b7b88748ab0810097a9e2976eaa34b4f3" +dependencies = [ + "chrono", + "chrono-tz-build", + "phf", +] + +[[package]] +name = "chrono-tz-build" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f10f8c9340e31fc120ff885fcdb54a0b48e474bbd77cab557f0c30a3e569402" +dependencies = [ + "parse-zoneinfo", + "phf_codegen", +] + [[package]] name = "cipher" version = "0.4.4" @@ -432,6 +478,26 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "520fbf3c07483f94e3e3ca9d0cfd913d7718ef2483d2cfd91c0d9e91474ab913" +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.7", + "once_cell", + "tiny-keccak", +] + [[package]] name = "constant_time_eq" version = "0.3.0" @@ -543,6 +609,12 @@ version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +[[package]] +name = "crunchy" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" + [[package]] name = "crypto-bigint" version = "0.5.5" @@ -792,6 +864,7 @@ name = "faux-mgs" version = "0.1.1" dependencies = [ "anyhow", + "async-recursion", "async-trait", "clap", "futures", @@ -799,10 +872,17 @@ dependencies = [ "gateway-sp-comms", "glob", "hex", + "hubtools", "humantime", + "lpc55_areas", + "lpc55_sign", "nix", "parse_int", "rand", + "rhai", + "rhai-chrono", + "rhai-env", + "rhai-fs", "serde", "serde_json", "sha2", @@ -812,9 +892,11 @@ dependencies = [ "ssh-agent-client-rs", "ssh-key", "termios", + "thiserror", "tokio", "tokio-stream", "tokio-util", + "toml", "uuid", "zerocopy 0.8.25", ] @@ -1317,7 +1399,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "socket2 0.5.10", "tokio", "tower-service", "tracing", @@ -1526,7 +1608,7 @@ dependencies = [ [[package]] name = "lpc55_areas" version = "0.2.5" -source = "git+https://github.com/oxidecomputer/lpc55_support#17d04af60b3a4fd82c77b1a33ca5370943cd25d9" +source = "git+https://github.com/oxidecomputer/lpc55_support/#17d04af60b3a4fd82c77b1a33ca5370943cd25d9" dependencies = [ "bitfield", "clap", @@ -1537,7 +1619,7 @@ dependencies = [ [[package]] name = "lpc55_sign" version = "0.3.4" -source = "git+https://github.com/oxidecomputer/lpc55_support#17d04af60b3a4fd82c77b1a33ca5370943cd25d9" +source = "git+https://github.com/oxidecomputer/lpc55_support/#17d04af60b3a4fd82c77b1a33ca5370943cd25d9" dependencies = [ "byteorder", "const-oid", @@ -1808,6 +1890,9 @@ name = "once_cell" version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +dependencies = [ + "portable-atomic", +] [[package]] name = "p256" @@ -1892,6 +1977,15 @@ dependencies = [ "windows-sys 0.36.1", ] +[[package]] +name = "parse-zoneinfo" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f2a05b18d44e2957b88f96ba460715e295bc1d7510468a2f3d3b44535d26c24" +dependencies = [ + "regex", +] + [[package]] name = "parse_int" version = "0.6.0" @@ -1973,6 +2067,35 @@ dependencies = [ "sha2", ] +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand", +] + [[package]] name = "phf_shared" version = "0.11.3" @@ -2027,6 +2150,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" +[[package]] +name = "portable-atomic" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" + [[package]] name = "ppv-lite86" version = "0.2.16" @@ -2087,6 +2216,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pure-rust-locales" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1190fd18ae6ce9e137184f207593877e70f39b015040156b1e05081cdfe3733a" + [[package]] name = "quote" version = "1.0.35" @@ -2171,6 +2306,35 @@ dependencies = [ "thiserror", ] +[[package]] +name = "regex" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" + [[package]] name = "reqwest" version = "0.11.27" @@ -2224,6 +2388,69 @@ dependencies = [ "subtle", ] +[[package]] +name = "rhai" +version = "1.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2780e813b755850e50b178931aaf94ed24f6817f46aaaf5d21c13c12d939a249" +dependencies = [ + "ahash", + "bitflags 2.9.1", + "instant", + "num-traits", + "once_cell", + "rhai_codegen", + "serde", + "serde_json", + "smallvec 1.10.0", + "smartstring", + "thin-vec", +] + +[[package]] +name = "rhai-chrono" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22099214084b30d0fc59b98404ddaf0e9a6e9d85a5410f93d6e19639f166e524" +dependencies = [ + "chrono", + "chrono-tz", + "rhai", +] + +[[package]] +name = "rhai-env" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e328a0eea295867e893ee7888f161ed3197f8ab561b3cc050d493399c410bee8" +dependencies = [ + "rhai", + "serde", + "serde_json", +] + +[[package]] +name = "rhai-fs" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9af3f7717d7194033924473b3e4f0b3dbd569e3528837c3fbb00b4e1d1c6ff16" +dependencies = [ + "rhai", + "serde", + "serde_json", +] + +[[package]] +name = "rhai_codegen" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5a11a05ee1ce44058fa3d5961d05194fdbe3ad6b40f904af764d81b86450e6b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "ring" version = "0.16.20" @@ -2669,6 +2896,21 @@ name = "smallvec" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +dependencies = [ + "serde", +] + +[[package]] +name = "smartstring" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" +dependencies = [ + "autocfg", + "serde", + "static_assertions", + "version_check", +] [[package]] name = "smoltcp" @@ -2964,6 +3206,15 @@ dependencies = [ "libc", ] +[[package]] +name = "thin-vec" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "144f754d318415ac792f9d69fc87abbbfc043ce2ef041c60f16ad828f638717d" +dependencies = [ + "serde", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -3022,6 +3273,15 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinyvec" version = "1.8.0" diff --git a/Cargo.toml b/Cargo.toml index 67c037c..0a02dc9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ hubtools = { git = "https://github.com/oxidecomputer/hubtools.git", branch = "ma slog-error-chain = { git = "https://github.com/oxidecomputer/slog-error-chain.git", branch = "main", features = ["derive"] } anyhow = "1.0" +async-recursion = "1.1.0" async-trait = "0.1" backoff = { version = "0.4.0", features = ["tokio"] } base64 = "0.22.1" @@ -38,6 +39,8 @@ glob = "0.3.2" hex = "0.4.3" hubpack = "0.1.2" humantime = "2.2.0" +lpc55_areas = { git = "https://github.com/oxidecomputer/lpc55_support/", version = "0.2.3" } +lpc55_sign = { git = "https://github.com/oxidecomputer/lpc55_support/", version = "0.3.0" } lru-cache = "0.1.2" lzss = "0.8" nix = { version = "0.27.1", features = ["net"] } @@ -46,6 +49,10 @@ once_cell = "1.21.3" paste = "1.0.15" parse_int = "0.6" rand = "0.8.5" +rhai-chrono = { version = "^0" } +rhai-env = "0.1.2" +rhai-fs = { version = "0.1.3", features = ["metadata"] } +rhai = { version = "1.21.0", features = ["serde", "metadata", "debugging"]} serde = { version = "1.0", default-features = false, features = ["derive"] } serde-big-array = "0.5.1" serde_bytes = "0.11.17" @@ -65,10 +72,11 @@ strum = { version = "0.27.1", default-features = false } strum_macros = "0.27.1" string_cache = "0.8.9" termios = "0.3" -thiserror = "1.0.69" +thiserror = { version = "1.0.69", default-features = false } tokio = { version = "1.29", features = ["full"] } tokio-stream = { version = "0.1", features = ["fs"] } tokio-util = { version = "0.7", features = ["compat"] } +toml = { version = "0.7", default-features = false, features = ["parse", "display"] } usdt = "0.5.0" uuid = { version = "1.16", default-features = false } version_check = "0.9.5" diff --git a/faux-mgs/Cargo.toml b/faux-mgs/Cargo.toml index 75acd0f..1c97102 100644 --- a/faux-mgs/Cargo.toml +++ b/faux-mgs/Cargo.toml @@ -32,3 +32,18 @@ zerocopy.workspace = true gateway-messages = { workspace = true, features = ["std"] } gateway-sp-comms.workspace = true + +async-recursion = { workspace = true, optional = true } +hubtools = { workspace = true, optional = true } +lpc55_areas = { workspace = true, optional = true } +lpc55_sign= { workspace = true, optional = true } +rhai-chrono = { workspace = true, optional = true } +rhai-env = { workspace = true, optional = true } +rhai-fs = { workspace = true, optional = true } +rhai = { workspace = true, optional = true } +thiserror = { workspace = true, optional = true } +toml = { workspace = true, optional = true } + +[features] +default = ["rhaiscript"] +rhaiscript = [ "dep:async-recursion", "dep:hubtools", "dep:lpc55_areas", "dep:lpc55_sign", "dep:rhai", "dep:rhai-chrono", "dep:rhai-env", "dep:rhai-fs", "dep:thiserror", "dep:toml"] diff --git a/faux-mgs/src/main.rs b/faux-mgs/src/main.rs index c109bd0..4fb902d 100644 --- a/faux-mgs/src/main.rs +++ b/faux-mgs/src/main.rs @@ -65,6 +65,8 @@ use uuid::Uuid; use zerocopy::IntoBytes; mod picocom_map; +#[cfg(feature = "rhaiscript")] +mod rhaiscript; mod usart; /// Command line program that can send MGS messages to a single SP. @@ -137,6 +139,14 @@ struct Args { command: Command, } +/// Rhai program that can send MGS messages to a single SP. +#[cfg(feature = "rhaiscript")] +#[derive(Parser, Debug)] +struct RhaiArgs { + #[clap(subcommand)] + command: Command, +} + fn level_from_str(s: &str) -> Result { if let Ok(level) = s.parse() { Ok(level) @@ -407,6 +417,16 @@ enum Command { disable_watchdog: bool, }, + /// Run a Rhai script within faux-mgs + #[cfg(feature = "rhaiscript")] + Rhai { + /// Path to Rhia script + script: PathBuf, + /// Additional arguments passed to Rhia scripe + #[clap(trailing_var_arg = true, allow_hyphen_values = true)] + script_args: Vec, + }, + /// Controls the system LED SystemLed { #[clap(subcommand)] @@ -891,7 +911,7 @@ async fn main() -> Result<()> { .into_iter() .map(|sp| { let interface = sp.interface().to_string(); - run_command( + run_any_command( sp, args.command.clone(), args.json.is_some(), @@ -965,11 +985,31 @@ fn ssh_list_keys(socket: &PathBuf) -> Result> { client.list_identities().context("failed to list identities") } -async fn run_command( +/// This function exists to break recursive calls to the Rhai interpreter. +/// the faux-mgs main function calls here. However, the `rhai` subcommand +/// calls run_command() which does not include any calls to the +/// rhai subcommand. +async fn run_any_command( sp: SingleSp, command: Command, json: bool, log: Logger, +) -> Result { + match command { + #[cfg(feature = "rhaiscript")] + Command::Rhai { script, script_args } => { + rhaiscript::interpreter(&sp, log, script, script_args).await + } + _ => run_command(&sp, command, json, log).await, + } +} + +/// Run faux-mgs commands except for the `rhai` subcommand. +async fn run_command( + sp: &SingleSp, + command: Command, + json: bool, + log: Logger, ) -> Result { match command { // Skip special commands handled by `main()` above. @@ -1328,7 +1368,7 @@ async fn run_command( let data = fs::read(&image).with_context(|| { format!("failed to read {}", image.display()) })?; - update(&log, &sp, component, slot, data).await.with_context( + update(&log, sp, component, slot, data).await.with_context( || { format!( "updating {} slot {} to {} failed", @@ -1465,7 +1505,10 @@ async fn run_command( Ok(Output::Lines(vec!["reset complete".to_string()])) } } - + #[cfg(feature = "rhaiscript")] + Command::Rhai { script, script_args } => { + rhaiscript::interpreter(sp, log, script, script_args).await + } Command::ResetComponent { component, disable_watchdog } => { sp.reset_component_prepare(component).await?; info!(log, "SP is prepared to reset component {component}",); @@ -1554,14 +1597,8 @@ async fn run_command( if time_sec == 0 { bail!("--time must be >= 1 second"); } - monorail_unlock( - &log, - &sp, - time_sec, - ssh_auth_sock, - key, - ) - .await?; + monorail_unlock(&log, sp, time_sec, ssh_auth_sock, key) + .await?; } } } @@ -2062,6 +2099,7 @@ async fn populate_phase2_images( Ok(()) } +#[derive(Clone)] enum Output { Json(serde_json::Value), Lines(Vec), diff --git a/faux-mgs/src/rhaiscript.rs b/faux-mgs/src/rhaiscript.rs new file mode 100644 index 0000000..20a83a2 --- /dev/null +++ b/faux-mgs/src/rhaiscript.rs @@ -0,0 +1,406 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use slog::crit; +use slog::error; +use slog::trace; + +use crate::{anyhow, debug, info, warn}; +use crate::{ + fs, json, run_command, Context, Logger, Output, PathBuf, Result, RhaiArgs, + RotBootInfo, SingleSp, +}; +use clap::Parser; + +use async_recursion::async_recursion; +use rhai::packages::{Package, MoreStringPackage}; +use rhai::{ + Array, Dynamic, Engine, EvalAltResult, ImmutableString, Map, + NativeCallContext, Scope, +}; +use rhai_chrono::ChronoPackage; +use rhai_env::EnvironmentPackage; +use rhai_fs::FilesystemPackage; + +mod hubris; + +/// Use a Rhai interpreter per SingleSp that can maintain a connection. +#[async_recursion] +pub async fn interpreter( + sp: &SingleSp, + log: Logger, + script: PathBuf, + script_args: Vec, +) -> Result { + // Channel: Script -> Master + let (tx_script, rx_master) = std::sync::mpsc::sync_channel::(1); + // Channel: Master -> Script + let (tx_master, rx_script) = std::sync::mpsc::sync_channel::(1); + + let interface = sp.interface().to_string().to_owned(); + let reset_watchdog_timeout_ms = sp.reset_watchdog_timeout_ms() as i64; + + let thread_log = log.clone(); + let handle = std::thread::spawn(move || { + let log = thread_log; + // Create Engine + let mut engine = Engine::new(); + + // Setup file system access for scripts + let package = FilesystemPackage::new(); + package.register_into_engine(&mut engine); + + // Standard date formats + let package = ChronoPackage::new(); + package.register_into_engine(&mut engine); + + // Additional string functions + let package = MoreStringPackage::new(); + package.register_into_engine(&mut engine); + + // Setup env access for scripts + let package = EnvironmentPackage::new(); + package.register_into_engine(&mut engine); + + // Don't limit resources for now. + engine.set_max_expr_depths(0, 0); + + // Access RawHubrisArchives and their Cabooses + engine.build_type::(); + engine.build_type::(); + engine.register_fn("system", system); + + // Compile the script + let program = match fs::read_to_string(&script) { + Ok(content) => content, + Err(e) => { + return Err(anyhow!( + "failed to read {}: {}", + script.display(), + e + )); + } + }; + + let script_file_name: String = script.file_name().unwrap().to_string_lossy().into(); + + // Construct argv for the script and canonicalize the script path. + let pb = fs::canonicalize(&script) + .context("Cannot canonicalize {&script}")?; + let script_dir = pb + .parent() + .context("Cannot get parent dir of {&script}")? + .display() + .to_string(); + let argv0 = pb.display().to_string(); + + engine + // faux_mgs thread consumes and produces JSON + .register_fn("faux_mgs", move |v: Array| -> Dynamic { + match tx_script.send(serde_json::to_string(&v).unwrap()) { + Ok(()) => match rx_script.recv() { + Ok(v) => { + // println!("RECEIVED Ok: \"{:?}\"", v); + serde_json::from_str::(&v).unwrap() + } + Err(e) => { + // println!("RECEIVED Ok(Err): \"{:?}\"", v); + let err = format!("{{\"error\": \"{:?}\"}}", e) + .to_string(); + serde_json::from_str::(&err).unwrap() + } + }, + Err(e) => { + // println!("RECEIVED Err: \"{:?}\"", v); + let err = + format!("{{\"error\": \"{:?}\"}}", e).to_string(); + serde_json::from_str::(&err).unwrap() + } + } + }) + // Offer proper JSON to Dynamic::Map conversion + .register_fn("json_to_map", move |v: Dynamic| -> Dynamic { + match v.clone().into_string() { + Ok(s) => match serde_json::from_str::(&s) { + Ok(v) => v, + Err(e) => { + let err = json!(e.to_string()).to_string(); + serde_json::from_str::(&err).unwrap() + } + }, + Err(e) => { + let err = + format!("{{\"error\": \"{:?}\"}}", e).to_string(); + serde_json::from_str::(&err).unwrap() + } + } + }); + + // A script can log via debug at any level: + // debug("INFO|log message at INFO level"); + // debug("CRIT|log message at CRIT level"); + // etc. + let rhai_log = log.clone(); + engine.on_debug(move |x, src, pos| { + let src: String = if let Some(src) = src { + std::path::Path::new(src).file_name().unwrap().to_string_lossy().into() + } else { + script_file_name.clone().into() + }; + let location = format!("{src}@{pos:?}"); + let x: Vec<&str> = x.trim_matches('"').splitn(2, '|').collect(); + let (level, msg) = if x.len() == 1 { + ("info".to_string(), x[0].to_string()) + } else { + let level = x[0].to_string().to_lowercase(); + let msg = x[1].to_string(); + match level.as_str() { + "trace" => ("trace".to_string(), msg), + "debug" => ("debug".to_string(), msg), + "info" => ("info".to_string(), msg), + "warn" => ("warn".to_string(), msg), + "error" => ("error".to_string(), msg), + "crit" => ("crit".to_string(), msg), + _ => ("debug".to_string(), format!("{}|{}", level, msg)), + } + }; + let msg = format!("{location} {msg}"); + match level.as_str() { + "crit" => crit!(rhai_log, "{msg}"), + "debug" => debug!(rhai_log, "{msg}"), + "error" => error!(rhai_log, "{msg}"), + "info" => info!(rhai_log, "{msg}"), + "trace" => trace!(rhai_log, "{msg}"), + "warn" => warn!(rhai_log, "{msg}"), + _ => unreachable!(), + } + }); + + // Print registered functions if you're interested. + // engine.gen_fn_signatures(false).into_iter().for_each(|func| println!("{func}")); + + match engine.compile(program) { + Ok(ast) => { + // These variables are visible in the script main() + let mut scope = Scope::new(); + let mut argv = vec![]; + argv.push(argv0); + argv.extend(script_args); + scope.push_dynamic("argv", argv.clone().into()); + scope.push_dynamic( + "rbi_default", + RotBootInfo::HIGHEST_KNOWN_VERSION.to_string().into(), + ); + scope.push_dynamic("script_dir", script_dir.into()); + scope.push_dynamic("interface", interface.into()); + scope.push_dynamic( + "reset_watchdog_timeout_ms", + reset_watchdog_timeout_ms.into(), + ); + match engine.call_fn::(&mut scope, &ast, "main", ()) { + Ok(exit_value) => { + Ok(Output::Json(json!({"exit": exit_value}))) + } + Err(err) => Err(anyhow!("{err}")), + } + } + Err(e) => Err(anyhow!(format!( + "failed to parse {}: {:?}", + &script.display(), + e + ))), + } + }); + + while let Ok(command_args) = rx_master.recv() { + // Service the script's calls to "faux_mgs". + // The script can only send arrays of string and i64 values. + let response = if let Ok(serde_json::Value::Array(script_args)) = + serde_json::from_str(&command_args) + { + // TODO: Check for non-string non-i64 values in the + // script_args and return an error instead of executing the faux-mgs + // command. + let faux_mgs_args: Vec = script_args + .iter() + .map(|v| { + v.as_str() + .map(|s| s.to_string()) + .or_else(|| v.as_i64().map(|i| i.to_string())) + .unwrap() + }) + .collect(); + debug!(log, "vec string: {:?}", faux_mgs_args); + let mut ra = vec![]; + // The clap crate is expecting ARGV[0] as the program name, insert a dummy. + ra.push("faux-mgs".to_string()); + ra.append(&mut faux_mgs_args.clone()); + + let args = RhaiArgs::parse_from(&ra); + match run_command(sp, args.command.clone(), true, log.clone()).await + { + Ok(Output::Json(json)) => { + // Turn all results into a map for easy digestion + // println!("RESULT: Ok: {:?}", &json); + let obj = match json { + serde_json::Value::Object(map) => map, + _ => json!({ "Ok": json }) + .as_object() + .unwrap() + .to_owned(), + }; + match serde_json::to_string(&obj) { + Ok(s) => s, + // More verbose code, but don't need to worry about quoting. + Err(e) => serde_json::to_string(json!({ + "Err": serde_json::Value::String(format!("{:?}", e)) + }).as_object().unwrap()).unwrap(), + } + } + Ok(Output::Lines(_)) => { + // The --json=pretty option is hard-coded + unreachable!(); + } + Err(e) => { + // Create a proper JSON Value. `serde_json` will handle + // correctly escaping any special characters like newlines + // in the error message. + let json_err = serde_json::json!({ + "error": "failed", + "message": format!("{:?}", e) + }); + // `to_string` on a `serde_json::Value` is guaranteed to + // produce a valid JSON string. + serde_json::to_string(&json_err).unwrap() + } + + } + } else { + "{{\"error\": \"cannot serialize faux_mgs args to json\"}}" + .to_string() + }; + if tx_master.send(response).is_err() { + break; + } + } + + match handle.join() { + Ok(result) => result, + Err(err) => Err(anyhow!("{:?}", err)), + } +} + +// +// This function was generated with the following prompt to +// gemini.google.com: +// +// Write a Rust function, `system`, that can be registered with the Rhai +// scripting engine. The function should take an array of strings (`Array`) +// as input, representing a command and its arguments, execute the command +// using `std::process::Command`, and return a Rhai `Map` containing the +// command's exit code, standard output, and standard error. +// +// The function should handle the following: +// +// * Convert the input `Array` to a `Vec`. +// * Handle errors if the input `Array` is empty or if any element cannot +// be converted to a `String`. +// * Use `std::process::Command` with fully qualified names (e.g., +// `std::process::Command::new`). +// * Capture the command's standard output and standard error using +// `std::process::Stdio::piped()`. +// * Convert the captured output to Rhai `ImmutableString` values using +// `String::from_utf8_lossy`. +// * Return a Rhai `Map` with the keys "exit_code", "stdout", and "stderr". +// * Handle errors during command execution and output capture. +// * Use `EvalAltResult::ErrorInFunctionCall` for function call errors and +// `EvalAltResult::ErrorRuntime` for runtime errors. +// * Ensure that error messages passed to `EvalAltResult::ErrorRuntime` +// are converted to `Dynamic` using `.into()`. +// * Place the underlying error in the third position of the +// `EvalAltResult::ErrorInFunctionCall` variant. +// * Use `context.call_position()` to get the error position. +// * Do not use the `mut` keyword on the `child` variable when calling +// `command.spawn()`. +// +// Provide a complete Rust code example that includes the `system` function +// and a `main` function that registers it with a Rhai engine and runs a +// sample Rhai script. + +/// Allow Rhai scripts to run a command and capture the stdout, stderr, and +/// exit code. +fn system( + context: NativeCallContext, + argv: Array, +) -> Result> { + let mut string_argv: Vec = Vec::new(); + for arg in argv.iter() { + match arg.clone().into_string() { + Ok(s) => string_argv.push(s), + Err(_) => { + return Err(Box::new(EvalAltResult::ErrorRuntime( + "Arguments must be strings.".into(), + context.call_position(), + ))); + } + } + } + + if string_argv.is_empty() { + return Err(Box::new(EvalAltResult::ErrorInFunctionCall( + "system".to_string(), + "Expected at least one argument.".to_string(), + Box::new(EvalAltResult::ErrorRuntime( + "".into(), + context.call_position(), + )), + context.call_position(), + ))); + } + + let command_name = &string_argv[0]; + let args = &string_argv[1..]; + + let mut command = std::process::Command::new(command_name); + command.args(args); + + command + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()); + + let output = match command.spawn() { + Ok(child) => child.wait_with_output(), + Err(e) => { + return Err(Box::new(EvalAltResult::ErrorRuntime( + format!("Failed to spawn command: {}", e).into(), + context.call_position(), + ))); + } + }; + + let output = match output { + Ok(output) => output, + Err(e) => { + return Err(Box::new(EvalAltResult::ErrorRuntime( + format!("Failed to get command output: {}", e).into(), + context.call_position(), + ))); + } + }; + + let exit_code = output.status.code().unwrap_or(-1) as i64; + let stdout = ImmutableString::from( + String::from_utf8_lossy(&output.stdout).to_string(), + ); + let stderr = ImmutableString::from( + String::from_utf8_lossy(&output.stderr).to_string(), + ); + + let mut result = Map::new(); + result.insert("exit_code".into(), exit_code.into()); + result.insert("stdout".into(), stdout.into()); + result.insert("stderr".into(), stderr.into()); + + Ok(result) +} diff --git a/faux-mgs/src/rhaiscript/hubris.rs b/faux-mgs/src/rhaiscript/hubris.rs new file mode 100644 index 0000000..13a562b --- /dev/null +++ b/faux-mgs/src/rhaiscript/hubris.rs @@ -0,0 +1,256 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use crate::Path; +use crate::PathBuf; +use hubtools::{Caboose, RawHubrisArchive}; +use rhai::{CustomType, Dynamic, EvalAltResult, ImmutableString, TypeBuilder}; +use serde_json::Value as Json; +use std::sync::Arc; +use toml::Value as Toml; + +fn json_text_to_dynamic( + contents: &[u8], +) -> Result> { + let text = String::from_utf8_lossy(contents); + match serde_json::from_str::(&text) { + Ok(json) => Ok(Dynamic::from(json)), + // The Json error includes the original text with a marker + // indicating where the error is. + Err(e) => Err(format!("Failed to parse JSON: {}", e).into()), + } +} + +// Adapted from toml crate example toml2json +fn toml2json(tv: Toml) -> Json { + match tv { + Toml::String(s) => Json::String(s), + Toml::Integer(i) => Json::Number(i.into()), + Toml::Float(f) => { + if let Some(n) = serde_json::Number::from_f64(f) { + Json::Number(n) + } else { + Json::Null + } + } + Toml::Boolean(b) => Json::Bool(b), + Toml::Array(arr) => { + Json::Array(arr.into_iter().map(toml2json).collect()) + } + Toml::Table(table) => Json::Object( + table.into_iter().map(|(k, v)| (k, toml2json(v))).collect(), + ), + Toml::Datetime(dt) => Json::String(dt.to_string()), + } +} + +#[derive(Debug, CustomType)] +#[rhai_type(name = "Archive", extra = Self::build_archive_inspector)] +pub struct ArchiveInspector { + #[rhai_type(skip)] + inner: Arc, +} + +impl Clone for ArchiveInspector { + fn clone(&self) -> Self { + ArchiveInspector { inner: self.inner.clone() } + } +} + +impl ArchiveInspector { + fn new(inner: Arc) -> Self { + ArchiveInspector { inner } + } + + pub fn from_vec(contents: Vec) -> Result> { + match RawHubrisArchive::from_vec(contents) { + Ok(archive) => Ok(Self::new(Arc::new(archive))), + Err(e) => Err(format!("RawHubrisArchive::from_vec: {e}") + .to_string() + .into()), + } + } + + pub fn load(path: ImmutableString) -> Result> { + let path = PathBuf::from(path.into_owned()); + match RawHubrisArchive::load(&path) { + Ok(archive) => Ok(Self::new(Arc::new(archive))), + Err(e) => { + Err(format!("RawHubrisArchive::load: {e}").to_string().into()) + } + } + } + + fn extract_and_convert( + &self, + index: &str, + ) -> Result> { + match self.inner.extract_file(index) { + Ok(contents) => match Path::new(index) + .extension() + .and_then(|os| os.to_str()) + { + Some("bin") | Some("elf") => Ok(Dynamic::from_blob(contents)), + Some("toml") => Self::toml_to_dynamic(&contents), + Some("json") => json_text_to_dynamic(&contents), + _ => { + // All remaining files that start with "\x7fELF" or are not valid UTF8 + // are blobs, everything else is text. + if contents[0..4] == *b"\x7fELF" { + Ok(Dynamic::from_blob(contents)) + } else { + match String::from_utf8(contents.clone()) { + Ok(text) => Ok(Dynamic::from(text)), + Err(_) => Ok(Dynamic::from_blob(contents)), + } + } + } + }, + Err(e) => Err(format!("hubtools error: {}", e).into()), + } + } + + pub fn indexer( + &mut self, + index: &str, + ) -> Result> { + match index { + "caboose" => Ok(Dynamic::from::( + CabooseInspector::from_archive(&self.inner)?, + )), + "image_name" => self + .inner + .image_name() + .map(Dynamic::from) + .map_or(Ok(Dynamic::UNIT), Ok), + _ => self.extract_and_convert(index), + } + } + + fn toml_to_dynamic(contents: &[u8]) -> Result> { + let text = String::from_utf8_lossy(contents).to_string(); + let toml_value = text + .parse::() + .map_err(|e| format!("Failed to parse TOML: {}", e))?; + match toml2json(toml_value.clone()) { + Json::Object(json) => Ok(Dynamic::from(json)), + _ => Err(format!( + "Failed to convert TOML to JSON object: {:?}", + toml_value + ) + .into()), + } + } + + fn decode_blob( + blob: Dynamic, + name: &str, + ) -> Result<[u8; N], Box> { + let bytes = blob + .read_lock::() + .ok_or_else(|| format!("invalid type {}", name))? + .to_vec(); + if bytes.len() != N { + return Err(format!( + "invalid {} length {} != {}", + name, + bytes.len(), + N + ) + .into()); + } + bytes.try_into().map_err(|_| format!("invalid {}", name).into()) + } + + pub fn verify_rot_image( + &mut self, + cmpa: Dynamic, + cfpa: Dynamic, + ) -> Result> { + let cmpa = Self::decode_blob::<512>(cmpa, "CMPA")?; + let cfpa = Self::decode_blob::<512>(cfpa, "CFPA")?; + if let Err(e) = self.inner.verify(&cmpa, &cfpa) { + return Err(Box::new(EvalAltResult::from(format!("{:?}", e)))); + } + Ok(true.into()) + } + + pub fn build_archive_inspector(builder: &mut TypeBuilder) { + builder + .with_name("Archive") + .with_fn("new_archive", ArchiveInspector::from_vec) + .with_fn("new_archive", ArchiveInspector::load) + .with_fn("verify_rot_image", ArchiveInspector::verify_rot_image) + .with_indexer_get(ArchiveInspector::indexer); + } +} + +macro_rules! caboose_tag { + ($caboose: ident, $method:ident) => { + $caboose + .inner + .$method() + .map(|v| Ok(u8_to_string(v).into())) + .unwrap_or(Ok(Dynamic::UNIT)) + }; +} + +#[derive(Debug, CustomType)] +#[rhai_type(name = "Caboose", extra = Self::build_caboose_inspector)] +pub struct CabooseInspector { + #[rhai_type(skip)] + inner: Arc, +} + +impl Clone for CabooseInspector { + fn clone(&self) -> Self { + CabooseInspector { inner: self.inner.clone() } + } +} + +impl CabooseInspector { + fn new(inner: Arc) -> Self { + CabooseInspector { inner } + } + + pub fn from_archive( + archive: &RawHubrisArchive, + ) -> Result> { + let caboose = archive + .read_caboose() + .map_err(|e| format!("RawArchive::read_caboose: {:?}", e))?; + Ok(CabooseInspector::new(Arc::new(caboose))) + } + + pub fn indexer( + &mut self, + index: &str, + ) -> Result> { + match index { + "BORD" => caboose_tag!(self, board), + "GITC" => caboose_tag!(self, git_commit), + "NAME" => caboose_tag!(self, name), + "SIGN" => caboose_tag!(self, sign), + "VERS" => caboose_tag!(self, version), + _ => Err(format!("unknown index: {:?}", index).into()), + } + } + + pub fn build_caboose_inspector(builder: &mut TypeBuilder) { + builder + .with_name("Caboose") + .with_indexer_get(CabooseInspector::indexer); + } +} + +fn u8_to_string(array: &[u8]) -> String { + String::from_utf8_lossy( + if let Some(p) = array.iter().position(|&x| x == 0) { + &array[0..p] + } else { + &array[0..] + }, + ) + .to_string() +} diff --git a/gateway-messages/src/lib.rs b/gateway-messages/src/lib.rs index 7b5f376..9c8be97 100644 --- a/gateway-messages/src/lib.rs +++ b/gateway-messages/src/lib.rs @@ -69,7 +69,7 @@ pub const HF_PAGE_SIZE: usize = 256; /// for more detail and discussion. pub mod version { pub const MIN: u32 = 2; - pub const CURRENT: u32 = 19; + pub const CURRENT: u32 = 20; /// MGS protocol version in which SP watchdog messages were added pub const WATCHDOG_VERSION: u32 = 12; diff --git a/gateway-messages/src/sp_to_mgs.rs b/gateway-messages/src/sp_to_mgs.rs index c5d1da7..d442767 100644 --- a/gateway-messages/src/sp_to_mgs.rs +++ b/gateway-messages/src/sp_to_mgs.rs @@ -1294,6 +1294,7 @@ pub enum UpdateError { ImageMismatch, SignatureNotValidated, VersionNotSupported, + InvalidPreferredSlotId, } impl fmt::Display for UpdateError { @@ -1353,6 +1354,12 @@ impl fmt::Display for UpdateError { Self::InvalidComponent => { write!(f, "invalid component for operation") } + Self::InvalidPreferredSlotId => { + write!( + f, + "updating a bootloader preferred slot is not permitted" + ) + } } } } diff --git a/gateway-messages/tests/versioning/mod.rs b/gateway-messages/tests/versioning/mod.rs index be3ff2d..ca9fe89 100644 --- a/gateway-messages/tests/versioning/mod.rs +++ b/gateway-messages/tests/versioning/mod.rs @@ -25,6 +25,7 @@ mod v16; mod v17; mod v18; mod v19; +mod v20; pub fn assert_serialized( expected: &[u8], diff --git a/gateway-messages/tests/versioning/v19.rs b/gateway-messages/tests/versioning/v19.rs index f724fa6..845abe9 100644 --- a/gateway-messages/tests/versioning/v19.rs +++ b/gateway-messages/tests/versioning/v19.rs @@ -16,6 +16,9 @@ //! tests can be removed as we will stop supporting $VERSION. use super::assert_serialized; +use gateway_messages::ImageError; +use gateway_messages::SpStateV3; +use gateway_messages::UpdateError; use gateway_messages::{HfError, MgsRequest, SpError, SpResponse}; #[test] @@ -60,3 +63,84 @@ fn read_host_flash() { assert_serialized(&[38, i as u8], &request); } } + +#[test] +fn sp_response() { + let response = SpResponse::SpStateV3(SpStateV3 { + hubris_archive_id: [1, 2, 3, 4, 5, 6, 7, 8], + serial_number: [ + 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + ], + model: [ + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, + 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, + ], + revision: 0xf0f1f2f3, + base_mac_address: [73, 74, 75, 76, 77, 78], + power_state: gateway_messages::PowerState::A0, + }); + + #[rustfmt::skip] + let expected = vec![ + 44, // SpStateV3 + 1, 2, 3, 4, 5, 6, 7, 8, // hubris_archive_id + + 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, + 39, 40, // serial_number + + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, + 71, 72, // model + + 0xf3, 0xf2, 0xf1, 0xf0, // revision + 73, 74, 75, 76, 77, 78, // base_mac_address + 0, // power_state + ]; + + assert_serialized(&expected, &response); +} + +#[test] +fn host_request() { + let request = MgsRequest::VersionedRotBootInfo { version: 3 }; + + #[rustfmt::skip] + let expected = vec![ + 45, // VersionedRotBootInfo + 3, // version + ]; + + assert_serialized(&expected, &request); +} + +#[test] +fn error_enums() { + let response: [ImageError; 13] = [ + ImageError::Unchecked, + ImageError::FirstPageErased, + ImageError::PartiallyProgrammed, + ImageError::InvalidLength, + ImageError::HeaderNotProgrammed, + ImageError::BootloaderTooSmall, + ImageError::BadMagic, + ImageError::HeaderImageSize, + ImageError::UnalignedLength, + ImageError::UnsupportedType, + ImageError::ResetVectorNotThumb2, + ImageError::ResetVector, + ImageError::Signature, + ]; + let expected = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; + assert_serialized(&expected, &response); + + let response: [UpdateError; 4] = [ + UpdateError::BlockOutOfOrder, + UpdateError::InvalidComponent, + UpdateError::InvalidSlotIdForOperation, + UpdateError::InvalidPreferredSlotId, + ]; + let expected = vec![27, 28, 29, 34]; + assert_serialized(&expected, &response); +} diff --git a/gateway-messages/tests/versioning/v20.rs b/gateway-messages/tests/versioning/v20.rs new file mode 100644 index 0000000..3bc765e --- /dev/null +++ b/gateway-messages/tests/versioning/v20.rs @@ -0,0 +1,29 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! This source file is named after the protocol version being tested, +//! e.g. v01.rs implements tests for protocol version 1. +//! The tested protocol version is represented by "$VERSION" below. +//! +//! The tests in this module check that the serialized form of messages from MGS +//! protocol version $VERSION have not changed. +//! +//! If a test in this module fails, _do not change the test_! This means you +//! have changed, deleted, or reordered an existing message type or enum +//! variant, and you should revert that change. This will remain true until we +//! bump the `version::MIN` to a value higher than $VERSION, at which point these +//! tests can be removed as we will stop supporting $VERSION. + +use super::assert_serialized; +use gateway_messages::UpdateError; + +#[test] +fn error_enums() { + + let response: [UpdateError; 1] = [ + UpdateError::InvalidPreferredSlotId, + ]; + let expected = vec![34]; + assert_serialized(&expected, &response); +} diff --git a/gateway-sp-comms/src/error.rs b/gateway-sp-comms/src/error.rs index 8c45978..1b3273b 100644 --- a/gateway-sp-comms/src/error.rs +++ b/gateway-sp-comms/src/error.rs @@ -116,6 +116,8 @@ pub enum UpdateError { InvalidComponent, #[error("an image was not found")] ImageNotFound, + #[error("updating a bootloader preferred slot is not permitted")] + InvalidPreferredSlotId, } #[derive(Debug, thiserror::Error, SlogInlineError)] diff --git a/gateway-sp-comms/src/single_sp.rs b/gateway-sp-comms/src/single_sp.rs index 6f3298c..8b649fa 100644 --- a/gateway-sp-comms/src/single_sp.rs +++ b/gateway-sp-comms/src/single_sp.rs @@ -430,6 +430,10 @@ impl SingleSp { &self.interface } + pub fn reset_watchdog_timeout_ms(&self) -> u32 { + self.reset_watchdog_timeout_ms + } + /// Retrieve the [`watch::Receiver`] for notifications of discovery of an /// SP's address. pub fn sp_addr_watch( diff --git a/scripts/FMR b/scripts/FMR new file mode 100755 index 0000000..c154777 --- /dev/null +++ b/scripts/FMR @@ -0,0 +1,78 @@ +#!/usr/bin/bash + +# This is a helper script to run `faux-mgs` with the `rhaiscript` feature +# enabled. +# Linking this script as "FMR-info", etc. will select that log level (info) +# as a filter. + +set -u + +fatal() { + printf "Fatal: %s" "$*" 1>&2 + exit 1 +} + +# Assuming that this script is in $REPO/scripts/FMR +REPO="$(dirname "$(dirname "$(readlink -f "$0")")")" + +if [[ "${1:-}" == "link" ]]; then + if [[ -r "./FMR" ]]; then + fatal There is already a local file named FMR + fi + ln -sf $REPO/scripts/FMR "./FMR" + for level in off crit error warn info debug trace; do + ln -sf ./FMR "./FMR-${level}" + done + echo Linked: $(ls FMR*) + exit +fi + +get_connection() ( + # TODO: Need use simulator if that is running + if [[ -n "${INTERFACE:-}" ]]; then + printf -- "--interface %s" "${INTERFACE}" + fi + # Assuming typical Linux networking commands and configuration to extract + # the interface name. + set $(ip route get 8.8.8.8) + printf -- "--interface %s" "$5" +) +CONNECTION="$(get_connection)" +[[ -z "${CONNECTION}" ]] && fatal Did not find connection to SP + +case "$(basename "$0")" in +*-off) LOG_LEVEL=off ;; +*-crit) LOG_LEVEL=crit ;; +*-error) LOG_LEVEL=error ;; +*-warn) LOG_LEVEL=warn ;; +*-info) LOG_LEVEL=info ;; +*-debug) LOG_LEVEL=debug ;; +*-trace) LOG_LEVEL=trace ;; +*) LOG_LEVEL=crit ;; +esac +if [[ "${1:-}" == "--too-quick" ]]; then + shift + # too fast for update to succeed. try to trigger watchdog + MAXATTEMPTS=5 + MAXATTEMPTS_RESET=1 + PER_ATTEMPT_MS=2000 +else + # Normal values + MAXATTEMPTS=5 + MAXATTEMPTS_RESET=30 + # PER_ATTEMPT_MS=2000 + # 2165 is on the edge 10825 + PER_ATTEMPT_MS=2200 +fi + +cd "${REPO}" || fatal "Cannot cd to ${REPO}" + +set -x +cargo -q run --bin faux-mgs --features=rhaiscript -- \ + --log-level="${LOG_LEVEL}" \ + ${CONNECTION} \ + --json=pretty \ + --max-attempts=${MAXATTEMPTS} \ + --max-attempts-reset=${MAXATTEMPTS_RESET} \ + --per-attempt-timeout-millis=${PER_ATTEMPT_MS} \ + "$@" diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..3e3be49 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,175 @@ +# Scripting in faux-mgs + +The `faux-mgs` utility is useful for testing the APIs and protocols used +between the control plane and service processor (SP) on Oxide hardware. + +Testing complex scenarios involving multiple commands, conditional logic based +on hardware state, and interactions across resets can be challenging when +using individual `faux-mgs` commands manually or via simple shell scripts. + +`faux-mgs` can be extended with new commands in Rust. For code of +general utility, that is encouraged. + +However, it is sometimes desirable to code in a scripting language. +This directory provides the ability to use the embedded **Rhai** +scripting language to automate one-off procedures, test sequences, +and personal workflows. + +## Benefits of Rhai Scripting + +* **Stateful Interaction:** Scripts maintain the connection (discovery, etc.) + to the SP across multiple command invocations within the same `faux-mgs` + process, leading to faster execution compared to separate calls. +* **Structured Data:** Command output (using the required `--json=pretty` + internal format) is automatically available to scripts as native Rhai maps + or other types, simplifying data extraction and conditional logic. +* **Portability:** Test logic is contained within Rhai scripts, which are + as portable as the `faux-mgs` binary itself. +* **Multi-Target Execution:** Scripts can leverage `faux-mgs`'s ability to + run against multiple SPs simultaneously by providing multiple `--target` + arguments to `faux-mgs`. +* **Extensibility:** Provides access to file system, environment variables, + and time functions within the script. + +## Rhai Integration Details + +* **Entry Point:** These Rhai scripts must define a `main()` function that + returns an integer (`fn main() -> i64`), typically `0` for success and + non-zero for failure. +* **Available Globals:** The following globals are available within the + script's scope: + * `argv`: An array of string arguments passed to the script after `--` + on the `faux-mgs` command line (e.g., `["script_name", "arg1", ...]`) + * `interface`: The value of the `faux-mgs --interface` argument (string). + * `reset_watchdog_timeout_ms`: The configured watchdog timeout (integer). + * `rbi_default`: The highest known RoT Boot Info version (integer string). + * `script_dir`: The canonical path to the directory containing the + main script file (string). +* **Extra Rhai Packages:** The following packages are enabled: + * `rhai_env::EnvironmentPackage`: Access user environment variables (`env`, `envs`). + * `rhai_fs::FilesystemPackage`: Access the file system (`open_file`, `path`, etc.). + * `rhai_chrono::ChronoPackage`: Standard time/date functions (`timestamp`, `datetime_local`, `sleep`). +* **Logging:** The standard Rhai `debug()` function is routed to the `faux-mgs` + logging system (`slog`). Prefixing the message with `"level|"` (e.g., + `"warn|Message"`, `"info|Message"`, `"error|Message"`, `"trace|Message"`) + logs at the corresponding level. Unprefixed messages default to `info`. +* **Custom Functions/Types (Built-in):** Core functions provided by the + `faux-mgs` Rust integration: + * `faux_mgs(["arg0", .., "argN"]) -> map`: Runs any `faux-mgs` command + internally (using `--json=pretty`) and returns the result as a Rhai map. + *Do not call this directly in test scripts; use wrappers from `util.rhai`.* + * `new_archive(path) -> ArchiveInspector`: Loads a Hubris archive (.zip). + * `ArchiveInspector[]`: Access files within the archive (returns + string/blob based on extension). Use `.caboose` property to get a + `CabooseInspector`. + * `ArchiveInspector.verify_rot_image(cmpa_blob, cfpa_blob) -> bool`: + Verifies RoT image signature against provided CMPA/CFPA blobs. + * `CabooseInspector[]`: Access caboose tags (e.g., `GITC`, `VERS`). + * `json_to_map(string) -> map`: Converts a JSON string to a Rhai map. + * `system(["cmd", .., "argN"]) -> map`: Runs an external OS command + (no shell expansion) and returns `#{ exit_code: i64, stdout: str, stderr: str }`. + *Use with caution.* + +## Utility Scripts (`scripts/util.rhai`) + +Common helper functions, constants, and wrappers around direct `faux_mgs` calls +are placed in `scripts/util.rhai`. Scripts should import this using: + +```rhai +import `${script_dir}/util` as util; +``` + +### Key utilities provided: + +- **Constants:** `ROT_FLASH_PAGE_SIZE`. +- **Formatting:** `to_hexstring`, `cstring_to_string`, `array_to_mac`. +- **Data Handling:** `env_expand`, `array_to_blob`, `ab_to_01`. +- **Hardware Info:** `get_cmpa`, `get_cfpa`, `get_rot_keyset`, `state`, + `caboose_value`, `get_device_cabooses`. +- **`faux_mgs` Wrappers:** + - `rot_boot_info()`: Gets formatted RoT Boot Info. + - `check_update_in_progress(component)`: Checks SP/RoT update status. + - `update_rot_image_file(slot, path, label)`: Updates RoT image. + - `set_rot_boot_preference(slot, use_transient, label)`: Sets RoT pref. + - `reset_rot_and_get_rbi(desc, label)`: Resets RoT and gets RBI. + - `update_sp_image(path)`: Updates SP image. + - `reset_sp()`: Resets SP. + - *(More wrappers can be added for other commands like `update-abort`,* + *`reset-component`, etc.)* +- **Argument Parsing:** `getopts(argv, options_string)`: Parses script arguments. + +## Example Script (`scripts/upgrade-rollback.rhai`) + +This script automates testing firmware upgrades and subsequent rollbacks between +two specified sets of firmware builds (a "baseline" and an "under-test" version). + +### Configuration: + +- Uses `scripts/targets.json` to define paths to firmware repositories, + specific image zip files, board names, and potentially other settings like + IPCC or power control commands. +- Supports `${VAR}` expansion in paths within `targets.json`, referencing + environment variables or other keys within the JSON file itself via + `util::env_expand`. + +### Key Command-Line Options: + +- `-b `: **Optional.** Path to the baseline Hubris repo` +- `-c `: **Required.** Path to the JSON configuration file + (e.g., `scripts/targets.json`). +- `-t`: Enable testing using the RoT "transient boot preference" feature. + The script attempts to use this mechanism if the active RoT supports it + and handles differences in behavior when targeting older "baseline" RoTs + that may not fully support the feature. +- `-u `: **Optional.** Path to the under-test Hubris repo` +- `-v`: Verbose output (enables more `debug("info|...")` messages). +- `-h`: Show help message. + +### Example Invocation: + +Assumes `faux-mgs` is built with `rhaiscript` feature enabled. + +```bash +# Set environment variable used in targets.json for the 'under-test' repo, e.g. +export UT_WORKTREE=my-feature-branch + +# Run faux-mgs, targeting the script, providing config and transient flag, +# and overriding repo paths via positional arguments after '--' +# Ensure the shell expands UT_WORKTREE in the path argument +cargo run --bin faux-mgs --features=rhaiscript -- \ + --interface=enp5s0 \ + --log-level=info \ + rhai scripts/upgrade-rollback.rhai \ + -c scripts/targets.json -t \ + -b $HOME/Oxide/src/hubris/master \ + -u $HOME/Oxide/src/hubris/${UT_WORKTREE} + +# Check exit code +echo $? +``` + +## Running Scripts Generally + +Use the `rhai` subcommand of `faux_mgs`: + +```bash +faux-mgs [faux-mgs options] rhai [script options] -- [script arguments] +``` + +- **`faux-mgs options`**: Standard options like `--interface`, `--target`, + `--log-level`. Note that `--json=pretty` is used *internally* for commands + called via the `faux_mgs()` function within Rhai, but you might set + `--log-level` for the overall execution. +- **`script_path.rhai`**: Path to the main Rhai script. +- **`script options`**: Currently none defined globally, but scripts might parse + their own using `util::getopts`. +- **`--`**: Separates `faux_mgs` options from arguments intended for the script. +- **`script arguments`**: Arguments passed to the script's `argv` global array. + +## Contributing / TODO + +See [`scripts/TODO.md`](TODO.md) for more details. + +## SP and RoT Update/Rollback test plan + +See [`scripts/TEST_PLAN.md`](TEST_PLAN.md) for more details. diff --git a/scripts/TEST_PLAN.md b/scripts/TEST_PLAN.md new file mode 100644 index 0000000..1b0b0c3 --- /dev/null +++ b/scripts/TEST_PLAN.md @@ -0,0 +1,131 @@ +# Final Validation Test Plan + +This document outlines a full set of tests to validate the `upgrade-rollback.rhai` test harness. It is split into two parts: +1. **Part 1**: Tests the current, real-world scenario where the new `baseline` firmware has features that the older `under-test` firmware lacks. +2. **Part 2**: Describes tests for a future state where both `baseline` and `under-test` firmware are fully compliant with the transient boot preference feature. + +## Prerequisites and Setup + +### 1. Environment Variables + +Before running these tests, you must set two environment variables to point to your local Hubris build repositories: +```bash +export REPO_BL=/path/to/your/baseline/hubris +export REPO_UT=/path/to/your/under-test/hubris +``` + +These repositories must have SP and RoT Hubris build products in their +respective `target/` directories. + +Examine and edit the `scripts/targets.json` file or make your own if you need +to use images from other locations. + +### 2. The `FMR` Wrapper Script + +The test commands use a helper script named `FMR`, which is a wrapper around the main `cargo run --bin faux-mgs` command. Its purpose is to simplify running tests by automatically including common arguments. + +* **Functionality**: The script automatically adds required arguments like `--features=rhaiscript`, `--json=pretty`, timeouts, and attempts to discover the correct network `--interface` setting. +* **Log Levels**: The name used to call the script sets the log level for the test run. For example, `FMR-info` sets `--log-level=info`, while `FMR-trace` sets `--log-level=trace`. +* **Setup**: To create the convenient `FMR-info`, `FMR-debug`, etc. symlinks in your working directory, you can run the following command from the repository root: + ```bash + ./scripts/FMR link + ``` + +### 3. Clean State + +Each numbered test case should be run from a known-clean state. Before starting a test, please perform two RoT resets: +```bash +FMR-info reset-component rot +FMR-info reset-component rot +``` + +--- + +## Part 1: Testing Asymmetric Feature Support (Current State) + +**Assumption**: For this set of tests, `$REPO_BL` points to an **old** firmware build that **does not** support the transient boot preference feature, and `$REPO_UT` points to a **new** build that does. + +### Test 1.1: Standard Workflow (Golden Path) + +* **Purpose**: To verify that the primary upgrade and rollback functionality works correctly without using any of the new features. +* **Command**: + ```bash + FMR-info rhai scripts/upgrade-rollback.rhai -c scripts/targets.json \ + -b $REPO_BL -u $REPO_UT + ``` +* **Expected Outcome**: The script should complete successfully with an exit code of 0. It will upgrade to the `under-test` image and then roll back to the `baseline` image using persistent updates. + +### Test 1.2: Transient Boot Path (`-t` flag) + +* **Purpose**: To verify the script correctly handles the feature asymmetry when the transient update path is requested. +* **Command**: + ```bash + FMR-info rhai scripts/upgrade-rollback.rhai -c scripts/targets.json \ + -b $REPO_BL -u $REPO_UT -t + ``` +* **Expected Outcome**: The script should complete successfully with an exit code of 0. The log should show: + * **Upgrade**: The active `baseline` firmware does not support the feature. The script will log a warning and use a persistent update. + * **Rollback**: The now-active `under-test` firmware supports the feature. The script will correctly use a transient update for the rollback. + +### Test 1.3: Negative Test (`-N`) Workflow + +* **Purpose**: To verify the logic that runs (or skips) the `test_and_recover...` negative test based on feature support. +* **Command**: + ```bash + FMR-info rhai scripts/upgrade-rollback.rhai -c scripts/targets.json \ + -b $REPO_BL -u $REPO_UT -N + ``` +* **Expected Outcome**: The script will **fail with exit code 1**. This is the correct behavior. + * **Upgrade**: The `baseline` firmware is active and does not support the transient feature. The script will detect this and, because the test is for the `ut` branch, it will log a `FATAL` error stating the `under-test` image must support the feature. This check is known to be flawed for this specific asymmetric case but correctly protects against regressions. + +### Test 1.4: Fault Injection - Conflicting `pending` Preference + +* **Purpose**: To verify that the test harness can recover from a pre-existing `pending_persistent` preference fault. +* **Command**: + ```bash + FMR-info rhai scripts/upgrade-rollback.rhai -c scripts/targets.json \ + -b $REPO_BL -u $REPO_UT \ + --inject-fault=pending --hubris-2093 + ``` +* **Expected Outcome**: The script should run the "pending" fault injection test and exit with code 0. The log will show the sanitizer detecting the fault and using the reset-based workaround to clear it before the main test flow runs successfully. + +### Test 1.5: Fault Injection - Conflicting `transient` Preference + +* **Purpose**: To verify the test harness correctly handles the inability to inject a fault into non-compliant firmware. +* **Command**: + ```bash + FMR-info rhai scripts/upgrade-rollback.rhai -c scripts/targets.json \ + -b $REPO_BL -u $REPO_UT \ + --inject-fault=transient + ``` +* **Expected Outcome**: The script is **expected to fail with exit code 1**. This is the correct outcome. The log will show: + 1. The script first installs the `baseline` (`master`) firmware. + 2. It then attempts to run the `transient` fault injection test. + 3. The `helper::inject_conflicting_transient_preference()` function will fail because the active `baseline` firmware does not support the transient preference command. + 4. The script will log an error like "Failed to inject transient preference fault" and exit. This proves the test harness correctly identifies that the fault cannot be created. + +--- + +## Part 2: Testing Symmetric Feature Support (Future State) + +**Assumption**: For this set of tests, assume **both** `$REPO_BL` and `$REPO_UT` point to firmware builds that support the transient boot preference feature. + +### Test 2.1: Transient Boot Path (`-t` flag) + +* **Purpose**: To verify that when both images are compliant, the script uses the transient update path for both the upgrade and the rollback. +* **Command**: + ```bash + FMR-info rhai scripts/upgrade-rollback.rhai -c scripts/targets.json \ + -b $REPO_BL -u $REPO_UT -t + ``` +* **Expected Outcome**: The script should complete successfully with an exit code of 0. The log should show a transient update is used for **both** the upgrade to `ut` and the subsequent rollback to `base`. + +### Test 2.2: Negative Test (`-N`) Workflow + +* **Purpose**: To verify that the negative test runs successfully in both directions when all firmware is compliant. +* **Command**: + ```bash + FMR-info rhai scripts/upgrade-rollback.rhai -c scripts/targets.json \ + -b $REPO_BL -u $REPO_UT -N + ``` +* **Expected Outcome**: The script should complete successfully with an exit code of 0. The `test_and_recover_from_preferred_slot_update_failure` function should be executed and pass for the `ut` branch during the upgrade, and then be executed and pass **again** for the `base` branch during the rollback. diff --git a/scripts/TODO.md b/scripts/TODO.md new file mode 100644 index 0000000..7a22af8 --- /dev/null +++ b/scripts/TODO.md @@ -0,0 +1,43 @@ +# TODO List for Rhai Test Suite + +This document tracks known issues, planned features, and refactoring opportunities for the `upgrade-rollback` test suite. + +## High Priority / Bugs & Workarounds + +* **Remove `--hubris-2093` Workaround** + * **Issue**: The `lpc55-update-server` firmware has a bug where setting a persistent preference does not correctly clear a pre-existing pending preference. This is tracked as "Hubris issue #2093". + * **Workaround**: The `sanitize_boot_preferences` function in `update-helper.rhai` uses a reset to reliably clear a pending preference when the `--hubris-2093` flag is active. + * **Action**: Once the firmware bug is fixed, the workaround logic should be removed from `sanitize_boot_preferences` and the `--hubris-2093` flag should be removed from `upgrade-rollback.rhai`. The "ideal" logic path should become the only path. + +* **Fix `faux-mgs` Error Reporting for `reset-component`** + * **Issue**: When the SP debugger is attached, the `reset-component sp` command fails. However, the `faux-mgs` Rust code does not gracefully package the detailed error message (`watchdog: RoT error: the SP programming dongle is connected`) into the JSON passed to Rhai. It returns a generic error. + * **Action**: Modify the error handling in `faux-mgs/src/rhaiscript.rs` to ensure the full, detailed error string from a failed `run_command` is always serialized into the JSON message passed to the Rhai script. *(Self-correction: We have since fixed this by changing the error formatting to `{:?}`, but this note is kept for historical context).* + +## Feature Enhancements & New Tests + +* **Implement Image Corruption Fault Injection** + * **Goal**: Add a fault injection test to simulate an incomplete image write, as might happen during a power failure. + * **Blocker**: This is difficult with the current script API. It would likely require a new `faux-mgs` command to be added in Rust, for example: `faux-mgs update-partial --bytes `. + * **Action**: + 1. Add the new debug command to `faux-mgs`. + 2. Create a corresponding wrapper in `util.rhai`. + 3. Add `inject_incomplete_update_fault()` to `update-helper.rhai`. + 4. Add `"incomplete-write"` to the `--inject-fault` options in `upgrade-rollback.rhai` and implement the test case. + +* **Add a Definitive Debugger Check** + * **Issue**: The current `check_for_sp_debugger` function is heuristic and only works if an update is pending on the SP. + * **Action**: Investigate if there is a more reliable, deterministic command or register read that can definitively report the presence of an attached and powered-on SWD probe, and update the function accordingly. + +* **Add Configuration Schema Validation** + * **Goal**: Improve robustness by validating the structure of the `targets.json` file when it is parsed in `process_cli`. + * **Action**: Add checks in `process_cli` to ensure required keys (`images`, `base`, `ut`, etc.) exist to prevent runtime errors later in the script. + +## Code Refactoring & Cleanup + +* **Improve `util::getopts` for Long Options** + * **Issue**: The `getopts` function does not support space-separated arguments for long options (e.g., `--option value`), only the `--option=value` format. + * **Action**: Refactor the long-option parsing logic in `util::getopts` to handle space-separated arguments, which would make it behave more like the standard GNU getopt. + +* **Validate Transient Boot with Bootloader Log** + * **Context**: The `update_rot_hubris` function has a `TODO` referencing "Hubris issue #2066". + * **Action**: When the bootloader provides a decision log in `RotBootInfo`, update the `rot_validate_initial_transient_boot_state` function to parse this log and confirm that the boot was a result of the transient preference, not a fallback. diff --git a/scripts/grapefruit-power b/scripts/grapefruit-power new file mode 100755 index 0000000..f6344a6 --- /dev/null +++ b/scripts/grapefruit-power @@ -0,0 +1,44 @@ +#!/usr/bin/bash + +# This script uses Home Assistant and a Zigbee controlled power outlet to +# control the power to the Grapefruit development board. This is analagous to +# the sled power control feature of "Ignition" on a compute sled or other +# device. +# See [hass-cli](https://github.com/home-assistant-ecosystem/home-assistant-cli) + +set -u -e + +fatal() { + printf "Fatal: %s" "$*" 1>&2 + exit 1 +} + +CONF="$HOME/.config/$(basename $0)/config.sh" +ls -l "${CONF}" +[[ -r "${CONF}" ]] || fatal "Cannot source config file $CONF" +source "${CONF}" +if [[ -z "${HASS_TOKEN}" || -z "${HASS_SERVER}" || -z "${ENTITY_ID}" ]]; then + fatal "HASS_SERVER, HASS_TOKEN, or ENTITY_ID not set after sourcing $CONF" +fi +# Ensure that HASS_* vars are exported. +export HASS_SERVER +export HASS_TOKEN + +STATE=$( + case ${1^^} in + ON) + hass-cli -o json service call switch.turn_on --arguments entity_id="${ENTITY_ID}" | jq -r '.[].state' + ;; + OFF) + hass-cli -o json service call switch.turn_off --arguments entity_id="${ENTITY_ID}" | jq -r '.[].state' + ;; + STATE) + hass-cli -o json state get "${ENTITY_ID}" | jq -r '.[].state' + ;; + *) + fatal 'Invalid request. Use ON|OFF|STATE' + ;; + esac +) +echo ${STATE^^} +exit $? diff --git a/scripts/targets-reset-fail.json b/scripts/targets-reset-fail.json new file mode 100644 index 0000000..df6bd75 --- /dev/null +++ b/scripts/targets-reset-fail.json @@ -0,0 +1,57 @@ +{ + "rot_hubris_power_cycle_on_failure": true, + "power_off_delay_secs": 2, + "dut_boot_delay_secs": 10, + + "repo-home": "${HOME}/Oxide/src", + "base_repo": "${repo-home}/hubris/transient-boot-selection", + "ut_repo": "${repo-home}/hubris/rot-hubris-fault-insertion", + + "keyset": "bart", + "keyset-dvt-dock": "${repo-home}/dvt-dock/${keyset}", + "base-b-ver": "v1.3.1", + "ut-b-ver": "v1.3.3", + + "bord": { + "sp": "grapefruit-standalone", + "rot": "oxide-rot-1-selfsigned" + }, + + "images": { + "base": { + "sp": "${base_repo}/target/${bord.sp}/dist/default/build-${bord.sp}-image-default.zip", + "rot_a": "${base_repo}/target/${bord.rot}/dist/a/build-${bord.rot}-image-a.zip", + "rot_b": "${base_repo}/target/${bord.rot}/dist/b/build-${bord.rot}-image-b.zip", + "stage0": "${keyset-dvt-dock}/gimlet/bootleby-${base-b-ver}-${keyset}-gimlet.zip" + }, + "ut": { + "sp": "${ut_repo}/target/${bord.sp}/dist/default/build-${bord.sp}-image-default.zip", + "rot_a": "${ut_repo}/target/${bord.rot}/dist/a/build-${bord.rot}-image-a.zip", + "rot_b": "${ut_repo}/target/${bord.rot}/dist/b/build-${bord.rot}-image-b.zip", + "stage0": "${keyset-dvt-dock}/gimlet/bootleby-${ut-b-ver}-${keyset}-gimlet.zip" + } + }, + + "ipcc": { + "faux_ipcc": "${HOME}/.cargo/bin/faux-ipcc", + "port": "/dev/ttyUSB0" + }, + + "power_control": { + "dut": { + "comment": "Commands for the main Device Under Test (e.g. grapefruit)", + "on_cmd": ["grapefruit-power", "on"], + "off_cmd": ["grapefruit-power", "off"], + "status_cmd": ["grapefruit-power", "state"], + "status_on_stdout_contains": "ON" + }, + + "stlink": { + "comment": "Commands for the STLINK debugger (optional)", + "off_cmd": ["echo", "'STLINK power off command placeholder'"], + "on_cmd": ["echo", "'STLINK power on command placeholder'"], + "status_cmd": ["echo", "'STLINK power status command placeholder'"], + "status_on_stdout_contains": "STLINK" + } + } +} diff --git a/scripts/targets.json b/scripts/targets.json new file mode 100644 index 0000000..989b88e --- /dev/null +++ b/scripts/targets.json @@ -0,0 +1,57 @@ +{ + "rot_hubris_power_cycle_on_failure": true, + "power_off_delay_secs": 2, + "dut_boot_delay_secs": 10, + + "repo-home": "${HOME}/Oxide/src", + "base_repo": "${repo-home}/hubris/master", + "ut_repo": "${repo-home}/hubris/${UT_WORKTREE}", + + "keyset": "bart", + "keyset-dvt-dock": "${repo-home}/dvt-dock/${keyset}", + "base-b-ver": "v1.3.1", + "ut-b-ver": "v1.3.3", + + "bord": { + "sp": "grapefruit-standalone", + "rot": "oxide-rot-1-selfsigned" + }, + + "images": { + "base": { + "sp": "${base_repo}/target/${bord.sp}/dist/default/build-${bord.sp}-image-default.zip", + "rot_a": "${base_repo}/target/${bord.rot}/dist/a/build-${bord.rot}-image-a.zip", + "rot_b": "${base_repo}/target/${bord.rot}/dist/b/build-${bord.rot}-image-b.zip", + "stage0": "${keyset-dvt-dock}/gimlet/bootleby-${base-b-ver}-${keyset}-gimlet.zip" + }, + "ut": { + "sp": "${ut_repo}/target/${bord.sp}/dist/default/build-${bord.sp}-image-default.zip", + "rot_a": "${ut_repo}/target/${bord.rot}/dist/a/build-${bord.rot}-image-a.zip", + "rot_b": "${ut_repo}/target/${bord.rot}/dist/b/build-${bord.rot}-image-b.zip", + "stage0": "${keyset-dvt-dock}/gimlet/bootleby-${ut-b-ver}-${keyset}-gimlet.zip" + } + }, + + "ipcc": { + "faux_ipcc": "${HOME}/.cargo/bin/faux-ipcc", + "port": "/dev/ttyUSB0" + }, + + "power_control": { + "dut": { + "comment": "Commands for the main Device Under Test (e.g. grapefruit)", + "on_cmd": ["grapefruit-power", "on"], + "off_cmd": ["grapefruit-power", "off"], + "status_cmd": ["grapefruit-power", "state"], + "status_on_stdout_contains": "ON" + }, + + "stlink": { + "comment": "Commands for the STLINK debugger (optional)", + "off_cmd": ["echo", "'STLINK power off command placeholder'"], + "on_cmd": ["echo", "'STLINK power on command placeholder'"], + "status_cmd": ["echo", "'STLINK power status command placeholder'"], + "status_on_stdout_contains": "STLINK" + } + } +} diff --git a/scripts/test-power.rhai b/scripts/test-power.rhai new file mode 100644 index 0000000..1b4a25d --- /dev/null +++ b/scripts/test-power.rhai @@ -0,0 +1,156 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// Copyright 2025 Oxide Computer Company + +import `${script_dir}/util` as util; +import `${script_dir}/update-helper` as helper; + +/// Print command line usage +fn usage(prog, error) { + if error != () { + print(`Error: ${error}`); + } + print( + `Usage: faux-mgs ... rhai ${prog} [-v] [-h] [-c config.json] ` + + `[path0] [path1]` + ); + print(" -c CONFIG.JSON # Path to configuration"); + print(" -v # be verbose"); + print(" -h # Help. Print this message"); + print(""); +} + +/// Parse command line options including the required / JSON configuration file. +/// Return an exit code or the configuration map +fn process_cli(argv) { + let prog = argv[0]; + let options = "b:c:htu:v"; + let parsed = util::getopts(argv, options); + if parsed?["error"] != () { + usage(prog, parsed.error); + return 1; + } + + if parsed.result?["h"] == true { + usage(prog, ()); + return 0; + } + + let conf = #{}; + conf["verbose"] = parsed.result?["v"] == true; + + let conf_path_str = parsed.result?["c"]; + if conf_path_str == () { + usage(prog, "Missing required option: -c config.json"); + return 1; + } + let conf_path = path(conf_path_str); + if !conf_path.is_file { + usage(prog, `Config file not found or not a file: ${conf_path}`); + return 1; + } + conf["conf_path"] = conf_path; + + let conf_json = ""; + let read_ok = true; + try { + let conf_file = open_file(conf_path); + conf_json = conf_file.read_string(); + debug(`Successfully read config file: ${conf_path}`); + } catch (err) { + debug(`error|Cannot open or read config file ${conf_path}: ${err}`); + read_ok = false; + } + if !read_ok { return 1; } + + let config_from_file = json_to_map(conf_json); // Global custom Rhai function + if config_from_file?.error != () { + debug( + `error|Failed to parse JSON from config file: ${conf_path}` + + `: ${config_from_file.error}` + ); + return 1; + } + if conf.verbose { + print(""); + print(`Parsed JSON config from ${conf_path}: ${config_from_file}`); + } + // The base configurartion is in `config_from_file`, command line options + // and positional parameters override the base. + let merged_config = config_from_file; + + // Add CLI flags to the config map passed around + conf += merged_config; // Add contents of merged_config to conf + // Now 'conf' contains original JSON + CLI overrides + verbose/transient flags + + if conf.verbose { + print("\nFully resolved configuration map (process_cli):"); + print(conf); + } + return conf; +} + +fn main() { + let start_ts = timestamp(); + let start_time = datetime_local(); + debug(`info|Starting upgrade-rollback script at ${start_time}`); + + let conf = process_cli(argv); + if type_of(conf) == "i64" { + // This is an error code. + return conf; + } + + print(""); + print(`Elapsed time after setup: ${start_ts.elapsed}`); + print(`Current time: ${datetime_local()}`); + print(""); + + // Turn power on + if util::control_power("dut", "on", conf) { + print("Power on success"); + } + + // Turn power off + if util::control_power("dut", "off", conf) { + print("Power off success"); + } + + let r = util::control_power("dut", "status", conf); + if type_of(r) == "map" { + print(`Error: cannot get DUT power state: ${r}`); + return 1; + } + // Else, state is an informative string, expected to be "OFF" + if r != "OFF" { + print(`Error: unexpected power state for dut: ${r}`); + return 1; + } else { + print(`Power state is ${r}`); + } + + // Turn power on + if util::control_power("dut", "on", conf) { + print("Power on success"); + } + + let r = util::control_power("dut", "status", conf); + if type_of(r) == "map" { + print(`Error: cannot get DUT power state: ${r}`); + return 1; + } + // Else, state is an informative string, expected to be "ON" + if r != "ON" { + print(`Error: unexpected power state for dut: ${r}`); + return 1; + } else { + print(`Power state is ${r}`); + } + + + print(`Elapsed time total: ${start_ts.elapsed}`); + print(`Current time: ${datetime_local()}`); + print("Done"); + return 0; +} diff --git a/scripts/test_getopts.rhai b/scripts/test_getopts.rhai new file mode 100644 index 0000000..9d08e2b --- /dev/null +++ b/scripts/test_getopts.rhai @@ -0,0 +1,288 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// Copyright 2025 Oxide Computer Company + +// Test script for the getopts function in util.rhai +// Run this as: +// `cargo -q run --bin faux-mgs --features=rhaiscript -- rhai scripts/test_getopts.rhai` + +// Example Output: +// $ cargo -q run --bin faux-mgs -- -l crit --interface lo rhai scripts/test_getopts.rhai +// --- Running getopts tests --- +// PASS: test_simple_flags: -a +// PASS: test_simple_flags: -b +// PASS: test_simple_flags: no positionals +// PASS: test_opt_with_sep_arg: -f value +// PASS: test_opt_with_sep_arg: positional +// PASS: test_opt_with_sep_arg: positional count +// PASS: test_opt_with_att_arg: -fvalue +// PASS: test_opt_with_att_arg: positional +// PASS: test_opt_with_att_arg: positional count +// PASS: test_combined_flags: -a +// PASS: test_combined_flags: -b +// PASS: test_combined_flags: -c +// PASS: test_combined_flags: positional +// PASS: test_combined_opt_att_arg: -a +// PASS: test_combined_opt_att_arg: -b value +// PASS: test_combined_opt_att_arg: positional +// PASS: test_combined_opt_sep_arg: -a +// PASS: test_combined_opt_sep_arg: -b value +// PASS: test_combined_opt_sep_arg: positional +// PASS: test_long_flag: --verbose +// PASS: test_long_flag: positional +// PASS: test_long_opt_eq_val: --file=value +// PASS: test_long_opt_eq_val: positional +// PASS: test_double_dash: -a +// PASS: test_double_dash: -b not parsed +// PASS: test_double_dash: positional count +// PASS: test_double_dash: first positional is -b +// PASS: test_double_dash: second positional +// PASS: test_err_unknown_short +// PASS: test_err_missing_arg +// --- getopts tests complete --- + +import `${script_dir}/util` as util; + +/// Basic assertion helper for tests. Throws on failure. +fn assert_equal(actual, expected, test_name) { + if actual != expected { + let fail_msg = + `FAIL: ${test_name}. Expected: ${expected}, Got: ${actual}`; + print(fail_msg); + throw fail_msg; // Throw error to stop execution on failure + } else { + // Optional: Only print PASS on verbose mode? + print(`PASS: ${test_name}`); + } +} + +/// Asserts that a value is specifically true. +fn assert_true(actual, test_name) { + assert_equal(actual, true, test_name); +} + +/// Asserts that a key exists within a map. +fn assert_key_exists(key, map, test_name) { + if !(key in map) { + let fail_msg = `FAIL: ${test_name}. Key '${key}' not found in map: ${map}`; + print(fail_msg); + throw fail_msg; + } + // Keep PASS silent or print if verbose needed + // print(`PASS: ${test_name}`); +} + +/// Asserts that a key does NOT exist within a map. +fn assert_key_not_exists(key, map, test_name) { + if key in map { + let fail_msg = `FAIL: ${test_name}. Key '${key}' unexpectedly found in map: ${map}`; + print(fail_msg); + throw fail_msg; + } + // Keep PASS silent or print if verbose needed + // print(`PASS: ${test_name}`); +} + +/// Wrapper to run a test function and catch/report errors. +fn run_test(func_ptr) { + // Assumes func_ptr.name is available in future Rhai or passed manually + // For now, use manual name logging if needed inside the test function. + try { + call(func_ptr); + } catch(e) { + print(`ERROR running test: ${e}`); + // Optionally re-throw or exit if any test failure should stop all tests + // throw e; + } +} + +// --- Test Cases --- + +fn test_simple_flags() { + let test_name = "test_simple_flags"; + let argv = ["script", "-a", "-b"]; + let options = "ab"; + let result = util::getopts(argv, options); + + assert_key_exists("a", result.result, `${test_name}: map has a`); + assert_true(result.result.a, `${test_name}: -a is true`); + assert_key_exists("b", result.result, `${test_name}: map has b`); + assert_true(result.result.b, `${test_name}: -b is true`); + assert_equal(result.positional.len(), 0, `${test_name}: positional count`); +} + +fn test_opt_with_sep_arg() { + let test_name = "test_opt_with_sep_arg"; + let argv = ["script", "-f", "file.txt", "pos1"]; + let options = "f:"; // -f requires an argument + let result = util::getopts(argv, options); + + assert_equal(result.result.f, "file.txt", `${test_name}: -f value`); + assert_equal(result.positional.len(), 1, `${test_name}: positional count`); + assert_equal(result.positional[0], "pos1", `${test_name}: positional value`); +} + +fn test_opt_with_att_arg() { + let test_name = "test_opt_with_att_arg"; + let argv = ["script", "-ffile.txt", "pos1"]; + let options = "f:"; // -f requires an argument + let result = util::getopts(argv, options); + + assert_equal(result.result.f, "file.txt", `${test_name}: -fvalue`); + assert_equal(result.positional.len(), 1, `${test_name}: positional count`); + assert_equal(result.positional[0], "pos1", `${test_name}: positional value`); +} + +fn test_combined_flags() { + let test_name = "test_combined_flags"; + let argv = ["script", "-abc", "pos1"]; + let options = "abc"; // All are flags + let result = util::getopts(argv, options); + + assert_true(result.result.a, `${test_name}: -a`); + assert_true(result.result.b, `${test_name}: -b`); + assert_true(result.result.c, `${test_name}: -c`); + assert_equal(result.positional.len(), 1, `${test_name}: positional count`); + assert_equal(result.positional[0], "pos1", `${test_name}: positional value`); +} + +fn test_combined_opt_att_arg() { + let test_name = "test_combined_opt_att_arg"; + let argv = ["script", "-abfile.txt", "pos1"]; + let options = "ab:"; // -b requires argument + let result = util::getopts(argv, options); + + assert_true(result.result.a, `${test_name}: -a`); + assert_equal(result.result.b, "file.txt", `${test_name}: -b value`); + assert_equal(result.positional.len(), 1, `${test_name}: positional count`); + assert_equal(result.positional[0], "pos1", `${test_name}: positional value`); +} + +fn test_combined_opt_sep_arg() { + let test_name = "test_combined_opt_sep_arg"; + let argv = ["script", "-ab", "file.txt", "pos1"]; + let options = "ab:"; // -b requires argument + let result = util::getopts(argv, options); + + assert_true(result.result.a, `${test_name}: -a`); + assert_equal(result.result.b, "file.txt", `${test_name}: -b value`); + assert_equal(result.positional.len(), 1, `${test_name}: positional count`); + assert_equal(result.positional[0], "pos1", `${test_name}: positional value`); +} + +fn test_long_flag() { + let test_name = "test_long_flag"; + let argv = ["script", "--verbose", "pos1"]; + let options = "abc"; // Long opts not defined/validated by 'options' string + let result = util::getopts(argv, options); + + assert_true(result.result.verbose, `${test_name}: --verbose`); + assert_equal(result.positional.len(), 1, `${test_name}: positional count`); + assert_equal(result.positional[0], "pos1", `${test_name}: positional value`); +} + +fn test_long_opt_eq_val() { + let test_name = "test_long_opt_eq_val"; + let argv = ["script", "--file=out.txt", "pos1"]; + let options = "abc"; + let result = util::getopts(argv, options); + + assert_equal(result.result.file, "out.txt", `${test_name}: --file=value`); + assert_equal(result.positional.len(), 1, `${test_name}: positional count`); + assert_equal(result.positional[0], "pos1", `${test_name}: positional value`); +} + +fn test_double_dash() { + let test_name = "test_double_dash"; + let argv = ["script", "-a", "--", "-b", "pos1"]; + let options = "ab"; // -b is a valid option + let result = util::getopts(argv, options); + + assert_true(result.result.a, `${test_name}: -a`); + assert_key_not_exists("b", result.result, `${test_name}: -b not parsed`); + assert_equal(result.positional.len(), 2, `${test_name}: positional count`); + assert_equal(result.positional[0], "-b", `${test_name}: first positional is -b`); + assert_equal(result.positional[1], "pos1", `${test_name}: second positional`); +} + +fn test_positional_only() { + let test_name = "test_positional_only"; + let argv = ["script", "arg1", "arg2"]; + let options = "ab"; + let result = util::getopts(argv, options); + + assert_equal(result.result.len(), 0, `${test_name}: result map empty`); + assert_equal(result.positional.len(), 2, `${test_name}: positional count`); + assert_equal(result.positional[0], "arg1", `${test_name}: positional 1`); + assert_equal(result.positional[1], "arg2", `${test_name}: positional 2`); +} + +fn test_mixed_opts_positional() { + let test_name = "test_mixed_opts_positional"; + // Note: Basic getopts might treat non-option args as positional anywhere + let argv = ["script", "pos1", "-a", "pos2", "-bval", "--long=lval"]; + let options = "ab:"; + let result = util::getopts(argv, options); + + assert_true(result.result.a, `${test_name}: -a`); + assert_equal(result.result.b, "val", `${test_name}: -bval`); + assert_equal(result.result.long, "lval", `${test_name}: --long=lval`); + assert_equal(result.positional.len(), 2, `${test_name}: positional count`); + assert_equal(result.positional[0], "pos1", `${test_name}: positional 1`); + assert_equal(result.positional[1], "pos2", `${test_name}: positional 2`); +} + +fn test_err_unknown_short() { + let test_name = "test_err_unknown_short"; + let argv = ["script", "-ax"]; // -x is the unknown option + let options = "a"; + let expected_err = "Unknown option '-x'."; + + // Call getopts and check the returned map directly + let result = util::getopts(argv, options); + + // Assert that the returned map has an 'error' key + assert_key_exists("error", result, `${test_name}: error key exists`); + // Assert that the value of the 'error' key matches the expected message + assert_equal(result.error, expected_err, `${test_name}: error message`); +} + +fn test_err_missing_arg() { + let test_name = "test_err_missing_arg"; + let argv = ["script", "-a"]; // Argument missing for -a + let options = "a:"; // -a requires an argument + let expected_err = "Option '-a' requires an argument."; + + // Call getopts and check the returned map directly + let result = util::getopts(argv, options); + + // Assert that the returned map has an 'error' key + assert_key_exists("error", result, `${test_name}: error key exists`); + // Assert that the value of the 'error' key matches the expected message + assert_equal(result.error, expected_err, `${test_name}: error message`); +} + +fn main() { + print("--- Running getopts tests ---"); + + // Call each test function using the run_test wrapper + run_test(test_simple_flags); + run_test(test_opt_with_sep_arg); + run_test(test_opt_with_att_arg); + run_test(test_combined_flags); + run_test(test_combined_opt_att_arg); + run_test(test_combined_opt_sep_arg); + run_test(test_long_flag); + run_test(test_long_opt_eq_val); + run_test(test_double_dash); + run_test(test_positional_only); + run_test(test_mixed_opts_positional); + run_test(test_err_unknown_short); + run_test(test_err_missing_arg); + + print("--- getopts tests complete ---"); + // Indicate overall success/failure based on whether any test threw? + // For now, just return 0 if it completes. + return 0; +} diff --git a/scripts/test_rot_power_recovery.rhai b/scripts/test_rot_power_recovery.rhai new file mode 100644 index 0000000..b5010a0 --- /dev/null +++ b/scripts/test_rot_power_recovery.rhai @@ -0,0 +1,241 @@ +import `${script_dir}/util` as util; +import `${script_dir}/update-helper` as helper; + +/// Print command line usage +fn usage(prog, error) { + if error != () { + print(`Error: ${error}`); + } + print( + `Usage: faux-mgs ... rhai ${prog} [-v] [-h] [-c config.json] ` + + `[path0] [path1]` + ); + print(" -b BASELINE_PATH # Path to baseline hubris repo"); + print(" -c CONFIG.JSON # Path to configuration"); + print(" -u UNDER_TEST_PATH # Path to a repo with faulty images to test"); + print(" -v # be verbose"); + print(" -h # Help. Print this message"); +} + +/// Parse command line options including the required / JSON configuration file. +/// Return an exit code or the configuration map +fn process_test_cli(argv) { + let prog = argv[0]; + let options = "b:c:hu:v"; + + let parsed = util::getopts(argv, options); + if parsed?["error"] != () { + usage(prog, parsed.error); + return 1; + } + + if parsed.result?["h"] == true { + usage(prog, ()); + return 0; + } + + let conf = #{}; + conf["verbose"] = parsed.result?["v"] == true; + + let conf_path_str = parsed.result?["c"]; + if conf_path_str == () { + usage(prog, "Missing required option: -c config.json"); + return 1; + } + let conf_path = path(conf_path_str); + if !conf_path.is_file { + usage(prog, `Config file not found or not a file: ${conf_path}`); + return 1; + } + conf["conf_path"] = conf_path; + + let conf_json = ""; + let read_ok = true; + try { + let conf_file = open_file(conf_path); + conf_json = conf_file.read_string(); + debug(`Successfully read config file: ${conf_path}`); + } catch (err) { + debug(`error|Cannot open or read config file ${conf_path}: ${err}`); + read_ok = false; + } + if !read_ok { return 1; } + + let config_from_file = json_to_map(conf_json); // Global custom Rhai function + if config_from_file?.error != () { + debug( + `error|Failed to parse JSON from config file: ${conf_path}` + + `: ${config_from_file.error}` + ); + return 1; + } + if conf.verbose { + print(`Parsed JSON config from ${conf_path}: ${config_from_file}`); + } + + let base_repo_override = parsed?.result?["b"]; + if base_repo_override != () { + conf.base_repo = base_repo_override; + } + + let ut_repo_override = parsed?.result?["u"]; + if ut_repo_override != () { + conf.ut_repo = ut_repo_override; + } + + // The base configurartion is in `config_from_file`. + // Some command line options override the config file. + + // CLI flags override the config file. + let conf = config_from_file + conf; + + // Ensure power cycle settings are present or provide defaults + if conf?.power_off_delay_secs == () { + conf.power_off_delay_secs = 3; + } + + if conf?.dut_boot_delay_secs == () { + conf.dut_boot_delay_secs = 10; + } + + // Explicitly enable the power cycle on failure for this test script's context + conf.rot_hubris_power_cycle_on_failure = true; + debug(`info|rot_hubris_power_cycle_on_failure=${conf.rot_hubris_power_cycle_on_failure}`); + + for branch in ["base", "ut"] { + conf[branch] = #{}; + for image_type in ["sp", "rot_a", "rot_b", "stage0"] { + let image_path_template = conf?.images?[branch]?[image_type]; + if image_path_template == () { + debug(`warn|process_cli: No config path for ${branch}.${image_type}`); + conf[branch][image_type] = (); + continue; + } + let zip_path = util::env_expand(image_path_template, conf); + if zip_path == () { + debug( + `error|process_cli: Failed expanding path for ${branch}.${image_type}: ` + + `'${image_path_template}'` + ); + conf[branch][image_type] = (); + } else { + conf[branch][image_type] = zip_path; + debug(`info|process_cli: Expanded path ${branch}.${image_type} = ${zip_path}`); + } + } + } + + + return conf; +} + +fn main() { + let start_ts = timestamp(); + let start_time = datetime_local(); + print(`info|Starting RoT Power Cycle Recovery Test at ${start_time}`); + debug(`info|Starting RoT Power Cycle Recovery Test at ${start_time}`); + let conf = (); + let images = (); + + try { + conf = process_test_cli(argv); + if type_of(conf) == "int" { + return conf; + } + + images = helper::get_image_info(conf); + if type_of(images) == "int" { return images; } + + // 1. Ensure DUT is on a known-good baseline state initially + debug("info|Ensuring DUT is on baseline firmware before test..."); + if (!helper::ensure_initial_baseline_state(conf, images)) { + debug("error|Failed to set initial baseline state. Cannot proceed with test."); + return 1; + } + let initial_rbi = util::rot_boot_info(); + if initial_rbi?.error != () { + debug(`error|Failed to get RBI of baseline: ${initial_rbi.error}`); + return 1; + } + let pre_test_active_slot = initial_rbi.active; + debug(`info|DUT is on baseline. Pre-test active RoT slot: ${pre_test_active_slot}`); + + // 2. Attempt to update to the "faulty" under-test RoT image. + // This update is EXPECTED to fail and trigger a power cycle. + // Assume conf.ut.rot_a/b point to your faulty images. + debug( + "info|Attempting to update to FAULTY under-test RoT image. " + + "Expecting this update to fail and trigger power cycle recovery." + ); + let update_to_faulty_image_succeeded = helper::update_rot_hubris( + conf.ut.rot_a, // Path to FAULTY RoT A image + conf.ut.rot_b, // Path to FAULTY RoT B image + true, // Attempt using transient for maximum effect + "faulty-ut", // Label for this attempt + conf + ); + + if update_to_faulty_image_succeeded { + debug("error|TEST FAILED: Update to faulty RoT image SUCCEEDED, but was expected to fail."); + return 1; + } + + debug("info|Update to faulty RoT image failed as expected, and power cycle was attempted."); + + // 3. Verify recovery: Check the state of the RoT after the power cycle. + // The power_cycle_dut function (called within update_rot_hubris) already logs + // the state after power cycling. Here, we add explicit checks. + print("INFO: Verifying RoT state after power cycle recovery attempt..."); + let rbi_after_recovery = util::rot_boot_info(); + + if rbi_after_recovery?.error != () { + debug(`error|TEST FAILED: Could not get RoT state after power cycle recovery: ${rbi_after_recovery.error}`); + return 1; + } + let rbi_recovered = rbi_after_recovery; // rot_boot_info returns RBI map directly on success + + debug(`info|RoT state after power cycle: active=${rbi_recovered.active}, p_pref=${rbi_recovered.persistent_boot_preference}`); + + // Define "successful recovery": + // - Is it back on the original baseline slot (pre_test_active_slot)? + // - Or, if the faulty image was on the other bank, did it revert to 'pre_test_active_slot'? + // This depends heavily on the bootloader's behavior after an interrupted/failed update and power loss. + // For now, let's check if it's on the pre_test_active_slot and that slot has the baseline GITC. + if rbi_recovered.active == pre_test_active_slot { + debug(`info|Recovery check: RoT correctly booted back to original slot ${pre_test_active_slot}.`); + // Further check: is the GITC on this slot actually the baseline's? + let gitc_after_recovery = util::caboose_value("rot", `${rbi_recovered.active}`, "GITC"); + let expected_baseline_gitc = if rbi_recovered.active == 0 { + images.base?.rot_a?.caboose?.GITC; + } else { + images.base?.rot_b?.caboose?.GITC; + }; + + + if gitc_after_recovery == expected_baseline_gitc && expected_baseline_gitc != () { + debug( + `info|TEST PASSED: RoT recovered to baseline slot ${pre_test_active_slot} `+ + `with correct GITC ${gitc_after_recovery}.` + ); + // Test success! + return 0; + } else { + debug( + `error|TEST FAILED: RoT recovered to slot ${pre_test_active_slot}, `+ + `but GITC mismatch. Expected: ${expected_baseline_gitc}, Got: ${gitc_after_recovery}.` + ); + return 1; + } + } else { + debug( + `error|TEST FAILED: RoT did not recover to original slot ${pre_test_active_slot}. ` + + `Currently active: ${rbi_recovered.active}.` + ); + return 1; + } + + } catch (err) { + debug(`error|FATAL ERROR in RoT Power Recovery Test: ${err}`); + return 1; + } +} diff --git a/scripts/update-helper.rhai b/scripts/update-helper.rhai new file mode 100644 index 0000000..16082e9 --- /dev/null +++ b/scripts/update-helper.rhai @@ -0,0 +1,1108 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// Copyright 2025 Oxide Computer Company + +// Helper module for the upgrade-rollback script. +// Contains functions for image processing, firmware updates, state checks, +// and other operational logic that may be useful with various higher level +// scripts. + +import `${script_dir}/util` as util; + +// --- Functions related to Image Info and Verification --- + +/// Checks and organizes image information from configuration. +/// Reads device info (CMPA, CFPA), verifies RoT images, builds GITC map. +/// +/// Args: +/// conf (map): The fully resolved configuration map. +/// +/// Returns: +/// Map: Image details, caboose info, verification status, GITC mapping. +/// int: `1` if a critical error occurred. +fn get_image_info(conf) { + let images = #{}; + + images["cmpa"] = util::get_cmpa(); + images["cfpa"] = util::get_cfpa(); + if images.cmpa == () || images.cfpa == () { + debug("error|Failed to read CMPA or CFPA from device."); + return 1; + } + images["keyset"] = util::get_rot_keyset(images.cmpa); + debug(`info|Detected RoT keyset: ${images.keyset}`); + + let gitc = #{}; + let error_found = false; + let failed_verify_list = []; + + for branch in ["base", "ut"] { + images[branch] = #{}; + for image_type in ["sp", "rot_a", "rot_b", "stage0"] { + let zip_path = conf?[branch]?[image_type]; + if zip_path == () { + debug(`warn|Skipping missing path for ${branch}.${image_type}`); + continue; + } + images[branch][image_type] = #{"path": zip_path}; + + // Setup to extract information from Hubris archive files using + // global custom Rhai object: `Archive` and `Caboose` + let current_ar = (); + let archive_ok = true; + try { + current_ar = new_archive(zip_path); + } catch(err) { + debug(`error|Failed creating archive for ${zip_path}: ${err}`); + error_found = true; + archive_ok = false; + } + if !archive_ok { + continue; + } + if type_of(current_ar) != "Archive" { + debug(`error|Invalid archive object for path ${zip_path}`); + error_found = true; + continue; + } + images[branch][image_type]["caboose"] = #{}; + + // Now get the desired information + let caboose_obj = current_ar?["caboose"]; + if caboose_obj == () || type_of(caboose_obj) == "()" { + debug(`warn|Could not get caboose object from ${zip_path}`); + } else if type_of(caboose_obj) == "map" && caboose_obj?.error != () { + debug( + `warn|Error getting caboose object from ${zip_path}: ` + + `${caboose_obj.error}` + ); + } else { + let caboose_read_ok = true; + for key in ["BORD", "GITC", "NAME", "SIGN", "VERS"] { + let val = (); + try { val = caboose_obj[key]; } catch (err) { + debug(`warn|Failed reading caboose key '${key}' from ${zip_path}: ${err}`); + caboose_read_ok = false; + } + if val != () { + images[branch][image_type]["caboose"][key] = val; + } + } + if images[branch][image_type]["caboose"]?.GITC == () { + debug( + `warn|Could not read essential GITC from caboose in ${zip_path}. ` + + `Read OK: ${caboose_read_ok}` + ); + } + } + + if image_type != "sp" { + // Assume that RoT or Stage0 signatures will verify + let verified = true; + let verify_error = (); + try { + verified = current_ar.verify_rot_image(images.cmpa, images.cfpa); + } catch (err) { + verified = false; + verify_error = err; + } + images[branch][image_type]["verified"] = verified; + if !verified { + failed_verify_list += images[branch][image_type]["path"]; + debug( + `warn|Sig verification FAILED: ` + + `${images[branch][image_type]["path"]}. Error: ${verify_error}` + ); + } else { + debug( + `info|Sig verified OK: ${images[branch][image_type]["path"]}` + ); + } + } else { // SP image + images[branch][image_type]["verified"] = true; + // TODO: Add SP-specific BORD/NAME checks if needed + } + + let image_gitc = images[branch][image_type]?.caboose?["GITC"]; + if image_gitc != () { + if gitc?[image_gitc] == () { + gitc[image_gitc] = []; + } + // TODO: Warn if base and ut GITCs match for sp/rot + gitc[image_gitc] += [ `${branch}_${image_type}` ]; + } + } + } + debug(`info|Image GITC reverse map: ${gitc}`); + images["by_gitc"] = gitc; + images["failed_to_verify"] = failed_verify_list; + + if images.failed_to_verify.len() > 0 { + debug("warn|Some RoT images FAILED signature verification:"); + for path_str in images.failed_to_verify { + debug(`warn|failed to verify: ${path_str}`); + } + } + debug("debug|Image Info Map (get_image_info):\n" + images); + if error_found { + return 1; + } else { + return images; + } +} + +/// Reads caboose for a given component and slot using util::caboose_value. +fn get_caboose(component, slot) { + let caboose = #{}; + for key in ["BORD", "GITC", "VERS", "NAME", "SIGN"] { + let value = util::caboose_value(component, slot, key); + if value != () { + caboose[key] = value; + } + } + return caboose; +} + +/// Checks if SP needs flashing (active image != desired image). +// +// TODO: Should this check that alternate image == desired image and +// thereby avoid erase/flash cycle? This would be a 'check_alternate' flag +// and would require returning additional information in the map. +fn sp_needs_flashing(target_image_label, gitc_map) { + let sp_gitc = util::caboose_value("sp", "active", "GITC"); + if sp_gitc == () { + return #{"error": "Could not read active SP GITC."}; + } + debug(`info|SP is running GITC=${sp_gitc}`); + let known_images = gitc_map?[sp_gitc]; + debug(`info|Known images for SP GITC ${sp_gitc}: ${known_images}`); + if known_images == () { + debug("warn|SP GITC does not match known base/ut images."); + return #{"ok": true}; + } else if target_image_label in known_images { + debug(`info|Target ${target_image_label} IS IN known images.`); + return #{"ok": false}; + } else { + debug(`warn|Target ${target_image_label} IS NOT in known images.`); + return #{"ok": true}; + } +} + +/// Checks if RoT needs flashing. +// +// TODO: Should this check that alternate image == desired image and +// thereby avoid erase/flash cycle? This would be a 'check_alternate' flag +// and would require returning additional information in the map. +fn rot_needs_flashing(target_branch_prefix, gitc_map) { + let rot_active_result = get_rot_active(); + if rot_active_result?.error != () { + return #{"error": `Cannot get active RoT slot: ${rot_active_result.error}`}; + } + let rot_active_slot = rot_active_result.ok; + let target_image_label = `${target_branch_prefix}_rot_a`; + if rot_active_slot == 1 { + target_image_label = `${target_branch_prefix}_rot_b`; + } + let rot_gitc = util::caboose_value("rot", `${rot_active_slot}`, "GITC"); + if rot_gitc == () { + return #{"error": `Could not read RoT GITC for slot ${rot_active_slot}.`}; + } + debug(`info|RoT is running GITC=${rot_gitc} on active slot ${rot_active_slot}`); + let known_images = gitc_map?[rot_gitc]; + debug(`info|Known images for RoT GITC ${rot_gitc}: ${known_images}`); + if known_images == () { + debug("info|RoT GITC does not match known base/ut images."); + return #{"ok": true}; + } else if target_image_label in known_images { + debug(`info|Target ${target_image_label} IS IN known images.`); + return #{"ok": false}; + } else { + debug(`info|Target ${target_image_label} IS NOT in known images.`); + return #{"ok": true}; + } +} + +/// Performs overall image check for SP and RoT against target branch. +// +// TODO: Add option to check if alternate partition has the desired image or +// always include that information. e.g. might return `#{ "sp": false, +// "sp_alternate": true, "rot": true, "rot_alternate": false}` indicating +// that the active sp image does not need to be flashed, the alternate sp image +// should be flashed to make it equal to the active image, the active rot image +// is needs flashing, but that the alternate rot image is the correct image. +// If the alternate rot image were made active, then the correct images would be +// running. The client could then work on matching active==alternate as their +// next step. +fn image_check(branch, images) { + let ok = #{ "sp": false, "rot": false }; + let error = #{}; + let sp_result = sp_needs_flashing(`${branch}_sp`, images.by_gitc); + if sp_result?.error != () { + error["sp"] = `${sp_result.error}`; + } else { + ok["sp"] = sp_result.ok; + } + let rot_result = rot_needs_flashing(`${branch}`, images.by_gitc); + if rot_result?.error != () { + error["rot"] = `${rot_result.error}`; + } else { + ok["rot"] = rot_result.ok; + } + if error.len() > 0 { + return #{"error": error}; + } else { + return #{"ok": ok}; + } +} + +/// Gets the currently active RoT slot (0 or 1). +fn get_rot_active() { + let rbi = util::rot_boot_info(); + if rbi?.error != () { + return #{"error": `Cannot get RBI: ${rbi.error}`}; + } + let active_slot = rbi?.active; + if active_slot == () { + return #{"error": `Cannot determine active slot from RBI: ${rbi}`}; + } + return #{"ok": active_slot}; +} + +/// Determine the *preferred* next RoT boot slot based on RBI preferences. +/// +/// Note that if only one image is valid, it will be selected. +/// An update attempt can transmit and flash an incomplete or improperly signed +/// image. +/// The signature validity is only known after an RoT reset. +fn get_rot_active_etc() { + let rbi = util::rot_boot_info(); + if rbi?.error != () { + return #{"error": `Cannot get RBI: ${rbi.error}`}; + } + if rbi?.active == () { + return #{"error": `Invalid RBI: ${rbi}`}; + } + + let next_preferred = if rbi.transient_boot_preference != () { + debug("info|Next boot by transient"); + rbi.transient_boot_preference + } else if rbi.pending_persistent_boot_preference != () { + debug("info|Next boot by pending persistent"); + rbi.pending_persistent_boot_preference + } else { + debug("info|Next boot by persistent"); + rbi.persistent_boot_preference + }; + + return #{ + "ok": #{ + "active": rbi.active, + "next": next_preferred, + "transient": rbi.transient_boot_preference, + "pending_persistent": rbi.pending_persistent_boot_preference, + "persistent": rbi.persistent_boot_preference, + } + }; +} + +/// Checks if the currently running RoT firmware supports transient boot commands. +/// This is a state-modifying function that attempts to set the transient +/// preference to the currently active slot. For compliant firmware, this action +/// succeeds and clears any existing transient preference. For non-compliant +/// firmware, it fails. +fn rot_supports_transient_boot_preference() { + let r = get_rot_active_etc(); + if r?.error != () { + debug(`error|Cannot get active RoT slot. Error: ${r.error}`); + return false; + } + if r.ok.transient != () { + // Evidence shows that the feature is supported. + // Don't alter the state + return true; + } + // For compliant firmware, this command succeeds and ensures the transient + // preference is cleared. For non-compliant firmware, it will fail. + if util::set_rot_boot_preference(r.ok.active, true, "transient_support_test") { + // Setting the transient preference to the active slot is a no-op + // but shows that the feature is supported. + debug("info|transient boot preference feature is supported."); + return true; + } + debug("warn|transient boot preference feature is not supported."); + return false; +} + +/// Resets both SP and RoT components sequentially. +fn reset_sp_and_rot() { + debug("info|Beginning reset sequence for RoT and SP."); + for params in [["rot", 3], ["sp", 5]] { + let component_to_reset = params[0]; + let sleep_duration = params[1]; + debug(`info|Reset ${component_to_reset}`); + if !util::reset_component(component_to_reset) { + return #{ "error": `Failed to reset component '${component_to_reset}'.` }; + } + debug( + `info|Reset '${component_to_reset}'. Sleeping ${sleep_duration}s.` + ); + sleep(sleep_duration); + } + debug("info|RoT and SP reset sequence complete."); + return #{"ok": ()}; +} + + +/// Updates SP firmware, resets, and polls for readiness. +fn update_sp(sp_zip) { + debug(`info|update_sp: ${sp_zip}`); + if (!util::update_sp_image(sp_zip)) { + return false; + } + debug(`info|SP update command acknowledged.`); + let r_status1 = util::check_update_in_progress("sp"); + debug(`info|SP status immediately after update command: ${r_status1}`); + debug("info|Resetting SP to boot new image."); + if (!util::reset_sp()) { + return false; + } + debug("info|SP reset acknowledged. Polling for readiness."); + + let max_poll_attempts = 15; + let poll_interval_secs = 1; + let sp_ready_and_clean = false; + for attempt in 1..=max_poll_attempts { + debug( + `info|Polling SP readiness, attempt ${attempt}` + + `/${max_poll_attempts}...` + ); + let sp_status = util::check_update_in_progress("sp"); + if sp_status == () { + debug("info|SP is ready and no update in progress."); + sp_ready_and_clean = true; + break; + } else if sp_status?.Err != () { + debug( + `warn|Readiness check attempt ${attempt}: ` + + `Error getting SP status: ${sp_status}. Retrying.` + ); + } else { + debug( + `warn|Readiness check attempt ${attempt}: ` + + `SP reported unexpected status: ${sp_status}. Retrying.` + ); + } + if attempt < max_poll_attempts { + sleep(poll_interval_secs); + } + } + + if !sp_ready_and_clean { + debug( + `error|SP did not become ready and clean after reset and `+ + `${max_poll_attempts} attempts.` + ); + debug(`error|SP did not successfully reset and clear update status.\n`); + return false; + } + debug("info|SP readiness and clean status confirmed."); + debug(`info|SUCCESS update_sp: ${sp_zip}`); + return true; +} + +fn rot_log_initial_params( + initial_slot, target_slot, image_path, target_label, use_transient_flag +) { + debug(`info|--- rot_log_initial_params (update-helper) ---`); + debug(`info|Target Label: ${target_label}, Use Transient: ${use_transient_flag}`); + debug(`info|Initial Active RoT Slot (before op): ${initial_slot}`); + debug(`info|Target RoT Update Slot (for op): ${target_slot}`); + debug(`info|Target RoT Image (for op): ${image_path}`); +} + +fn rot_log_rbi_details( + rbi, context_description, initial_slot, + target_slot, use_transient_flag_for_op +) { + debug(`info|--- rot_log_rbi_details (${context_description}, update-helper) ---`); + debug(` Initial Active Slot (before op): ${initial_slot}`); + debug(` Target Update Slot (for op): ${target_slot}`); + debug(` Use Transient Flag (for op): ${use_transient_flag_for_op}`); + debug(` Current Active Slot (from RBI): ${rbi.active}`); + debug(` Persistent Pref (from RBI): ${rbi.persistent_boot_preference}`); + debug(` Transient Pref (from RBI): ${rbi.transient_boot_preference}`); + debug(` Pending Persistent Pref (from RBI): ${rbi.pending_persistent_boot_preference}`); +} + +/// Check that the post-reset state after using the transient_boot_preference is +// as expected. +fn rot_validate_initial_transient_boot_state( + rbi, initial_slot, target_update_slot, target_label +) { + let failure_details = ""; + + // The desired hubris image should be the active image. + if rbi.active != target_update_slot { + failure_details += ` Active(${rbi.active}) not target(${target_update_slot}).`; + } + + // Transient_boot_preference will be None + if rbi.transient_boot_preference != () { + failure_details += ` transient_boot_preference(${rbi.transient_boot_preference}) not null.`; + } + + // Pending_persistent_boot_preference will be None + if rbi.pending_persistent_boot_preference != () { + failure_details += ` pp_pref (${rbi.pending_persistent_boot_preference}) not null.`; + } + + if failure_details.len() != 0 { + debug(`error|Initial transient boot validation FAILED: ` + + `${failure_details} RBI=${rbi}`); + return false; + } else { + debug(`info|Initial transient boot validation PASSED for '${target_label}'.`); + } + return true; +} + +fn rot_ensure_persistence_after_transient_boot( + rbi_after_first_boot, target_update_slot, target_label +) { + debug("info|--- rot_ensure_persistence_after_transient_boot (update-helper) ---"); + let active_after_first_reset = rbi_after_first_boot.active; + let p_pref_from_newly_active = rbi_after_first_boot.persistent_boot_preference; + let set_persistent_needed = true; + + if target_label == "baseline" { + if p_pref_from_newly_active == active_after_first_reset { + debug( + `info|Target '${target_label}': p_pref already matches active. ` + + `No explicit set_persistent needed.` + ); + set_persistent_needed = false; + } else { + debug( + `info|Target '${target_label}': p_pref differs from active. ` + + `Will align by setting persistent.` + ); + } + } + if set_persistent_needed { + let reason = if target_label == "baseline" { + "Align for baseline." + } else { + "Standard." + }; + debug(`info|Setting persistent for slot ${target_update_slot} ('${target_label}'). Reason: ${reason}`); + if !util::set_rot_boot_preference(target_update_slot, false, target_label) { // false -> '-p' + debug(`error|util::set_rot_boot_preference failed.`); + return false; + } + debug(`debug|util::set_rot_boot_preference called successfully.`); + } else { + debug(`debug|Skipping set persistent for '${target_label}'.`); + } + return true; +} + +fn rot_validate_final_persistent_boot_state( + final_rbi, target_update_slot, target_label +) { + debug("info|--- rot_validate_final_persistent_boot_state (update-helper) ---"); + let final_active_slot = final_rbi.active; + let final_p_pref = final_rbi.persistent_boot_preference; + debug(`info|Final RBI: active=${final_active_slot}, p_pref=${final_p_pref}`); + + if final_active_slot != target_update_slot { + debug(`error|Validation FAILED: Target '${target_label}' NOT on persistent slot.`); + return false; + } + if final_p_pref != target_update_slot { + debug(`warn|Validation NOTE: Target '${target_label}': final p_pref is ${final_p_pref}.`); + } + debug(`info|Validation PASSED: Booted '${target_label}' on persistent slot ${final_active_slot}.`); + return true; +} + +fn rot_validate_direct_persistent_boot_state( + rbi, target_update_slot, target_label +) { + debug("info|--- rot_validate_direct_persistent_boot_state (update-helper) ---"); + if rbi.active != target_update_slot { + debug(`error|Validation FAILED: Unexpected active slot for '${target_label}'.`); + return false; + } + debug(`info|Validation PASSED: RoT correctly booted for '${target_label}'.`); + return true; +} + +/// Attempts to power cycle the DUT and log its RoT state afterwards. +/// This is a helper for failure recovery testing. +/// +/// Args: +/// conf (map): The main configuration map. +/// context_msg (string): A message describing when this power cycle is happening. +/// +/// Returns: +/// None. Logs information about the process. +fn power_cycle_dut(conf, context_msg) { + if conf?.rot_hubris_power_cycle_on_failure != true { + debug(`info|Power cycle on RoT Hubris update failure not enabled.`); + return; + } + + debug(`info|${context_msg} - Initiating DUT power cycle for recovery test.`); + let power_off_delay = conf?.power_off_delay_secs; + if power_off_delay == () || type_of(power_off_delay) != "int" { + power_off_delay = 2; + } + + let dut_boot_delay = conf?.dut_boot_delay_secs; + if dut_boot_delay == () || type_of(dut_boot_delay) != "int" { + dut_boot_delay = 8; + } + + if (!util::control_power("dut", "off", conf)) { + debug(`warn|Failed to power OFF DUT.`); + // Continue anyway to see if it's already off or attempt power on + } else { + debug(`info|DUT power off command sent. Waiting ${power_off_delay}s.`); + sleep(power_off_delay); + } + + if (!util::control_power("dut", "on", conf)) { + debug(`error|Failed to power ON DUT. Recovery state unknown.`); + return; + } + debug(`info|DUT power on command sent. Waiting ${dut_boot_delay}s for boot.`); + sleep(dut_boot_delay); + + debug("info|Attempting to get RoT state after power cycle..."); + let rbi_after_cycle = util::reset_rot_and_get_rbi("after power cycle recovery", "recovery_check"); + if rbi_after_cycle?.error != () { + debug(`error|Could not get RBI after power cycle: ${rbi_after_cycle.error}`); + return; + } + + let active_slot_after_cycle = rbi_after_cycle.ok.active; + debug(`info|RoT active slot after power cycle: ${active_slot_after_cycle}`); + if active_slot_after_cycle != () { + let gitc_after_cycle = util::caboose_value("rot", `${active_slot_after_cycle}`, "GITC"); + debug(`info|RoT GITC on slot ${active_slot_after_cycle} ` + + `after power cycle: ${gitc_after_cycle}` + ); + } else { + debug(`warn|Could not determine active RoT slot after power cycle.`); + } +} + +/// Updates RoT firmware, handles transient vs persistent, performs resets, +/// validates the final state, and optionally power cycles on failure for recovery testing. +/// +/// Args: +/// path_a (string): Path to image for RoT slot A. +/// path_b (string): Path to image for RoT slot B. +/// use_transient (bool): Whether to attempt using transient preference mechanism. +/// target_label (string): Descriptive label (e.g., "baseline", "under-test"). +/// conf (map): The main configuration map. +/// +/// Returns: +/// bool: `true` on successful original update and validation, `false` otherwise. +fn update_rot_hubris( + path_a, + path_b, + use_transient, + target_label, + conf +) { + debug(`info|update_rot_hubris target=${target_label}`); + debug(`info|transient=${use_transient}`); + debug(`info|Image A: ${path_a}`); + debug(`info|Image B: ${path_b}`); + + // Determine the non-active partition and select the appropriate update + // image. + let r = get_rot_active(); + if r?.error != () { + debug(`error|get_rot_active failed: ${r.error}`); + return false; + } + let initial_slot = r.ok; + let target_slot = if initial_slot == 0 { 1 } else { 0 }; + let target_image = if target_slot == 0 { path_a } else { path_b }; + + rot_log_initial_params( + initial_slot, + target_slot, + target_image, + target_label, + use_transient + ); + + if (!util::update_rot_image_file(target_slot, target_image, target_label)) { + // Power cycle only if recovery test for this type of failure is enabled + if conf?.rot_hubris_power_cycle_on_failure == true { + power_cycle_dut(conf, "RoT image file update command failed"); + } + return false; + } + + if (!util::set_rot_boot_preference(target_slot, use_transient, target_label)) { + if conf?.rot_hubris_power_cycle_on_failure == true { + power_cycle_dut(conf, "RoT set boot preference command failed"); + } + return false; + } + + let r = util::reset_rot_and_get_rbi( + "after initial preference set", target_label + ); + if r?.error != () { // Checks for the "error" key + debug( + `error|RoT first reset or RBI read failed. Error details: ` + + `${r.error}` + ); + // Decide to power cycle based on the error_type + if conf?.rot_hubris_power_cycle_on_failure == true { + let error_type = r?.error_type; + // Only power cycle on errors where device might be unresponsive or in bad state + if error_type == "reset_command_failed" || + error_type == "rbi_fetch_failed" + { + debug( + `info|Error type '${error_type}' suggests power cycle might help.` + ); + power_cycle_dut(conf, `RoT reset/RBI read failed (${error_type})`); + } else { + debug( + `info|Error type '${error_type}' - power cycle ` + + `likely won't resolve. Skipping.` + ); + } + } + return false; + } + let rbi = r.ok; // Get the actual RBI data + + rot_log_rbi_details( + rbi, "after first reset", initial_slot, + target_slot, use_transient + ); + + if use_transient { + // TODO: When Hubris issue #2066 is fixed, then the boot loader + // decision log will be available in RotBootInfo. We will then be able + // to confirm that the Hubris image was chosen due to the transient + // preference feature. + + if (!rot_validate_initial_transient_boot_state( + rbi, initial_slot, + target_slot, target_label + )) { + if (conf?.rot_hubris_power_cycle_on_failure == true) { + power_cycle_dut(conf, "RoT initial transient boot validation failed"); + } + return false; + } + if (!rot_ensure_persistence_after_transient_boot( + rbi, target_slot, target_label + )) { + if (conf?.rot_hubris_power_cycle_on_failure == true) { + power_cycle_dut(conf, "RoT ensure persistence failed"); + } + return false; + } + + let final_rbi_result = util::reset_rot_and_get_rbi( + "after setting persistence", target_label + ); + if final_rbi_result?.error != () { + if (conf?.rot_hubris_power_cycle_on_failure == true) { + let error_type = final_rbi_result?.error_type; + if error_type == "reset_command_failed" || error_type == "rbi_fetch_failed" { + power_cycle_dut(conf, `RoT final reset/RBI read failed (${error_type})`); + } + } + return false; + } + let final_rbi = final_rbi_result.ok; + + rot_log_rbi_details( + final_rbi, "after final reset (persistence check)", + initial_slot, target_slot, use_transient + ); + if (!rot_validate_final_persistent_boot_state( + final_rbi, target_slot, target_label + )) { + if (conf?.rot_hubris_power_cycle_on_failure == true) { + power_cycle_dut(conf, "RoT final persistent boot validation failed"); + } + return false; + } + } else { + // Direct Persistent Boot Flow + if (!rot_validate_direct_persistent_boot_state( + rbi, target_slot, target_label + )) { + // No specific error type from this local helper yet to condition power cycle + if (conf?.rot_hubris_power_cycle_on_failure == true) { + power_cycle_dut(conf, "RoT direct persistent boot validation failed"); + } + return false; + } + } + + debug(`info|updated and booted target:${target_label} slot=${target_slot}`); + return true; +} + +/// Ensures the device is in a clean state and running baseline firmware. +/// This is a high-cost function that may use resets to sanitize the state. +/// It should be called once at the beginning of a test run. +fn ensure_initial_baseline_state(conf, images) { + // PRE-FLIGHT CHECK + if (!check_for_sp_debugger_and_sp_pending_update()) { + return false; + } + + debug("info|Phase 1: Sanitize state to establish a clean baseline."); + + // Step 1: Sanitize any lingering preferences from previous runs. + // This may reset the RoT. + if (!sanitize_boot_preferences(conf)) { + debug("error|Failed to sanitize boot preferences during initial setup."); + return false; + } + + // Step 2: Clear any stuck "Complete" update states. + // This may also reset the RoT and SP. + let problems = 0; + for attempt in 1..=2 { + problems = 0; + for component in ["rot", "sp"] { + let r_check = util::check_update_in_progress(component); + if r_check?.Err != () { + debug(`error|util::check_update_in_progress for ${component}: ${r_check}`); + return false; + } + if r_check != () { + problems += 1; + debug(`info|Component ${component} has update status: ${r_check}.`); + let id = r_check?.id; + if (id != ()) { + util::abort_update(component, `${id}`); + } + } + } + if problems == 0 { + break; + } + debug(`warn|Attempt ${attempt}: Found ${problems} issues. Resetting SP and RoT.`); + if (reset_sp_and_rot()?.error != ()) { + debug(`error|Cannot reset RoT/SP to clear update status.`); + return false; + } + } + if problems > 0 { + debug(`error|Devices stuck with update status after resets.`); + return false; + } + + // --- STABLE STATE --- + // At this point, all sanitizing operations and potential resets are complete. + // We can now safely check the device's state and act on it. + + debug("info|Phase 2: State is clean. Checking firmware versions against baseline."); + let initial_check = image_check("base", images); + if initial_check?.error != () { + debug(`error|Initial image_check for baseline: ${initial_check.error}`); + return false; + } + let flash_sp = initial_check.ok?.sp; + let flash_rot = initial_check.ok?.rot; + + if !flash_sp && !flash_rot { + debug("info|Device is already running correct baseline firmware."); + } else { + debug(`info|Device needs baseline flashing. SP: ${flash_sp}, RoT: ${flash_rot}`); + if flash_rot { + debug("info|Updating RoT to baseline (persistent)."); + if !update_rot_hubris(conf.base.rot_a, conf.base.rot_b, false, "baseline_setup", conf) { + debug("error|Failed to update RoT to baseline."); + return false; + } + } + if flash_sp { + debug("info|Updating SP to baseline."); + if !update_sp(conf.base.sp) { + debug("error|Failed to update SP to baseline."); + return false; + } + } + + // Final verification after flashing. + debug("info|Re-checking images after baseline updates."); + let post_update_check = image_check("base", images); + if post_update_check?.error != () || post_update_check.ok?.sp || post_update_check.ok?.rot { + debug(`warn|Post-baseline image_check FAILED or images are still not correct: ${post_update_check}`); + return false; + } + } + + debug("info|ensure_initial_baseline_state completed successfully."); + return true; +} + +/// Tests that a RoT update to a preferred slot fails, then recovers by +/// clearing the transient preference (without a reset) and successfully +/// completes the update. +/// +/// Args: +/// conf (map): The main configuration map. +/// target_branch_name (string): "base" or "ut", to select the correct image paths. +/// +/// Returns: +/// bool: `true` if the entire test passes, `false` otherwise. +fn test_and_recover_from_preferred_slot_update_failure(conf, target_branch_name) { + debug(`info|--- Starting Negative Test for '${target_branch_name}' (Reset-less Recovery) ---`); + + // 1. SETUP + let r = get_rot_active(); + if r?.error != () { return false; } + let initial_active_slot = r.ok; + let preferred_slot = if initial_active_slot == 0 { 1 } else { 0 }; + debug!(`//// conf=${conf}`); + debug!(`//// target_branch_name=${target_branch_name}`); + let branch_conf = conf[target_branch_name]; + debug!(`//// branch_conf=${branch_conf}`); + let image_path = if preferred_slot == 0 { branch_conf.rot_a } else { branch_conf.rot_b }; + let test_label = `neg_test_${target_branch_name}`; + + debug(`info|Negative Test: Initial active=${initial_active_slot}, setting transient pref to=${preferred_slot}`); + + // 2. INDUCE FAILURE: Set transient preference, then confirm update fails. + if (!util::set_rot_boot_preference(preferred_slot, true, test_label)) { + debug(`error|Negative Test: Could not set transient pref for '${test_label}'.`); + return false; + } + if (util::update_rot_image_file(preferred_slot, image_path, test_label)) { + debug(`error|Negative Test FAILED: Update to preferred slot '${preferred_slot}' SUCCEEDED.`); + return false; + } + debug("info|Negative Test PASSED: Update to preferred slot failed as expected."); + + // 3. RECOVER: Use the new helper to clear the preference without a reset. + if (!clear_transient_preference_without_reset()) { + debug("error|Negative Test FAILED: Could not recover by clearing transient preference."); + return false; + } + debug("info|Negative Test: Recovery successful. Conflicting preference cleared."); + + // 4. RETRY UPDATE: Attempt the same update again. It should now succeed. + debug(`info|Negative Test: Retrying update on slot ${preferred_slot}.`); + if (!util::update_rot_image_file(preferred_slot, image_path, test_label)) { + debug(`error|Negative Test FAILED: Update retry on slot ${preferred_slot} failed.`); + return false; + } + debug("info|Negative Test PASSED: Update retry succeeded."); + + // 5. FINALIZE (Optional but good practice): + // To leave the device in a predictable state, we can now set the preference + // and reset to actually boot the image we just flashed. + debug(`info|Negative Test: Finalizing by setting persistent pref to ${preferred_slot} and resetting.`); + if (!util::set_rot_boot_preference(preferred_slot, false, test_label)) { + return false; + } + let rbi_result = util::reset_rot_and_get_rbi("finalizing negative test", test_label); + if (rbi_result?.error != () || rbi_result.ok.active != preferred_slot) { + debug(`error|Negative Test FAILED: Could not boot into newly updated image. RBI: ${rbi_result}`); + return false; + } + + debug(`info|--- Negative Test for '${target_branch_name}' COMPLETED SUCCESSFULLY ---`); + return true; +} + +/// Clears an active transient boot preference without a reset by setting the +/// transient preference to the already-active slot. It then verifies that +/// the transient preference field in RotBootInfo is null. +/// +/// Returns: +/// bool: `true` if the preference was successfully cleared, `false` otherwise. +fn clear_transient_preference_without_reset() { + debug("info|Attempting to clear transient preference without reset."); + let r_active = get_rot_active(); + if r_active?.error != () { + debug(`error|clear_transient_preference: Could not get active slot: ${r_active.error}`); + return false; + } + let active_slot = r_active.ok; + + debug(`info|clearing transient pref by setting it to current active slot: ${active_slot}`); + if (!util::set_rot_boot_preference(active_slot, true, "clearing_transient")) { + debug("error|clear_transient_preference: Call to set_rot_boot_preference failed."); + return false; + } + + // Verify it was cleared + let rbi = util::rot_boot_info(); + if rbi?.error != () || rbi.transient_boot_preference != () { + debug(`error|clear_transient_preference: Verification failed. RBI: ${rbi}`); + return false; + } + + debug("info|Successfully cleared transient boot preference."); + return true; +} + +/// Ensures there are no conflicting boot preferences set for the inactive slot. +/// This function is intended to be called before any update operation to ensure +/// the device is in a clean state. +/// - If a conflicting transient preference is found, it is cleared without a reset. +/// - If a conflicting pending persistent preference is found, it uses a +/// workaround (#2093) or the ideal method based on the conf flag. +/// +/// Args: +/// conf (map): The configuration map, used to check for workarounds. +/// +/// Returns: +/// bool: `true` if the state is clean or was successfully cleaned, `false` otherwise. +fn sanitize_boot_preferences(conf) { + debug("info|Sanitizing boot preferences..."); + let rbi = util::rot_boot_info(); + if rbi?.error != () { + debug(`error|sanitize_boot_preferences: Could not get initial RBI: ${rbi.error}`); + return false; + } + + let active = rbi.active; + let inactive = if active == 0 { 1 } else { 0 }; + + // Case 1: Conflicting transient preference (clear is the same for both cases) + if rbi.transient_boot_preference == inactive { + debug(`warn|Found conflicting transient preference for inactive slot ${inactive}. Clearing it.`); + if (!clear_transient_preference_without_reset()) { + debug("error|sanitize_boot_preferences: Failed to clear conflicting transient preference."); + return false; + } + } + + // Case 2: Conflicting pending persistent preference + let rbi = util::rot_boot_info(); // Re-fetch state + if rbi.pending_persistent_boot_preference == inactive { + debug(`warn|Found conflicting pending persistent preference for inactive slot ${inactive}.`); + + if (conf?.hubris_2093_workaround) { + // --- WORKAROUND for Hubris issue #2093 --- + // The current firmware has a bug preventing a pending preference + // from being cleared by setting a new persistent one. + // The only reliable way to clear it is to reset the RoT. + debug("info|Hubris #2093 workaround enabled. Resetting RoT to clear pending preference."); + if !util::reset_component("rot") { + debug("error|sanitize_boot_preferences: Failed to reset RoT to clear pending preference."); + return false; + } + sleep(5); // Wait for RoT to come back up. + } else { + // --- IDEAL a.k.a. POST-BUGFIX LOGIC --- + // Once the firmware is fixed, we should be able to clear a pending + // preference for the inactive slot by simply re-setting the + // persistent preference to the already-active slot. + debug("info|Hubris #2093 workaround disabled. Attempting to clear pending preference without reset."); + if !util::set_rot_boot_preference(active, false, "clearing_pending") { + debug("error|sanitize_boot_preferences: Failed to send command to clear pending preference."); + return false; + } + } + + // Final verification + let final_rbi = util::rot_boot_info(); + if final_rbi?.error != () || final_rbi.pending_persistent_boot_preference != () { + debug(`error|sanitize_boot_preferences: Failed to verify pending pref was cleared. RBI: ${final_rbi}`); + return false; + } + } + + debug("info|Boot preferences sanitized successfully."); + return true; +} + +/// Checks for the specific case of an an SWD debugger attached to the SP and +/// the SP having a pending update. +/// Returns: +/// bool: `true` if no debugger and pending update detected, `false` otherwise. +// +/// Fixing Hubris issue 2066 will give us more definitive information to use in +/// testing. +fn check_for_sp_debugger_and_sp_pending_update() { + debug("info|Performing pre-flight check for attached SP debugger..."); + + // We call faux_mgs directly here instead of util::reset_component + // so we can inspect the specific error message. + let result = faux_mgs(["reset-component", "sp"]); + + if result?.ack == "reset" { + debug("info|SP reset successful. No debugger detected."); + // A reset here is fine and helps ensure a clean state. + return true; + } + + if result?.error != () { + // Use the 'message' key to get the detailed error string + let error_string = `${result.message}`; + if "the SP programming dongle is connected" in error_string { + debug("error|FATAL: SP Debugger Detected."); + debug("error|A JTAG/SWD debugger probe appears to be attached to the SP."); + debug("error|Please disconnect or power-off the debugger and restart the test."); + return false; + } else { + debug(`error|An unexpected error occurred while trying to reset the SP: ${result}`); + return false; + } + } + + // Should not be reached if the command returns a well-formed map. + debug(`warn|Unexpected result from SP reset command: ${result}`); + return false; +} + +/// Injects a fault by setting a conflicting pending persistent preference. +/// This sets up a state that should cause the next update attempt to fail. +/// +/// Returns: bool - true on success +fn inject_conflicting_pending_preference() { + debug("info|FAULT INJECTION: Setting a conflicting pending persistent preference."); + let rbi = util::rot_boot_info(); + if rbi?.error != () { return false; } + let inactive_slot = if rbi.active == 0 { 1 } else { 0 }; + // Set persistent preference for the INACTIVE slot. + // This creates a pending preference that conflicts with any update to that slot. + return util::set_rot_boot_preference(inactive_slot, false, "inject_pending_fault"); +} + +/// Injects a fault by setting a conflicting transient preference. +fn inject_conflicting_transient_preference() { + debug("info|FAULT INJECTION: Setting a conflicting transient preference."); + let rbi = util::rot_boot_info(); + if rbi?.error != () { return false; } + let inactive_slot = if rbi.active == 0 { 1 } else { 0 }; + // Set transient preference for the INACTIVE slot. + return util::set_rot_boot_preference(inactive_slot, true, "inject_transient_fault"); +} + +/// Checks if a specific fault injection test is enabled in the config. +/// +/// Args: +/// conf (map): The configuration map. +/// fault_name (string): The name of the fault to check (e.g., "pending"). +/// +/// Returns: +/// bool: `true` if the fault is in the list of tests to run. +fn is_fault_enabled(conf, fault_name) { + if conf?.faults_to_inject == () { + return false; + } + return fault_name in conf.faults_to_inject; +} diff --git a/scripts/upgrade-rollback.rhai b/scripts/upgrade-rollback.rhai new file mode 100644 index 0000000..f998008 --- /dev/null +++ b/scripts/upgrade-rollback.rhai @@ -0,0 +1,314 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// Copyright 2025 Oxide Computer Company + +import `${script_dir}/util` as util; +import `${script_dir}/update-helper` as helper; + +/// Print command line usage +fn usage(prog, error) { + if error != () { + debug(`error|${error}`); + } + // Standard help text is better suited for print() + print( + `Usage: faux-mgs ... rhai ${prog} [-v] [-h] [-c config.json] ` + + `[path0] [path1]` + ); + print(" -b BASELINE_PATH # Path to baseline hubris repo"); + print(" -c CONFIG.JSON # Path to configuration"); + print(" -h # Help. Print this message"); + print(" --hubris-2093 # Enable workaround for issue #2093 (pending pref bug)"); + print(" -f , --inject-fault # Inject fault(s). T is a comma-separated list,"); + print(" # e.g., 'pending,transient'"); + print(" -N # Run negative test for RoT update failure and recovery"); + print(" -t # Use transient boot preference to vet RoT Hubris"); + print(" -u UNDER_TEST_PATH # Path to under-test hubris repo"); + print(" -v # Verbose logging"); +} + +/// Parse command line options including the required / JSON configuration file. +fn process_cli(argv) { + let prog = argv[0]; + let options = "b:c:f:hNtu:v"; + let parsed = util::getopts(argv, options); + if parsed?["error"] != () { + usage(prog, parsed.error); + return 1; + } + + if parsed.result?["h"] == true { + usage(prog, ()); + return 0; + } + + let conf = #{}; + conf["verbose"] = parsed.result?["v"] == true; + conf["use_transient_boot_preference"] = parsed.result?["t"] == true; + conf["run_negative_tests"] = parsed.result?["N"] == true; + conf["hubris_2093_workaround"] = parsed.result?["hubris-2093"] == true; + + let conf_path_str = parsed.result?["c"]; + if conf_path_str == () { + usage(prog, "Missing required option: -c config.json"); + return 1; + } + let conf_path = path(conf_path_str); + if !conf_path.is_file { + usage(prog, `Config file not found or not a file: ${conf_path}`); + return 1; + } + conf["conf_path"] = conf_path; + + let conf_json = ""; + let read_ok = true; + try { + let conf_file = open_file(conf_path); + conf_json = conf_file.read_string(); + } catch (err) { + debug(`error|Cannot open or read config file ${conf_path}: ${err}`); + read_ok = false; + } + if !read_ok { return 1; } + + let config_from_file = json_to_map(conf_json); + if config_from_file?.error != () { + debug( + `error|Failed to parse JSON from config file: ${conf_path}` + + `: ${config_from_file.error}` + ); + return 1; + } + + conf += config_from_file; + + // --- FAULT INJECTION CONFIG --- + let faults_to_inject = conf?.fault_injection?.tests; + if faults_to_inject == () || type_of(faults_to_inject) != "array" { + faults_to_inject = []; + } + let cli_faults = parsed.result?["inject-fault"] ?? parsed.result?["f"]; + if cli_faults != () { + faults_to_inject = []; // CLI overrides config file. + for fault in cli_faults.split(',') { + let f = fault; + f.trim(); + if f != "" && f !in faults_to_inject { + faults_to_inject.push(f); + } + } + } + conf["faults_to_inject"] = faults_to_inject; + + let base_repo_override = parsed?.result?["b"]; + if base_repo_override != () { + debug(`info|Overriding config 'base_repo' from command line:`); + debug(`info| was: ${conf?.base_repo}`); + conf.base_repo = base_repo_override; + debug(`info| now: ${conf.base_repo}`); + } + + let ut_repo_override = parsed?.result?["u"]; + if ut_repo_override != () { + debug(`info|Overriding config 'ut_repo' from command line:`); + debug(`info| was: ${conf?.ut_repo}`); + conf.ut_repo = ut_repo_override; + debug(`info| now: ${conf.ut_repo}`); + } + + // --- Expand paths --- + conf["sp_bord"] = util::env_expand(conf?.bord?.sp, conf); + conf["rot_bord"] = util::env_expand(conf?.bord?.rot, conf); + + for branch in ["base", "ut"] { + conf[branch] = #{}; + for image_type in ["sp", "rot_a", "rot_b", "stage0"] { + let image_path_template = conf?.images?[branch]?[image_type]; + if image_path_template == () { + conf[branch][image_type] = (); + continue; + } + let zip_path = util::env_expand(image_path_template, conf); + if zip_path == () { + debug( + `error|process_cli: Failed expanding path for ${branch}.${image_type}: ` + + `'${image_path_template}'` + ); + conf[branch][image_type] = (); + } else { + conf[branch][image_type] = zip_path; + } + } + } + + debug("trace|Fully resolved configuration map (process_cli): " + conf); + return conf; +} + +/// Contains the main upgrade/rollback test procedure. +fn run_standard_test_flow(conf, images) { + let start_ts = timestamp(); + + if (!helper::ensure_initial_baseline_state(conf, images)) { + debug("error|Failed to ensure initial baseline state. Exiting test run."); + return 1; + } + + debug(`info|Elapsed time after setup: ${start_ts.elapsed}`); + debug(`info|Current time: ${datetime_local()}`); + + // Main upgrade/rollback loop + for v_params in [ + #{ + "up_down": "upgrade", "label": "under-test", "branch": "ut", + "sp_path": conf.ut.sp, "rot_a_path": conf.ut.rot_a, "rot_b_path": conf.ut.rot_b, + }, + #{ + "up_down": "rollback", "label": "baseline", "branch": "base", + "sp_path": conf.base.sp, "rot_a_path": conf.base.rot_a, "rot_b_path": conf.base.rot_b, + } + ] { + debug( + `info|\n--- Starting ${v_params.up_down} to ${v_params.label} ` + + `(${v_params.branch} branch versions) ---` + ); + + let supports_transient = helper::rot_supports_transient_boot_preference(); + + if (conf.run_negative_tests) { + debug(`info|Checking transient support on current RoT for '${v_params.label}' negative test.`); + if (supports_transient) { + if (!helper::test_and_recover_from_preferred_slot_update_failure(conf, v_params.branch)) { + debug(`error|Negative test for '${v_params.label}' FAILED.`); + return 1; + } + debug(`info|Negative test for '${v_params.label}' PASSED. Resetting to known good state.`); + if (!helper::ensure_initial_baseline_state(conf, images)) { return 1; } + } else { + if (v_params.up_down == "rollback") { + debug(`error|FATAL: The under-test image was active but does not support transient preference.`); + return 1; + } else { + debug(`warn|Skipping negative test for '${v_params.label}': currently running baseline firmware does not support transient preference.`); + } + } + } + + let use_transient_for_this_op = false; + if conf.use_transient_boot_preference { + if (supports_transient) { + debug(`info|Attempting transient update to ${v_params.label}.`); + use_transient_for_this_op = true; + } else { + debug(`warn|Current RoT does NOT support transient commands. Using persistent update.`); + } + } else { + debug(`info|Transient preference not enabled via -t. Using persistent update.`); + } + + if !helper::update_rot_hubris( + v_params.rot_a_path, v_params.rot_b_path, + use_transient_for_this_op, v_params.label, conf + ) { + debug(`error|Failed to ${v_params.up_down} RoT to ${v_params.label}.`); + return 1; + } + + if !helper::update_sp(v_params.sp_path) { + debug(`error|Failed to ${v_params.up_down} SP to ${v_params.label}.`); + return 1; + } + + let result_loop_check = helper::image_check(v_params.branch, images); + if result_loop_check?.error != () || + result_loop_check.ok?.sp || + result_loop_check.ok?.rot + { + debug( + `error|image_check error or failed ${v_params.label} updates: ` + + `${result_loop_check}` + ); + return 1; + } + debug( + `info|SUCCESS: ${v_params.up_down} to SP and RoT ${v_params.label} ` + + `images completed successfully.` + ); + } + + debug(`info|\nElapsed time total: ${start_ts.elapsed}`); + debug(`info|Current time: ${datetime_local()}`); + debug("info|Done."); + return 0; +} + +/// Main entry point and test runner. +fn main() { + debug(`info|Starting upgrade-rollback script at ${datetime_local()}`); + + let conf = process_cli(argv); + if type_of(conf) == "i64" { return conf; } + + let images = helper::get_image_info(conf); + if type_of(images) == "i64" { return images; } + + if (conf.faults_to_inject.len() > 0) { + // --- FAULT INJECTION MODE --- + debug("info|--- Executing in Fault Injection & Recovery Mode ---"); + debug(`info|Faults to inject: ${conf.faults_to_inject}`); + let failed_tests = []; + let passed_tests = []; + + if (helper::is_fault_enabled(conf, "pending")) { + debug("info|\n--- FAULT INJECTION: PENDING PREFERENCE ---"); + if (!helper::ensure_initial_baseline_state(conf, images)) { return 1; } + if (!helper::inject_conflicting_pending_preference()) { + debug("error|Failed to inject pending preference fault."); + failed_tests.push("pending"); + } else { + debug("info|Fault injected. Running standard test flow to verify recovery."); + if (run_standard_test_flow(conf, images) == 0) { + debug("info|SUCCESS: System recovered from 'pending' fault and passed tests."); + passed_tests.push("pending"); + } else { + debug("error|FAILURE: System did NOT recover from 'pending' fault."); + failed_tests.push("pending"); + } + } + } + + if (helper::is_fault_enabled(conf, "transient")) { + debug("info|\n--- FAULT INJECTION: TRANSIENT PREFERENCE ---"); + if (!helper::ensure_initial_baseline_state(conf, images)) { return 1; } + if (!helper::inject_conflicting_transient_preference()) { + debug("error|Failed to inject transient preference fault."); + failed_tests.push("transient"); + } else { + debug("info|Fault injected. Running standard test flow to verify recovery."); + if (run_standard_test_flow(conf, images) == 0) { + debug("info|SUCCESS: System recovered from 'transient' fault and passed tests."); + passed_tests.push("transient"); + } else { + debug("error|FAILURE: System did NOT recover from 'transient' fault."); + failed_tests.push("transient"); + } + } + } + + debug("info|\n--- FAULT INJECTION SUMMARY ---"); + if (passed_tests.len() > 0) { + debug(`info|PASSED: ${passed_tests}`); + } + if (failed_tests.len() > 0) { + debug(`error|FAILED: ${failed_tests}`); + return 1; + } + return 0; + + } else { + // --- STANDARD MODE --- + debug("info|--- Executing in Standard Upgrade/Rollback Mode ---"); + return run_standard_test_flow(conf, images); + } +} diff --git a/scripts/util.rhai b/scripts/util.rhai new file mode 100644 index 0000000..bfbf1ca --- /dev/null +++ b/scripts/util.rhai @@ -0,0 +1,1178 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// Copyright 2025 Oxide Computer Company + +// Define constants used in this module +const ROT_FLASH_PAGE_SIZE = 512; + +/// Converts an array or blob of byte-like values into a hexadecimal string. +/// Each byte is represented by two hexadecimal characters. +/// Throws an error if any element cannot be converted to hex or if .to_hex() +/// does not produce a string. +/// +/// Args: +/// a (array | blob): The input array or blob. Elements are expected to +/// have a `.to_hex()` method. +/// +/// Returns: +/// string: The resulting hexadecimal string on success. +/// string: "invalid_input_type" if 'a' is not an array or blob. +/// string: An empty string if 'a' is empty or null. +/// +/// Throws: +/// Map: `#{ error: "message", value: original_value }` if an element +/// cannot be converted. +fn to_hexstring(a) { + let s = ""; + let type_of_a = type_of(a); + if type_of_a != "array" && type_of_a != "blob" { + debug(`warn|to_hexstring: Input is not array or blob: ${type_of_a}`); + return "invalid_input_type"; + } + if a == () || a.len() == 0 { + return ""; + } + + for b in a { + let h = (); + try { + h = b.to_hex(); + } catch (err) { + let error_msg = + `Value cannot be converted to hex: ${b} (original error: ${err})`; + debug(`error|to_hexstring: ${error_msg}`); + throw #{ + "error": "to_hexstring: " + error_msg, + "value": b + }; + } + + if type_of(h) == "string" { + if h.len() == 1 { + s += "0"; // Pad with leading zero if necessary + } + s += h; + } else { + let error_msg = + `.to_hex() did not return a string for value: ${b}. Got type: ${type_of(h)}`; + debug(`error|to_hexstring: ${error_msg}`); + throw #{ + "error": "to_hexstring: " + error_msg, + "value": b, + "hex_result_type": type_of(h) + }; + } + } + return s.to_string(); +} + +/// Converts a C-style null-terminated string from an array or blob of bytes +/// into a Rhai string. Processing stops at the first NUL (0x00) character. +/// +/// Args: +/// a (array | blob): The input array or blob containing byte-like values. +/// +/// Returns: +/// string: The resulting string. Returns an empty string if the input is +/// invalid, null, or if non-integer/out-of-byte-range values +/// are encountered before a NUL terminator. +fn cstring_to_string(a) { + let type_of_a = type_of(a); + if type_of_a != "array" && type_of_a != "blob" { + debug(`warn|cstring_to_string: Input not array or blob: ${type_of_a}`); + return ""; + } + if a == () { + return ""; + } + + let data = blob(); + for b_val in a { + let int_b = b_val; + if type_of(b_val) != "int" { + debug(`warn|cstring_to_string: Non-integer value in array: ${b_val}, skipping.`); + continue; + } + if int_b == 0 { // NUL terminator + break; + } + if int_b < 0 || int_b > 255 { // Ensure it's a valid byte value + debug(`error|cstring_to_string: Value out of byte range: ${int_b}, skipping.`); + continue; + } + data += int_b; + } + return data.as_string(); +} + +/// Formats a byte array or blob into an ASCII MAC address string. +/// Example: "00:11:22:AA:BB:CC". +/// +/// Args: +/// a (array | blob): The input array/blob, typically 6 bytes long, +/// containing byte values. +/// +/// Returns: +/// string: The formatted MAC address string. Returns an empty string if input +/// is invalid or "format_error:..." if elements are not integers. +fn array_to_mac(a) { + let type_of_a = type_of(a); + if type_of_a != "array" && type_of_a != "blob" { + debug(`warn|array_to_mac: Input not array or blob: ${type_of_a}`); + return ""; + } + if a == () || a.len() == 0 { + return ""; + } + + let first_byte = a[0]; + if type_of(first_byte) != "int" { + debug(`error|array_to_mac: First element is not an integer.`); + return "format_error:non_integer_element"; + } + let mac_str = first_byte.to_hex(); + if mac_str.len() == 1 { + mac_str = "0" + mac_str; + } + + for i in 1..a.len() { + mac_str += ":"; + let byte_val = a.get(i); + if type_of(byte_val) != "int" { + debug(`error|array_to_mac: Element at index ${i} is not an integer.`); + return "format_error:non_integer_element"; + } + let hex_part = byte_val.to_hex(); + if hex_part.len() == 1 { + mac_str += "0"; + } + mac_str += hex_part; + } + return mac_str; +} + +/// Translates RoT bank designators ("A", "B", or null/unit type) to +/// numeric (0, 1) or null/unit type `()`. +/// +/// Args: +/// v (string | ()): The bank designator, typically "A", "B", or `()`. +/// +/// Returns: +/// int | (): `0` for "A", `1` for "B", or `()` if input is `()` or unrecognized. +fn ab_to_01(v) { + switch v { + "A" => { + return 0; + } + "B" => { + return 1; + } + () => { + return (); + } + _ => { + debug(`warn|ab_to_01: Received unexpected value: ${v}, returning ().`); + return (); + } + } +} + +/// Expands variables in a string using environment variables and an override map. +/// Variables are denoted by `${VAR_NAME}` (for environment or top-level override) +/// or `${map.nested.key}` (for nested keys in the override map). +/// Values from `override_map` take precedence over environment variables. +/// +/// Args: +/// s (string): The input string containing variables to expand. +/// override_map (map): A map of key-value pairs for overrides. +/// +/// Returns: +/// string: The string with variables expanded. +/// (): Returns `()` if the input string `s` is null or not a string, or if +/// output string grows beyond a safety limit (2048 chars). +/// Unfound variables are treated as literal text (e.g., "${UNFOUND_VAR}"). +fn env_expand(s, override_map) { + if s == () { + debug("error|env_expand: Input string is null."); + return (); + } + if type_of(s) != "string" { + debug(`error|env_expand: Requires a string input, got: ${type_of(s)}`); + return (); + } + + let out = ""; + let remain = s; + let envmap = envs(); // Cache environment variables map + + while remain.len() > 0 { + if out.len() > 2048 { // Basic guard against runaway expansion + debug( + `error|env_expand: Output exceeded safety limit during ` + + `expansion of original string: ${s}` + ); + return (); + } + + let start_index = remain.index_of("${"); + if start_index == -1 { + out += remain; + return out; // No more variables found + } + + out += remain.sub_string(0, start_index); + remain = remain.sub_string(start_index + 2, remain.len()); // Skip "${" + + let end_index = remain.index_of("}"); + if end_index == -1 { + debug(`warn|env_expand: Missing closing '}' in expansion: ${s}`); + out += "${" + remain; // Treat as literal + remain = ""; // Stop processing + continue; + } + + let key = remain.sub_string(0, end_index); + remain = remain.sub_string(end_index + 1, remain.len()); // Skip "}" + + let value = (); + let found = false; + + // Check override_map (supports nested keys like "path.to.value") + if key.contains(".") { + let key_parts = key.split("."); + let current_val = override_map; + let path_valid = true; + for part in key_parts { + if type_of(current_val) == "map" && (part in current_val) { + current_val = current_val[part]; + } else { + path_valid = false; + break; + } + } + if path_valid { + value = current_val; + found = true; + } + } else { + // Check top-level in override_map + if type_of(override_map) == "map" && (key in override_map) { + value = override_map[key]; + found = true; + } + } + + // If not in override_map, check environment variables + if !found { + if key in envmap { + value = envmap[key]; // Use cached envmap + found = true; + } + } + + if found { + if type_of(value) != "string" { + value = value.to_string(); + } + remain = value + remain; // Prepend value for further expansion + } else { + debug(`warn|env_expand: Cannot expand variable '${key}' in: ${s}`); + out += "${" + key + "}"; // Treat unfound variable as literal + } + } + return out; +} + + +// --- Wrapped faux_mgs functions --- + +/// Reads RoT Boot Information (RBI) using `faux_mgs rot-boot-info --version 3`. +/// Parses the V3 structure into a more script-friendly map. +/// +/// Returns: +/// Map: A map containing RBI fields (`active`, `persistent_boot_preference`, +/// `slot_a.fwid`, etc.) on success. +/// Map: An error map like `#{ error: "message" }` or the direct error map +/// from `faux_mgs` on failure. +fn rot_boot_info() { + let r = faux_mgs(["rot-boot-info", "--version", "3"]); + if r?.error != () { + debug(`error|util::rot_boot_info: faux_mgs command failed: ${r.error}`); + return r; + } + if r?.V3 == () || r.V3?.active == () { + let err_msg = "Failed to parse V3 rot-boot-info or missing key fields."; + debug(`error|util::rot_boot_info: ${err_msg} Raw Response: ${r}`); + return #{"error": err_msg}; + } + + let v3 = r.V3; + let rbi = #{ + active: ab_to_01(v3.active), + persistent_boot_preference: ab_to_01(v3.persistent_boot_preference), + pending_persistent_boot_preference: + ab_to_01(v3.pending_persistent_boot_preference), + transient_boot_preference: ab_to_01(v3.transient_boot_preference), + slot_a: #{ + fwid: to_hexstring(v3?.slot_a_fwid?.Sha3_256), + status: v3?.slot_a_status, + }, + slot_b: #{ + fwid: to_hexstring(v3?.slot_b_fwid?.Sha3_256), + status: v3?.slot_b_status, + }, + stage0: #{ + fwid: to_hexstring(v3?.stage0_fwid?.Sha3_256), + status: v3?.stage0_status, + }, + stage0next: #{ + fwid: to_hexstring(v3?.stage0next_fwid?.Sha3_256), + status: v3?.stage0next_status, + }, + }; + if rbi.active == () { + debug(`error|util::rot_boot_info: Parsed RBI is missing a valid 'active' slot.`); + return #{"error": "Parsed RBI missing valid 'active' slot."}; + } + return rbi; +} + +/// Reads a specific key from a component's caboose. +/// Wraps `faux_mgs read-component-caboose ...`. +/// +/// Args: +/// component (string): The component name (e.g., "sp", "rot"). +/// slot (string | int): The slot identifier (e.g., "0", "1", "active"). +/// key (string): The caboose key to read (e.g., "GITC", "VERS"). +/// +/// Returns: +/// string: The value of the caboose key if found. +/// (): Returns null/unit `()` if the key is not found or an error occurs. +fn caboose_value(component, slot, key) { + let r = faux_mgs([ + "read-component-caboose", "--component", component, + "--slot", `${slot}`, key // Ensure slot is stringified + ]); + if r?.error != () || r?.value == () { + debug( + `warn|util::caboose_value: Failed to read key '${key}' for ` + + `${component} slot ${slot}. Result: ${r}` + ); + return (); + } + return r.value; +} + +/// Reads caboose information for standard components (RoT, SP, Stage0). +/// Iteratively calls `util::caboose_value`. +/// +/// Returns: +/// Map: A nested map structured as `#{ component: #{ slot: #{ key: value } } }`. +/// Missing values will not be present in the inner maps. +fn get_device_cabooses() { + let caboose_data = #{}; + let components_slots = #{ + "rot": ["0", "1"], + "sp": ["0"], + "stage0": ["0"], + }; + + for component in components_slots.keys() { + caboose_data[component] = #{}; + for slot in components_slots[component] { + caboose_data[component][slot] = #{}; + for key in ["BORD", "GITC", "VERS", "NAME", "SIGN"] { + let value = caboose_value(component, slot, key); + if value != () { + caboose_data[component][slot][key] = value; + } + } + } + } + return caboose_data; +} + +/// Translates a Root Key Table Hash (RKTH) string to a well-known keyset name. +/// +/// Args: +/// rkth (string): The RKTH hex string. +/// +/// Returns: +/// string: The human-readable keyset name if known; otherwise, the original rkth. +/// Returns the input if it's not a string. +fn rkth_to_key_name(rkth) { + if type_of(rkth) != "string" { + debug(`warn|rkth_to_key_name: Expected string input, got ${type_of(rkth)}`); + return rkth; + } + let known_rkths = #{ + "84332ef8279df87fbb759dc3866cbc50cd246fbb5a64705a7e60ba86bf01c27d": "Bart", + "11594bb5548a757e918e6fe056e2ad9e084297c9555417a025d8788eacf55daf": "StagingDevGimlet", + "1432cc4cfe5688c51b55546fe37837c753cfbc89e8c3c6aabcf977fdf0c41e27": "StagingDevSidecar", + "f592d8f109b81881221eed5af6438abad9b5df8c220b9129c03763e7e10b22c7": "StagingDevPSC", + "31942f8d53dc908c5cb338bdcecb204785fa87834e8b18f706fc972a42886c8b": "ProdRelPSC", + "5796ee3433f840519c3bcde73e19ee82ccb6af3857eddaabb928b8d9726d93c0": "ProdRelGimlet", + "5c69a42ee1f1e6cd5f356d14f81d46f8dbee783bb28777334226c689f169c0eb": "ProdRelSidecar" + }; + + if rkth in known_rkths { + return known_rkths[rkth]; + } else { + debug(`info|rkth_to_key_name: Unrecognized RKTH: ${rkth}`); + return rkth; + } +} + +/// Converts an array of numbers (typically from JSON) into a Rhai blob. +/// +/// Args: +/// a (array): The input array of numbers. +/// +/// Returns: +/// blob: The created blob. Returns an empty blob if input is not an array. +fn array_to_blob(a) { + if type_of(a) != "array" { + debug(`warn|array_to_blob: Input is not an array, type=${type_of(a)}`); + return blob(); + } + let out = blob(); + for byte_val in a { + // Assumes elements are directly compatible with blob.push (e.g., integers) + out.push(byte_val); + } + return out; +} + +/// Reads the RoT Customer Manufacturing Page Area (CMPA) as a blob. +/// Calls `faux_mgs read-cmpa`. Validates the size. +/// +/// Returns: +/// blob: The CMPA data as a blob on success. +/// (): Returns null/unit `()` on failure (e.g., command error, wrong size). +fn get_cmpa() { + let r = faux_mgs(["read-cmpa"]); + let cmpa_array = r?["cmpa"]; + + if r?.error != () { + debug(`warn|util::get_cmpa: Failed to read CMPA. Error: ${r.error}`); + return (); + } + if cmpa_array == () || type_of(cmpa_array) != "array" { + debug( + `error|util::get_cmpa: Missing or invalid 'cmpa' field in response. ` + + `Result: ${r}` + ); + return (); + } + + let cmpa_blob = array_to_blob(cmpa_array); + if cmpa_blob.len() != global::ROT_FLASH_PAGE_SIZE { + debug( + `warn|util::get_cmpa: Invalid CMPA blob size ` + + `(${cmpa_blob.len()} != ${global::ROT_FLASH_PAGE_SIZE}).` + ); + return (); + } + return cmpa_blob; +} + +/// Reads the RoT Customer Factory Page Area (CFPA) as a blob. +/// Calls `faux_mgs read-cfpa`. Validates the size. +/// +/// Returns: +/// blob: The CFPA data as a blob on success. +/// (): Returns null/unit `()` on failure. +fn get_cfpa() { + let r = faux_mgs(["read-cfpa"]); + let cfpa_array = r?["cfpa"]; + + if r?.error != () { + debug(`warn|util::get_cfpa: Failed to read CFPA. Error: ${r.error}`); + return (); + } + if cfpa_array == () || type_of(cfpa_array) != "array" { + debug( + `error|util::get_cfpa: Missing or invalid 'cfpa' field in response.`+ + ` Result: ${r}` + ); + return (); + } + + let cfpa_blob = array_to_blob(cfpa_array); + if cfpa_blob.len() != global::ROT_FLASH_PAGE_SIZE { + debug( + `error|util::get_cfpa: Invalid CFPA blob size `+ + `(${cfpa_blob.len()} != ${global::ROT_FLASH_PAGE_SIZE}).` + ); + return (); + } + return cfpa_blob; +} + +/// Extracts the RoT keyset name from a provided CMPA blob. +/// +/// Args: +/// cmpa_blob (blob): The CMPA data, expected to be `ROT_FLASH_PAGE_SIZE` bytes. +/// +/// Returns: +/// string: The well-known keyset name, the RKTH hex string if unrecognized, +/// or "unknown" if extraction fails or CMPA is invalid. +fn get_rot_keyset(cmpa_blob) { + if type_of(cmpa_blob) != "blob" || + cmpa_blob.len() != global::ROT_FLASH_PAGE_SIZE + { + debug( + `warn|util::get_rot_keyset: Invalid CMPA blob provided ` + + `(type: ${type_of(cmpa_blob)}, len: ${cmpa_blob.len()}).` + ); + return "unknown_cmpa_format"; + } + // RKTH is 32 bytes starting at offset 80 in CMPA + let rkth_blob = cmpa_blob.extract(80, 32); + if rkth_blob.len() != 32 { + debug(`error|util::get_rot_keyset: Extracted RKTH blob is not 32 bytes.`); + return "unknown_rkth_extraction"; + } + return rkth_to_key_name(to_hexstring(rkth_blob)); +} + + +// --- Functions for specific script logic (e.g., upgrade-rollback) --- + +/// Checks for an update in progress for a given component. +/// Wraps `faux_mgs update-status `. +/// +/// Args: +/// component (string): The component name (e.g., "sp", "rot"). +/// +/// Returns: +/// Map: Parsed status like `#{ state: "InProgress", id: "...", ... }` or +/// `#{ state: "Complete", id: "..." }`. +/// (): If no update is in progress (typically `{"Ok": "None"}`). +/// Map: An error map (e.g., `#{ Err: ... }` or `#{ "Err": ... }`) on +/// command failure or if the response structure is unrecognized. +fn check_update_in_progress(component) { + let r = faux_mgs(["update-status", component]); + debug( + `info|util::check_update_in_progress: raw update_status(${component}) `+ + `result = ${r}` + ); + + if r?.Err != () { + debug( + `error|util::check_update_in_progress: faux_mgs error for `+ + `${component}: ${r}` + ); + return r; + } + + if r?.Ok != () { + let ok_value = r.Ok; + if type_of(ok_value) == "string" && ok_value == "None" { + debug( + `info|util::check_update_in_progress: no update for `+ + `${component} (Ok: "None").` + ); + return (); + } + if type_of(ok_value) == "map" { + if ok_value?.InProgress != () { + let details = ok_value.InProgress; + debug( + `info|util::check_update_in_progress: update InProgress `+ + `(Ok.InProgress) for ${component}: ${details}` + ); + return #{ + "state": "InProgress", "id": to_hexstring(details.id), + "bytes_received": details.bytes_received, + "total_size": details.total_size + }; + } + if ok_value?.Complete != () { + let id = ok_value.Complete; + debug( + `info|util::check_update_in_progress: update Complete `+ + `(Ok.Complete) for ${component}: ${id}` + ); + return #{ "state": "Complete", "id": to_hexstring(id) }; + } + } + } else { // No "Ok" key, check for top-level status + if r?.InProgress != () { + let details = r.InProgress; + debug( + `info|util::check_update_in_progress: update InProgress `+ + `(top-level) for ${component}: ${details}` + ); + return #{ + "state": "InProgress", "id": to_hexstring(details.id), + "bytes_received": details.bytes_received, + "total_size": details.total_size + }; + } + if r?.Complete != () { + let id = r.Complete; + debug( + `info|util::check_update_in_progress: update Complete `+ + `(top-level) for ${component}: ${id}` + ); + return #{ "state": "Complete", "id": to_hexstring(id) }; + } + } + + debug( + `error|util::check_update_in_progress: unrecognized structure for ` + + `${component}: ${r}` + ); + return #{"Err": `unrecognized update-status structure: ${r}`}; +} + +/// Updates a RoT image file on a specified slot. +/// Wraps `faux_mgs update rot `. +/// +/// Args: +/// slot (int): The RoT slot to update (0 or 1). +/// image_path (string): Path to the RoT image file. +/// label (string): Descriptive label for logging (e.g., "baseline"). +/// +/// Returns: +/// true if the command was acknowledged as "updated", false otherwise. +fn update_rot_image_file(slot, image_path, label) { + debug(`info|Updating RoT slot ${slot} with ${image_path} (target '${label}').`); + let r = faux_mgs(["update", "rot", `${slot}`, image_path]); + debug(`info|Result for '${label}': ${r}`); + if r?.ack == "updated" { + return true; + } else { + debug(`error|RoT update failed for '${label}' slot ${slot}): ${r}`); + return false; + } +} + +/// Sets the RoT boot preference (transient or persistent). +/// Wraps `faux_mgs component-active-slot -s rot`. +/// +/// Args: +/// target_slot (int): The RoT slot to set preference for. +/// use_transient_flag (bool): If true, sets transient ('-t') preference; +/// otherwise, persistent ('-p'). +/// target_label (string): Descriptive label for logging. +/// +/// Returns: +/// true if command acknowledged as "set" for correct slot, false otherwise. +fn set_rot_boot_preference(target_slot, use_transient_flag, target_label) { + let duration_param = if use_transient_flag { "-t" } else { "-p" }; + debug( + `info|util::set_rot_boot_preference: Setting RoT pref: `+ + `slot ${target_slot}, type ${duration_param} (target '${target_label}').` + ); + let r_set_slot = faux_mgs([ + "component-active-slot", duration_param, "-s", `${target_slot}`, "rot" + ]); + debug( + `info|util::set_rot_boot_preference: Result for '${target_label}': `+ + `${r_set_slot}` + ); + if r_set_slot?.ack == "set" && r_set_slot?.slot == target_slot { + return true; + } else { + debug( + `error|util::set_rot_boot_preference: Failed for '${target_label}' `+ + `(slot ${target_slot}, type ${duration_param}). Result: ${r_set_slot}` + ); + return false; + } +} + +/// Reset the RoT, wait, get RotBootInfo (RBI), and validate RBI. +/// Wraps `faux_mgs reset-component rot` and calls `util::rot_boot_info()`. +/// +/// Args: +/// reset_description (string): Description for logging (e.g., "after update"). +/// target_label (string): Descriptive label for logging. +/// +/// Returns: +/// Map: `#{ "ok": rbi_map }` on full success, where `rbi_map` is the parsed RBI. +/// Map: An error map on any failure, structured as +/// `#{ "error": "message", "error_type": "type", "details": original_map_if_any }`. +/// Possible `error_type` values include: "reset_command_failed", +/// "reset_ack_mismatch", "rbi_fetch_failed", "rbi_invalid_structure". +fn reset_rot_and_get_rbi(reset_description, target_label) { + debug(`info|Resetting RoT: ${reset_description} (target '${target_label}')`); + + let r_reset = faux_mgs(["reset-component", "rot"]); + + // Check 1: Did the faux_mgs command itself return an error? + if r_reset?.Err != () { + let err_msg = `RoT reset command failed. Error: ${r_reset.Err}`; + debug(`error|${err_msg}`); + return #{ + "error": err_msg, + "error_type": "reset_command_failed", + "details": r_reset // Contains the original "Err" from faux_mgs + }; + } + + // Check 2: Did the command succeed but not give the expected ack? + if r_reset?.ack != "reset" { + let err_msg = + `RoT reset command did not return expected 'ack: "reset"'. ` + + `Received: ${r_reset}`; + debug(`error|${err_msg}`); + return #{ + "error": err_msg, + "error_type": "reset_ack_mismatch", + "details": r_reset + }; + } + + debug( + `info|RoT reset command acknowledged. Waiting for RoT to boot ` + + `(context: ${reset_description}).` + ); + // TODO: Consider replacing this fixed sleep with active polling for RoT readiness + // if a lightweight "ping" or status command for RoT becomes available. + sleep(5); + + // util::rot_boot_info() itself returns either an error map or the RBI map. + let rbi_result = util::rot_boot_info(); + + if rbi_result?.error != () { + // This means rot_boot_info detected an error (either its own parsing + // or an error propagated from its internal faux_mgs call). + let err_msg = + `Failed to get valid RBI after reset (${reset_description}, ` + + `target '${target_label}'). Underlying error: ${rbi_result.error}`; + debug(`error|${err_msg}`); + let error_type = if rbi_result?.error_type != () { + rbi_result.error_type + } else { + "rbi_fetch_failed" + }; + return #{ + "error": err_msg, + // Attempt to preserve original error type if possible, or use a general one + "error_type": error_type, + "details": rbi_result // Contains the original error map from rot_boot_info + }; + } + // If no 'error' key, rbi_result *is* the RBI map. + // The current util::rot_boot_info already checks rbi.active internally + // and returns an error map if it's missing/invalid. + + debug( + `info|Successfully reset RoT and fetched RBI ` + + `(${reset_description}, target '${target_label}'). Active: ${rbi_result.active}` + ); + // Standardize success return + return #{ "ok": rbi_result }; +} + +/// Updates the SP image file (assumes slot 0). +/// Wraps `faux_mgs update sp 0 `. +/// +/// Args: +/// image_path (string): Path to the SP image file. +/// +/// Returns: +/// true if acknowledged as "updated", false otherwise. +fn update_sp_image(image_path) { + debug(`info|util::update_sp_image: Updating SP with image ${image_path}`); + let r_update = faux_mgs(["update", "sp", "0", image_path]); + debug(`info|util::update_sp_image: Result of 'update sp': ${r_update}`); + if r_update?.ack == "updated" { + return true; + } else { + debug(`error|SP image update command failed. Result: ${r_update}`); + return false; + } +} + +/// Resets the Service Processor (SP). +/// Wraps `faux_mgs reset`. +/// +/// Returns: +/// true if acknowledged as "reset", false otherwise. +fn reset_sp() { + debug(`info|util::reset_sp: Resetting SP.`); + let r_reset = faux_mgs(["reset"]); + debug(`info|util::reset_sp: Result of 'reset': ${r_reset}`); + if r_reset?.ack == "reset" { + return true; + } else { + let err_msg = + `SP reset command failed or gave unexpected ack. Result: ${r_reset}`; + debug(`error|util::reset_sp: ${err_msg}`); + return false; + } +} + +/// Aborts an update in progress for a given component and update ID. +/// Wraps `faux_mgs update-abort `. +/// +/// Args: +/// component (string): Component (e.g., "sp", "rot"). +/// id (string): Hex string ID of the update to abort. +/// +/// Returns: +/// Map: The result map from the `faux_mgs` command (includes `ack` or `error`). +fn abort_update(component, id) { + debug( + `info|util::abort_update: Aborting update for component '${component}' ` + + `with ID '${id}'.` + ); + let result = faux_mgs(["update-abort", component, id]); + debug( + `info|util::abort_update: Result for component '${component}', `+ + `ID '${id}': ${result}` + ); + return result; +} + +/// Resets a specified component (e.g., "sp", "rot"). +/// Wraps `faux_mgs reset-component `. +/// +/// Args: +/// component_name (string): The name of the component to reset. +/// +/// Returns: +/// true if acknowledged as "reset", false otherwise (errors logged/printed). +fn reset_component(component_name) { + debug( + `info|util::reset_component: Attempting to reset ` + + `component '${component_name}'.` + ); + let result = faux_mgs(["reset-component", component_name]); + debug( + `info|util::reset_component: Result for 'reset-component ` + + `${component_name}': ${result}` + ); + + if result?.ack == "reset" { + debug( + `info|util::reset_component: Component '${component_name}' ` + + `reset acknowledged.` + ); + return true; + } else if result?.error != () { + debug(`error|reset component ${component_name}: ${result.error}`); + return false; + } else { + debug(`error|reset ${component_name}: result=${result}`); + return false; + } +} + +/// Retrieves and expands a power control command array from the configuration. +/// Each element of the command array will be expanded using `util::env_expand`. +/// (This function remains largely the same as proposed in Turn 52, ensuring +/// it returns () on errors which control_power will then handle.) +/// +/// Args: +/// conf (map): The main configuration map. +/// target_device (string): Key for the target device in `conf.power_control`. +/// action (string): Power action ("on", "off", "status"). +/// +/// Returns: +/// array: An array of strings representing the expanded command and its arguments. +/// (): Returns null/unit `()` if the command is not defined, not an array, +/// empty, contains non-string elements, or if any part fails expansion. +fn get_power_command(conf, target_device, action) { + let cmd_key = action + "_cmd"; + let command_template_array = conf?.power_control?[target_device]?[cmd_key]; + + if command_template_array == () { + debug( + `warn|util::get_power_command: No command array defined for device ` + + `'${target_device}', action '${action}' (expected key '${cmd_key}').` + ); + return (); + } + if type_of(command_template_array) != "array" { + debug( + `error|util::get_power_command: Command for ` + + `'${target_device}.${cmd_key}' is not an array.` + ); + return (); + } + if command_template_array.len() == 0 { + debug( + `error|util::get_power_command: Command array for ` + + `'${target_device}.${cmd_key}' is empty.` + ); + return (); + } + + let expanded_command_array = []; + for part_template in command_template_array { + if type_of(part_template) != "string" { + debug( + `error|util::get_power_command: Non-string element ` + + `'${part_template}' in command array for '${target_device}.${cmd_key}'.` + ); + return (); + } + let expanded_part = env_expand(part_template, conf); + if expanded_part == () || type_of(expanded_part) != "string" { + debug( + `error|util::get_power_command: Failed to expand or got non-string ` + + `for part '${part_template}' in command for '${target_device}.${cmd_key}'.` + ); + return (); + } + expanded_command_array.push(expanded_part); + } + debug( + `info|util::get_power_command: Expanded for '${target_device}' action '${action}': `+ + `${expanded_command_array}` + ); + return expanded_command_array; +} + +/// Controls power for a specified target device using external commands as arrays. +/// Uses the Rhai `system()` function to execute configured commands. +/// +/// Args: +/// target_device (string): Key for the target device in `conf.power_control`. +/// action (string): Power action: "on", "off", or "status". +/// conf (map): Main configuration map including the `power_control` section. +/// +/// Returns: +/// For "on", "off" actions: +/// true: If command executes with exit code 0. +/// false: If command not defined, fails execution, or has non-zero exit code. +/// For "status" action: +/// "ON" (string): Power is confirmed to be ON. +/// "OFF" (string): Power is confirmed to be OFF. +/// "UNKNOWN" (string): Command ran successfully but output was ambiguous. +/// Map `#{ "error": "message", ... }`: If status could not be determined due +/// to configuration error or command failure. +fn control_power(target_device, action, conf) { + debug( + `info|util::control_power: Requesting action '${action}' ` + + `for device '${target_device}'.` + ); + let command_array = get_power_command(conf, target_device, action); + + if command_array == () { // Checks for null from get_power_command indicating config error + let err_msg = + `Configuration error: Could not retrieve command for ` + + `'${target_device}' action '${action}'.`; + debug(`error|util::control_power: ${err_msg}`); + return #{ "error": err_msg }; + } + // No need to check type_of(command_array) != "array" or len == 0 here + // as get_power_command would have returned () in those cases. + + debug(`info|util::control_power: Executing command array: ${command_array}`); + let system_result = system(command_array); + + if system_result?.error != () { // Error from system() call itself + let err_msg = + `Command execution failed: system() call for '${command_array[0]}' ` + + `failed: ${system_result.error}`; + debug(`error|util::control_power: ${err_msg}`); + return #{ "error": err_msg, "details": system_result }; + } + + debug(`info|util::control_power: System result for '${action}' on '${target_device}': ${system_result}`); + + if action == "on" || action == "off" { + if system_result.exit_code == 0 { + debug( + `info|util::control_power: Action '${action}' for ` + + `'${target_device}' SUCCEEDED (exit code 0).` + ); + return true; + } else { + let err_msg = + `Action '${action}' for '${target_device}' FAILED ` + + `(command exit code ${system_result.exit_code}).`; + debug( + `error|util::control_power: ${err_msg} `+ + `Stdout: [${system_result.stdout}], Stderr: [${system_result.stderr}]` + ); + return false; // Explicit false for command failure for on/off actions + } + } else if action == "status" { + let cmd_name = if command_array.len() > 0 { command_array[0] } else { "unknown_command" }; + if system_result.exit_code != 0 { + let err_msg = + `Status command '${cmd_name}' for '${target_device}' indicated failure ` + + `with exit code ${system_result.exit_code}.`; + debug( + `error|util::control_power: ${err_msg} Stdout: [${system_result.stdout}], ` + + `Stderr: [${system_result.stderr}]` + ); + return #{ "error": err_msg, "details": system_result }; + } + + // Status command executed successfully (exit code 0). Now determine power state. + let expected_on_text = conf?.power_control?[target_device]?.status_on_stdout_contains; + + if expected_on_text != () && type_of(expected_on_text) == "string" && expected_on_text.len() > 0 { + // status_on_stdout_contains is configured, use it + if system_result.stdout.contains(expected_on_text) { + debug( + `info|util::control_power: Status for '${target_device}' is ON ` + + `(stdout matched '${expected_on_text}').` + ); + return "ON"; + } else { + debug( + `info|util::control_power: Status for '${target_device}' is OFF `+ + `(stdout did not match '${expected_on_text}'). Stdout: [${system_result.stdout}]` + ); + return "OFF"; + } + } else { + // No status_on_stdout_contains configured. Exit code 0 from status_cmd means ON. + debug( + `info|util::control_power: Status for '${target_device}' is ON ` + + `(exit code 0 from command, no specific stdout check configured).` + ); + return "ON"; + } + } else { + let err_msg = `Unknown action '${action}' requested for power control.`; + debug(`error|util::control_power: ${err_msg}`); + return #{ "error": err_msg }; // Should not happen if called correctly + } +} + +/// Parses command-line arguments from an argv-style array based on an options string. +/// This function is inspired by the behavior of POSIX getopts and GNU getopt, +/// supporting short options, combined short options, options with arguments +/// (attached or separate), and basic long option support (flag or with '='). +/// +/// Args: +/// argv (array): An array of strings representing the command-line arguments. +/// Typically, `argv[0]` is the script name, and parsing starts +/// from `argv[1]`. +/// options (string): A string specifying the valid short options. +/// - Each character is a short option (e.g., "a", "b", "c"). +/// - If a character is followed by a colon `:`, it signifies +/// that this short option requires an argument +/// (e.g., "ab:c" means -a and -c are flags, -b takes an argument). +/// - Long options are not defined in this string but are parsed +/// if they start with "--". Their argument requirement is +/// determined by the presence of an '=' sign. +/// +/// Returns: +/// Map: +/// On success: +/// #{ +/// "result": #{ // Map of parsed options +/// // e.g., "a": true (for flag -a) +/// // "b": "value" (for option -b value or -bvalue) +/// // "long-option": true (for --long-option) +/// // "another-opt": "val" (for --another-opt=val) +/// }, +/// "positional": [ // Array of positional arguments +/// // e.g., "arg1", "arg2" +/// ] +/// } +/// On failure (e.g., unknown short option, missing argument for short option): +/// #{ "error": "Descriptive error message string" } +/// +/// Behavior Notes: +/// - Parsing stops at the first non-option argument if options are not intermingled +/// with positional arguments, or when "--" is encountered. Any subsequent +/// arguments are treated as positional. (Current basic implementation might +/// collect all non-options as positional as it finds them). +/// - Combined short options (e.g., `-abc`) are treated as `-a -b -c`. +/// - If a short option in a combined group requires an argument (e.g., `options = "ab:c"`, +/// and `-cbvalue` is passed), `value` is taken as the argument for `b`. +/// - Long options (`--option`): +/// - Are not validated against the `options` string. +/// - If in the form `--option=value`, `option` is set to `"value"`. +/// - If in the form `--option` (no `=`), `option` is set to `true` (boolean flag). +/// - `--`: All subsequent arguments are treated as positional. +// +fn getopts(argv, options) { + let result = #{}; + let positional = []; + let i = 1; // Start parsing from the first argument after script name + + while i < argv.len() { + let arg = argv[i]; + + if arg == "--" { + // End of options, all subsequent are positional + i += 1; + while i < argv.len() { + positional.push(argv[i]); + i += 1; + } + break; + } else if arg.starts_with("--") && arg.len() > 2 { + // Long option (e.g., --option, --option=value) + let opt_full = arg.sub_string(2, arg.len()); + let opt_name = opt_full; + let opt_val = true; // Default for flag if no '=' + + let eq_idx = opt_full.index_of("="); + if eq_idx != -1 { + opt_name = opt_full.sub_string(0, eq_idx); + opt_val = opt_full.sub_string(eq_idx + 1, opt_full.len()); + } + result[opt_name] = opt_val; + i += 1; + } else if arg.starts_with("-") && arg.len() > 1 { + // Short options (e.g., -a, -b val, -bval, -abc) + let short_opts_str = arg.sub_string(1, arg.len()); + let current_opt_char_idx = 0; + // True if the *next* argv element was consumed as an argument + // to the last short option in this group. + let consumed_next_argv_for_opt = false; + + while current_opt_char_idx < short_opts_str.len() { + let opt_char = short_opts_str[current_opt_char_idx]; + let opt_str = opt_char.to_string(); + let opt_spec_index = options.index_of(opt_str); + + if opt_spec_index == -1 { + return #{ "error": `Unknown option '-${opt_str}'.` }; + } + + let requires_arg = (opt_spec_index + 1 < options.len() && + options[opt_spec_index + 1] == ":"); + + if requires_arg { + // This short option requires an argument + if current_opt_char_idx + 1 < short_opts_str.len() { + // Argument is attached (e.g., -fFILENAME) + result[opt_str] = short_opts_str.sub_string( + current_opt_char_idx + 1, short_opts_str.len() + ); + // Consumed the rest of this combined arg token + current_opt_char_idx = short_opts_str.len(); + } else if i + 1 < argv.len() { + // Argument is the next argv element (e.g., -f FILENAME) + result[opt_str] = argv[i + 1]; + consumed_next_argv_for_opt = true; + current_opt_char_idx += 1; // Move past this option char + } else { + return #{ "error": `Option '-${opt_str}' requires an argument.` }; + } + } else { + // Option is a flag (does not require an argument) + result[opt_str] = true; + current_opt_char_idx += 1; // Move to next char in combined opts + } + } + + i += 1; // Consumed the current argv element (e.g., "-abc" or "-o") + if consumed_next_argv_for_opt { + i += 1; // Also consumed the next argv element if it was an opt arg + } + } else { + // Positional argument + positional.push(arg); + i += 1; + } + } + + return #{ "result": result, "positional": positional }; +}