From 16d5f6df14c3f45f9fc7a72eb5bb36d4eb9d16c6 Mon Sep 17 00:00:00 2001 From: Vinod Chitrali Date: Sat, 30 May 2026 07:47:55 +0000 Subject: [PATCH] feat(api): poll power state for PowerShelf in Ready While a PowerShelf idles in Ready, perform a best-effort GetPowerStateByDeviceList RPC against RMS and persist the observed pstate to power_shelves.status. The lookup mirrors the NodeSet-with-inline-BMC-endpoint shape used by the Maintenance handler's SetPowerStateByDeviceList path, so build_power_shelf_node_info is bumped to pub(super) for reuse. Missing prerequisites (no RMS client, no rack association, no BMC details, no credentials) and transport / status failures are logged but never transition the controller out of Ready, so a transient RMS outage cannot bounce the shelf into Error. Also bumps librms from v0.0.12-rc1 to v0.0.12-rc4 (adds the prost-types dependency and the new GetPowerStateByDeviceList, firmware-object management, and SetScaleUpFabricState RPCs), and updates the in-tree mock RmsApi implementations to cover the new trait methods. --- Cargo.lock | 4 +- Cargo.toml | 2 +- crates/api-test-helper/src/mock_rms.rs | 16 ++ .../power-shelf-controller/src/maintenance.rs | 11 +- crates/power-shelf-controller/src/ready.rs | 206 +++++++++++++++++- crates/rack/src/rms_client.rs | 23 ++ 6 files changed, 246 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 39c8535c11..b2d13f876a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6424,7 +6424,7 @@ dependencies = [ [[package]] name = "librms" version = "0.0.12" -source = "git+https://github.com/NVIDIA/nv-rms-client.git?tag=v0.0.12-rc3#88797e64f4fbf072b5844ed7b8d7e1e987d4aef6" +source = "git+https://github.com/NVIDIA/nv-rms-client.git?tag=v0.0.12-rc4#a870c06ee1483a46ef3fb01cc4ce7d73470c7d3c" dependencies = [ "async-trait", "chrono", @@ -11375,7 +11375,7 @@ dependencies = [ [[package]] name = "tonic-client-wrapper" version = "1.0.0" -source = "git+https://github.com/NVIDIA/nv-rms-client.git?tag=v0.0.12-rc3#88797e64f4fbf072b5844ed7b8d7e1e987d4aef6" +source = "git+https://github.com/NVIDIA/nv-rms-client.git?tag=v0.0.12-rc4#a870c06ee1483a46ef3fb01cc4ce7d73470c7d3c" dependencies = [ "async-trait", "heck", diff --git a/Cargo.toml b/Cargo.toml index 10f933605a..1e13f9b0e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ authors = ["NVIDIA Carbide Engineering "] [workspace.dependencies] clap = { version = "4", features = ["derive", "env"] } libredfish = { git = "https://github.com/NVIDIA/libredfish.git", tag = "v0.44.4" } -librms = { git = "https://github.com/NVIDIA/nv-rms-client.git", tag = "v0.0.12-rc3" } +librms = { git = "https://github.com/NVIDIA/nv-rms-client.git", tag = "v0.0.12-rc4" } ansi-to-html = "0.2.2" tokio = { version = "1", features = ["full", "tracing"] } diff --git a/crates/api-test-helper/src/mock_rms.rs b/crates/api-test-helper/src/mock_rms.rs index fd862d7b4e..925461449c 100644 --- a/crates/api-test-helper/src/mock_rms.rs +++ b/crates/api-test-helper/src/mock_rms.rs @@ -67,6 +67,10 @@ pub struct MockRmsApi { Mutex>>, get_power_state_calls: Mutex>, + get_power_state_by_device_list_responses: + Mutex>>, + get_power_state_by_device_list_calls: Mutex>, + sequence_rack_power_responses: Mutex>>, sequence_rack_power_calls: Mutex>, @@ -244,6 +248,8 @@ impl MockRmsApi { set_power_state_calls: Default::default(), set_power_state_by_device_list_responses: Default::default(), set_power_state_by_device_list_calls: Default::default(), + get_power_state_by_device_list_responses: Default::default(), + get_power_state_by_device_list_calls: Default::default(), get_power_state_responses: Default::default(), get_power_state_calls: Default::default(), sequence_rack_power_responses: Default::default(), @@ -809,6 +815,16 @@ fn pop_or_err( #[async_trait::async_trait] impl RmsApi for MockRmsApi { + async fn get_power_state_by_device_list( + &self, + cmd: rms::GetPowerStateByDeviceListRequest, + ) -> Result { + self.get_power_state_by_device_list_calls + .lock() + .await + .push(cmd); + pop_or_err(&mut self.get_power_state_by_device_list_responses.lock().await) + } async fn set_power_state( &self, cmd: rms::SetPowerStateRequest, diff --git a/crates/power-shelf-controller/src/maintenance.rs b/crates/power-shelf-controller/src/maintenance.rs index 907e884bc9..0f8e4bbb7d 100644 --- a/crates/power-shelf-controller/src/maintenance.rs +++ b/crates/power-shelf-controller/src/maintenance.rs @@ -245,11 +245,12 @@ async fn invoke_rms_power_operation( } /// Build the `rms::NewNodeInfo` describing this power shelf for inclusion -/// in a `SetPowerStateByDeviceList` request. Resolves the BMC IP from the -/// database and BMC credentials via the credential manager, since the -/// caller-supplied variant of the RPC requires the BMC connection details -/// inline rather than relying on RMS's inventory. -async fn build_power_shelf_node_info( +/// in any caller-supplied `NodeSet` request (`SetPowerStateByDeviceList` +/// from `Maintenance`, `GetDeviceInfoByDeviceList` from `Ready`). Resolves +/// the BMC IP from the database and BMC credentials via the credential +/// manager, since these RPCs require the BMC connection details inline +/// rather than relying on RMS's inventory. +pub(super) async fn build_power_shelf_node_info( power_shelf_id: &PowerShelfId, state: &PowerShelf, rack_id: String, diff --git a/crates/power-shelf-controller/src/ready.rs b/crates/power-shelf-controller/src/ready.rs index b3324813da..db6de5dc64 100644 --- a/crates/power-shelf-controller/src/ready.rs +++ b/crates/power-shelf-controller/src/ready.rs @@ -17,27 +17,33 @@ //! Handler for PowerShelfControllerState::Ready. +use carbide_rack::rack_manager_error; use carbide_uuid::power_shelf::PowerShelfId; -use model::power_shelf::{PowerShelf, PowerShelfControllerState}; +use db::power_shelf as db_power_shelf; +use librms::protos::rack_manager as rms; +use model::power_shelf::{PowerShelf, PowerShelfControllerState, PowerShelfStatus}; +use sqlx::PgTransaction; use state_controller::state_handler::{ StateHandlerContext, StateHandlerError, StateHandlerOutcome, }; use crate::context::PowerShelfStateHandlerContextObjects; +use crate::maintenance::build_power_shelf_node_info; /// Handles the Ready state for a power shelf. /// /// If the power shelf is marked for deletion, transitions to `Deleting`. /// If a maintenance request has been posted via /// `power_shelf_maintenance_requested`, transitions to `Maintenance` with the -/// requested operation (PowerOn / PowerOff). Otherwise idles. +/// requested operation (PowerOn / PowerOff). Otherwise polls RMS for the +/// current power state (best-effort observation) and idles. /// /// TODO: Implement PowerShelf monitoring (health checks, status updates, /// power consumption / efficiency tracking). pub async fn handle_ready( power_shelf_id: &PowerShelfId, state: &mut PowerShelf, - _ctx: &mut StateHandlerContext<'_, PowerShelfStateHandlerContextObjects>, + ctx: &mut StateHandlerContext<'_, PowerShelfStateHandlerContextObjects>, ) -> Result, StateHandlerError> { if state.is_marked_as_deleted() { return Ok(StateHandlerOutcome::transition( @@ -58,9 +64,193 @@ pub async fn handle_ready( )); } - tracing::info!("PowerShelf {} is ready", power_shelf_id,); - Ok(StateHandlerOutcome::wait(format!( - "PowerShelf {} is ready", - power_shelf_id - ))) + let txn = poll_rms_power_state(power_shelf_id, state, ctx).await; + + Ok(StateHandlerOutcome::do_nothing().with_txn_opt(txn)) +} +/// +/// On a successful response, the observed `pstate` for this power shelf is +/// persisted to the `power_shelves.status` column and the in-memory `state` +/// is updated to match. The returned `PgTransaction` (if any) carries that +/// status write so the caller can attach it to the `Ready` outcome and have +/// the state-controller framework commit it alongside the usual outcome +/// bookkeeping. +async fn poll_rms_power_state( + power_shelf_id: &PowerShelfId, + state: &mut PowerShelf, + ctx: &mut StateHandlerContext<'_, PowerShelfStateHandlerContextObjects>, +) -> Option> { + let Some(rms_client) = ctx.services.rms_client.as_ref() else { + tracing::debug!( + power_shelf_id = %power_shelf_id, + "PowerShelf Ready: skipping RMS GetPowerStateByDeviceList; RMS client not configured", + ); + return None; + }; + + let Some(rack_id) = state.rack_id.as_ref() else { + tracing::debug!( + power_shelf_id = %power_shelf_id, + "PowerShelf Ready: skipping RMS GetPowerStateByDeviceList; power shelf has no rack association", + ); + return None; + }; + + let device = match build_power_shelf_node_info( + power_shelf_id, + state, + rack_id.to_string(), + &ctx.services.db_pool, + ctx.services.credential_manager.as_ref(), + ) + .await + { + Ok(device) => device, + Err(cause) => { + tracing::debug!( + power_shelf_id = %power_shelf_id, + rack_id = %rack_id, + cause = %cause, + "PowerShelf Ready: skipping RMS GetPowerStateByDeviceList; unable to build NodeSet", + ); + return None; + } + }; + + let request = rms::GetPowerStateByDeviceListRequest { + nodes: Some(rms::NodeSet { + devices: vec![device], + }), + ..Default::default() + }; + + let rack_id_str = rack_id.to_string(); + let response = match rms_client.get_power_state_by_device_list(request).await { + Ok(response) => response, + Err(error) => { + let error = rack_manager_error("get_power_state_by_device_list", error); + tracing::warn!( + power_shelf_id = %power_shelf_id, + rack_id = %rack_id_str, + error = %error, + "RMS GetPowerStateByDeviceList transport error", + ); + return None; + } + }; + + let batch = response.response.clone().unwrap_or_default(); + if !(batch.status == rms::ReturnCode::Success as i32 && batch.failed_nodes == 0) { + tracing::warn!( + power_shelf_id = %power_shelf_id, + rack_id = %rack_id_str, + batch_status = batch.status, + successful_nodes = batch.successful_nodes, + failed_nodes = batch.failed_nodes, + message = %batch.message, + "RMS GetPowerStateByDeviceList returned non-Success result", + ); + return None; + } + + tracing::info!( + power_shelf_id = %power_shelf_id, + rack_id = %rack_id_str, + successful_nodes = batch.successful_nodes, + pstates = ?response + .node_power_states + .iter() + .map(|node| (node.node_id.as_str(), node.pstate.as_str())) + .collect::>(), + "RMS GetPowerStateByDeviceList succeeded", + ); + + persist_observed_power_state(power_shelf_id, state, ctx, &response.node_power_states).await +} + +/// Look up the `NodePowerState` for this power shelf in the RMS response, +/// stamp the value into `state.status`, and persist it via +/// `db_power_shelf::update`. Returns the open `PgTransaction` so the caller +/// can attach it to the `Ready` outcome. +/// +/// Status persistence is best-effort: if RMS did not echo a result for this +/// node, or if the DB write fails, the in-memory state is left untouched +/// and `None` is returned — `Ready` must stay in `Ready` regardless. +async fn persist_observed_power_state( + power_shelf_id: &PowerShelfId, + state: &mut PowerShelf, + ctx: &mut StateHandlerContext<'_, PowerShelfStateHandlerContextObjects>, + node_power_states: &[rms::NodePowerState], +) -> Option> { + let node_id = power_shelf_id.to_string(); + let Some(observed) = node_power_states + .iter() + .find(|node| node.node_id == node_id) + else { + tracing::debug!( + power_shelf_id = %power_shelf_id, + "RMS GetPowerStateByDeviceList: no NodePowerState echoed for this power shelf; skipping status update", + ); + return None; + }; + + let new_power_state = observed.pstate.to_lowercase(); + let new_status = match state.status.as_ref() { + Some(existing) => PowerShelfStatus { + shelf_name: existing.shelf_name.clone(), + power_state: new_power_state.clone(), + health_status: existing.health_status.clone(), + }, + None => PowerShelfStatus { + shelf_name: state.config.name.clone(), + power_state: new_power_state.clone(), + health_status: String::new(), + }, + }; + + if state + .status + .as_ref() + .is_some_and(|s| s.power_state == new_status.power_state) + { + tracing::debug!( + power_shelf_id = %power_shelf_id, + power_state = %new_status.power_state, + "PowerShelf status power_state unchanged; skipping DB write", + ); + return None; + } + + let previous_status = state.status.replace(new_status); + + let mut txn = match ctx.services.db_pool.begin().await { + Ok(txn) => txn, + Err(error) => { + state.status = previous_status; + tracing::warn!( + power_shelf_id = %power_shelf_id, + error = %error, + "PowerShelf Ready: failed to begin txn while persisting observed power state", + ); + return None; + } + }; + + if let Err(error) = db_power_shelf::update(state, &mut txn).await { + state.status = previous_status; + tracing::warn!( + power_shelf_id = %power_shelf_id, + error = %error, + "PowerShelf Ready: failed to persist observed power state to DB", + ); + return None; + } + + tracing::info!( + power_shelf_id = %power_shelf_id, + power_state = %new_power_state, + "PowerShelf Ready: persisted observed power state from RMS", + ); + + Some(txn) } diff --git a/crates/rack/src/rms_client.rs b/crates/rack/src/rms_client.rs index 999333654f..1f6e97b9e0 100644 --- a/crates/rack/src/rms_client.rs +++ b/crates/rack/src/rms_client.rs @@ -158,6 +158,10 @@ pub mod test_support { fn build_mock_client(&self) -> MockRmsClient { MockRmsClient { + submitted_get_power_state_by_device_list_requests: Arc::new(Mutex::new(Vec::new())), + queued_get_power_state_by_device_list_responses: Arc::new(Mutex::new( + VecDeque::new(), + )), fail_add_node: self.fail_add_node.clone(), fail_inventory_get: self.fail_inventory_get.clone(), registered_nodes: self.registered_nodes.clone(), @@ -452,6 +456,10 @@ pub mod test_support { switch_system_image_job_statuses: Arc>>, switch_system_image_job_errors: Arc>>, + submitted_get_power_state_by_device_list_requests: + Arc>>, + queued_get_power_state_by_device_list_responses: + Arc>>>, submitted_get_device_info_by_device_list_requests: Arc>>, queued_get_device_info_by_device_list_responses: @@ -473,6 +481,21 @@ pub mod test_support { #[async_trait::async_trait] impl RmsApi for MockRmsClient { + async fn get_power_state_by_device_list( + &self, + cmd: rms::GetPowerStateByDeviceListRequest, + ) -> Result { + self.submitted_get_power_state_by_device_list_requests + .lock() + .await + .push(cmd); + self.queued_get_power_state_by_device_list_responses + .lock() + .await + .pop_front() + .unwrap_or(Ok(rms::GetPowerStateByDeviceListResponse::default())) + } + async fn get_device_info_by_device_list( &self, cmd: rms::GetDeviceInfoByDeviceListRequest,