diff --git a/src/firecracker/src/api_server/request/hotplug/memory.rs b/src/firecracker/src/api_server/request/hotplug/memory.rs index 59755d4d541..4bdeec73a6d 100644 --- a/src/firecracker/src/api_server/request/hotplug/memory.rs +++ b/src/firecracker/src/api_server/request/hotplug/memory.rs @@ -25,8 +25,9 @@ pub(crate) fn parse_get_memory_hotplug() -> Result #[cfg(test)] mod tests { - use vmm::devices::virtio::mem::{VIRTIO_MEM_DEFAULT_BLOCK_SIZE, VIRTIO_MEM_DEFAULT_SLOT_SIZE}; - use vmm::utils::bytes_to_mib; + use vmm::devices::virtio::mem::{ + VIRTIO_MEM_DEFAULT_BLOCK_SIZE_MIB, VIRTIO_MEM_DEFAULT_SLOT_SIZE_MIB, + }; use super::*; use crate::api_server::parsed_request::tests::vmm_action_from_request; @@ -47,8 +48,8 @@ mod tests { }"#; let expected_config = MemoryHotplugConfig { total_size_mib: 2048, - block_size_mib: bytes_to_mib(VIRTIO_MEM_DEFAULT_BLOCK_SIZE), - slot_size_mib: bytes_to_mib(VIRTIO_MEM_DEFAULT_SLOT_SIZE), + block_size_mib: VIRTIO_MEM_DEFAULT_BLOCK_SIZE_MIB, + slot_size_mib: VIRTIO_MEM_DEFAULT_SLOT_SIZE_MIB, }; assert_eq!( vmm_action_from_request(parse_put_memory_hotplug(&Body::new(body)).unwrap()), diff --git a/src/vmm/src/device_manager/pci_mngr.rs b/src/vmm/src/device_manager/pci_mngr.rs index 5373e3d510a..a6f01742256 100644 --- a/src/vmm/src/device_manager/pci_mngr.rs +++ b/src/vmm/src/device_manager/pci_mngr.rs @@ -20,6 +20,8 @@ use crate::devices::virtio::block::device::Block; use crate::devices::virtio::block::persist::{BlockConstructorArgs, BlockState}; use crate::devices::virtio::device::VirtioDevice; use crate::devices::virtio::generated::virtio_ids; +use crate::devices::virtio::mem::VirtioMem; +use crate::devices::virtio::mem::persist::{VirtioMemConstructorArgs, VirtioMemState}; use crate::devices::virtio::net::Net; use crate::devices::virtio::net::persist::{NetConstructorArgs, NetState}; use crate::devices::virtio::rng::Entropy; @@ -240,6 +242,8 @@ pub struct PciDevicesState { pub mmds: Option, /// Entropy device state. pub entropy_device: Option>, + /// Memory device state. + pub memory_device: Option>, } pub struct PciDevicesConstructorArgs<'a> { @@ -388,6 +392,20 @@ impl<'a> Persist<'a> for PciDevices { transport_state, }) } + virtio_ids::VIRTIO_ID_MEM => { + let mem_dev = locked_virtio_dev + .as_mut_any() + .downcast_mut::() + .unwrap(); + let device_state = mem_dev.save(); + + state.memory_device = Some(VirtioDeviceState { + device_id: mem_dev.id().to_string(), + pci_device_bdf, + device_state, + transport_state, + }) + } _ => unreachable!(), } } @@ -566,6 +584,29 @@ impl<'a> Persist<'a> for PciDevices { .unwrap() } + if let Some(memory_device) = &state.memory_device { + let ctor_args = VirtioMemConstructorArgs::new(Arc::clone(constructor_args.vm)); + + let device = Arc::new(Mutex::new( + VirtioMem::restore(ctor_args, &memory_device.device_state).unwrap(), + )); + + constructor_args + .vm_resources + .update_from_restored_device(SharedDeviceType::VirtioMem(device.clone())) + .unwrap(); + + pci_devices + .restore_pci_device( + constructor_args.vm, + device, + &memory_device.device_id, + &memory_device.transport_state, + constructor_args.event_manager, + ) + .unwrap() + } + Ok(pci_devices) } } @@ -583,6 +624,7 @@ mod tests { use crate::snapshot::Snapshot; use crate::vmm_config::balloon::BalloonDeviceConfig; use crate::vmm_config::entropy::EntropyDeviceConfig; + use crate::vmm_config::memory_hotplug::MemoryHotplugConfig; use crate::vmm_config::net::NetworkInterfaceConfig; use crate::vmm_config::vsock::VsockDeviceConfig; @@ -645,6 +687,18 @@ mod tests { let entropy_config = EntropyDeviceConfig::default(); insert_entropy_device(&mut vmm, &mut cmdline, &mut event_manager, entropy_config); + let memory_hotplug_config = MemoryHotplugConfig { + total_size_mib: 1024, + block_size_mib: 2, + slot_size_mib: 128, + }; + insert_virtio_mem_device( + &mut vmm, + &mut cmdline, + &mut event_manager, + memory_hotplug_config, + ); + Snapshot::new(vmm.device_manager.save()) .save(&mut buf.as_mut_slice()) .unwrap(); @@ -674,7 +728,6 @@ mod tests { let _restored_dev_manager = PciDevices::restore(restore_args, &device_manager_state.pci_state).unwrap(); - // TODO(virtio-mem): add memory-hotplug device when snapshot-restore is implemented let expected_vm_resources = format!( r#"{{ "balloon": {{ @@ -734,7 +787,11 @@ mod tests { "entropy": {{ "rate_limiter": null }}, - "memory-hotplug": null + "memory-hotplug": {{ + "total_size_mib": 1024, + "block_size_mib": 2, + "slot_size_mib": 128 + }} }}"#, _block_files.last().unwrap().as_path().to_str().unwrap(), tmp_sock_file.as_path().to_str().unwrap() diff --git a/src/vmm/src/device_manager/persist.rs b/src/vmm/src/device_manager/persist.rs index c9e465bfc46..c9525365872 100644 --- a/src/vmm/src/device_manager/persist.rs +++ b/src/vmm/src/device_manager/persist.rs @@ -25,6 +25,10 @@ use crate::devices::virtio::block::device::Block; use crate::devices::virtio::block::persist::{BlockConstructorArgs, BlockState}; use crate::devices::virtio::device::VirtioDevice; use crate::devices::virtio::generated::virtio_ids; +use crate::devices::virtio::mem::VirtioMem; +use crate::devices::virtio::mem::persist::{ + VirtioMemConstructorArgs, VirtioMemPersistError, VirtioMemState, +}; use crate::devices::virtio::net::Net; use crate::devices::virtio::net::persist::{ NetConstructorArgs, NetPersistError as NetError, NetState, @@ -72,6 +76,8 @@ pub enum DevicePersistError { MmdsConfig(#[from] MmdsConfigError), /// Entropy: {0} Entropy(#[from] EntropyError), + /// virtio-mem: {0} + VirtioMem(#[from] VirtioMemPersistError), /// Resource misconfiguration: {0}. Is the snapshot file corrupted? ResourcesError(#[from] ResourcesError), /// Could not activate device: {0} @@ -125,6 +131,8 @@ pub struct DeviceStates { pub mmds: Option, /// Entropy device state. pub entropy_device: Option>, + /// Memory device state. + pub memory_device: Option>, } /// A type used to extract the concrete `Arc>` for each of the device @@ -136,6 +144,7 @@ pub enum SharedDeviceType { Balloon(Arc>), Vsock(Arc>>), Entropy(Arc>), + VirtioMem(Arc>), } pub struct MMIODevManagerConstructorArgs<'a> { @@ -335,6 +344,20 @@ impl<'a> Persist<'a> for MMIODeviceManager { device_info, }); } + virtio_ids::VIRTIO_ID_MEM => { + let mem = locked_device + .as_mut_any() + .downcast_mut::() + .unwrap(); + let device_state = mem.save(); + + states.memory_device = Some(VirtioDeviceState { + device_id, + device_state, + transport_state, + device_info, + }); + } _ => unreachable!(), }; @@ -549,6 +572,30 @@ impl<'a> Persist<'a> for MMIODeviceManager { )?; } + if let Some(memory_state) = &state.memory_device { + let ctor_args = VirtioMemConstructorArgs::new(Arc::clone(vm)); + + let device = Arc::new(Mutex::new(VirtioMem::restore( + ctor_args, + &memory_state.device_state, + )?)); + + constructor_args + .vm_resources + .update_from_restored_device(SharedDeviceType::VirtioMem(device.clone()))?; + + restore_helper( + device.clone(), + memory_state.device_state.virtio_state.activated, + false, + device, + &memory_state.device_id, + &memory_state.transport_state, + &memory_state.device_info, + constructor_args.event_manager, + )?; + } + Ok(dev_manager) } } @@ -565,6 +612,7 @@ mod tests { use crate::snapshot::Snapshot; use crate::vmm_config::balloon::BalloonDeviceConfig; use crate::vmm_config::entropy::EntropyDeviceConfig; + use crate::vmm_config::memory_hotplug::MemoryHotplugConfig; use crate::vmm_config::net::NetworkInterfaceConfig; use crate::vmm_config::vsock::VsockDeviceConfig; @@ -582,6 +630,7 @@ mod tests { && self.net_devices == other.net_devices && self.vsock_device == other.vsock_device && self.entropy_device == other.entropy_device + && self.memory_device == other.memory_device } } @@ -666,6 +715,18 @@ mod tests { let entropy_config = EntropyDeviceConfig::default(); insert_entropy_device(&mut vmm, &mut cmdline, &mut event_manager, entropy_config); + let memory_hotplug_config = MemoryHotplugConfig { + total_size_mib: 1024, + block_size_mib: 2, + slot_size_mib: 128, + }; + insert_virtio_mem_device( + &mut vmm, + &mut cmdline, + &mut event_manager, + memory_hotplug_config, + ); + Snapshot::new(vmm.device_manager.save()) .save(&mut buf.as_mut_slice()) .unwrap(); @@ -691,7 +752,6 @@ mod tests { let _restored_dev_manager = MMIODeviceManager::restore(restore_args, &device_manager_state.mmio_state).unwrap(); - // TODO(virtio-mem): add memory-hotplug device when snapshot-restore is implemented let expected_vm_resources = format!( r#"{{ "balloon": {{ @@ -751,7 +811,11 @@ mod tests { "entropy": {{ "rate_limiter": null }}, - "memory-hotplug": null + "memory-hotplug": {{ + "total_size_mib": 1024, + "block_size_mib": 2, + "slot_size_mib": 128 + }} }}"#, _block_files.last().unwrap().as_path().to_str().unwrap(), tmp_sock_file.as_path().to_str().unwrap() diff --git a/src/vmm/src/devices/virtio/mem/device.rs b/src/vmm/src/devices/virtio/mem/device.rs index 95346b88681..0a1a5e0580a 100644 --- a/src/vmm/src/devices/virtio/mem/device.rs +++ b/src/vmm/src/devices/virtio/mem/device.rs @@ -23,6 +23,7 @@ use crate::devices::virtio::generated::virtio_mem::{ VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE, virtio_mem_config, }; use crate::devices::virtio::iov_deque::IovDequeError; +use crate::devices::virtio::mem::metrics::METRICS; use crate::devices::virtio::mem::{VIRTIO_MEM_DEV_ID, VIRTIO_MEM_GUEST_ADDRESS}; use crate::devices::virtio::queue::{FIRECRACKER_MAX_QUEUE_SIZE, InvalidAvailIdx, Queue}; use crate::devices::virtio::transport::{VirtioInterrupt, VirtioInterruptType}; @@ -125,12 +126,12 @@ impl VirtioMem { /// Gets the total hotpluggable size. pub fn total_size_mib(&self) -> usize { - bytes_to_mib(self.config.region_size.try_into().unwrap()) + bytes_to_mib(u64_to_usize(self.config.region_size)) } /// Gets the block size. pub fn block_size_mib(&self) -> usize { - bytes_to_mib(self.config.block_size.try_into().unwrap()) + bytes_to_mib(u64_to_usize(self.config.block_size)) } /// Gets the block size. @@ -140,12 +141,12 @@ impl VirtioMem { /// Gets the total size of the plugged memory blocks. pub fn plugged_size_mib(&self) -> usize { - bytes_to_mib(self.config.plugged_size.try_into().unwrap()) + bytes_to_mib(u64_to_usize(self.config.plugged_size)) } /// Gets the requested size pub fn requested_size_mib(&self) -> usize { - bytes_to_mib(self.config.requested_size.try_into().unwrap()) + bytes_to_mib(u64_to_usize(self.config.requested_size)) } pub fn status(&self) -> VirtioMemStatus { @@ -170,12 +171,15 @@ impl VirtioMem { } pub(crate) fn process_mem_queue_event(&mut self) { + METRICS.queue_event_count.inc(); if let Err(err) = self.queue_events[MEM_QUEUE].read() { + METRICS.queue_event_fails.inc(); error!("Failed to read mem queue event: {err}"); return; } if let Err(err) = self.process_mem_queue() { + METRICS.queue_event_fails.inc(); error!("virtio-mem: Failed to process queue: {err}"); } } @@ -233,7 +237,7 @@ impl VirtioDevice for VirtioMem { } fn read_config(&self, offset: u64, data: &mut [u8]) { - let offset: usize = u64_to_usize(offset); + let offset = u64_to_usize(offset); self.config .as_slice() .get(offset..offset + data.len()) @@ -263,7 +267,7 @@ impl VirtioDevice for VirtioMem { error!( "virtio-mem: VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE feature not acknowledged by guest" ); - // TODO(virtio-mem): activation failed metric + METRICS.activate_fails.inc(); return Err(ActivateError::RequiredFeatureNotAcked( "VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE", )); @@ -276,7 +280,7 @@ impl VirtioDevice for VirtioMem { self.device_state = DeviceState::Activated(ActiveState { mem, interrupt }); if self.activate_event.write(1).is_err() { - // TODO(virtio-mem): activation failed metric + METRICS.activate_fails.inc(); self.device_state = DeviceState::Inactive; return Err(ActivateError::EventFd); } diff --git a/src/vmm/src/devices/virtio/mem/metrics.rs b/src/vmm/src/devices/virtio/mem/metrics.rs new file mode 100644 index 00000000000..443e9a8b8f1 --- /dev/null +++ b/src/vmm/src/devices/virtio/mem/metrics.rs @@ -0,0 +1,78 @@ +// Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Defines the metrics system for memory devices. +//! +//! # Metrics format +//! The metrics are flushed in JSON when requested by vmm::logger::metrics::METRICS.write(). +//! +//! ## JSON example with metrics: +//! ```json +//! "memory_hotplug": { +//! "activate_fails": "SharedIncMetric", +//! "queue_event_fails": "SharedIncMetric", +//! "queue_event_count": "SharedIncMetric", +//! ... +//! } +//! } +//! ``` +//! Each `memory` field in the example above is a serializable `VirtioMemDeviceMetrics` structure +//! collecting metrics such as `activate_fails`, `queue_event_fails` etc. for the memoty hotplug +//! device. +//! Since Firecrakcer only supports one virtio-mem device, there is no per device metrics and +//! `memory_hotplug` represents the aggregate entropy metrics. + +use serde::ser::SerializeMap; +use serde::{Serialize, Serializer}; + +use crate::logger::{LatencyAggregateMetrics, SharedIncMetric}; + +/// Stores aggregated virtio-mem metrics +pub(super) static METRICS: VirtioMemDeviceMetrics = VirtioMemDeviceMetrics::new(); + +/// Called by METRICS.flush(), this function facilitates serialization of virtio-mem device metrics. +pub fn flush_metrics(serializer: S) -> Result { + let mut seq = serializer.serialize_map(Some(1))?; + seq.serialize_entry("memory_hotplug", &METRICS)?; + seq.end() +} + +#[derive(Debug, Serialize)] +pub(super) struct VirtioMemDeviceMetrics { + /// Number of device activation failures + pub activate_fails: SharedIncMetric, + /// Number of queue event handling failures + pub queue_event_fails: SharedIncMetric, + /// Number of queue events handled + pub queue_event_count: SharedIncMetric, +} + +impl VirtioMemDeviceMetrics { + /// Const default construction. + const fn new() -> Self { + Self { + activate_fails: SharedIncMetric::new(), + queue_event_fails: SharedIncMetric::new(), + queue_event_count: SharedIncMetric::new(), + } + } +} + +#[cfg(test)] +pub mod tests { + use super::*; + use crate::logger::IncMetric; + + #[test] + fn test_memory_hotplug_metrics() { + let mem_metrics: VirtioMemDeviceMetrics = VirtioMemDeviceMetrics::new(); + let mem_metrics_local: String = serde_json::to_string(&mem_metrics).unwrap(); + // the 1st serialize flushes the metrics and resets values to 0 so that + // we can compare the values with local metrics. + serde_json::to_string(&METRICS).unwrap(); + let mem_metrics_global: String = serde_json::to_string(&METRICS).unwrap(); + assert_eq!(mem_metrics_local, mem_metrics_global); + mem_metrics.queue_event_count.inc(); + assert_eq!(mem_metrics.queue_event_count.count(), 1); + } +} diff --git a/src/vmm/src/devices/virtio/mem/mod.rs b/src/vmm/src/devices/virtio/mem/mod.rs index e4db655328e..1c9e98f98a6 100644 --- a/src/vmm/src/devices/virtio/mem/mod.rs +++ b/src/vmm/src/devices/virtio/mem/mod.rs @@ -3,6 +3,8 @@ mod device; mod event_handler; +pub mod metrics; +pub mod persist; use vm_memory::GuestAddress; @@ -13,8 +15,8 @@ pub(crate) const MEM_NUM_QUEUES: usize = 1; pub(crate) const MEM_QUEUE: usize = 0; -pub const VIRTIO_MEM_DEFAULT_BLOCK_SIZE: usize = 2 << 20; // 2MiB -pub const VIRTIO_MEM_DEFAULT_SLOT_SIZE: usize = 128 << 20; // 128 MiB +pub const VIRTIO_MEM_DEFAULT_BLOCK_SIZE_MIB: usize = 2; +pub const VIRTIO_MEM_DEFAULT_SLOT_SIZE_MIB: usize = 128; pub const VIRTIO_MEM_GUEST_ADDRESS: GuestAddress = GuestAddress(FIRST_ADDR_PAST_64BITS_MMIO); // 512GiB pub const VIRTIO_MEM_DEV_ID: &str = "mem"; diff --git a/src/vmm/src/devices/virtio/mem/persist.rs b/src/vmm/src/devices/virtio/mem/persist.rs new file mode 100644 index 00000000000..e48246de500 --- /dev/null +++ b/src/vmm/src/devices/virtio/mem/persist.rs @@ -0,0 +1,136 @@ +// Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Defines the structures needed for saving/restoring virtio-mem devices. + +use std::sync::Arc; + +use serde::{Deserialize, Serialize}; +use vm_memory::Address; + +use crate::Vm; +use crate::devices::virtio::generated::virtio_ids::VIRTIO_ID_MEM; +use crate::devices::virtio::generated::virtio_mem::virtio_mem_config; +use crate::devices::virtio::mem::{ + MEM_NUM_QUEUES, VIRTIO_MEM_GUEST_ADDRESS, VirtioMem, VirtioMemError, +}; +use crate::devices::virtio::persist::{PersistError as VirtioStateError, VirtioDeviceState}; +use crate::devices::virtio::queue::FIRECRACKER_MAX_QUEUE_SIZE; +use crate::snapshot::Persist; +use crate::vstate::memory::{GuestMemoryMmap, GuestRegionMmap}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VirtioMemState { + pub virtio_state: VirtioDeviceState, + region_size: u64, + block_size: u64, + usable_region_size: u64, + plugged_size: u64, + requested_size: u64, + slot_size: usize, +} + +#[derive(Debug)] +pub struct VirtioMemConstructorArgs { + vm: Arc, +} + +impl VirtioMemConstructorArgs { + pub fn new(vm: Arc) -> Self { + Self { vm } + } +} + +#[derive(Debug, thiserror::Error, displaydoc::Display)] +pub enum VirtioMemPersistError { + /// Create virtio-mem: {0} + CreateVirtioMem(#[from] VirtioMemError), + /// Virtio state: {0} + VirtioState(#[from] VirtioStateError), +} + +impl Persist<'_> for VirtioMem { + type State = VirtioMemState; + type ConstructorArgs = VirtioMemConstructorArgs; + type Error = VirtioMemPersistError; + + fn save(&self) -> Self::State { + VirtioMemState { + virtio_state: VirtioDeviceState::from_device(self), + region_size: self.config.region_size, + block_size: self.config.block_size, + usable_region_size: self.config.usable_region_size, + plugged_size: self.config.plugged_size, + requested_size: self.config.requested_size, + slot_size: self.slot_size, + } + } + + fn restore( + constructor_args: Self::ConstructorArgs, + state: &Self::State, + ) -> Result { + let queues = state.virtio_state.build_queues_checked( + constructor_args.vm.guest_memory(), + VIRTIO_ID_MEM, + MEM_NUM_QUEUES, + FIRECRACKER_MAX_QUEUE_SIZE, + )?; + + let config = virtio_mem_config { + addr: VIRTIO_MEM_GUEST_ADDRESS.raw_value(), + region_size: state.region_size, + block_size: state.block_size, + usable_region_size: state.usable_region_size, + plugged_size: state.plugged_size, + requested_size: state.requested_size, + ..Default::default() + }; + + let mut virtio_mem = + VirtioMem::from_state(constructor_args.vm, queues, config, state.slot_size)?; + virtio_mem.set_avail_features(state.virtio_state.avail_features); + virtio_mem.set_acked_features(state.virtio_state.acked_features); + + Ok(virtio_mem) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::devices::virtio::device::VirtioDevice; + use crate::devices::virtio::mem::device::test_utils::default_virtio_mem; + use crate::vstate::vm::tests::setup_vm_with_memory; + + #[test] + fn test_save_state() { + let dev = default_virtio_mem(); + let state = dev.save(); + + assert_eq!(state.region_size, dev.config.region_size); + assert_eq!(state.block_size, dev.config.block_size); + assert_eq!(state.usable_region_size, dev.config.usable_region_size); + assert_eq!(state.plugged_size, dev.config.plugged_size); + assert_eq!(state.requested_size, dev.config.requested_size); + assert_eq!(state.slot_size, dev.slot_size); + } + + #[test] + fn test_save_restore_state() { + let mut original_dev = default_virtio_mem(); + original_dev.set_acked_features(original_dev.avail_features()); + let state = original_dev.save(); + + // Create a "new" VM for restore + let (_, vm) = setup_vm_with_memory(0x1000); + let vm = Arc::new(vm); + let constructor_args = VirtioMemConstructorArgs::new(vm); + let restored_dev = VirtioMem::restore(constructor_args, &state).unwrap(); + + assert_eq!(original_dev.config, restored_dev.config); + assert_eq!(original_dev.slot_size, restored_dev.slot_size); + assert_eq!(original_dev.avail_features(), restored_dev.avail_features()); + assert_eq!(original_dev.acked_features(), restored_dev.acked_features()); + } +} diff --git a/src/vmm/src/logger/metrics.rs b/src/vmm/src/logger/metrics.rs index deb1dfda868..74e3fb6eb01 100644 --- a/src/vmm/src/logger/metrics.rs +++ b/src/vmm/src/logger/metrics.rs @@ -74,6 +74,7 @@ use super::FcLineWriter; use crate::devices::legacy; use crate::devices::virtio::balloon::metrics as balloon_metrics; use crate::devices::virtio::block::virtio::metrics as block_metrics; +use crate::devices::virtio::mem::metrics as virtio_mem_metrics; use crate::devices::virtio::net::metrics as net_metrics; use crate::devices::virtio::rng::metrics as entropy_metrics; use crate::devices::virtio::vhost_user_metrics; @@ -873,6 +874,7 @@ create_serialize_proxy!(BalloonMetricsSerializeProxy, balloon_metrics); create_serialize_proxy!(EntropyMetricsSerializeProxy, entropy_metrics); create_serialize_proxy!(VsockMetricsSerializeProxy, vsock_metrics); create_serialize_proxy!(LegacyDevMetricsSerializeProxy, legacy); +create_serialize_proxy!(MemoryHotplugSerializeProxy, virtio_mem_metrics); /// Structure storing all metrics while enforcing serialization support on them. #[derive(Debug, Default, Serialize)] @@ -923,6 +925,9 @@ pub struct FirecrackerMetrics { #[serde(flatten)] /// Vhost-user device related metrics. pub vhost_user_ser: VhostUserMetricsSerializeProxy, + #[serde(flatten)] + /// Virtio-mem device related metrics (memory hotplugging) + pub memory_hotplug_ser: MemoryHotplugSerializeProxy, } impl FirecrackerMetrics { /// Const default construction. @@ -948,6 +953,7 @@ impl FirecrackerMetrics { vsock_ser: VsockMetricsSerializeProxy {}, entropy_ser: EntropyMetricsSerializeProxy {}, vhost_user_ser: VhostUserMetricsSerializeProxy {}, + memory_hotplug_ser: MemoryHotplugSerializeProxy {}, } } } diff --git a/src/vmm/src/resources.rs b/src/vmm/src/resources.rs index a6f3b50dea4..82912144309 100644 --- a/src/vmm/src/resources.rs +++ b/src/vmm/src/resources.rs @@ -257,6 +257,14 @@ impl VmResources { SharedDeviceType::Entropy(entropy) => { self.entropy.set_device(entropy); } + SharedDeviceType::VirtioMem(mem) => { + let mem = mem.lock().unwrap(); + self.memory_hotplug = Some(MemoryHotplugConfig { + total_size_mib: mem.total_size_mib(), + block_size_mib: mem.block_size_mib(), + slot_size_mib: mem.slot_size_mib(), + }) + } } Ok(()) diff --git a/src/vmm/src/vmm_config/memory_hotplug.rs b/src/vmm/src/vmm_config/memory_hotplug.rs index 689301c0eac..d09141c1b66 100644 --- a/src/vmm/src/vmm_config/memory_hotplug.rs +++ b/src/vmm/src/vmm_config/memory_hotplug.rs @@ -3,8 +3,9 @@ use serde::{Deserialize, Serialize}; -use crate::devices::virtio::mem::{VIRTIO_MEM_DEFAULT_BLOCK_SIZE, VIRTIO_MEM_DEFAULT_SLOT_SIZE}; -use crate::utils::bytes_to_mib; +use crate::devices::virtio::mem::{ + VIRTIO_MEM_DEFAULT_BLOCK_SIZE_MIB, VIRTIO_MEM_DEFAULT_SLOT_SIZE_MIB, +}; /// Errors associated with memory hotplug configuration. #[derive(Debug, thiserror::Error, displaydoc::Display)] @@ -24,11 +25,11 @@ pub enum MemoryHotplugConfigError { } fn default_block_size_mib() -> usize { - bytes_to_mib(VIRTIO_MEM_DEFAULT_BLOCK_SIZE) + VIRTIO_MEM_DEFAULT_BLOCK_SIZE_MIB } fn default_slot_size_mib() -> usize { - bytes_to_mib(VIRTIO_MEM_DEFAULT_SLOT_SIZE) + VIRTIO_MEM_DEFAULT_SLOT_SIZE_MIB } /// Configuration for memory hotplug device. @@ -48,7 +49,7 @@ pub struct MemoryHotplugConfig { impl MemoryHotplugConfig { /// Validates the configuration. pub fn validate(&self) -> Result<(), MemoryHotplugConfigError> { - let min_block_size_mib = bytes_to_mib(VIRTIO_MEM_DEFAULT_BLOCK_SIZE); + let min_block_size_mib = VIRTIO_MEM_DEFAULT_BLOCK_SIZE_MIB; if self.block_size_mib < min_block_size_mib { return Err(MemoryHotplugConfigError::BlockSizeTooSmall( min_block_size_mib, @@ -58,7 +59,7 @@ impl MemoryHotplugConfig { return Err(MemoryHotplugConfigError::BlockSizeNotPowerOfTwo); } - let min_slot_size_mib = bytes_to_mib(VIRTIO_MEM_DEFAULT_SLOT_SIZE); + let min_slot_size_mib = VIRTIO_MEM_DEFAULT_SLOT_SIZE_MIB; if self.slot_size_mib < min_slot_size_mib { return Err(MemoryHotplugConfigError::SlotSizeTooSmall( min_slot_size_mib, diff --git a/tests/host_tools/fcmetrics.py b/tests/host_tools/fcmetrics.py index 7e91e04f8c8..00da02423ec 100644 --- a/tests/host_tools/fcmetrics.py +++ b/tests/host_tools/fcmetrics.py @@ -300,6 +300,11 @@ def validate_fc_metrics(metrics): "entropy_rate_limiter_throttled", "rate_limiter_event_count", ], + "memory_hotplug": [ + "activate_fails", + "queue_event_fails", + "queue_event_count", + ], } # validate timestamp before jsonschema validation which some more time diff --git a/tests/integration_tests/functional/test_api.py b/tests/integration_tests/functional/test_api.py index f268a079add..81454559990 100644 --- a/tests/integration_tests/functional/test_api.py +++ b/tests/integration_tests/functional/test_api.py @@ -1006,7 +1006,7 @@ def test_api_memory_hotplug(uvm_plain): test_microvm.start() # Put API should be rejected after boot - with pytest.raises(RuntimeError): + with pytest.raises(RuntimeError, match=NOT_SUPPORTED_AFTER_START): test_microvm.api.memory_hotplug.put(total_size_mib=1024) # Get API should work after boot @@ -1130,8 +1130,12 @@ def test_get_full_config_after_restoring_snapshot(microvm_factory, uvm_nano): uvm_nano.api.vsock.put(guest_cid=15, uds_path="vsock.sock") setup_cfg["vsock"] = {"guest_cid": 15, "uds_path": "vsock.sock"} - # TODO(virtio-mem): add memory hotplug when snapshot support is added. - setup_cfg["memory-hotplug"] = None + setup_cfg["memory-hotplug"] = { + "total_size_mib": 1024, + "block_size_mib": 128, + "slot_size_mib": 1024, + } + uvm_nano.api.memory_hotplug.put(**setup_cfg["memory-hotplug"]) setup_cfg["logger"] = None setup_cfg["metrics"] = None