diff --git a/resources/chroot.sh b/resources/chroot.sh index 82061700b4a..d3b333079b5 100755 --- a/resources/chroot.sh +++ b/resources/chroot.sh @@ -64,6 +64,10 @@ rm -vf /etc/systemd/system/timers.target.wants/* systemctl enable var-lib-systemd.mount +# disable Predictable Network Interface Names to keep ethN names +# even with PCI enabled +ln -s /dev/null /etc/systemd/network/99-default.link + #### trim image https://wiki.ubuntu.com/ReducingDiskFootprint # this does not save much, but oh well rm -rf /usr/share/{doc,man,info,locale} diff --git a/src/vmm/src/arch/aarch64/fdt.rs b/src/vmm/src/arch/aarch64/fdt.rs index 429153669fa..6a50c0257a9 100644 --- a/src/vmm/src/arch/aarch64/fdt.rs +++ b/src/vmm/src/arch/aarch64/fdt.rs @@ -379,7 +379,7 @@ fn create_virtio_node(fdt: &mut FdtWriter, dev_info: &MMIODeviceInfo) -> Result< "interrupts", &[ GIC_FDT_IRQ_TYPE_SPI, - dev_info.irq.unwrap(), + dev_info.gsi.unwrap(), IRQ_TYPE_EDGE_RISING, ], )?; @@ -400,7 +400,7 @@ fn create_serial_node(fdt: &mut FdtWriter, dev_info: &MMIODeviceInfo) -> Result< "interrupts", &[ GIC_FDT_IRQ_TYPE_SPI, - dev_info.irq.unwrap(), + dev_info.gsi.unwrap(), IRQ_TYPE_EDGE_RISING, ], )?; diff --git a/src/vmm/src/arch/aarch64/gic/gicv2/mod.rs b/src/vmm/src/arch/aarch64/gic/gicv2/mod.rs index dfa2302d6be..01fd4b4d73d 100644 --- a/src/vmm/src/arch/aarch64/gic/gicv2/mod.rs +++ b/src/vmm/src/arch/aarch64/gic/gicv2/mod.rs @@ -135,9 +135,9 @@ impl GICv2 { // On arm there are 3 types of interrupts: SGI (0-15), PPI (16-31), SPI (32-1020). // SPIs are used to signal interrupts from various peripherals accessible across // the whole system so these are the ones that we increment when adding a new virtio device. - // KVM_DEV_ARM_VGIC_GRP_NR_IRQS sets the highest SPI number. Consequently, we will have a - // total of `super::layout::IRQ_MAX - 32` usable SPIs in our microVM. - let nr_irqs: u32 = super::layout::IRQ_MAX; + // KVM_DEV_ARM_VGIC_GRP_NR_IRQS sets the number of interrupts (SGI, PPI, and SPI). + // Consequently, we need to add 32 to the number of SPIs ("legacy GSI"). + let nr_irqs: u32 = crate::arch::GSI_LEGACY_NUM + super::layout::SPI_START; let nr_irqs_ptr = &nr_irqs as *const u32; Self::set_device_attribute( gic_device.device_fd(), diff --git a/src/vmm/src/arch/aarch64/gic/gicv2/regs/dist_regs.rs b/src/vmm/src/arch/aarch64/gic/gicv2/regs/dist_regs.rs index 21a404b302b..09a33a4a1ff 100644 --- a/src/vmm/src/arch/aarch64/gic/gicv2/regs/dist_regs.rs +++ b/src/vmm/src/arch/aarch64/gic/gicv2/regs/dist_regs.rs @@ -8,7 +8,7 @@ use kvm_ioctls::DeviceFd; use crate::arch::aarch64::gic::GicError; use crate::arch::aarch64::gic::regs::{GicRegState, MmioReg, SimpleReg, VgicRegEngine}; -use crate::arch::{IRQ_BASE, IRQ_MAX}; +use crate::arch::{GSI_LEGACY_NUM, SPI_START}; // Distributor registers as detailed at page 75 from // https://developer.arm.com/documentation/ihi0048/latest/. @@ -62,9 +62,9 @@ impl MmioReg for SharedIrqReg { // read-as-zero/write-ignore (RAZ/WI) policy. // The first part of a shared-irq register, the one corresponding to the // SGI and PPI IRQs (0-32) is RAZ/WI, so we skip it. - let start = self.offset + u64::from(IRQ_BASE) * u64::from(self.bits_per_irq) / 8; + let start = self.offset + u64::from(SPI_START) * u64::from(self.bits_per_irq) / 8; - let size_in_bits = u64::from(self.bits_per_irq) * u64::from(IRQ_MAX - IRQ_BASE); + let size_in_bits = u64::from(self.bits_per_irq) * u64::from(GSI_LEGACY_NUM); let mut size_in_bytes = size_in_bits / 8; if size_in_bits % 8 > 0 { size_in_bytes += 1; diff --git a/src/vmm/src/arch/aarch64/gic/gicv3/mod.rs b/src/vmm/src/arch/aarch64/gic/gicv3/mod.rs index 075687bc23e..5d131cf7b76 100644 --- a/src/vmm/src/arch/aarch64/gic/gicv3/mod.rs +++ b/src/vmm/src/arch/aarch64/gic/gicv3/mod.rs @@ -184,9 +184,9 @@ impl GICv3 { // On arm there are 3 types of interrupts: SGI (0-15), PPI (16-31), SPI (32-1020). // SPIs are used to signal interrupts from various peripherals accessible across // the whole system so these are the ones that we increment when adding a new virtio device. - // KVM_DEV_ARM_VGIC_GRP_NR_IRQS sets the highest SPI number. Consequently, we will have a - // total of `super::layout::IRQ_MAX - 32` usable SPIs in our microVM. - let nr_irqs: u32 = super::layout::IRQ_MAX; + // KVM_DEV_ARM_VGIC_GRP_NR_IRQS sets the number of interrupts (SGI, PPI, and SPI). + // Consequently, we need to add 32 to the number of SPIs ("legacy GSI"). + let nr_irqs: u32 = crate::arch::GSI_LEGACY_NUM + super::layout::SPI_START; let nr_irqs_ptr = &nr_irqs as *const u32; Self::set_device_attribute( gic_device.device_fd(), diff --git a/src/vmm/src/arch/aarch64/gic/gicv3/regs/dist_regs.rs b/src/vmm/src/arch/aarch64/gic/gicv3/regs/dist_regs.rs index 96c617dcc17..5a6eafb7003 100644 --- a/src/vmm/src/arch/aarch64/gic/gicv3/regs/dist_regs.rs +++ b/src/vmm/src/arch/aarch64/gic/gicv3/regs/dist_regs.rs @@ -8,7 +8,7 @@ use kvm_ioctls::DeviceFd; use crate::arch::aarch64::gic::GicError; use crate::arch::aarch64::gic::regs::{GicRegState, MmioReg, SimpleReg, VgicRegEngine}; -use crate::arch::{IRQ_BASE, IRQ_MAX}; +use crate::arch::{GSI_LEGACY_NUM, SPI_START}; // Distributor registers as detailed at page 456 from // https://static.docs.arm.com/ihi0069/c/IHI0069C_gic_architecture_specification.pdf. @@ -64,9 +64,9 @@ impl MmioReg for SharedIrqReg { // read-as-zero/write-ignore (RAZ/WI) policy. // The first part of a shared-irq register, the one corresponding to the // SGI and PPI IRQs (0-32) is RAZ/WI, so we skip it. - let start = self.offset + u64::from(IRQ_BASE) * u64::from(self.bits_per_irq) / 8; + let start = self.offset + u64::from(SPI_START) * u64::from(self.bits_per_irq) / 8; - let size_in_bits = u64::from(self.bits_per_irq) * u64::from(IRQ_MAX - IRQ_BASE); + let size_in_bits = u64::from(self.bits_per_irq) * u64::from(GSI_LEGACY_NUM); let mut size_in_bytes = size_in_bits / 8; if size_in_bits % 8 > 0 { size_in_bytes += 1; diff --git a/src/vmm/src/arch/aarch64/layout.rs b/src/vmm/src/arch/aarch64/layout.rs index 4b1f6ecda5b..c4937e43c92 100644 --- a/src/vmm/src/arch/aarch64/layout.rs +++ b/src/vmm/src/arch/aarch64/layout.rs @@ -76,19 +76,25 @@ pub const FDT_MAX_SIZE: usize = 0x20_0000; // * bigger than 32 // * less than 1023 and // * a multiple of 32. -/// The highest usable SPI on aarch64. -pub const IRQ_MAX: u32 = 128; - -/// First usable interrupt on aarch64. -pub const IRQ_BASE: u32 = 32; - -// The Linux kernel automatically shifts the GSI by 32 if it is an SPI, -// allowing us to start numbering from 0 instead of 32. -/// The first usable GSI on aarch64. -pub const GSI_BASE: u32 = 0; - -/// The maximum usable GSI on aarch64. -pub const GSI_MAX: u32 = IRQ_MAX - IRQ_BASE - 1; +// The first 32 SPIs are reserved, but KVM already shifts the gsi we +// pass, so we go from 0 to 95 for legacy gsis ("irq") and the remaining +// we use for MSI. +/// Offset of first SPI in the GIC +pub const SPI_START: u32 = 32; +/// Last possible SPI in the GIC (128 total SPIs) +pub const SPI_END: u32 = 127; +/// First usable GSI id on aarch64 (corresponds to SPI #32). +pub const GSI_LEGACY_START: u32 = 0; +/// There are 128 SPIs available, but the first 32 are reserved +pub const GSI_LEGACY_NUM: u32 = SPI_END - SPI_START + 1; +/// Last available GSI +pub const GSI_LEGACY_END: u32 = GSI_LEGACY_START + GSI_LEGACY_NUM - 1; +/// First GSI used by MSI after legacy GSI +pub const GSI_MSI_START: u32 = GSI_LEGACY_END + 1; +/// The highest available GSI in KVM (KVM_MAX_IRQ_ROUTES=4096) +pub const GSI_MSI_END: u32 = 4095; +/// Number of GSI available for MSI. +pub const GSI_MSI_NUM: u32 = GSI_MSI_END - GSI_MSI_START + 1; /// The start of the memory area reserved for MMIO 32-bit accesses. /// Below this address will reside the GIC, above this address will reside the MMIO devices. diff --git a/src/vmm/src/arch/mod.rs b/src/vmm/src/arch/mod.rs index fbeb9fa0ce0..6d33ce461b9 100644 --- a/src/vmm/src/arch/mod.rs +++ b/src/vmm/src/arch/mod.rs @@ -21,13 +21,14 @@ pub use aarch64::vm::{ArchVm, ArchVmError, VmState}; #[cfg(target_arch = "aarch64")] pub use aarch64::{ ConfigurationError, arch_memory_regions, configure_system_for_boot, get_kernel_start, - initrd_load_addr, layout::BOOT_DEVICE_MEM_START, layout::CMDLINE_MAX_SIZE, layout::GSI_BASE, - layout::GSI_MAX, layout::IRQ_BASE, layout::IRQ_MAX, layout::MEM_32BIT_DEVICES_SIZE, + initrd_load_addr, layout::BOOT_DEVICE_MEM_START, layout::CMDLINE_MAX_SIZE, + layout::GSI_LEGACY_END, layout::GSI_LEGACY_NUM, layout::GSI_LEGACY_START, layout::GSI_MSI_END, + layout::GSI_MSI_NUM, layout::GSI_MSI_START, layout::MEM_32BIT_DEVICES_SIZE, layout::MEM_32BIT_DEVICES_START, layout::MEM_64BIT_DEVICES_SIZE, layout::MEM_64BIT_DEVICES_START, layout::MMIO32_MEM_SIZE, layout::MMIO32_MEM_START, layout::PCI_MMCONFIG_SIZE, layout::PCI_MMCONFIG_START, layout::PCI_MMIO_CONFIG_SIZE_PER_SEGMENT, layout::RTC_MEM_START, layout::SERIAL_MEM_START, - layout::SYSTEM_MEM_SIZE, layout::SYSTEM_MEM_START, load_kernel, + layout::SPI_START, layout::SYSTEM_MEM_SIZE, layout::SYSTEM_MEM_START, load_kernel, }; /// Module for x86_64 related functionality. @@ -45,7 +46,8 @@ pub use x86_64::vm::{ArchVm, ArchVmError, VmState}; pub use crate::arch::x86_64::{ ConfigurationError, arch_memory_regions, configure_system_for_boot, get_kernel_start, initrd_load_addr, layout::APIC_ADDR, layout::BOOT_DEVICE_MEM_START, layout::CMDLINE_MAX_SIZE, - layout::GSI_BASE, layout::GSI_MAX, layout::IOAPIC_ADDR, layout::IRQ_BASE, layout::IRQ_MAX, + layout::GSI_LEGACY_END, layout::GSI_LEGACY_NUM, layout::GSI_LEGACY_START, layout::GSI_MSI_END, + layout::GSI_MSI_NUM, layout::GSI_MSI_START, layout::IOAPIC_ADDR, layout::MEM_32BIT_DEVICES_SIZE, layout::MEM_32BIT_DEVICES_START, layout::MEM_64BIT_DEVICES_SIZE, layout::MEM_64BIT_DEVICES_START, layout::MMIO32_MEM_SIZE, layout::MMIO32_MEM_START, layout::PCI_MMCONFIG_SIZE, layout::PCI_MMCONFIG_START, diff --git a/src/vmm/src/arch/x86_64/layout.rs b/src/vmm/src/arch/x86_64/layout.rs index b7d5eb6dc5f..34ad343af2a 100644 --- a/src/vmm/src/arch/x86_64/layout.rs +++ b/src/vmm/src/arch/x86_64/layout.rs @@ -21,17 +21,21 @@ pub const CMDLINE_MAX_SIZE: usize = 2048; /// Start of the high memory. pub const HIMEM_START: u64 = 0x0010_0000; // 1 MB. -// Typically, on x86 systems 24 IRQs are used (0-23). -/// First usable IRQ ID for virtio device interrupts on x86_64. -pub const IRQ_BASE: u32 = 5; -/// Last usable IRQ ID for virtio device interrupts on x86_64. -pub const IRQ_MAX: u32 = 23; - -/// The first usable GSI on x86_64 is the same as the first usable IRQ ID. -pub const GSI_BASE: u32 = IRQ_BASE; - -/// The maximum usable GSI on x86_64 is the same as the last usable IRQ ID. -pub const GSI_MAX: u32 = IRQ_MAX; +// Typically, on x86 systems 24 IRQs are used for legacy devices (0-23). +// However, the first 5 are reserved. +// We allocate the remaining GSIs to MSIs. +/// First usable GSI for legacy interrupts (IRQ) on x86_64. +pub const GSI_LEGACY_START: u32 = 5; +/// Last usable GSI for legacy interrupts (IRQ) on x86_64. +pub const GSI_LEGACY_END: u32 = 23; +/// Number of legacy GSI (IRQ) available on x86_64. +pub const GSI_LEGACY_NUM: u32 = GSI_LEGACY_END - GSI_LEGACY_START + 1; +/// First GSI used by MSI after legacy GSI. +pub const GSI_MSI_START: u32 = GSI_LEGACY_END + 1; +/// The highest available GSI in KVM (KVM_MAX_IRQ_ROUTES=4096). +pub const GSI_MSI_END: u32 = 4095; +/// Number of GSI available for MSI. +pub const GSI_MSI_NUM: u32 = GSI_MSI_END - GSI_MSI_START + 1; /// Address for the TSS setup. pub const KVM_TSS_ADDRESS: u64 = 0xfffb_d000; diff --git a/src/vmm/src/arch/x86_64/mptable.rs b/src/vmm/src/arch/x86_64/mptable.rs index a4b1e2fa632..99fb202c8d8 100644 --- a/src/vmm/src/arch/x86_64/mptable.rs +++ b/src/vmm/src/arch/x86_64/mptable.rs @@ -13,7 +13,7 @@ use libc::c_char; use log::debug; use vm_allocator::AllocPolicy; -use crate::arch::IRQ_MAX; +use crate::arch::GSI_LEGACY_END; use crate::arch::x86_64::generated::mpspec; use crate::vstate::memory::{ Address, ByteValued, Bytes, GuestAddress, GuestMemory, GuestMemoryMmap, @@ -109,7 +109,7 @@ fn compute_mp_size(num_cpus: u8) -> usize { + mem::size_of::() * (num_cpus as usize) + mem::size_of::() + mem::size_of::() - + mem::size_of::() * (IRQ_MAX as usize + 1) + + mem::size_of::() * (GSI_LEGACY_END as usize + 1) + mem::size_of::() * 2 } @@ -225,7 +225,7 @@ pub fn setup_mptable( mp_num_entries += 1; } // Per kvm_setup_default_irq_routing() in kernel - for i in 0..=u8::try_from(IRQ_MAX).map_err(|_| MptableError::TooManyIrqs)? { + for i in 0..=u8::try_from(GSI_LEGACY_END).map_err(|_| MptableError::TooManyIrqs)? { let size = mem::size_of::() as u64; let mpc_intsrc = mpspec::mpc_intsrc { type_: mpspec::MP_INTSRC.try_into().unwrap(), @@ -406,7 +406,7 @@ mod tests { // ISA Bus + 1 // IRQ - + u16::try_from(IRQ_MAX).unwrap() + 1 + + u16::try_from(GSI_LEGACY_END).unwrap() + 1 // Interrupt source ExtINT + 1 // Interrupt source NMI diff --git a/src/vmm/src/device_manager/acpi.rs b/src/vmm/src/device_manager/acpi.rs index 3f0af80c7aa..874443fcc5c 100644 --- a/src/vmm/src/device_manager/acpi.rs +++ b/src/vmm/src/device_manager/acpi.rs @@ -64,7 +64,7 @@ impl Aml for ACPIDeviceManager { // We know that the maximum IRQ number fits in a u8. We have up to // 32 IRQs in x86 and up to 128 in // ARM (look into - // `vmm::crate::arch::layout::IRQ_MAX`) + // `vmm::crate::arch::layout::GSI_LEGACY_END`) #[allow(clippy::cast_possible_truncation)] &aml::Equal::new(&aml::Arg(0), &(vmgenid.gsi as u8)), vec![&aml::Notify::new( diff --git a/src/vmm/src/device_manager/mmio.rs b/src/vmm/src/device_manager/mmio.rs index d0c116ce20d..a87646b11cf 100644 --- a/src/vmm/src/device_manager/mmio.rs +++ b/src/vmm/src/device_manager/mmio.rs @@ -73,8 +73,8 @@ pub struct MMIODeviceInfo { pub addr: u64, /// Mmio addr range length. pub len: u64, - /// Used Irq line for the device. - pub irq: Option, + /// Used GSI (interrupt line) for the device. + pub gsi: Option, } #[cfg(target_arch = "x86_64")] @@ -82,12 +82,12 @@ fn add_virtio_aml( dsdt_data: &mut Vec, addr: u64, len: u64, - irq: u32, + gsi: u32, ) -> Result<(), aml::AmlError> { - let dev_id = irq - crate::arch::GSI_BASE; + let dev_id = gsi - crate::arch::GSI_LEGACY_START; debug!( - "acpi: Building AML for VirtIO device _SB_.V{:03}. memory range: {:#010x}:{} irq: {}", - dev_id, addr, len, irq + "acpi: Building AML for VirtIO device _SB_.V{:03}. memory range: {:#010x}:{} gsi: {}", + dev_id, addr, len, gsi ); aml::Device::new( format!("V{:03}", dev_id).as_str().try_into()?, @@ -103,7 +103,7 @@ fn add_virtio_aml( addr.try_into().unwrap(), len.try_into().unwrap(), ), - &aml::Interrupt::new(true, true, false, false, irq), + &aml::Interrupt::new(true, true, false, false, gsi), ]), )?, ], @@ -156,9 +156,9 @@ impl MMIODeviceManager { resource_allocator: &mut ResourceAllocator, irq_count: u32, ) -> Result { - let irq = match resource_allocator.allocate_gsi(irq_count)?[..] { + let gsi = match resource_allocator.allocate_gsi_legacy(irq_count)?[..] { [] => None, - [irq] => Some(irq), + [gsi] => Some(gsi), _ => return Err(MmioError::InvalidIrqConfig), }; @@ -169,7 +169,7 @@ impl MMIODeviceManager { AllocPolicy::FirstMatch, )?, len: MMIO_LEN, - irq, + gsi, }; Ok(device_info) } @@ -183,7 +183,7 @@ impl MMIODeviceManager { ) -> Result<(), MmioError> { // Our virtio devices are currently hardcoded to use a single IRQ. // Validate that requirement. - let irq = device.resources.irq.ok_or(MmioError::InvalidIrqConfig)?; + let gsi = device.resources.gsi.ok_or(MmioError::InvalidIrqConfig)?; let identifier; { let mmio_device = device.inner.lock().expect("Poisoned lock"); @@ -197,7 +197,7 @@ impl MMIODeviceManager { .register_ioevent(queue_evt, &io_addr, u32::try_from(i).unwrap()) .map_err(MmioError::RegisterIoEvent)?; } - vm.register_irq(&mmio_device.interrupt.irq_evt, irq) + vm.register_irq(&mmio_device.interrupt.irq_evt, gsi) .map_err(MmioError::RegisterIrqFd)?; } @@ -226,7 +226,7 @@ impl MMIODeviceManager { .add_virtio_mmio_device( device_info.len, GuestAddress(device_info.addr), - device_info.irq.unwrap(), + device_info.gsi.unwrap(), None, ) .map_err(MmioError::Cmdline) @@ -255,7 +255,7 @@ impl MMIODeviceManager { device.resources.len, // We are sure that `irqs` has at least one element; allocate_mmio_resources makes // sure of it. - device.resources.irq.unwrap(), + device.resources.gsi.unwrap(), )?; } self.register_mmio_virtio(vm, device_id, device)?; @@ -276,17 +276,17 @@ impl MMIODeviceManager { let device_info = if let Some(device_info) = device_info_opt { device_info } else { - let gsi = vm.resource_allocator().allocate_gsi(1)?; + let gsi = vm.resource_allocator().allocate_gsi_legacy(1)?; MMIODeviceInfo { addr: SERIAL_MEM_START, len: MMIO_LEN, - irq: Some(gsi[0]), + gsi: Some(gsi[0]), } }; vm.register_irq( serial.lock().expect("Poisoned lock").serial.interrupt_evt(), - device_info.irq.unwrap(), + device_info.gsi.unwrap(), ) .map_err(MmioError::RegisterIrqFd)?; @@ -335,11 +335,11 @@ impl MMIODeviceManager { let device_info = if let Some(device_info) = device_info_opt { device_info } else { - let gsi = vm.resource_allocator().allocate_gsi(1)?; + let gsi = vm.resource_allocator().allocate_gsi_legacy(1)?; MMIODeviceInfo { addr: RTC_MEM_START, len: MMIO_LEN, - irq: Some(gsi[0]), + gsi: Some(gsi[0]), } }; @@ -367,7 +367,7 @@ impl MMIODeviceManager { let device_info = MMIODeviceInfo { addr: BOOT_DEVICE_MEM_START, len: MMIO_LEN, - irq: None, + gsi: None, }; let device = MMIODevice { @@ -496,7 +496,7 @@ pub(crate) mod tests { pub fn used_irqs_count(&self) -> usize { self.virtio_devices .iter() - .filter(|(_, mmio_dev)| mmio_dev.resources.irq.is_some()) + .filter(|(_, mmio_dev)| mmio_dev.resources.gsi.is_some()) .count() } } @@ -612,7 +612,7 @@ pub(crate) mod tests { let dev = device_manager.get_virtio_device(0, "dummy").unwrap(); assert_eq!(dev.resources.addr, arch::MEM_32BIT_DEVICES_START); assert_eq!(dev.resources.len, MMIO_LEN); - assert_eq!(dev.resources.irq, Some(arch::GSI_BASE)); + assert_eq!(dev.resources.gsi, Some(arch::GSI_LEGACY_START)); device_manager .for_each_virtio_device(|virtio_type, device_id, mmio_device| { @@ -620,7 +620,7 @@ pub(crate) mod tests { assert_eq!(device_id, "dummy"); assert_eq!(mmio_device.resources.addr, arch::MEM_32BIT_DEVICES_START); assert_eq!(mmio_device.resources.len, MMIO_LEN); - assert_eq!(mmio_device.resources.irq, Some(arch::GSI_BASE)); + assert_eq!(mmio_device.resources.gsi, Some(arch::GSI_LEGACY_START)); Ok::<(), ()>(()) }) .unwrap(); @@ -643,7 +643,7 @@ pub(crate) mod tests { #[cfg(target_arch = "aarch64")] vm.setup_irqchip(1).unwrap(); - for _i in crate::arch::GSI_BASE..=crate::arch::GSI_MAX { + for _i in crate::arch::GSI_LEGACY_START..=crate::arch::GSI_LEGACY_END { device_manager .register_virtio_test_device( &vm, @@ -711,10 +711,10 @@ pub(crate) mod tests { .addr ); assert_eq!( - crate::arch::GSI_BASE, + crate::arch::GSI_LEGACY_START, device_manager.virtio_devices[&(type_id, id)] .resources - .irq + .gsi .unwrap() ); @@ -751,7 +751,7 @@ pub(crate) mod tests { let device_info = device_manager .allocate_mmio_resources(&mut resource_allocator, 0) .unwrap(); - assert!(device_info.irq.is_none()); + assert!(device_info.gsi.is_none()); } #[test] @@ -762,7 +762,7 @@ pub(crate) mod tests { let device_info = device_manager .allocate_mmio_resources(&mut resource_allocator, 1) .unwrap(); - assert_eq!(device_info.irq.unwrap(), crate::arch::GSI_BASE); + assert_eq!(device_info.gsi.unwrap(), crate::arch::GSI_LEGACY_START); } #[test] diff --git a/src/vmm/src/device_manager/mod.rs b/src/vmm/src/device_manager/mod.rs index cfc7fe44d79..c7f6acabfe1 100644 --- a/src/vmm/src/device_manager/mod.rs +++ b/src/vmm/src/device_manager/mod.rs @@ -81,6 +81,17 @@ pub enum AttachDeviceError { PciTransport(#[from] PciManagerError), } +#[derive(Debug, thiserror::Error, displaydoc::Display)] +/// Error while searching for a VirtIO device +pub enum FindDeviceError { + /// Device type is invalid + InvalidDeviceType, + /// Device not found + DeviceNotFound, + /// Internal Device error: {0} + InternalDeviceError(String), +} + #[derive(Debug)] /// A manager of all peripheral devices of Firecracker pub struct DeviceManager { @@ -342,6 +353,30 @@ impl DeviceManager { ) } } + + /// Run fn `f()` for the virtio device matching `virtio_type` and `id`. + pub fn with_virtio_device_with_id( + &self, + virtio_type: u32, + id: &str, + f: F, + ) -> Result<(), FindDeviceError> + where + T: VirtioDevice + 'static + Debug, + F: FnOnce(&mut T) -> Result<(), String>, + { + if let Some(device) = self.get_virtio_device(virtio_type, id) { + let mut dev = device.lock().expect("Poisoned lock"); + f(dev + .as_mut_any() + .downcast_mut::() + .ok_or(FindDeviceError::InvalidDeviceType)?) + .map_err(FindDeviceError::InternalDeviceError)?; + } else { + return Err(FindDeviceError::DeviceNotFound); + } + Ok(()) + } } #[derive(Debug, Default, Clone, Serialize, Deserialize)] diff --git a/src/vmm/src/devices/acpi/vmgenid.rs b/src/vmm/src/devices/acpi/vmgenid.rs index 6d096007193..8dc89289c98 100644 --- a/src/vmm/src/devices/acpi/vmgenid.rs +++ b/src/vmm/src/devices/acpi/vmgenid.rs @@ -88,7 +88,7 @@ impl VmGenId { mem: &GuestMemoryMmap, resource_allocator: &mut ResourceAllocator, ) -> Result { - let gsi = resource_allocator.allocate_gsi(1)?; + let gsi = resource_allocator.allocate_gsi_legacy(1)?; // The generation ID needs to live in an 8-byte aligned buffer let addr = resource_allocator.allocate_system_memory( VMGENID_MEM_SIZE, diff --git a/src/vmm/src/lib.rs b/src/vmm/src/lib.rs index 4549c79857a..046e624b971 100644 --- a/src/vmm/src/lib.rs +++ b/src/vmm/src/lib.rs @@ -123,7 +123,6 @@ use std::time::Duration; use device_manager::DeviceManager; use devices::acpi::vmgenid::VmGenIdError; -use devices::virtio::device::VirtioDevice; use event_manager::{EventManager as BaseEventManager, EventOps, Events, MutEventSubscriber}; use seccomp::BpfProgram; use snapshot::Persist; @@ -252,6 +251,8 @@ pub enum VmmError { VmmObserverTeardown(vmm_sys_util::errno::Error), /// VMGenID error: {0} VMGenID(#[from] VmGenIdError), + /// Failed perform action on device: {0} + FindDeviceError(#[from] device_manager::FindDeviceError), } /// Shorthand type for KVM dirty page bitmap. @@ -327,20 +328,6 @@ impl Vmm { self.shutdown_exit_code } - /// Gets the specified bus device. - pub fn get_virtio_device( - &self, - device_type: u32, - device_id: &str, - ) -> Option>> { - let device = self - .device_manager - .mmio_devices - .get_virtio_device(device_type, device_id)?; - - Some(device.inner.lock().expect("Poisoned lock").device().clone()) - } - /// Starts the microVM vcpus. /// /// # Errors @@ -537,13 +524,12 @@ impl Vmm { path_on_host: String, ) -> Result<(), VmmError> { self.device_manager - .mmio_devices .with_virtio_device_with_id(TYPE_BLOCK, drive_id, |block: &mut Block| { block .update_disk_image(path_on_host) .map_err(|err| err.to_string()) }) - .map_err(VmmError::MmioDeviceManager) + .map_err(VmmError::FindDeviceError) } /// Updates the rate limiter parameters for block device with `drive_id` id. @@ -554,23 +540,21 @@ impl Vmm { rl_ops: BucketUpdate, ) -> Result<(), VmmError> { self.device_manager - .mmio_devices .with_virtio_device_with_id(TYPE_BLOCK, drive_id, |block: &mut Block| { block .update_rate_limiter(rl_bytes, rl_ops) .map_err(|err| err.to_string()) }) - .map_err(VmmError::MmioDeviceManager) + .map_err(VmmError::FindDeviceError) } /// Updates the rate limiter parameters for block device with `drive_id` id. pub fn update_vhost_user_block_config(&mut self, drive_id: &str) -> Result<(), VmmError> { self.device_manager - .mmio_devices .with_virtio_device_with_id(TYPE_BLOCK, drive_id, |block: &mut Block| { block.update_config().map_err(|err| err.to_string()) }) - .map_err(VmmError::MmioDeviceManager) + .map_err(VmmError::FindDeviceError) } /// Updates the rate limiter parameters for net device with `net_id` id. @@ -583,17 +567,19 @@ impl Vmm { tx_ops: BucketUpdate, ) -> Result<(), VmmError> { self.device_manager - .mmio_devices .with_virtio_device_with_id(TYPE_NET, net_id, |net: &mut Net| { net.patch_rate_limiters(rx_bytes, rx_ops, tx_bytes, tx_ops); Ok(()) }) - .map_err(VmmError::MmioDeviceManager) + .map_err(VmmError::FindDeviceError) } /// Returns a reference to the balloon device if present. pub fn balloon_config(&self) -> Result { - if let Some(virtio_device) = self.get_virtio_device(TYPE_BALLOON, BALLOON_DEV_ID) { + if let Some(virtio_device) = self + .device_manager + .get_virtio_device(TYPE_BALLOON, BALLOON_DEV_ID) + { let config = virtio_device .lock() .expect("Poisoned lock") @@ -610,7 +596,10 @@ impl Vmm { /// Returns the latest balloon statistics if they are enabled. pub fn latest_balloon_stats(&self) -> Result { - if let Some(virtio_device) = self.get_virtio_device(TYPE_BALLOON, BALLOON_DEV_ID) { + if let Some(virtio_device) = self + .device_manager + .get_virtio_device(TYPE_BALLOON, BALLOON_DEV_ID) + { let latest_stats = virtio_device .lock() .expect("Poisoned lock") @@ -635,7 +624,10 @@ impl Vmm { return Err(BalloonError::TooManyPagesRequested); } - if let Some(virtio_device) = self.get_virtio_device(TYPE_BALLOON, BALLOON_DEV_ID) { + if let Some(virtio_device) = self + .device_manager + .get_virtio_device(TYPE_BALLOON, BALLOON_DEV_ID) + { { virtio_device .lock() @@ -657,7 +649,10 @@ impl Vmm { &mut self, stats_polling_interval_s: u16, ) -> Result<(), BalloonError> { - if let Some(virtio_device) = self.get_virtio_device(TYPE_BALLOON, BALLOON_DEV_ID) { + if let Some(virtio_device) = self + .device_manager + .get_virtio_device(TYPE_BALLOON, BALLOON_DEV_ID) + { { virtio_device .lock() diff --git a/src/vmm/src/persist.rs b/src/vmm/src/persist.rs index 2c9b1a0eea6..80acd9d5fbe 100644 --- a/src/vmm/src/persist.rs +++ b/src/vmm/src/persist.rs @@ -321,18 +321,23 @@ pub fn restore_from_snapshot( ) -> Result>, RestoreFromSnapshotError> { let mut microvm_state = snapshot_state_from_file(¶ms.snapshot_path)?; for entry in ¶ms.network_overrides { - let net_devices = &mut microvm_state.device_states.mmio_state.net_devices; - if let Some(device) = net_devices + microvm_state + .device_states + .mmio_state + .net_devices .iter_mut() - .find(|x| x.device_state.id == entry.iface_id) - { - device - .device_state - .tap_if_name - .clone_from(&entry.host_dev_name); - } else { - return Err(SnapshotStateFromFileError::UnknownNetworkDevice.into()); - } + .map(|device| &mut device.device_state) + .chain( + microvm_state + .device_states + .pci_state + .net_devices + .iter_mut() + .map(|device| &mut device.device_state), + ) + .find(|x| x.id == entry.iface_id) + .map(|device_state| device_state.tap_if_name.clone_from(&entry.host_dev_name)) + .ok_or(SnapshotStateFromFileError::UnknownNetworkDevice)?; } let track_dirty_pages = params.enable_diff_snapshots; diff --git a/src/vmm/src/vstate/resources.rs b/src/vmm/src/vstate/resources.rs index 6571abf8cfc..545b211699f 100644 --- a/src/vmm/src/vstate/resources.rs +++ b/src/vmm/src/vstate/resources.rs @@ -10,6 +10,29 @@ use vm_allocator::{AddressAllocator, IdAllocator}; use crate::arch; use crate::snapshot::Persist; +/// Helper function to allocate many ids from an id allocator +fn allocate_many_ids( + id_allocator: &mut IdAllocator, + count: u32, +) -> Result, vm_allocator::Error> { + let mut ids = Vec::with_capacity(count as usize); + + for _ in 0..count { + match id_allocator.allocate_id() { + Ok(id) => ids.push(id), + Err(err) => { + // It is ok to unwrap here, we just allocated the GSI + ids.into_iter().for_each(|id| { + id_allocator.free_id(id).unwrap(); + }); + return Err(err); + } + } + } + + Ok(ids) +} + /// A resource manager for (de)allocating interrupt lines (GSIs) and guest memory /// /// At the moment, we support: @@ -19,8 +42,10 @@ use crate::snapshot::Persist; /// * Memory allocations in the MMIO address space #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ResourceAllocator { - /// Allocator for device interrupt lines - pub gsi_allocator: IdAllocator, + /// Allocator for legacy device interrupt lines + pub gsi_legacy_allocator: IdAllocator, + /// Allocator for PCI device GSIs + pub gsi_msi_allocator: IdAllocator, /// Allocator for memory in the 32-bit MMIO address space pub mmio32_memory: AddressAllocator, /// Allocator for memory in the 64-bit MMIO address space @@ -41,7 +66,9 @@ impl ResourceAllocator { // It is fine for us to unwrap the following since we know we are passing valid ranges for // all allocators Self { - gsi_allocator: IdAllocator::new(arch::GSI_BASE, arch::GSI_MAX).unwrap(), + gsi_legacy_allocator: IdAllocator::new(arch::GSI_LEGACY_START, arch::GSI_LEGACY_END) + .unwrap(), + gsi_msi_allocator: IdAllocator::new(arch::GSI_MSI_START, arch::GSI_MSI_END).unwrap(), mmio32_memory: AddressAllocator::new( arch::MEM_32BIT_DEVICES_START, arch::MEM_32BIT_DEVICES_SIZE, @@ -57,28 +84,22 @@ impl ResourceAllocator { } } - /// Allocate a number of GSIs + /// Allocate a number of legacy GSIs /// /// # Arguments /// - /// * `gsi_count` - The number of GSIs to allocate - pub fn allocate_gsi(&mut self, gsi_count: u32) -> Result, vm_allocator::Error> { - let mut gsis = Vec::with_capacity(gsi_count as usize); - - for _ in 0..gsi_count { - match self.gsi_allocator.allocate_id() { - Ok(gsi) => gsis.push(gsi), - Err(err) => { - // It is ok to unwrap here, we just allocated the GSI - gsis.into_iter().for_each(|gsi| { - self.gsi_allocator.free_id(gsi).unwrap(); - }); - return Err(err); - } - } - } + /// * `gsi_count` - The number of legacy GSIs to allocate + pub fn allocate_gsi_legacy(&mut self, gsi_count: u32) -> Result, vm_allocator::Error> { + allocate_many_ids(&mut self.gsi_legacy_allocator, gsi_count) + } - Ok(gsis) + /// Allocate a number of GSIs for MSI + /// + /// # Arguments + /// + /// * `gsi_count` - The number of GSIs to allocate + pub fn allocate_gsi_msi(&mut self, gsi_count: u32) -> Result, vm_allocator::Error> { + allocate_many_ids(&mut self.gsi_msi_allocator, gsi_count) } /// Allocate a memory range in 32-bit MMIO address space @@ -167,47 +188,93 @@ mod tests { use vm_allocator::AllocPolicy; use super::ResourceAllocator; - use crate::arch::{self, GSI_BASE}; + use crate::arch::{self, GSI_LEGACY_NUM, GSI_LEGACY_START, GSI_MSI_NUM, GSI_MSI_START}; use crate::snapshot::{Persist, Snapshot}; - const MAX_IRQS: u32 = arch::GSI_MAX - arch::GSI_BASE + 1; + #[test] + fn test_allocate_irq() { + let mut allocator = ResourceAllocator::new(); + // asking for 0 IRQs should return us an empty vector + assert_eq!(allocator.allocate_gsi_legacy(0), Ok(vec![])); + // We cannot allocate more GSIs than available + assert_eq!( + allocator.allocate_gsi_legacy(GSI_LEGACY_NUM + 1), + Err(vm_allocator::Error::ResourceNotAvailable) + ); + // But allocating all of them at once should work + assert_eq!( + allocator.allocate_gsi_legacy(GSI_LEGACY_NUM), + Ok((arch::GSI_LEGACY_START..=arch::GSI_LEGACY_END).collect::>()) + ); + // And now we ran out of GSIs + assert_eq!( + allocator.allocate_gsi_legacy(1), + Err(vm_allocator::Error::ResourceNotAvailable) + ); + // But we should be able to ask for 0 GSIs + assert_eq!(allocator.allocate_gsi_legacy(0), Ok(vec![])); + + let mut allocator = ResourceAllocator::new(); + // We should be able to allocate 1 GSI + assert_eq!( + allocator.allocate_gsi_legacy(1), + Ok(vec![arch::GSI_LEGACY_START]) + ); + // We can't allocate MAX_IRQS any more + assert_eq!( + allocator.allocate_gsi_legacy(GSI_LEGACY_NUM), + Err(vm_allocator::Error::ResourceNotAvailable) + ); + // We can allocate another one and it should be the second available + assert_eq!( + allocator.allocate_gsi_legacy(1), + Ok(vec![arch::GSI_LEGACY_START + 1]) + ); + // Let's allocate the rest in a loop + for i in arch::GSI_LEGACY_START + 2..=arch::GSI_LEGACY_END { + assert_eq!(allocator.allocate_gsi_legacy(1), Ok(vec![i])); + } + } #[test] fn test_allocate_gsi() { let mut allocator = ResourceAllocator::new(); // asking for 0 IRQs should return us an empty vector - assert_eq!(allocator.allocate_gsi(0), Ok(vec![])); + assert_eq!(allocator.allocate_gsi_msi(0), Ok(vec![])); // We cannot allocate more GSIs than available assert_eq!( - allocator.allocate_gsi(MAX_IRQS + 1), + allocator.allocate_gsi_msi(GSI_MSI_NUM + 1), Err(vm_allocator::Error::ResourceNotAvailable) ); // But allocating all of them at once should work assert_eq!( - allocator.allocate_gsi(MAX_IRQS), - Ok((arch::GSI_BASE..=arch::GSI_MAX).collect::>()) + allocator.allocate_gsi_msi(GSI_MSI_NUM), + Ok((arch::GSI_MSI_START..=arch::GSI_MSI_END).collect::>()) ); // And now we ran out of GSIs assert_eq!( - allocator.allocate_gsi(1), + allocator.allocate_gsi_msi(1), Err(vm_allocator::Error::ResourceNotAvailable) ); // But we should be able to ask for 0 GSIs - assert_eq!(allocator.allocate_gsi(0), Ok(vec![])); + assert_eq!(allocator.allocate_gsi_msi(0), Ok(vec![])); let mut allocator = ResourceAllocator::new(); // We should be able to allocate 1 GSI - assert_eq!(allocator.allocate_gsi(1), Ok(vec![arch::GSI_BASE])); + assert_eq!(allocator.allocate_gsi_msi(1), Ok(vec![arch::GSI_MSI_START])); // We can't allocate MAX_IRQS any more assert_eq!( - allocator.allocate_gsi(MAX_IRQS), + allocator.allocate_gsi_msi(GSI_MSI_NUM), Err(vm_allocator::Error::ResourceNotAvailable) ); // We can allocate another one and it should be the second available - assert_eq!(allocator.allocate_gsi(1), Ok(vec![arch::GSI_BASE + 1])); + assert_eq!( + allocator.allocate_gsi_msi(1), + Ok(vec![arch::GSI_MSI_START + 1]) + ); // Let's allocate the rest in a loop - for i in arch::GSI_BASE + 2..=arch::GSI_MAX { - assert_eq!(allocator.allocate_gsi(1), Ok(vec![i])); + for i in arch::GSI_MSI_START + 2..=arch::GSI_MSI_END { + assert_eq!(allocator.allocate_gsi_msi(1), Ok(vec![i])); } } @@ -221,12 +288,16 @@ mod tests { #[test] fn test_save_restore() { let mut allocator0 = ResourceAllocator::new(); - let gsi_0 = allocator0.allocate_gsi(1).unwrap()[0]; - assert_eq!(gsi_0, GSI_BASE); + let irq_0 = allocator0.allocate_gsi_legacy(1).unwrap()[0]; + assert_eq!(irq_0, GSI_LEGACY_START); + let gsi_0 = allocator0.allocate_gsi_msi(1).unwrap()[0]; + assert_eq!(gsi_0, GSI_MSI_START); let mut allocator1 = clone_allocator(&allocator0); - let gsi_1 = allocator1.allocate_gsi(1).unwrap()[0]; - assert_eq!(gsi_1, GSI_BASE + 1); + let irq_1 = allocator1.allocate_gsi_legacy(1).unwrap()[0]; + assert_eq!(irq_1, GSI_LEGACY_START + 1); + let gsi_1 = allocator1.allocate_gsi_msi(1).unwrap()[0]; + assert_eq!(gsi_1, GSI_MSI_START + 1); let mmio32_mem = allocator1 .allocate_32bit_mmio_memory(0x42, 1, AllocPolicy::FirstMatch) .unwrap(); @@ -251,8 +322,10 @@ mod tests { .allocate_system_memory(0x42, 1, AllocPolicy::ExactMatch(system_mem)) .unwrap_err(); - let gsi_2 = allocator2.allocate_gsi(1).unwrap()[0]; - assert_eq!(gsi_2, GSI_BASE + 2); + let irq_2 = allocator2.allocate_gsi_legacy(1).unwrap()[0]; + assert_eq!(irq_2, GSI_LEGACY_START + 2); + let gsi_2 = allocator2.allocate_gsi_msi(1).unwrap()[0]; + assert_eq!(gsi_2, GSI_MSI_START + 2); let mmio32_mem = allocator1 .allocate_32bit_mmio_memory(0x42, 1, AllocPolicy::FirstMatch) .unwrap(); diff --git a/src/vmm/src/vstate/vm.rs b/src/vmm/src/vstate/vm.rs index 0f6c7c0b50d..fa46e2f32de 100644 --- a/src/vmm/src/vstate/vm.rs +++ b/src/vmm/src/vstate/vm.rs @@ -587,7 +587,7 @@ impl Vm { let mut irq_routes = HashMap::with_capacity(count as usize); for (gsi, i) in vm .resource_allocator() - .allocate_gsi(count as u32)? + .allocate_gsi_msi(count as u32)? .iter() .zip(0u32..) { @@ -877,7 +877,7 @@ pub(crate) mod tests { } for i in 0..4 { - let gsi = crate::arch::GSI_BASE + i; + let gsi = crate::arch::GSI_MSI_START + i; let interrupts = vm.common.interrupts.lock().unwrap(); let kvm_route = interrupts.get(&gsi).unwrap(); assert!(kvm_route.masked); @@ -894,7 +894,7 @@ pub(crate) mod tests { // Simply enabling the vectors should not update the registered IRQ routes msix_group.enable().unwrap(); for i in 0..4 { - let gsi = crate::arch::GSI_BASE + i; + let gsi = crate::arch::GSI_MSI_START + i; let interrupts = vm.common.interrupts.lock().unwrap(); let kvm_route = interrupts.get(&gsi).unwrap(); assert!(kvm_route.masked); @@ -914,7 +914,7 @@ pub(crate) mod tests { .update(0, InterruptSourceConfig::MsiIrq(config), false, true) .unwrap(); for i in 0..4 { - let gsi = crate::arch::GSI_BASE + i; + let gsi = crate::arch::GSI_MSI_START + i; let interrupts = vm.common.interrupts.lock().unwrap(); let kvm_route = interrupts.get(&gsi).unwrap(); assert_eq!(kvm_route.masked, i != 0); @@ -989,7 +989,7 @@ pub(crate) mod tests { let (gsi, range) = { let mut resource_allocator = vm.resource_allocator(); - let gsi = resource_allocator.allocate_gsi(1).unwrap()[0]; + let gsi = resource_allocator.allocate_gsi_msi(1).unwrap()[0]; let range = resource_allocator .allocate_32bit_mmio_memory(1024, 1024, AllocPolicy::FirstMatch) .unwrap(); @@ -1003,7 +1003,7 @@ pub(crate) mod tests { vm.restore_state(&restored_state).unwrap(); let mut resource_allocator = vm.resource_allocator(); - let gsi_new = resource_allocator.allocate_gsi(1).unwrap()[0]; + let gsi_new = resource_allocator.allocate_gsi_msi(1).unwrap()[0]; assert_eq!(gsi + 1, gsi_new); resource_allocator diff --git a/tests/conftest.py b/tests/conftest.py index aad4016b554..4cf28e0e6d2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -507,9 +507,21 @@ def rootfs_rw(): @pytest.fixture -def uvm_plain(microvm_factory, guest_kernel_linux_5_10, rootfs): +def uvm_plain(microvm_factory, guest_kernel_linux_5_10, rootfs, pci_enabled): """Create a vanilla VM, non-parametrized""" - return microvm_factory.build(guest_kernel_linux_5_10, rootfs) + return microvm_factory.build(guest_kernel_linux_5_10, rootfs, pci=pci_enabled) + + +@pytest.fixture +def uvm_plain_6_1(microvm_factory, guest_kernel_linux_6_1, rootfs, pci_enabled): + """Create a vanilla VM, non-parametrized""" + return microvm_factory.build(guest_kernel_linux_6_1, rootfs, pci=pci_enabled) + + +@pytest.fixture +def uvm_plain_acpi(microvm_factory, guest_kernel_acpi, rootfs, pci_enabled): + """Create a vanilla VM, non-parametrized""" + return microvm_factory.build(guest_kernel_acpi, rootfs, pci=pci_enabled) @pytest.fixture @@ -535,12 +547,12 @@ def artifact_dir(): @pytest.fixture -def uvm_plain_any(microvm_factory, guest_kernel, rootfs): +def uvm_plain_any(microvm_factory, guest_kernel, rootfs, pci_enabled): """All guest kernels kernel: all rootfs: Ubuntu 24.04 """ - return microvm_factory.build(guest_kernel, rootfs) + return microvm_factory.build(guest_kernel, rootfs, pci=pci_enabled) guest_kernel_6_1_debug = pytest.fixture( @@ -583,8 +595,8 @@ def uvm_booted( mem_size_mib=256, ): """Return a booted uvm""" - uvm = microvm_factory.build(guest_kernel, rootfs) - uvm.spawn(pci=pci_enabled) + uvm = microvm_factory.build(guest_kernel, rootfs, pci=pci_enabled) + uvm.spawn() uvm.basic_config(vcpu_count=vcpu_count, mem_size_mib=mem_size_mib) uvm.set_cpu_template(cpu_template) uvm.add_net_iface() diff --git a/tests/framework/microvm.py b/tests/framework/microvm.py index 5c936fd8395..5c37dc2734d 100644 --- a/tests/framework/microvm.py +++ b/tests/framework/microvm.py @@ -191,6 +191,7 @@ def __init__( jailer_kwargs: Optional[dict] = None, numa_node=None, custom_cpu_template: Path = None, + pci: bool = False, ): """Set up microVM attributes, paths, and data structures.""" # pylint: disable=too-many-statements @@ -198,7 +199,6 @@ def __init__( assert microvm_id is not None self._microvm_id = microvm_id - self.pci_enabled = False self.kernel_file = None self.rootfs_file = None self.ssh_key = None @@ -222,6 +222,10 @@ def __init__( **jailer_kwargs, ) + self.pci_enabled = pci + if pci: + self.jailer.extra_args["enable-pci"] = None + # Copy the /etc/localtime file in the jailer root self.jailer.jailed_path("/etc/localtime", subdir="etc") @@ -612,7 +616,6 @@ def spawn( log_show_origin=False, metrics_path="fc.ndjson", emit_metrics: bool = False, - pci: bool = False, ): """Start a microVM as a daemon or in a screen session.""" # pylint: disable=subprocess-run-check @@ -658,10 +661,6 @@ def spawn( # Checking the timings requires DEBUG level log messages self.time_api_requests = False - if pci: - self.pci_enabled = True - self.jailer.extra_args["enable-pci"] = None - cmd = [ *self._pre_cmd, str(self.jailer_binary_path), diff --git a/tests/framework/vm_config.json b/tests/framework/vm_config.json index 5df673308d9..6948002e245 100644 --- a/tests/framework/vm_config.json +++ b/tests/framework/vm_config.json @@ -1,7 +1,7 @@ { "boot-source": { "kernel_image_path": "vmlinux.bin", - "boot_args": "console=ttyS0 reboot=k panic=1 pci=off", + "boot_args": "console=ttyS0 reboot=k panic=1", "initrd_path": null }, "drives": [ diff --git a/tests/framework/vm_config_cpu_template_C3.json b/tests/framework/vm_config_cpu_template_C3.json index 3b842594a18..b6dbf124022 100644 --- a/tests/framework/vm_config_cpu_template_C3.json +++ b/tests/framework/vm_config_cpu_template_C3.json @@ -1,7 +1,7 @@ { "boot-source": { "kernel_image_path": "vmlinux.bin", - "boot_args": "console=ttyS0 reboot=k panic=1 pci=off" + "boot_args": "console=ttyS0 reboot=k panic=1" }, "drives": [ { diff --git a/tests/framework/vm_config_missing_mem_size_mib.json b/tests/framework/vm_config_missing_mem_size_mib.json index 15ff19fa1b3..ea20d152473 100644 --- a/tests/framework/vm_config_missing_mem_size_mib.json +++ b/tests/framework/vm_config_missing_mem_size_mib.json @@ -1,7 +1,7 @@ { "boot-source": { "kernel_image_path": "vmlinux.bin", - "boot_args": "console=ttyS0 reboot=k panic=1 pci=off" + "boot_args": "console=ttyS0 reboot=k panic=1" }, "drives": [ { diff --git a/tests/framework/vm_config_missing_vcpu_count.json b/tests/framework/vm_config_missing_vcpu_count.json index b5aac05ddd2..39bb6a38954 100644 --- a/tests/framework/vm_config_missing_vcpu_count.json +++ b/tests/framework/vm_config_missing_vcpu_count.json @@ -1,7 +1,7 @@ { "boot-source": { "kernel_image_path": "vmlinux.bin", - "boot_args": "console=ttyS0 reboot=k panic=1 pci=off" + "boot_args": "console=ttyS0 reboot=k panic=1" }, "drives": [ { diff --git a/tests/framework/vm_config_network.json b/tests/framework/vm_config_network.json index a081e4f6990..7e25823cd66 100644 --- a/tests/framework/vm_config_network.json +++ b/tests/framework/vm_config_network.json @@ -1,7 +1,7 @@ { "boot-source": { "kernel_image_path": "vmlinux.bin", - "boot_args": "console=ttyS0 reboot=k panic=1 pci=off", + "boot_args": "console=ttyS0 reboot=k panic=1", "initrd_path": null }, "drives": [ diff --git a/tests/framework/vm_config_smt_true.json b/tests/framework/vm_config_smt_true.json index 3a1b79a1752..383bf68519a 100644 --- a/tests/framework/vm_config_smt_true.json +++ b/tests/framework/vm_config_smt_true.json @@ -1,7 +1,7 @@ { "boot-source": { "kernel_image_path": "vmlinux.bin", - "boot_args": "console=ttyS0 reboot=k panic=1 pci=off" + "boot_args": "console=ttyS0 reboot=k panic=1" }, "drives": [ { diff --git a/tests/framework/vm_config_with_mmdsv1.json b/tests/framework/vm_config_with_mmdsv1.json index 6c30e535b1d..30f67ff5bfa 100644 --- a/tests/framework/vm_config_with_mmdsv1.json +++ b/tests/framework/vm_config_with_mmdsv1.json @@ -1,7 +1,7 @@ { "boot-source": { "kernel_image_path": "vmlinux.bin", - "boot_args": "console=ttyS0 reboot=k panic=1 pci=off", + "boot_args": "console=ttyS0 reboot=k panic=1", "initrd_path": null }, "drives": [ diff --git a/tests/framework/vm_config_with_mmdsv2.json b/tests/framework/vm_config_with_mmdsv2.json index 2bc70d1ef3f..340c41cab1f 100644 --- a/tests/framework/vm_config_with_mmdsv2.json +++ b/tests/framework/vm_config_with_mmdsv2.json @@ -1,7 +1,7 @@ { "boot-source": { "kernel_image_path": "vmlinux.bin", - "boot_args": "console=ttyS0 reboot=k panic=1 pci=off", + "boot_args": "console=ttyS0 reboot=k panic=1", "initrd_path": null }, "drives": [ diff --git a/tests/host_tools/change_net_config_space.c b/tests/host_tools/change_net_config_space.c index 7b803bdc878..592a0cfe6af 100644 --- a/tests/host_tools/change_net_config_space.c +++ b/tests/host_tools/change_net_config_space.c @@ -14,7 +14,7 @@ #include int show_usage() { - printf("Usage: ./change_net_config_space.bin [dev_addr_start] [mac_addr]\n"); + printf("Usage: ./change_net_config_space.bin [dev_addr] [mac_addr]\n"); printf("Example:\n"); printf("> ./change_net_config_space.bin 0xd00001000 0x060504030201\n"); return 0; @@ -25,18 +25,17 @@ int main(int argc, char *argv[]) { uint8_t *map_base; volatile uint8_t *virt_addr; - uint64_t mapped_size, page_size, offset_in_page, target; + uint64_t mapped_size, page_size, page_addr, offset_in_page; uint64_t width = 6; - uint64_t config_offset = 0x100; - uint64_t device_start_addr = 0x00000000; + uint64_t dev_addr = 0x00000000; uint64_t mac = 0; if (argc != 3) { return show_usage(); } - device_start_addr = strtoull(argv[1], NULL, 0); + dev_addr = strtoull(argv[1], NULL, 0); mac = strtoull(argv[2], NULL, 0); fd = open("/dev/mem", O_RDWR | O_SYNC); @@ -45,11 +44,11 @@ int main(int argc, char *argv[]) { return 1; } - target = device_start_addr + config_offset; // Get the page size. mapped_size = page_size = getpagesize(); // Get the target address physical frame page offset. - offset_in_page = (unsigned) target & (page_size - 1); + offset_in_page = (unsigned) dev_addr & (page_size - 1); + page_addr = dev_addr & ~(page_size - 1); /* If the data length goes out of the current page, * double the needed map size. */ if (offset_in_page + width > page_size) { @@ -64,7 +63,8 @@ int main(int argc, char *argv[]) { PROT_READ | PROT_WRITE, MAP_SHARED, fd, - target & ~(off_t)(page_size - 1)); + page_addr + ); if (map_base == MAP_FAILED) { perror("Failed to mmap '/dev/mem'."); return 2; diff --git a/tests/integration_tests/functional/test_api.py b/tests/integration_tests/functional/test_api.py index ab2546c861f..ce54a7ccbbc 100644 --- a/tests/integration_tests/functional/test_api.py +++ b/tests/integration_tests/functional/test_api.py @@ -772,7 +772,7 @@ def test_send_ctrl_alt_del(uvm_plain_any): def _drive_patch(test_microvm, io_engine): """Exercise drive patch test scenarios.""" # Patches without mandatory fields for virtio block are not allowed. - expected_msg = "Unable to patch the block device: MMIO Device manager error: Running method expected different backend. Please verify the request arguments" + expected_msg = "Running method expected different backend." with pytest.raises(RuntimeError, match=expected_msg): test_microvm.api.drive.patch(drive_id="scratch") @@ -814,7 +814,7 @@ def _drive_patch(test_microvm, io_engine): ) # Updates to `path_on_host` with an invalid path are not allowed. - expected_msg = f"Unable to patch the block device: MMIO Device manager error: Virtio backend error: Error manipulating the backing file: No such file or directory (os error 2) {drive_path} Please verify the request arguments" + expected_msg = f"Error manipulating the backing file: No such file or directory (os error 2) {drive_path}" with pytest.raises(RuntimeError, match=re.escape(expected_msg)): test_microvm.api.drive.patch(drive_id="scratch", path_on_host=drive_path) diff --git a/tests/integration_tests/functional/test_balloon.py b/tests/integration_tests/functional/test_balloon.py index 2bfa82f4f1c..46f7a8a278c 100644 --- a/tests/integration_tests/functional/test_balloon.py +++ b/tests/integration_tests/functional/test_balloon.py @@ -449,11 +449,11 @@ def test_stats_update(uvm_plain_any): assert next_stats["available_memory"] != final_stats["available_memory"] -def test_balloon_snapshot(microvm_factory, guest_kernel, rootfs): +def test_balloon_snapshot(uvm_plain_any, microvm_factory): """ Test that the balloon works after pause/resume. """ - vm = microvm_factory.build(guest_kernel, rootfs) + vm = uvm_plain_any vm.spawn() vm.basic_config( vcpu_count=2, @@ -531,11 +531,11 @@ def test_balloon_snapshot(microvm_factory, guest_kernel, rootfs): assert stats_after_snap["available_memory"] > latest_stats["available_memory"] -def test_memory_scrub(microvm_factory, guest_kernel, rootfs): +def test_memory_scrub(uvm_plain_any): """ Test that the memory is zeroed after deflate. """ - microvm = microvm_factory.build(guest_kernel, rootfs) + microvm = uvm_plain_any microvm.spawn() microvm.basic_config(vcpu_count=2, mem_size_mib=256) microvm.add_net_iface() diff --git a/tests/integration_tests/functional/test_concurrency.py b/tests/integration_tests/functional/test_concurrency.py index e4756729f2b..15394ec6ada 100644 --- a/tests/integration_tests/functional/test_concurrency.py +++ b/tests/integration_tests/functional/test_concurrency.py @@ -7,13 +7,13 @@ NO_OF_MICROVMS = 20 -def test_run_concurrency(microvm_factory, guest_kernel, rootfs): +def test_run_concurrency(microvm_factory, guest_kernel, rootfs, pci_enabled): """ Check we can spawn multiple microvms. """ def launch1(): - microvm = microvm_factory.build(guest_kernel, rootfs) + microvm = microvm_factory.build(guest_kernel, rootfs, pci=pci_enabled) microvm.time_api_requests = False # is flaky because of parallelism microvm.spawn() microvm.basic_config(vcpu_count=1, mem_size_mib=128) diff --git a/tests/integration_tests/functional/test_cpu_template_helper.py b/tests/integration_tests/functional/test_cpu_template_helper.py index 6ff2db1f7f0..2b52047a2f6 100644 --- a/tests/integration_tests/functional/test_cpu_template_helper.py +++ b/tests/integration_tests/functional/test_cpu_template_helper.py @@ -261,9 +261,7 @@ def get_guest_msrs(microvm, msr_index_list): ), ) def test_cpu_config_dump_vs_actual( - microvm_factory, - guest_kernel, - rootfs, + uvm_plain_any, cpu_template_helper, tmp_path, ): @@ -277,7 +275,7 @@ def test_cpu_config_dump_vs_actual( dump_cpu_config = build_cpu_config_dict(cpu_config_path) # Retrieve actual CPU config from guest - microvm = microvm_factory.build(guest_kernel, rootfs) + microvm = uvm_plain_any microvm.spawn() microvm.basic_config(vcpu_count=1) microvm.add_net_iface() diff --git a/tests/integration_tests/functional/test_drive_vhost_user.py b/tests/integration_tests/functional/test_drive_vhost_user.py index 79cc41b0f3a..07fcafb715e 100644 --- a/tests/integration_tests/functional/test_drive_vhost_user.py +++ b/tests/integration_tests/functional/test_drive_vhost_user.py @@ -6,11 +6,62 @@ import shutil from pathlib import Path +import pytest + import host_tools.drive as drive_tools from framework.utils_drive import partuuid_and_disk_path from host_tools.fcmetrics import FcDeviceMetrics +@pytest.fixture +def uvm_vhost_user_plain_any(microvm_factory, guest_kernel, pci_enabled): + """Builds a plain VM with no root volume""" + return microvm_factory.build( + guest_kernel, None, pci=pci_enabled, monitor_memory=False + ) + + +@pytest.fixture +def uvm_vhost_user_booted_ro(uvm_vhost_user_plain_any, rootfs): + """Returns a VM with a vhost-user rootfs""" + vm = uvm_vhost_user_plain_any + + # We need to setup ssh keys manually because we did not specify rootfs + # in microvm_factory.build method + ssh_key = rootfs.with_suffix(".id_rsa") + vm.ssh_key = ssh_key + vm.spawn() + vm.basic_config(add_root_device=False) + vm.add_vhost_user_drive("rootfs", rootfs, is_root_device=True, is_read_only=True) + vm.add_net_iface() + vm.start() + + return vm + + +@pytest.fixture +def uvm_vhost_user_booted_rw(uvm_vhost_user_plain_any, rootfs): + """Returns a VM with a vhost-user rootfs""" + vm = uvm_vhost_user_plain_any + + # We need to setup ssh keys manually because we did not specify rootfs + # in microvm_factory.build method + ssh_key = rootfs.with_suffix(".id_rsa") + vm.ssh_key = ssh_key + vm.spawn() + vm.basic_config(add_root_device=False) + # Create a rw rootfs file that is unique to the microVM + rootfs_rw = Path(vm.chroot()) / "rootfs" + shutil.copy(rootfs, rootfs_rw) + vm.add_vhost_user_drive( + "rootfs", rootfs_rw, is_root_device=True, is_read_only=False + ) + vm.add_net_iface() + vm.start() + + return vm + + def _check_block_size(ssh_connection, dev_path, size): """ Checks the size of the block device. @@ -34,26 +85,16 @@ def _check_drives(test_microvm, assert_dict, keys_array): assert blockdev_out_line_cols[col] == assert_dict[key] -def test_vhost_user_block(microvm_factory, guest_kernel, rootfs): +def test_vhost_user_block(uvm_vhost_user_booted_ro): """ This test simply tries to boot a VM with vhost-user-block as a root device. """ - vm = microvm_factory.build(guest_kernel, None, monitor_memory=False) - - # We need to setup ssh keys manually because we did not specify rootfs - # in microvm_factory.build method - ssh_key = rootfs.with_suffix(".id_rsa") - vm.ssh_key = ssh_key - vm.spawn() - vm.basic_config(add_root_device=False) - vm.add_vhost_user_drive("rootfs", rootfs, is_root_device=True, is_read_only=True) - vm.add_net_iface() + vm = uvm_vhost_user_booted_ro vhost_user_block_metrics = FcDeviceMetrics( "vhost_user_block", 1, aggr_supported=False ) - vm.start() # Now check that vhost-user-block with rw is last. # 1-0 means line 1, column 0. @@ -65,29 +106,14 @@ def test_vhost_user_block(microvm_factory, guest_kernel, rootfs): vhost_user_block_metrics.validate(vm) -def test_vhost_user_block_read_write(microvm_factory, guest_kernel, rootfs): +def test_vhost_user_block_read_write(uvm_vhost_user_booted_rw): """ This test simply tries to boot a VM with vhost-user-block as a root device. This test configures vhost-user-block to be read write. """ - vm = microvm_factory.build(guest_kernel, None, monitor_memory=False) - - # We need to setup ssh keys manually because we did not specify rootfs - # in microvm_factory.build method - ssh_key = rootfs.with_suffix(".id_rsa") - vm.ssh_key = ssh_key - vm.spawn() - vm.basic_config(add_root_device=False) - - # Create a rw rootfs file that is unique to the microVM - rootfs_rw = Path(vm.chroot()) / "rootfs" - shutil.copy(rootfs, rootfs_rw) - - vm.add_vhost_user_drive("rootfs", rootfs_rw, is_root_device=True) - vm.add_net_iface() - vm.start() + vm = uvm_vhost_user_booted_rw # Now check that vhost-user-block with rw is last. # 1-0 means line 1, column 0. @@ -98,22 +124,12 @@ def test_vhost_user_block_read_write(microvm_factory, guest_kernel, rootfs): _check_drives(vm, assert_dict, assert_dict.keys()) -def test_vhost_user_block_disconnect(microvm_factory, guest_kernel, rootfs): +def test_vhost_user_block_disconnect(uvm_vhost_user_booted_ro): """ Test that even if backend is killed, Firecracker is still responsive. """ - vm = microvm_factory.build(guest_kernel, None, monitor_memory=False) - - # We need to set up ssh keys manually because we did not specify rootfs - # in microvm_factory.build method - ssh_key = rootfs.with_suffix(".id_rsa") - vm.ssh_key = ssh_key - vm.spawn() - vm.basic_config(add_root_device=False) - vm.add_vhost_user_drive("rootfs", rootfs, is_root_device=True, is_read_only=True) - vm.add_net_iface() - vm.start() + vm = uvm_vhost_user_booted_ro # Killing the backend vm.disks_vhost_user["rootfs"].kill() @@ -123,7 +139,7 @@ def test_vhost_user_block_disconnect(microvm_factory, guest_kernel, rootfs): _config = vm.api.vm_config.get().json() -def test_device_ordering(microvm_factory, guest_kernel, rootfs): +def test_device_ordering(uvm_vhost_user_plain_any, rootfs): """ Verify device ordering. @@ -131,7 +147,7 @@ def test_device_ordering(microvm_factory, guest_kernel, rootfs): the order of the other devices should match their configuration order. """ - vm = microvm_factory.build(guest_kernel, None, monitor_memory=False) + vm = uvm_vhost_user_plain_any # We need to setup ssh keys manually because we did not specify rootfs # in microvm_factory.build method @@ -194,16 +210,12 @@ def test_device_ordering(microvm_factory, guest_kernel, rootfs): vhost_user_block_metrics.validate(vm) -def test_partuuid_boot( - microvm_factory, - guest_kernel, - rootfs, -): +def test_partuuid_boot(uvm_vhost_user_plain_any, rootfs): """ Test the output reported by blockdev when booting with PARTUUID. """ - vm = microvm_factory.build(guest_kernel, None, monitor_memory=False) + vm = uvm_vhost_user_plain_any # We need to setup ssh keys manually because we did not specify rootfs # in microvm_factory.build method @@ -230,12 +242,12 @@ def test_partuuid_boot( _check_drives(vm, assert_dict, assert_dict.keys()) -def test_partuuid_update(microvm_factory, guest_kernel, rootfs): +def test_partuuid_update(uvm_vhost_user_plain_any, rootfs): """ Test successful switching from PARTUUID boot to /dev/vda boot. """ - vm = microvm_factory.build(guest_kernel, None, monitor_memory=False) + vm = uvm_vhost_user_plain_any # We need to setup ssh keys manually because we did not specify rootfs # in microvm_factory.build method @@ -272,7 +284,7 @@ def test_partuuid_update(microvm_factory, guest_kernel, rootfs): vhost_user_block_metrics.validate(vm) -def test_config_change(microvm_factory, guest_kernel, rootfs): +def test_config_change(uvm_plain_any): """ Verify handling of block device resize. We expect that the guest will start reporting the updated size @@ -283,7 +295,7 @@ def test_config_change(microvm_factory, guest_kernel, rootfs): new_sizes = [20, 10, 30] # MB mkfs_mount_cmd = "mkfs.ext4 /dev/vdb && mkdir -p /tmp/tmp && mount /dev/vdb /tmp/tmp && umount /tmp/tmp" - vm = microvm_factory.build(guest_kernel, rootfs, monitor_memory=False) + vm = uvm_plain_any vm.spawn(log_level="Info") vm.basic_config() vm.add_net_iface() diff --git a/tests/integration_tests/functional/test_error_code.py b/tests/integration_tests/functional/test_error_code.py index d1a74b6f418..171d3853460 100644 --- a/tests/integration_tests/functional/test_error_code.py +++ b/tests/integration_tests/functional/test_error_code.py @@ -25,7 +25,7 @@ def test_enosys_error_code(uvm_plain): vm.memory_monitor = None vm.basic_config( vcpu_count=1, - boot_args="reboot=k panic=1 pci=off init=/usr/local/bin/devmemread", + boot_args="reboot=k panic=1 init=/usr/local/bin/devmemread", ) vm.start() diff --git a/tests/integration_tests/functional/test_feat_parity.py b/tests/integration_tests/functional/test_feat_parity.py index 1eadbc6d29c..9fc89ffcd2c 100644 --- a/tests/integration_tests/functional/test_feat_parity.py +++ b/tests/integration_tests/functional/test_feat_parity.py @@ -28,16 +28,11 @@ def inst_set_cpu_template_fxt(request): @pytest.fixture(name="vm") -def vm_fxt( - microvm_factory, - inst_set_cpu_template, - guest_kernel, - rootfs, -): +def vm_fxt(uvm_plain_any, inst_set_cpu_template): """ Create a VM, using the normal CPU templates """ - vm = microvm_factory.build(guest_kernel, rootfs) + vm = uvm_plain_any vm.spawn() vm.basic_config(vcpu_count=1, mem_size_mib=1024, cpu_template=inst_set_cpu_template) vm.add_net_iface() diff --git a/tests/integration_tests/functional/test_kernel_cmdline.py b/tests/integration_tests/functional/test_kernel_cmdline.py index 9707eb8a92c..7ba345f2111 100644 --- a/tests/integration_tests/functional/test_kernel_cmdline.py +++ b/tests/integration_tests/functional/test_kernel_cmdline.py @@ -21,8 +21,7 @@ def test_init_params(uvm_plain): # Ubuntu version from the /etc/issue file. vm.basic_config( vcpu_count=1, - boot_args="console=ttyS0 reboot=k panic=1 pci=off" - " init=/bin/cat -- /etc/issue", + boot_args="console=ttyS0 reboot=k panic=1 init=/bin/cat -- /etc/issue", ) vm.start() diff --git a/tests/integration_tests/functional/test_max_devices.py b/tests/integration_tests/functional/test_max_devices.py index 85cf2f1399c..7cf9922c77b 100644 --- a/tests/integration_tests/functional/test_max_devices.py +++ b/tests/integration_tests/functional/test_max_devices.py @@ -6,63 +6,80 @@ import pytest -# On x86_64, IRQs are available from 5 to 23. We always use one IRQ for VMGenID -# device, so the maximum number of devices supported at the same time is 18. -# On aarch64, IRQs are available from 32 to 127. We always use one IRQ each for -# the VMGenID and RTC devices, so the maximum number of devices supported -# at the same time is 94. -MAX_DEVICES_ATTACHED = {"x86_64": 18, "aarch64": 94}.get(platform.machine()) - - -def test_attach_maximum_devices(microvm_factory, guest_kernel, rootfs): +def max_devices(uvm): + """ + Returns the maximum number of devices supported by the platform. + """ + if uvm.pci_enabled: + # On PCI, we only have one bus, so 32 minus the bus itself + return 31 + + match platform.machine(): + case "aarch64": + # On aarch64, IRQs are available from 32 to 127. We always use one IRQ each for + # the VMGenID and RTC devices, so the maximum number of devices supported + # at the same time is 94. + return 94 + case "x86_64": + # IRQs are available from 5 to 23. We always use one IRQ for VMGenID device, so + # the maximum number of devices supported at the same time is 18. + return 18 + case _: + raise ValueError("Unknown platform") + + +def test_attach_maximum_devices(uvm_plain_any): """ Test attaching maximum number of devices to the microVM. """ - if MAX_DEVICES_ATTACHED is None: - pytest.skip("Unsupported platform for this test.") - - test_microvm = microvm_factory.build(guest_kernel, rootfs, monitor_memory=False) + test_microvm = uvm_plain_any + test_microvm.memory_monitor = None test_microvm.spawn() # The default 256mib is not enough for 94 ssh connections on aarch64. test_microvm.basic_config(mem_size_mib=512) + max_devices_attached = max_devices(test_microvm) # Add (`MAX_DEVICES_ATTACHED` - 1) devices because the rootfs # has already been configured in the `basic_config()`function. - for _ in range(MAX_DEVICES_ATTACHED - 1): + for _ in range(max_devices_attached - 1): test_microvm.add_net_iface() test_microvm.start() # Test that network devices attached are operational. - for i in range(MAX_DEVICES_ATTACHED - 1): + for i in range(max_devices_attached - 1): # Verify if guest can run commands. test_microvm.ssh_iface(i).check_output("sync") -def test_attach_too_many_devices(microvm_factory, guest_kernel, rootfs): +def test_attach_too_many_devices(uvm_plain): """ Test attaching to a microVM more devices than available IRQs. """ - if MAX_DEVICES_ATTACHED is None: - pytest.skip("Unsupported platform for this test.") - - test_microvm = microvm_factory.build(guest_kernel, rootfs, monitor_memory=False) + test_microvm = uvm_plain + test_microvm.memory_monitor = None test_microvm.spawn() # Set up a basic microVM. test_microvm.basic_config() + max_devices_attached = max_devices(test_microvm) + # Add `MAX_DEVICES_ATTACHED` network devices on top of the # already configured rootfs. - for _ in range(MAX_DEVICES_ATTACHED): + for _ in range(max_devices_attached): test_microvm.add_net_iface() # Attempting to start a microVM with more than # `MAX_DEVICES_ATTACHED` devices should fail. error_str = ( - "Failed to allocate requested resource: The requested resource" - " is not available." + ("Could not find an available device slot on the PCI bus.") + if test_microvm.pci_enabled + else ( + "Failed to allocate requested resource: The requested resource" + " is not available." + ) ) with pytest.raises(RuntimeError, match=error_str): test_microvm.start() diff --git a/tests/integration_tests/functional/test_net_config_space.py b/tests/integration_tests/functional/test_net_config_space.py index c4ddfea9189..d58b49b6d4a 100644 --- a/tests/integration_tests/functional/test_net_config_space.py +++ b/tests/integration_tests/functional/test_net_config_space.py @@ -2,9 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """Tests on devices config space.""" -import platform import random -import re import string import subprocess from threading import Thread @@ -64,6 +62,8 @@ def test_net_change_mac_address(uvm_plain_any, change_net_config_space_bin): net_addr_base = _get_net_mem_addr_base(ssh_conn, guest_if1_name) assert net_addr_base is not None + config_offset = 0x4000 if test_microvm.pci_enabled else 0x100 + dev_addr = net_addr_base + config_offset # Write into '/dev/mem' the same mac address, byte by byte. # This changes the MAC address physically, in the network device registers. @@ -72,7 +72,7 @@ def test_net_change_mac_address(uvm_plain_any, change_net_config_space_bin): # `tx_spoofed_mac_count` metric shouldn't be incremented later on. rmt_path = "/tmp/change_net_config_space" test_microvm.ssh.scp_put(change_net_config_space_bin, rmt_path) - cmd = f"chmod u+x {rmt_path} && {rmt_path} {net_addr_base} {mac_hex}" + cmd = f"chmod u+x {rmt_path} && {rmt_path} {dev_addr} {mac_hex}" # This should be executed successfully. _, stdout, _ = ssh_conn.check_output(cmd) @@ -219,8 +219,7 @@ def _find_iomem_range(ssh_connection, dev_name): # its contents and grep for the VirtIO device name, which # with ACPI is "LNRO0005:XY". cmd = f"cat /proc/iomem | grep -m 1 {dev_name}" - rc, stdout, stderr = ssh_connection.run(cmd) - assert rc == 0, stderr + _, stdout, _ = ssh_connection.check_output(cmd) # Take range in the form 'start-end' from line. The line looks like this: # d00002000-d0002fff : LNRO0005:02 @@ -231,89 +230,16 @@ def _find_iomem_range(ssh_connection, dev_name): return (int(tokens[0], 16), int(tokens[1], 16)) -def _get_net_mem_addr_base_x86_acpi(ssh_connection, if_name): - """Check for net device memory start address via ACPI info""" - # On x86 we define VirtIO devices through ACPI AML bytecode. VirtIO devices - # are identified as "LNRO0005" and appear under /sys/devices/platform - sys_virtio_mmio_cmdline = "/sys/devices/platform/" - cmd = "ls {}" - _, stdout, _ = ssh_connection.check_output(cmd.format(sys_virtio_mmio_cmdline)) - virtio_devs = list(filter(lambda x: "LNRO0005" in x, stdout.strip().split())) - - # For virtio-net LNRO0005 devices, we should have a path like: - # /sys/devices/platform/LNRO0005::XY/virtioXY/net which is a directory - # that includes a subdirectory `ethZ` which represents the network device - # that corresponds to the virtio-net device. - cmd = "ls {}/{}/virtio{}/net" - for idx, dev in enumerate(virtio_devs): - _, guest_if_name, _ = ssh_connection.run( - cmd.format(sys_virtio_mmio_cmdline, dev, idx) - ) - if guest_if_name.strip() == if_name: - return _find_iomem_range(ssh_connection, dev)[0] - - return None - - -def _get_net_mem_addr_base_x86_cmdline(ssh_connection, if_name): - """Check for net device memory start address via command line arguments""" - sys_virtio_mmio_cmdline = "/sys/devices/virtio-mmio-cmdline/" - cmd = "ls {} | grep virtio-mmio. | sed 's/virtio-mmio.//'" - exit_code, stdout, stderr = ssh_connection.run(cmd.format(sys_virtio_mmio_cmdline)) - assert exit_code == 0, stderr - virtio_devs_idx = stdout.strip().split() - - cmd = "cat /proc/cmdline" - _, cmd_line, _ = ssh_connection.check_output(cmd) - pattern_dev = re.compile("(virtio_mmio.device=4K@0x[0-9a-f]+:[0-9]+)+") - pattern_addr = re.compile("virtio_mmio.device=4K@(0x[0-9a-f]+):[0-9]+") - devs_addr = [] - for dev in re.findall(pattern_dev, cmd_line): - matched_addr = pattern_addr.search(dev) - # The 1st group which matches this pattern - # is the device start address. `0` group is - # full match - addr = matched_addr.group(1) - devs_addr.append(addr) - - cmd = "ls {}/virtio-mmio.{}/virtio{}/net" - for idx in virtio_devs_idx: - _, guest_if_name, _ = ssh_connection.run( - cmd.format(sys_virtio_mmio_cmdline, idx, idx) - ) - if guest_if_name.strip() == if_name: - return devs_addr[int(idx)] - - return None - - def _get_net_mem_addr_base(ssh_connection, if_name): """Get the net device memory start address.""" - if platform.machine() == "x86_64": - acpi_info = _get_net_mem_addr_base_x86_acpi(ssh_connection, if_name) - if acpi_info is not None: - return acpi_info - - return _get_net_mem_addr_base_x86_cmdline(ssh_connection, if_name) - - if platform.machine() == "aarch64": - sys_virtio_mmio_cmdline = "/sys/devices/platform" - cmd = "ls {} | grep .virtio_mmio".format(sys_virtio_mmio_cmdline) - rc, stdout, _ = ssh_connection.run(cmd) - assert rc == 0 - - virtio_devs = stdout.split() - devs_addr = list(map(lambda dev: dev.split(".")[0], virtio_devs)) - - cmd = "ls {}/{}/virtio{}/net" - # Device start addresses lack the hex prefix and are not interpreted - # accordingly when parsed inside `change_config_space.c`. - hex_prefix = "0x" - for idx, dev in enumerate(virtio_devs): - _, guest_if_name, _ = ssh_connection.run( - cmd.format(sys_virtio_mmio_cmdline, dev, idx) - ) - if guest_if_name.strip() == if_name: - return hex_prefix + devs_addr[int(idx)] - - return None + _, stdout, _ = ssh_connection.check_output(f"find /sys/devices -name {if_name}") + device_paths = stdout.strip().split("\n") + assert ( + len(device_paths) == 1 + ), f"No or multiple devices found for {if_name}:\n{stdout}" + device_path = device_paths[0] + parts = device_path.split("/") + assert len(parts) >= 6, f"Unexpected device path: {device_path}" + device = parts[-4] + start_addr, _ = _find_iomem_range(ssh_connection, device) + return start_addr diff --git a/tests/integration_tests/functional/test_rng.py b/tests/integration_tests/functional/test_rng.py index f2acf96735a..8719472a121 100644 --- a/tests/integration_tests/functional/test_rng.py +++ b/tests/integration_tests/functional/test_rng.py @@ -8,12 +8,11 @@ from host_tools.network import SSHConnection -def uvm_with_rng_booted( - microvm_factory, guest_kernel, rootfs, rate_limiter, pci_enabled -): +def uvm_with_rng_booted(uvm_plain_any, microvm_factory, rate_limiter): """Return a booted microvm with virtio-rng configured""" - uvm = microvm_factory.build(guest_kernel, rootfs) - uvm.spawn(log_level="INFO", pci=pci_enabled) + # pylint: disable=unused-argument + uvm = uvm_plain_any + uvm.spawn(log_level="INFO") uvm.basic_config(vcpu_count=2, mem_size_mib=256) uvm.add_net_iface() uvm.api.entropy.put(rate_limiter=rate_limiter) @@ -23,13 +22,9 @@ def uvm_with_rng_booted( return uvm -def uvm_with_rng_restored( - microvm_factory, guest_kernel, rootfs, rate_limiter, pci_enabled -): +def uvm_with_rng_restored(uvm_plain_any, microvm_factory, rate_limiter): """Return a restored uvm with virtio-rng configured""" - uvm = uvm_with_rng_booted( - microvm_factory, guest_kernel, rootfs, rate_limiter, pci_enabled - ) + uvm = uvm_with_rng_booted(uvm_plain_any, microvm_factory, rate_limiter) snapshot = uvm.snapshot_full() uvm.kill() uvm2 = microvm_factory.build_from_snapshot(snapshot) @@ -50,9 +45,9 @@ def rate_limiter(request): @pytest.fixture -def uvm_any(microvm_factory, uvm_ctor, guest_kernel, rootfs, rate_limiter, pci_enabled): +def uvm_any(microvm_factory, uvm_ctor, uvm_plain_any, rate_limiter): """Return booted and restored uvms""" - return uvm_ctor(microvm_factory, guest_kernel, rootfs, rate_limiter, pci_enabled) + return uvm_ctor(uvm_plain_any, microvm_factory, rate_limiter) def list_rng_available(ssh_connection: SSHConnection) -> list[str]: diff --git a/tests/integration_tests/functional/test_serial_io.py b/tests/integration_tests/functional/test_serial_io.py index 7a8c0b8c79d..0a1cebca8fe 100644 --- a/tests/integration_tests/functional/test_serial_io.py +++ b/tests/integration_tests/functional/test_serial_io.py @@ -55,7 +55,7 @@ def test_serial_after_snapshot(uvm_plain, microvm_factory): microvm.basic_config( vcpu_count=2, mem_size_mib=256, - boot_args="console=ttyS0 reboot=k panic=1 pci=off", + boot_args="console=ttyS0 reboot=k panic=1", ) serial = Serial(microvm) serial.open() @@ -99,9 +99,7 @@ def test_serial_console_login(uvm_plain_any): microvm.memory_monitor = None # Set up the microVM with 1 vCPU and a serial console. - microvm.basic_config( - vcpu_count=1, boot_args="console=ttyS0 reboot=k panic=1 pci=off" - ) + microvm.basic_config(vcpu_count=1, boot_args="console=ttyS0 reboot=k panic=1") microvm.start() @@ -144,7 +142,7 @@ def test_serial_dos(uvm_plain_any): # Set up the microVM with 1 vCPU and a serial console. microvm.basic_config( vcpu_count=1, - boot_args="console=ttyS0 reboot=k panic=1 pci=off", + boot_args="console=ttyS0 reboot=k panic=1", ) microvm.add_net_iface() microvm.start() @@ -176,7 +174,7 @@ def test_serial_block(uvm_plain_any): test_microvm.basic_config( vcpu_count=1, mem_size_mib=512, - boot_args="console=ttyS0 reboot=k panic=1 pci=off", + boot_args="console=ttyS0 reboot=k panic=1", ) test_microvm.add_net_iface() test_microvm.start() diff --git a/tests/integration_tests/functional/test_snapshot_basic.py b/tests/integration_tests/functional/test_snapshot_basic.py index 2af89b66a69..c4319ffd254 100644 --- a/tests/integration_tests/functional/test_snapshot_basic.py +++ b/tests/integration_tests/functional/test_snapshot_basic.py @@ -115,9 +115,8 @@ def test_snapshot_current_version(uvm_nano): def test_cycled_snapshot_restore( bin_vsock_path, tmp_path, + uvm_plain_any, microvm_factory, - guest_kernel, - rootfs, snapshot_type, use_snapshot_editor, cpu_template_any, @@ -133,7 +132,7 @@ def test_cycled_snapshot_restore( logger = logging.getLogger("snapshot_sequence") diff_snapshots = snapshot_type == SnapshotType.DIFF - vm = microvm_factory.build(guest_kernel, rootfs) + vm = uvm_plain_any vm.spawn() vm.basic_config( vcpu_count=2, @@ -249,7 +248,7 @@ def test_load_snapshot_failure_handling(uvm_plain): vm.mark_killed() -def test_cmp_full_and_first_diff_mem(microvm_factory, guest_kernel, rootfs): +def test_cmp_full_and_first_diff_mem(uvm_plain_any): """ Compare memory of 2 consecutive full and diff snapshots. @@ -260,7 +259,7 @@ def test_cmp_full_and_first_diff_mem(microvm_factory, guest_kernel, rootfs): """ logger = logging.getLogger("snapshot_sequence") - vm = microvm_factory.build(guest_kernel, rootfs) + vm = uvm_plain_any vm.spawn() vm.basic_config( vcpu_count=2, @@ -430,12 +429,12 @@ def test_create_large_diff_snapshot(uvm_plain): # process would have been taken down. -def test_diff_snapshot_overlay(guest_kernel, rootfs, microvm_factory): +def test_diff_snapshot_overlay(uvm_plain_any, microvm_factory): """ Tests that if we take a diff snapshot and direct firecracker to write it on top of an existing snapshot file, it will successfully merge them. """ - basevm = microvm_factory.build(guest_kernel, rootfs) + basevm = uvm_plain_any basevm.spawn() basevm.basic_config(track_dirty_pages=True) basevm.add_net_iface() @@ -467,7 +466,7 @@ def test_diff_snapshot_overlay(guest_kernel, rootfs, microvm_factory): # Check that the restored VM works -def test_snapshot_overwrite_self(guest_kernel, rootfs, microvm_factory): +def test_snapshot_overwrite_self(uvm_plain_any, microvm_factory): """Tests that if we try to take a snapshot that would overwrite the very file from which the current VM is stored, nothing happens. @@ -475,7 +474,7 @@ def test_snapshot_overwrite_self(guest_kernel, rootfs, microvm_factory): of mmap does not specify what should happen if the file is changed after being mmap'd (https://man7.org/linux/man-pages/man2/mmap.2.html). It seems that these changes can propagate to the mmap'd memory region.""" - base_vm = microvm_factory.build(guest_kernel, rootfs) + base_vm = uvm_plain_any base_vm.spawn() base_vm.basic_config() base_vm.add_net_iface() @@ -499,11 +498,11 @@ def test_snapshot_overwrite_self(guest_kernel, rootfs, microvm_factory): # restored, with a new snapshot of this vm, does not break the VM -def test_vmgenid(guest_kernel_linux_6_1, rootfs, microvm_factory, snapshot_type): +def test_vmgenid(uvm_plain_6_1, microvm_factory, snapshot_type): """ Test VMGenID device upon snapshot resume """ - base_vm = microvm_factory.build(guest_kernel_linux_6_1, rootfs) + base_vm = uvm_plain_6_1 base_vm.spawn() base_vm.basic_config(track_dirty_pages=True) base_vm.add_net_iface() diff --git a/tests/integration_tests/functional/test_vsock.py b/tests/integration_tests/functional/test_vsock.py index 5b6221c32a9..8c0d30700c6 100644 --- a/tests/integration_tests/functional/test_vsock.py +++ b/tests/integration_tests/functional/test_vsock.py @@ -37,7 +37,7 @@ TEST_WORKER_COUNT = 10 -def test_vsock(uvm_plain_any, pci_enabled, bin_vsock_path, test_fc_session_root_path): +def test_vsock(uvm_plain_any, bin_vsock_path, test_fc_session_root_path): """ Test guest and host vsock initiated connections. @@ -45,7 +45,7 @@ def test_vsock(uvm_plain_any, pci_enabled, bin_vsock_path, test_fc_session_root_ """ vm = uvm_plain_any - vm.spawn(pci=pci_enabled) + vm.spawn() vm.basic_config() vm.add_net_iface() @@ -102,12 +102,12 @@ def negative_test_host_connections(vm, blob_path, blob_hash): validate_fc_metrics(metrics) -def test_vsock_epipe(uvm_plain, pci_enabled, bin_vsock_path, test_fc_session_root_path): +def test_vsock_epipe(uvm_plain_any, bin_vsock_path, test_fc_session_root_path): """ Vsock negative test to validate SIGPIPE/EPIPE handling. """ - vm = uvm_plain - vm.spawn(pci=pci_enabled) + vm = uvm_plain_any + vm.spawn() vm.basic_config() vm.add_net_iface() vm.api.vsock.put(vsock_id="vsock0", guest_cid=3, uds_path=f"/{VSOCK_UDS_PATH}") @@ -129,7 +129,7 @@ def test_vsock_epipe(uvm_plain, pci_enabled, bin_vsock_path, test_fc_session_roo def test_vsock_transport_reset_h2g( - uvm_plain, pci_enabled, microvm_factory, bin_vsock_path, test_fc_session_root_path + uvm_plain_any, microvm_factory, bin_vsock_path, test_fc_session_root_path ): """ Vsock transport reset test. @@ -146,8 +146,8 @@ def test_vsock_transport_reset_h2g( 6. Close VM -> Load VM from Snapshot -> check that vsock device is still working. """ - test_vm = uvm_plain - test_vm.spawn(pci=pci_enabled) + test_vm = uvm_plain_any + test_vm.spawn() test_vm.basic_config(vcpu_count=2, mem_size_mib=256) test_vm.add_net_iface() test_vm.api.vsock.put(vsock_id="vsock0", guest_cid=3, uds_path=f"/{VSOCK_UDS_PATH}") @@ -215,12 +215,12 @@ def test_vsock_transport_reset_h2g( validate_fc_metrics(metrics) -def test_vsock_transport_reset_g2h(uvm_plain, pci_enabled, microvm_factory): +def test_vsock_transport_reset_g2h(uvm_plain_any, microvm_factory): """ Vsock transport reset test. """ - test_vm = uvm_plain - test_vm.spawn(pci=pci_enabled) + test_vm = uvm_plain_any + test_vm.spawn() test_vm.basic_config(vcpu_count=2, mem_size_mib=256) test_vm.add_net_iface() test_vm.api.vsock.put(vsock_id="vsock0", guest_cid=3, uds_path=f"/{VSOCK_UDS_PATH}") diff --git a/tests/integration_tests/performance/test_block.py b/tests/integration_tests/performance/test_block.py index 7fe9216e559..7fdd9576a3a 100644 --- a/tests/integration_tests/performance/test_block.py +++ b/tests/integration_tests/performance/test_block.py @@ -161,14 +161,11 @@ def emit_fio_metrics(logs_dir, metrics): @pytest.mark.parametrize("fio_block_size", [4096], ids=["bs4096"]) @pytest.mark.parametrize("fio_engine", ["libaio", "psync"]) def test_block_performance( - microvm_factory, - guest_kernel_acpi, - rootfs, + uvm_any_acpi, vcpus, fio_mode, fio_block_size, fio_engine, - pci_enabled, io_engine, metrics, results_dir, @@ -176,8 +173,8 @@ def test_block_performance( """ Execute block device emulation benchmarking scenarios. """ - vm = microvm_factory.build(guest_kernel_acpi, rootfs, monitor_memory=False) - vm.spawn(log_level="Info", emit_metrics=True, pci=pci_enabled) + vm = uvm_any_acpi + vm.spawn(log_level="Info", emit_metrics=True) vm.basic_config(vcpu_count=vcpus, mem_size_mib=GUEST_MEM_MIB) vm.add_net_iface() # Add a secondary block device for benchmark tests. @@ -214,9 +211,7 @@ def test_block_performance( @pytest.mark.parametrize("fio_mode", ["randread"]) @pytest.mark.parametrize("fio_block_size", [4096], ids=["bs4096"]) def test_block_vhost_user_performance( - microvm_factory, - guest_kernel_acpi, - rootfs, + uvm_any_acpi, vcpus, fio_mode, fio_block_size, @@ -227,7 +222,7 @@ def test_block_vhost_user_performance( Execute block device emulation benchmarking scenarios. """ - vm = microvm_factory.build(guest_kernel_acpi, rootfs, monitor_memory=False) + vm = uvm_any_acpi vm.spawn(log_level="Info", emit_metrics=True) vm.basic_config(vcpu_count=vcpus, mem_size_mib=GUEST_MEM_MIB) vm.add_net_iface() diff --git a/tests/integration_tests/performance/test_boottime.py b/tests/integration_tests/performance/test_boottime.py index 173e352f67d..f542a7d153e 100644 --- a/tests/integration_tests/performance/test_boottime.py +++ b/tests/integration_tests/performance/test_boottime.py @@ -98,14 +98,13 @@ def launch_vm_with_boot_timer( microvm_factory, guest_kernel_acpi, rootfs_rw, vcpu_count, mem_size_mib, pci_enabled ): """Launches a microVM with guest-timer and returns the reported metrics for it""" - boot_args = DEFAULT_BOOT_ARGS if pci_enabled else DEFAULT_BOOT_ARGS + " pci=off" - vm = microvm_factory.build(guest_kernel_acpi, rootfs_rw) + vm = microvm_factory.build(guest_kernel_acpi, rootfs_rw, pci=pci_enabled) vm.jailer.extra_args.update({"boot-timer": None}) - vm.spawn(pci=pci_enabled) + vm.spawn() vm.basic_config( vcpu_count=vcpu_count, mem_size_mib=mem_size_mib, - boot_args=boot_args + " init=/usr/local/bin/init", + boot_args=DEFAULT_BOOT_ARGS + " init=/usr/local/bin/init", enable_entropy_device=True, ) vm.add_net_iface() diff --git a/tests/integration_tests/performance/test_huge_pages.py b/tests/integration_tests/performance/test_huge_pages.py index ae1cb595311..0248fb6a3c7 100644 --- a/tests/integration_tests/performance/test_huge_pages.py +++ b/tests/integration_tests/performance/test_huge_pages.py @@ -68,13 +68,13 @@ def test_hugetlbfs_boot(uvm_plain): ) -def test_hugetlbfs_snapshot(microvm_factory, guest_kernel_linux_5_10, rootfs): +def test_hugetlbfs_snapshot(microvm_factory, uvm_plain): """ Test hugetlbfs snapshot restore via uffd """ ### Create Snapshot ### - vm = microvm_factory.build(guest_kernel_linux_5_10, rootfs) + vm = uvm_plain vm.memory_monitor = None vm.spawn() vm.basic_config(huge_pages=HugePagesConfig.HUGETLBFS_2MB, mem_size_mib=128) @@ -138,8 +138,7 @@ def test_hugetlbfs_diff_snapshot(microvm_factory, uvm_plain): @pytest.mark.parametrize("huge_pages", HugePagesConfig) def test_ept_violation_count( microvm_factory, - guest_kernel_linux_5_10, - rootfs, + uvm_plain, metrics, huge_pages, ): @@ -149,7 +148,7 @@ def test_ept_violation_count( """ ### Create Snapshot ### - vm = microvm_factory.build(guest_kernel_linux_5_10, rootfs) + vm = uvm_plain vm.memory_monitor = None vm.spawn() vm.basic_config(huge_pages=huge_pages, mem_size_mib=256) diff --git a/tests/integration_tests/performance/test_initrd.py b/tests/integration_tests/performance/test_initrd.py index 3845e5610c0..6cf133e373c 100644 --- a/tests/integration_tests/performance/test_initrd.py +++ b/tests/integration_tests/performance/test_initrd.py @@ -9,13 +9,15 @@ @pytest.fixture -def uvm_with_initrd(microvm_factory, guest_kernel, record_property, artifact_dir): +def uvm_with_initrd( + microvm_factory, guest_kernel, pci_enabled, record_property, artifact_dir +): """ See file:../docs/initrd.md """ fs = artifact_dir / "initramfs.cpio" record_property("rootfs", fs.name) - uvm = microvm_factory.build(guest_kernel) + uvm = microvm_factory.build(guest_kernel, pci=pci_enabled) uvm.initrd_file = fs yield uvm @@ -33,7 +35,7 @@ def test_microvm_initrd_with_serial(uvm_with_initrd, huge_pages): vm.basic_config( add_root_device=False, vcpu_count=1, - boot_args="console=ttyS0 reboot=k panic=1 pci=off", + boot_args="console=ttyS0 reboot=k panic=1", use_initrd=True, huge_pages=huge_pages, ) diff --git a/tests/integration_tests/performance/test_memory_overhead.py b/tests/integration_tests/performance/test_memory_overhead.py index 7935397cff4..2f4888c95ea 100644 --- a/tests/integration_tests/performance/test_memory_overhead.py +++ b/tests/integration_tests/performance/test_memory_overhead.py @@ -30,7 +30,13 @@ ) @pytest.mark.nonci def test_memory_overhead( - microvm_factory, guest_kernel_acpi, rootfs, vcpu_count, mem_size_mib, metrics + microvm_factory, + guest_kernel_acpi, + rootfs, + vcpu_count, + mem_size_mib, + pci_enabled, + metrics, ): """Track Firecracker memory overhead. @@ -38,7 +44,9 @@ def test_memory_overhead( """ for _ in range(5): - microvm = microvm_factory.build(guest_kernel_acpi, rootfs, monitor_memory=False) + microvm = microvm_factory.build( + guest_kernel_acpi, rootfs, pci=pci_enabled, monitor_memory=False + ) microvm.spawn(emit_metrics=True) microvm.basic_config(vcpu_count=vcpu_count, mem_size_mib=mem_size_mib) microvm.add_net_iface() diff --git a/tests/integration_tests/performance/test_network.py b/tests/integration_tests/performance/test_network.py index 4c2deba0041..cf22fb691f2 100644 --- a/tests/integration_tests/performance/test_network.py +++ b/tests/integration_tests/performance/test_network.py @@ -38,15 +38,15 @@ def consume_ping_output(ping_putput, request_per_round): @pytest.fixture -def network_microvm(request, microvm_factory, guest_kernel_acpi, rootfs, pci_enabled): +def network_microvm(request, uvm_plain_acpi): """Creates a microvm with the networking setup used by the performance tests in this file. This fixture receives its vcpu count via indirect parameterization""" guest_mem_mib = 1024 guest_vcpus = request.param - vm = microvm_factory.build(guest_kernel_acpi, rootfs, monitor_memory=False) - vm.spawn(log_level="Info", emit_metrics=True, pci=pci_enabled) + vm = uvm_plain_acpi + vm.spawn(log_level="Info", emit_metrics=True) vm.basic_config(vcpu_count=guest_vcpus, mem_size_mib=guest_mem_mib) vm.add_net_iface() vm.start() diff --git a/tests/integration_tests/performance/test_snapshot.py b/tests/integration_tests/performance/test_snapshot.py index 99307c76d24..f142b444dda 100644 --- a/tests/integration_tests/performance/test_snapshot.py +++ b/tests/integration_tests/performance/test_snapshot.py @@ -44,12 +44,13 @@ def id(self): """Computes a unique id for this test instance""" return "all_dev" if self.all_devices else f"{self.vcpus}vcpu_{self.mem}mb" - def boot_vm(self, microvm_factory, guest_kernel, rootfs) -> Microvm: + def boot_vm(self, microvm_factory, guest_kernel, rootfs, pci_enabled) -> Microvm: """Creates the initial snapshot that will be loaded repeatedly to sample latencies""" vm = microvm_factory.build( guest_kernel, rootfs, monitor_memory=False, + pci=pci_enabled, ) vm.spawn(log_level="Info", emit_metrics=True) vm.time_api_requests = False @@ -96,7 +97,7 @@ def boot_vm(self, microvm_factory, guest_kernel, rootfs) -> Microvm: ids=lambda x: x.id, ) def test_restore_latency( - microvm_factory, rootfs, guest_kernel_linux_5_10, test_setup, metrics + microvm_factory, guest_kernel_linux_5_10, rootfs, pci_enabled, test_setup, metrics ): """ Restores snapshots with vcpu/memory configuration, roughly scaling according to mem = (vcpus - 1) * 2048MB, @@ -105,7 +106,9 @@ def test_restore_latency( We only test a single guest kernel, as the guest kernel does not "participate" in snapshot restore. """ - vm = test_setup.boot_vm(microvm_factory, guest_kernel_linux_5_10, rootfs) + vm = test_setup.boot_vm( + microvm_factory, guest_kernel_linux_5_10, rootfs, pci_enabled + ) metrics.set_dimensions( { @@ -147,6 +150,7 @@ def test_post_restore_latency( microvm_factory, rootfs, guest_kernel_linux_5_10, + pci_enabled, metrics, uffd_handler, huge_pages, @@ -156,7 +160,9 @@ def test_post_restore_latency( pytest.skip("huge page snapshots can only be restored using uffd") test_setup = SnapshotRestoreTest(mem=1024, vcpus=2, huge_pages=huge_pages) - vm = test_setup.boot_vm(microvm_factory, guest_kernel_linux_5_10, rootfs) + vm = test_setup.boot_vm( + microvm_factory, guest_kernel_linux_5_10, rootfs, pci_enabled + ) metrics.set_dimensions( { @@ -204,6 +210,7 @@ def test_population_latency( microvm_factory, rootfs, guest_kernel_linux_5_10, + pci_enabled, metrics, huge_pages, vcpus, @@ -211,7 +218,9 @@ def test_population_latency( ): """Collects population latency metrics (e.g. how long it takes UFFD handler to fault in all memory)""" test_setup = SnapshotRestoreTest(mem=mem, vcpus=vcpus, huge_pages=huge_pages) - vm = test_setup.boot_vm(microvm_factory, guest_kernel_linux_5_10, rootfs) + vm = test_setup.boot_vm( + microvm_factory, guest_kernel_linux_5_10, rootfs, pci_enabled + ) metrics.set_dimensions( { @@ -255,15 +264,13 @@ def test_population_latency( @pytest.mark.nonci def test_snapshot_create_latency( - microvm_factory, - guest_kernel_linux_5_10, - rootfs, + uvm_plain, metrics, snapshot_type, ): """Measure the latency of creating a Full snapshot""" - vm = microvm_factory.build(guest_kernel_linux_5_10, rootfs, monitor_memory=False) + vm = uvm_plain vm.spawn() vm.basic_config( vcpu_count=2, diff --git a/tests/integration_tests/performance/test_vhost_user_metrics.py b/tests/integration_tests/performance/test_vhost_user_metrics.py index fd20b34a47b..a278ae79971 100644 --- a/tests/integration_tests/performance/test_vhost_user_metrics.py +++ b/tests/integration_tests/performance/test_vhost_user_metrics.py @@ -10,9 +10,7 @@ @pytest.mark.parametrize("vcpu_count", [1, 2], ids=["1vcpu", "2vcpu"]) -def test_vhost_user_block_metrics( - microvm_factory, guest_kernel_acpi, rootfs, vcpu_count, metrics -): +def test_vhost_user_block_metrics(uvm_plain_acpi, vcpu_count, metrics): """ This test tries to boot a VM with vhost-user-block as a scratch device, resize the vhost-user scratch drive to have @@ -28,7 +26,7 @@ def test_vhost_user_block_metrics( # low->high->low->high and so the numbers are not in monotonic sequence. new_sizes = [20, 10, 30] # MB - vm = microvm_factory.build(guest_kernel_acpi, rootfs, monitor_memory=False) + vm = uvm_plain_acpi vm.spawn(log_level="Info") vm.basic_config(vcpu_count=vcpu_count) vm.add_net_iface() diff --git a/tests/integration_tests/performance/test_vsock.py b/tests/integration_tests/performance/test_vsock.py index 5a023f53eea..402e7ff66b5 100644 --- a/tests/integration_tests/performance/test_vsock.py +++ b/tests/integration_tests/performance/test_vsock.py @@ -75,12 +75,9 @@ def guest_command(self, port_offset): @pytest.mark.parametrize("payload_length", ["64K", "1024K"], ids=["p64K", "p1024K"]) @pytest.mark.parametrize("mode", ["g2h", "h2g", "bd"]) def test_vsock_throughput( - microvm_factory, - guest_kernel_acpi, - rootfs, + uvm_plain_acpi, vcpus, payload_length, - pci_enabled, mode, metrics, results_dir, @@ -95,8 +92,8 @@ def test_vsock_throughput( pytest.skip("bidrectional test only done with at least 2 vcpus") mem_size_mib = 1024 - vm = microvm_factory.build(guest_kernel_acpi, rootfs, monitor_memory=False) - vm.spawn(log_level="Info", emit_metrics=True, pci=pci_enabled) + vm = uvm_plain_acpi + vm.spawn(log_level="Info", emit_metrics=True) vm.basic_config(vcpu_count=vcpus, mem_size_mib=mem_size_mib) vm.add_net_iface() # Create a vsock device