diff --git a/src/vmm/src/arch/aarch64/layout.rs b/src/vmm/src/arch/aarch64/layout.rs index bdecc712696..ecd9200e5dd 100644 --- a/src/vmm/src/arch/aarch64/layout.rs +++ b/src/vmm/src/arch/aarch64/layout.rs @@ -78,9 +78,12 @@ pub const FDT_MAX_SIZE: usize = 0x20_0000; // * a multiple of 32. /// The highest usable SPI on aarch64. pub const IRQ_MAX: u32 = 128; - /// First usable interrupt on aarch64. pub const IRQ_BASE: u32 = 32; +/// The highest available GSI in KVM (KVM_MAX_IRQ_ROUTES=4096) +pub const GSI_MAX: u32 = 4095; +/// First GSI after legacy IRQ on aarch64. +pub const GSI_BASE: u32 = 129; /// The start of the memory area reserved for MMIO 32-bit accesses. /// Below this address will reside the GIC, above this address will reside the MMIO devices. diff --git a/src/vmm/src/arch/mod.rs b/src/vmm/src/arch/mod.rs index 3693feed04b..fbeb9fa0ce0 100644 --- a/src/vmm/src/arch/mod.rs +++ b/src/vmm/src/arch/mod.rs @@ -21,10 +21,11 @@ pub use aarch64::vm::{ArchVm, ArchVmError, VmState}; #[cfg(target_arch = "aarch64")] pub use aarch64::{ ConfigurationError, arch_memory_regions, configure_system_for_boot, get_kernel_start, - initrd_load_addr, layout::BOOT_DEVICE_MEM_START, layout::CMDLINE_MAX_SIZE, layout::IRQ_BASE, - layout::IRQ_MAX, layout::MEM_32BIT_DEVICES_SIZE, layout::MEM_32BIT_DEVICES_START, - layout::MEM_64BIT_DEVICES_SIZE, layout::MEM_64BIT_DEVICES_START, layout::MMIO32_MEM_SIZE, - layout::MMIO32_MEM_START, layout::PCI_MMCONFIG_SIZE, layout::PCI_MMCONFIG_START, + initrd_load_addr, layout::BOOT_DEVICE_MEM_START, layout::CMDLINE_MAX_SIZE, layout::GSI_BASE, + layout::GSI_MAX, layout::IRQ_BASE, layout::IRQ_MAX, layout::MEM_32BIT_DEVICES_SIZE, + layout::MEM_32BIT_DEVICES_START, layout::MEM_64BIT_DEVICES_SIZE, + layout::MEM_64BIT_DEVICES_START, layout::MMIO32_MEM_SIZE, layout::MMIO32_MEM_START, + layout::PCI_MMCONFIG_SIZE, layout::PCI_MMCONFIG_START, layout::PCI_MMIO_CONFIG_SIZE_PER_SEGMENT, layout::RTC_MEM_START, layout::SERIAL_MEM_START, layout::SYSTEM_MEM_SIZE, layout::SYSTEM_MEM_START, load_kernel, }; @@ -44,10 +45,10 @@ pub use x86_64::vm::{ArchVm, ArchVmError, VmState}; pub use crate::arch::x86_64::{ ConfigurationError, arch_memory_regions, configure_system_for_boot, get_kernel_start, initrd_load_addr, layout::APIC_ADDR, layout::BOOT_DEVICE_MEM_START, layout::CMDLINE_MAX_SIZE, - layout::IOAPIC_ADDR, layout::IRQ_BASE, layout::IRQ_MAX, layout::MEM_32BIT_DEVICES_SIZE, - layout::MEM_32BIT_DEVICES_START, layout::MEM_64BIT_DEVICES_SIZE, - layout::MEM_64BIT_DEVICES_START, layout::MMIO32_MEM_SIZE, layout::MMIO32_MEM_START, - layout::PCI_MMCONFIG_SIZE, layout::PCI_MMCONFIG_START, + layout::GSI_BASE, layout::GSI_MAX, layout::IOAPIC_ADDR, layout::IRQ_BASE, layout::IRQ_MAX, + layout::MEM_32BIT_DEVICES_SIZE, layout::MEM_32BIT_DEVICES_START, + layout::MEM_64BIT_DEVICES_SIZE, layout::MEM_64BIT_DEVICES_START, layout::MMIO32_MEM_SIZE, + layout::MMIO32_MEM_START, layout::PCI_MMCONFIG_SIZE, layout::PCI_MMCONFIG_START, layout::PCI_MMIO_CONFIG_SIZE_PER_SEGMENT, layout::SYSTEM_MEM_SIZE, layout::SYSTEM_MEM_START, load_kernel, }; diff --git a/src/vmm/src/arch/x86_64/layout.rs b/src/vmm/src/arch/x86_64/layout.rs index 8ae558e91c3..59961d767d9 100644 --- a/src/vmm/src/arch/x86_64/layout.rs +++ b/src/vmm/src/arch/x86_64/layout.rs @@ -26,6 +26,10 @@ pub const HIMEM_START: u64 = 0x0010_0000; // 1 MB. pub const IRQ_BASE: u32 = 5; /// Last usable IRQ ID for virtio device interrupts on x86_64. pub const IRQ_MAX: u32 = 23; +/// First GSI after legacy interrupts/ +pub const GSI_BASE: u32 = 24; +/// The highest available GSI in KVM (KVM_MAX_IRQ_ROUTES=4096) +pub const GSI_MAX: u32 = 4095; /// Address for the TSS setup. pub const KVM_TSS_ADDRESS: u64 = 0xfffb_d000; diff --git a/src/vmm/src/device_manager/mmio.rs b/src/vmm/src/device_manager/mmio.rs index fe32376ebb4..e3fbfe5ce23 100644 --- a/src/vmm/src/device_manager/mmio.rs +++ b/src/vmm/src/device_manager/mmio.rs @@ -157,7 +157,7 @@ impl MMIODeviceManager { resource_allocator: &mut ResourceAllocator, irq_count: u32, ) -> Result { - let irq = match resource_allocator.allocate_gsi(irq_count)?[..] { + let irq = match resource_allocator.allocate_irq(irq_count)?[..] { [] => None, [irq] => NonZeroU32::new(irq), _ => return Err(MmioError::InvalidIrqConfig), @@ -277,11 +277,11 @@ impl MMIODeviceManager { let device_info = if let Some(device_info) = device_info_opt { device_info } else { - let gsi = vm.resource_allocator().allocate_gsi(1)?; + let irq = vm.resource_allocator().allocate_irq(1)?; MMIODeviceInfo { addr: SERIAL_MEM_START, len: MMIO_LEN, - irq: NonZeroU32::new(gsi[0]), + irq: NonZeroU32::new(irq[0]), } }; @@ -336,11 +336,11 @@ impl MMIODeviceManager { let device_info = if let Some(device_info) = device_info_opt { device_info } else { - let gsi = vm.resource_allocator().allocate_gsi(1)?; + let irq = vm.resource_allocator().allocate_irq(1)?; MMIODeviceInfo { addr: RTC_MEM_START, len: MMIO_LEN, - irq: NonZeroU32::new(gsi[0]), + irq: NonZeroU32::new(irq[0]), } }; diff --git a/src/vmm/src/devices/acpi/vmgenid.rs b/src/vmm/src/devices/acpi/vmgenid.rs index 6d096007193..029cc5376f6 100644 --- a/src/vmm/src/devices/acpi/vmgenid.rs +++ b/src/vmm/src/devices/acpi/vmgenid.rs @@ -88,7 +88,7 @@ impl VmGenId { mem: &GuestMemoryMmap, resource_allocator: &mut ResourceAllocator, ) -> Result { - let gsi = resource_allocator.allocate_gsi(1)?; + let irq = resource_allocator.allocate_irq(1)?; // The generation ID needs to live in an 8-byte aligned buffer let addr = resource_allocator.allocate_system_memory( VMGENID_MEM_SIZE, @@ -96,7 +96,7 @@ impl VmGenId { vm_allocator::AllocPolicy::LastMatch, )?; - Self::from_parts(GuestAddress(addr), gsi[0], mem) + Self::from_parts(GuestAddress(addr), irq[0], mem) } // Create a 16-bytes random number diff --git a/src/vmm/src/vstate/resources.rs b/src/vmm/src/vstate/resources.rs index 8b0cb4a67c4..4ad719a512f 100644 --- a/src/vmm/src/vstate/resources.rs +++ b/src/vmm/src/vstate/resources.rs @@ -19,7 +19,9 @@ use crate::snapshot::Persist; /// * Memory allocations in the MMIO address space #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ResourceAllocator { - /// Allocator for device interrupt lines + /// Allocator for legacy device interrupt lines + pub irq_allocator: IdAllocator, + /// Allocator for PCI device GSIs pub gsi_allocator: IdAllocator, /// Allocator for memory in the 32-bit MMIO address space pub mmio32_memory: AddressAllocator, @@ -41,7 +43,8 @@ impl ResourceAllocator { // It is fine for us to unwrap the following since we know we are passing valid ranges for // all allocators Self { - gsi_allocator: IdAllocator::new(arch::IRQ_BASE, arch::IRQ_MAX).unwrap(), + irq_allocator: IdAllocator::new(arch::IRQ_BASE, arch::IRQ_MAX).unwrap(), + gsi_allocator: IdAllocator::new(arch::GSI_BASE, arch::GSI_MAX).unwrap(), mmio32_memory: AddressAllocator::new( arch::MEM_32BIT_DEVICES_START, arch::MEM_32BIT_DEVICES_SIZE, @@ -57,6 +60,30 @@ impl ResourceAllocator { } } + /// Allocate a number of legacy IRQs + /// + /// # Arguments + /// + /// * `irq_count` - The number of legacy IRQs to allocate + pub fn allocate_irq(&mut self, irq_count: u32) -> Result, vm_allocator::Error> { + let mut irqs = Vec::with_capacity(irq_count as usize); + + for _ in 0..irq_count { + match self.irq_allocator.allocate_id() { + Ok(irq) => irqs.push(irq), + Err(err) => { + // It is ok to unwrap here, we just allocated the GSI + irqs.into_iter().for_each(|irq| { + self.irq_allocator.free_id(irq).unwrap(); + }); + return Err(err); + } + } + } + + Ok(irqs) + } + /// Allocate a number of GSIs /// /// # Arguments @@ -167,11 +194,52 @@ mod tests { use vm_allocator::AllocPolicy; use super::ResourceAllocator; - use crate::arch::{self, IRQ_BASE}; + use crate::arch::{self, GSI_BASE, IRQ_BASE}; use crate::snapshot::{Persist, Snapshot}; const MAX_IRQS: u32 = arch::IRQ_MAX - arch::IRQ_BASE + 1; + #[test] + fn test_allocate_irq() { + let mut allocator = ResourceAllocator::new(); + // asking for 0 IRQs should return us an empty vector + assert_eq!(allocator.allocate_irq(0), Ok(vec![])); + // We cannot allocate more GSIs than available + assert_eq!( + allocator.allocate_irq(MAX_IRQS + 1), + Err(vm_allocator::Error::ResourceNotAvailable) + ); + // But allocating all of them at once should work + assert_eq!( + allocator.allocate_irq(MAX_IRQS), + Ok((arch::IRQ_BASE..=arch::IRQ_MAX).collect::>()) + ); + // And now we ran out of GSIs + assert_eq!( + allocator.allocate_irq(1), + Err(vm_allocator::Error::ResourceNotAvailable) + ); + // But we should be able to ask for 0 GSIs + assert_eq!(allocator.allocate_irq(0), Ok(vec![])); + + let mut allocator = ResourceAllocator::new(); + // We should be able to allocate 1 GSI + assert_eq!(allocator.allocate_irq(1), Ok(vec![arch::IRQ_BASE])); + // We can't allocate MAX_IRQS any more + assert_eq!( + allocator.allocate_irq(MAX_IRQS), + Err(vm_allocator::Error::ResourceNotAvailable) + ); + // We can allocate another one and it should be the second available + assert_eq!(allocator.allocate_irq(1), Ok(vec![arch::IRQ_BASE + 1])); + // Let's allocate the rest in a loop + for i in arch::IRQ_BASE + 2..=arch::IRQ_MAX { + assert_eq!(allocator.allocate_irq(1), Ok(vec![i])); + } + } + + const MAX_GSIS: u32 = arch::GSI_MAX - arch::GSI_BASE + 1; + #[test] fn test_allocate_gsi() { let mut allocator = ResourceAllocator::new(); @@ -179,13 +247,13 @@ mod tests { assert_eq!(allocator.allocate_gsi(0), Ok(vec![])); // We cannot allocate more GSIs than available assert_eq!( - allocator.allocate_gsi(MAX_IRQS + 1), + allocator.allocate_gsi(MAX_GSIS + 1), Err(vm_allocator::Error::ResourceNotAvailable) ); // But allocating all of them at once should work assert_eq!( - allocator.allocate_gsi(MAX_IRQS), - Ok((arch::IRQ_BASE..=arch::IRQ_MAX).collect::>()) + allocator.allocate_gsi(MAX_GSIS), + Ok((arch::GSI_BASE..=arch::GSI_MAX).collect::>()) ); // And now we ran out of GSIs assert_eq!( @@ -197,16 +265,16 @@ mod tests { let mut allocator = ResourceAllocator::new(); // We should be able to allocate 1 GSI - assert_eq!(allocator.allocate_gsi(1), Ok(vec![arch::IRQ_BASE])); + assert_eq!(allocator.allocate_gsi(1), Ok(vec![arch::GSI_BASE])); // We can't allocate MAX_IRQS any more assert_eq!( - allocator.allocate_gsi(MAX_IRQS), + allocator.allocate_gsi(MAX_GSIS), Err(vm_allocator::Error::ResourceNotAvailable) ); // We can allocate another one and it should be the second available - assert_eq!(allocator.allocate_gsi(1), Ok(vec![arch::IRQ_BASE + 1])); + assert_eq!(allocator.allocate_gsi(1), Ok(vec![arch::GSI_BASE + 1])); // Let's allocate the rest in a loop - for i in arch::IRQ_BASE + 2..=arch::IRQ_MAX { + for i in arch::GSI_BASE + 2..=arch::GSI_MAX { assert_eq!(allocator.allocate_gsi(1), Ok(vec![i])); } } @@ -221,12 +289,16 @@ mod tests { #[test] fn test_save_restore() { let mut allocator0 = ResourceAllocator::new(); + let irq_0 = allocator0.allocate_irq(1).unwrap()[0]; + assert_eq!(irq_0, IRQ_BASE); let gsi_0 = allocator0.allocate_gsi(1).unwrap()[0]; - assert_eq!(gsi_0, IRQ_BASE); + assert_eq!(gsi_0, GSI_BASE); let mut allocator1 = clone_allocator(&allocator0); + let irq_1 = allocator1.allocate_irq(1).unwrap()[0]; + assert_eq!(irq_1, IRQ_BASE + 1); let gsi_1 = allocator1.allocate_gsi(1).unwrap()[0]; - assert_eq!(gsi_1, IRQ_BASE + 1); + assert_eq!(gsi_1, GSI_BASE + 1); let mmio32_mem = allocator1 .allocate_32bit_mmio_memory(0x42, 1, AllocPolicy::FirstMatch) .unwrap(); @@ -251,8 +323,10 @@ mod tests { .allocate_system_memory(0x42, 1, AllocPolicy::ExactMatch(system_mem)) .unwrap_err(); + let irq_2 = allocator2.allocate_irq(1).unwrap()[0]; + assert_eq!(irq_2, IRQ_BASE + 2); let gsi_2 = allocator2.allocate_gsi(1).unwrap()[0]; - assert_eq!(gsi_2, IRQ_BASE + 2); + assert_eq!(gsi_2, GSI_BASE + 2); let mmio32_mem = allocator1 .allocate_32bit_mmio_memory(0x42, 1, AllocPolicy::FirstMatch) .unwrap(); diff --git a/src/vmm/src/vstate/vm.rs b/src/vmm/src/vstate/vm.rs index 787cfa12cce..0f6c7c0b50d 100644 --- a/src/vmm/src/vstate/vm.rs +++ b/src/vmm/src/vstate/vm.rs @@ -877,7 +877,7 @@ pub(crate) mod tests { } for i in 0..4 { - let gsi = crate::arch::IRQ_BASE + i; + let gsi = crate::arch::GSI_BASE + i; let interrupts = vm.common.interrupts.lock().unwrap(); let kvm_route = interrupts.get(&gsi).unwrap(); assert!(kvm_route.masked); @@ -894,7 +894,7 @@ pub(crate) mod tests { // Simply enabling the vectors should not update the registered IRQ routes msix_group.enable().unwrap(); for i in 0..4 { - let gsi = crate::arch::IRQ_BASE + i; + let gsi = crate::arch::GSI_BASE + i; let interrupts = vm.common.interrupts.lock().unwrap(); let kvm_route = interrupts.get(&gsi).unwrap(); assert!(kvm_route.masked); @@ -914,7 +914,7 @@ pub(crate) mod tests { .update(0, InterruptSourceConfig::MsiIrq(config), false, true) .unwrap(); for i in 0..4 { - let gsi = crate::arch::IRQ_BASE + i; + let gsi = crate::arch::GSI_BASE + i; let interrupts = vm.common.interrupts.lock().unwrap(); let kvm_route = interrupts.get(&gsi).unwrap(); assert_eq!(kvm_route.masked, i != 0); diff --git a/tests/integration_tests/functional/test_max_devices.py b/tests/integration_tests/functional/test_max_devices.py index fc8397445c9..9242eeb495b 100644 --- a/tests/integration_tests/functional/test_max_devices.py +++ b/tests/integration_tests/functional/test_max_devices.py @@ -6,59 +6,82 @@ import pytest + # IRQs are available from 5 to 23. We always use one IRQ for VMGenID device, so # the maximum number of devices supported at the same time is 18. -MAX_DEVICES_ATTACHED = 18 +def max_devices(uvm): + """ + Returns the maximum number of devices supported by the platform. + """ + if uvm.pci_enabled: + return 31 + match platform.machine(): + case "aarch64": + return 64 + case "x86_64": + return 18 + case _: + raise ValueError("Unknown platform") -@pytest.mark.skipif( - platform.machine() != "x86_64", reason="Firecracker supports 24 IRQs on x86_64." -) -def test_attach_maximum_devices(uvm_plain_any): + +def test_attach_maximum_devices(microvm_factory, guest_kernel, rootfs, pci_enabled): """ Test attaching maximum number of devices to the microVM. """ - test_microvm = uvm_plain_any - test_microvm.spawn() + test_microvm = microvm_factory.build( + kernel=guest_kernel, rootfs=rootfs, monitor_memory=False + ) + test_microvm.spawn(pci=pci_enabled) # Set up a basic microVM. - test_microvm.basic_config() + test_microvm.basic_config(mem_size_mib=1024) + max_devices_attached = max_devices(test_microvm) # Add (`MAX_DEVICES_ATTACHED` - 1) devices because the rootfs # has already been configured in the `basic_config()`function. - for _ in range(MAX_DEVICES_ATTACHED - 1): + for _ in range(max_devices_attached - 1): test_microvm.add_net_iface() test_microvm.start() # Test that network devices attached are operational. - for i in range(MAX_DEVICES_ATTACHED - 1): + for i in range(max_devices_attached - 1): # Verify if guest can run commands. test_microvm.ssh_iface(i).check_output("sync") @pytest.mark.skipif( - platform.machine() != "x86_64", reason="Firecracker supports 24 IRQs on x86_64." + platform.machine() != "x86_64", + reason="Firecracker has a bug in aarch that limits devices to 64, from the theoretical 96.", ) -def test_attach_too_many_devices(uvm_plain): +def test_attach_too_many_devices(microvm_factory, guest_kernel, rootfs, pci_enabled): """ Test attaching to a microVM more devices than available IRQs. """ - test_microvm = uvm_plain - test_microvm.spawn() + test_microvm = microvm_factory.build( + kernel=guest_kernel, rootfs=rootfs, monitor_memory=False + ) + test_microvm.spawn(pci=pci_enabled) # Set up a basic microVM. - test_microvm.basic_config() + test_microvm.basic_config(mem_size_mib=1024) + + max_devices_attached = max_devices(test_microvm) # Add `MAX_DEVICES_ATTACHED` network devices on top of the # already configured rootfs. - for _ in range(MAX_DEVICES_ATTACHED): + for _ in range(max_devices_attached): test_microvm.add_net_iface() # Attempting to start a microVM with more than # `MAX_DEVICES_ATTACHED` devices should fail. error_str = ( - "Failed to allocate requested resource: The requested resource" - " is not available." + ("Could not find an available device slot on the PCI bus.") + if pci_enabled + else ( + "Failed to allocate requested resource: The requested resource" + " is not available." + ) ) with pytest.raises(RuntimeError, match=error_str): test_microvm.start()