fix(msi): allocate GSI for MSI and legacy IRQ from different ranges

Manciukic · Manciukic · commit d4e6f53a08a1 · 2025-07-11T13:38:56.000+01:00
Currently, we're limited to 24 GSI lines, which is too little for PCI
devices. Keep the current ranges as "legacy IRQ", and create a new range
for "GSI" that goes up to the kvm theoretical maximum of 4096 lines.

Signed-off-by: Riccardo Mancini &lt;mancio@amazon.com&gt;
diff --git a/src/vmm/src/arch/aarch64/gic/gicv2/regs/dist_regs.rs b/src/vmm/src/arch/aarch64/gic/gicv2/regs/dist_regs.rs
@@ -8,7 +8,7 @@ use kvm_ioctls::DeviceFd;
 
 use crate::arch::aarch64::gic::GicError;
 use crate::arch::aarch64::gic::regs::{GicRegState, MmioReg, SimpleReg, VgicRegEngine};
-use crate::arch::{IRQ_BASE, IRQ_MAX};
+use crate::arch::{NR_IRQ, SPI_BASE};
 
 // Distributor registers as detailed at page 75 from
 // https://developer.arm.com/documentation/ihi0048/latest/.
@@ -62,9 +62,9 @@ impl MmioReg for SharedIrqReg {
         // read-as-zero/write-ignore (RAZ/WI) policy.
         // The first part of a shared-irq register, the one corresponding to the
         // SGI and PPI IRQs (0-32) is RAZ/WI, so we skip it.
-        let start = self.offset + u64::from(IRQ_BASE) * u64::from(self.bits_per_irq) / 8;
+        let start = self.offset + u64::from(SPI_BASE) * u64::from(self.bits_per_irq) / 8;
 
-        let size_in_bits = u64::from(self.bits_per_irq) * u64::from(IRQ_MAX - IRQ_BASE);
+        let size_in_bits = u64::from(self.bits_per_irq) * u64::from(NR_IRQ);
         let mut size_in_bytes = size_in_bits / 8;
         if size_in_bits % 8 > 0 {
             size_in_bytes += 1;
diff --git a/src/vmm/src/arch/aarch64/gic/gicv3/regs/dist_regs.rs b/src/vmm/src/arch/aarch64/gic/gicv3/regs/dist_regs.rs
@@ -8,7 +8,7 @@ use kvm_ioctls::DeviceFd;
 
 use crate::arch::aarch64::gic::GicError;
 use crate::arch::aarch64::gic::regs::{GicRegState, MmioReg, SimpleReg, VgicRegEngine};
-use crate::arch::{IRQ_BASE, IRQ_MAX};
+use crate::arch::{NR_IRQ, SPI_BASE};
 
 // Distributor registers as detailed at page 456 from
 // https://static.docs.arm.com/ihi0069/c/IHI0069C_gic_architecture_specification.pdf.
@@ -64,9 +64,9 @@ impl MmioReg for SharedIrqReg {
         // read-as-zero/write-ignore (RAZ/WI) policy.
         // The first part of a shared-irq register, the one corresponding to the
         // SGI and PPI IRQs (0-32) is RAZ/WI, so we skip it.
-        let start = self.offset + u64::from(IRQ_BASE) * u64::from(self.bits_per_irq) / 8;
+        let start = self.offset + u64::from(SPI_BASE) * u64::from(self.bits_per_irq) / 8;
 
-        let size_in_bits = u64::from(self.bits_per_irq) * u64::from(IRQ_MAX - IRQ_BASE);
+        let size_in_bits = u64::from(self.bits_per_irq) * u64::from(NR_IRQ);
         let mut size_in_bytes = size_in_bits / 8;
         if size_in_bits % 8 > 0 {
             size_in_bytes += 1;
diff --git a/src/vmm/src/arch/aarch64/layout.rs b/src/vmm/src/arch/aarch64/layout.rs
@@ -76,19 +76,19 @@ pub const FDT_MAX_SIZE: usize = 0x20_0000;
 // * bigger than 32
 // * less than 1023 and
 // * a multiple of 32.
-/// The highest usable SPI on aarch64.
-pub const IRQ_MAX: u32 = 128;
-
-/// First usable interrupt on aarch64.
-pub const IRQ_BASE: u32 = 32;
-
-// The Linux kernel automatically shifts the GSI by 32 if it is an SPI,
-// allowing us to start numbering from 0 instead of 32.
-/// The first usable GSI on aarch64.
-pub const GSI_BASE: u32 = 0;
-
-/// The maximum usable GSI on aarch64.
-pub const GSI_MAX: u32 = IRQ_MAX - IRQ_BASE - 1;
+// The first 32 SPIs are reserved, but KVM already shifts the gsi we
+// pass, so we go from 0 to 95 for legacy gsis ("irq") and the remaining
+// we use for MSI.
+/// Offset of first legacy SPI in the GIC
+pub const SPI_BASE: u32 = 32;
+/// First usable interrupt on aarch64 (corresponds to SPI #32)
+pub const IRQ_BASE: u32 = 0;
+/// The highest usable SPI on aarch64 is 127 - 32 = 95
+pub const IRQ_MAX: u32 = 95;
+/// First GSI after legacy interrupts which will be used by MSI
+pub const GSI_BASE: u32 = IRQ_MAX + 1;
+/// The highest available GSI in KVM (KVM_MAX_IRQ_ROUTES=4096)
+pub const GSI_MAX: u32 = 4095;
 
 /// The start of the memory area reserved for MMIO 32-bit accesses.
 /// Below this address will reside the GIC, above this address will reside the MMIO devices.
diff --git a/src/vmm/src/arch/mod.rs b/src/vmm/src/arch/mod.rs
@@ -27,7 +27,7 @@ pub use aarch64::{
     layout::MEM_64BIT_DEVICES_START, layout::MMIO32_MEM_SIZE, layout::MMIO32_MEM_START,
     layout::PCI_MMCONFIG_SIZE, layout::PCI_MMCONFIG_START,
     layout::PCI_MMIO_CONFIG_SIZE_PER_SEGMENT, layout::RTC_MEM_START, layout::SERIAL_MEM_START,
-    layout::SYSTEM_MEM_SIZE, layout::SYSTEM_MEM_START, load_kernel,
+    layout::SPI_BASE, layout::SYSTEM_MEM_SIZE, layout::SYSTEM_MEM_START, load_kernel,
 };
 
 /// Module for x86_64 related functionality.
@@ -53,6 +53,11 @@ pub use crate::arch::x86_64::{
     load_kernel,
 };
 
+/// Total count of legacy interrupt lines available (IRQs)
+pub const NR_IRQ: u32 = IRQ_MAX - IRQ_BASE + 1;
+/// Total count of MSI GSIs available
+pub const NR_GSI: u32 = GSI_MAX - GSI_BASE + 1;
+
 /// Types of devices that can get attached to this platform.
 #[derive(Clone, Debug, PartialEq, Eq, Hash, Copy, Serialize, Deserialize)]
 pub enum DeviceType {
diff --git a/src/vmm/src/arch/x86_64/layout.rs b/src/vmm/src/arch/x86_64/layout.rs
@@ -21,17 +21,17 @@ pub const CMDLINE_MAX_SIZE: usize = 2048;
 /// Start of the high memory.
 pub const HIMEM_START: u64 = 0x0010_0000; // 1 MB.
 
-// Typically, on x86 systems 24 IRQs are used (0-23).
+// Typically, on x86 systems 24 IRQs are used for legacy devices (0-23).
+// However, the first 5 are reserved.
+// We allocate the remaining GSIs to MSIs.
 /// First usable IRQ ID for virtio device interrupts on x86_64.
 pub const IRQ_BASE: u32 = 5;
 /// Last usable IRQ ID for virtio device interrupts on x86_64.
 pub const IRQ_MAX: u32 = 23;
-
-/// The first usable GSI on x86_64 is the same as the first usable IRQ ID.
-pub const GSI_BASE: u32 = IRQ_BASE;
-
-/// The maximum usable GSI on x86_64 is the same as the last usable IRQ ID.
-pub const GSI_MAX: u32 = IRQ_MAX;
+/// First GSI after legacy interrupts which will be used by MSI
+pub const GSI_BASE: u32 = IRQ_MAX + 1;
+/// The highest available GSI in KVM (KVM_MAX_IRQ_ROUTES=4096)
+pub const GSI_MAX: u32 = 4095;
 
 /// Address for the TSS setup.
 pub const KVM_TSS_ADDRESS: u64 = 0xfffb_d000;
diff --git a/src/vmm/src/device_manager/mmio.rs b/src/vmm/src/device_manager/mmio.rs
@@ -84,7 +84,7 @@ fn add_virtio_aml(
     len: u64,
     irq: u32,
 ) -> Result<(), aml::AmlError> {
-    let dev_id = irq - crate::arch::GSI_BASE;
+    let dev_id = irq - crate::arch::IRQ_BASE;
     debug!(
         "acpi: Building AML for VirtIO device _SB_.V{:03}. memory range: {:#010x}:{} irq: {}",
         dev_id, addr, len, irq
@@ -156,7 +156,7 @@ impl MMIODeviceManager {
         resource_allocator: &mut ResourceAllocator,
         irq_count: u32,
     ) -> Result<MMIODeviceInfo, MmioError> {
-        let irq = match resource_allocator.allocate_gsi(irq_count)?[..] {
+        let irq = match resource_allocator.allocate_irq(irq_count)?[..] {
             [] => None,
             [irq] => Some(irq),
             _ => return Err(MmioError::InvalidIrqConfig),
@@ -276,11 +276,11 @@ impl MMIODeviceManager {
         let device_info = if let Some(device_info) = device_info_opt {
             device_info
         } else {
-            let gsi = vm.resource_allocator().allocate_gsi(1)?;
+            let irq = vm.resource_allocator().allocate_irq(1)?;
             MMIODeviceInfo {
                 addr: SERIAL_MEM_START,
                 len: MMIO_LEN,
-                irq: Some(gsi[0]),
+                irq: Some(irq[0]),
             }
         };
 
@@ -335,11 +335,11 @@ impl MMIODeviceManager {
         let device_info = if let Some(device_info) = device_info_opt {
             device_info
         } else {
-            let gsi = vm.resource_allocator().allocate_gsi(1)?;
+            let irq = vm.resource_allocator().allocate_irq(1)?;
             MMIODeviceInfo {
                 addr: RTC_MEM_START,
                 len: MMIO_LEN,
-                irq: Some(gsi[0]),
+                irq: Some(irq[0]),
             }
         };
 
@@ -612,15 +612,15 @@ pub(crate) mod tests {
         let dev = device_manager.get_virtio_device(0, "dummy").unwrap();
         assert_eq!(dev.resources.addr, arch::MEM_32BIT_DEVICES_START);
         assert_eq!(dev.resources.len, MMIO_LEN);
-        assert_eq!(dev.resources.irq, Some(arch::GSI_BASE));
+        assert_eq!(dev.resources.irq, Some(arch::IRQ_BASE));
 
         device_manager
             .for_each_virtio_device(|virtio_type, device_id, mmio_device| {
                 assert_eq!(*virtio_type, 0);
                 assert_eq!(device_id, "dummy");
                 assert_eq!(mmio_device.resources.addr, arch::MEM_32BIT_DEVICES_START);
                 assert_eq!(mmio_device.resources.len, MMIO_LEN);
-                assert_eq!(mmio_device.resources.irq, Some(arch::GSI_BASE));
+                assert_eq!(mmio_device.resources.irq, Some(arch::IRQ_BASE));
                 Ok::<(), ()>(())
             })
             .unwrap();
@@ -643,7 +643,7 @@ pub(crate) mod tests {
         #[cfg(target_arch = "aarch64")]
         vm.setup_irqchip(1).unwrap();
 
-        for _i in crate::arch::GSI_BASE..=crate::arch::GSI_MAX {
+        for _i in crate::arch::IRQ_BASE..=crate::arch::IRQ_MAX {
             device_manager
                 .register_virtio_test_device(
                     &vm,
@@ -711,7 +711,7 @@ pub(crate) mod tests {
                 .addr
         );
         assert_eq!(
-            crate::arch::GSI_BASE,
+            crate::arch::IRQ_BASE,
             device_manager.virtio_devices[&(type_id, id)]
                 .resources
                 .irq
@@ -762,7 +762,7 @@ pub(crate) mod tests {
         let device_info = device_manager
             .allocate_mmio_resources(&mut resource_allocator, 1)
             .unwrap();
-        assert_eq!(device_info.irq.unwrap(), crate::arch::GSI_BASE);
+        assert_eq!(device_info.irq.unwrap(), crate::arch::IRQ_BASE);
     }
 
     #[test]
diff --git a/src/vmm/src/devices/acpi/vmgenid.rs b/src/vmm/src/devices/acpi/vmgenid.rs
@@ -88,15 +88,15 @@ impl VmGenId {
         mem: &GuestMemoryMmap,
         resource_allocator: &mut ResourceAllocator,
     ) -> Result<Self, VmGenIdError> {
-        let gsi = resource_allocator.allocate_gsi(1)?;
+        let irq = resource_allocator.allocate_irq(1)?;
         // The generation ID needs to live in an 8-byte aligned buffer
         let addr = resource_allocator.allocate_system_memory(
             VMGENID_MEM_SIZE,
             8,
             vm_allocator::AllocPolicy::LastMatch,
         )?;
 
-        Self::from_parts(GuestAddress(addr), gsi[0], mem)
+        Self::from_parts(GuestAddress(addr), irq[0], mem)
     }
 
     // Create a 16-bytes random number
diff --git a/src/vmm/src/vstate/resources.rs b/src/vmm/src/vstate/resources.rs
@@ -19,7 +19,9 @@ use crate::snapshot::Persist;
 /// * Memory allocations in the MMIO address space
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct ResourceAllocator {
-    /// Allocator for device interrupt lines
+    /// Allocator for legacy device interrupt lines
+    pub irq_allocator: IdAllocator,
+    /// Allocator for PCI device GSIs
     pub gsi_allocator: IdAllocator,
     /// Allocator for memory in the 32-bit MMIO address space
     pub mmio32_memory: AddressAllocator,
@@ -41,6 +43,7 @@ impl ResourceAllocator {
         // It is fine for us to unwrap the following since we know we are passing valid ranges for
         // all allocators
         Self {
+            irq_allocator: IdAllocator::new(arch::IRQ_BASE, arch::IRQ_MAX).unwrap(),
             gsi_allocator: IdAllocator::new(arch::GSI_BASE, arch::GSI_MAX).unwrap(),
             mmio32_memory: AddressAllocator::new(
                 arch::MEM_32BIT_DEVICES_START,
@@ -57,6 +60,30 @@ impl ResourceAllocator {
         }
     }
 
+    /// Allocate a number of legacy IRQs
+    ///
+    /// # Arguments
+    ///
+    /// * `irq_count` - The number of legacy IRQs to allocate
+    pub fn allocate_irq(&mut self, irq_count: u32) -> Result<Vec<u32>, vm_allocator::Error> {
+        let mut irqs = Vec::with_capacity(irq_count as usize);
+
+        for _ in 0..irq_count {
+            match self.irq_allocator.allocate_id() {
+                Ok(irq) => irqs.push(irq),
+                Err(err) => {
+                    // It is ok to unwrap here, we just allocated the GSI
+                    irqs.into_iter().for_each(|irq| {
+                        self.irq_allocator.free_id(irq).unwrap();
+                    });
+                    return Err(err);
+                }
+            }
+        }
+
+        Ok(irqs)
+    }
+
     /// Allocate a number of GSIs
     ///
     /// # Arguments
@@ -167,10 +194,47 @@ mod tests {
     use vm_allocator::AllocPolicy;
 
     use super::ResourceAllocator;
-    use crate::arch::{self, GSI_BASE};
+    use crate::arch::{self, GSI_BASE, IRQ_BASE, NR_GSI, NR_IRQ};
     use crate::snapshot::{Persist, Snapshot};
 
-    const MAX_IRQS: u32 = arch::GSI_MAX - arch::GSI_BASE + 1;
+    #[test]
+    fn test_allocate_irq() {
+        let mut allocator = ResourceAllocator::new();
+        // asking for 0 IRQs should return us an empty vector
+        assert_eq!(allocator.allocate_irq(0), Ok(vec![]));
+        // We cannot allocate more GSIs than available
+        assert_eq!(
+            allocator.allocate_irq(NR_IRQ + 1),
+            Err(vm_allocator::Error::ResourceNotAvailable)
+        );
+        // But allocating all of them at once should work
+        assert_eq!(
+            allocator.allocate_irq(NR_IRQ),
+            Ok((arch::IRQ_BASE..=arch::IRQ_MAX).collect::<Vec<_>>())
+        );
+        // And now we ran out of GSIs
+        assert_eq!(
+            allocator.allocate_irq(1),
+            Err(vm_allocator::Error::ResourceNotAvailable)
+        );
+        // But we should be able to ask for 0 GSIs
+        assert_eq!(allocator.allocate_irq(0), Ok(vec![]));
+
+        let mut allocator = ResourceAllocator::new();
+        // We should be able to allocate 1 GSI
+        assert_eq!(allocator.allocate_irq(1), Ok(vec![arch::IRQ_BASE]));
+        // We can't allocate MAX_IRQS any more
+        assert_eq!(
+            allocator.allocate_irq(NR_IRQ),
+            Err(vm_allocator::Error::ResourceNotAvailable)
+        );
+        // We can allocate another one and it should be the second available
+        assert_eq!(allocator.allocate_irq(1), Ok(vec![arch::IRQ_BASE + 1]));
+        // Let's allocate the rest in a loop
+        for i in arch::IRQ_BASE + 2..=arch::IRQ_MAX {
+            assert_eq!(allocator.allocate_irq(1), Ok(vec![i]));
+        }
+    }
 
     #[test]
     fn test_allocate_gsi() {
@@ -179,12 +243,12 @@ mod tests {
         assert_eq!(allocator.allocate_gsi(0), Ok(vec![]));
         // We cannot allocate more GSIs than available
         assert_eq!(
-            allocator.allocate_gsi(MAX_IRQS + 1),
+            allocator.allocate_gsi(NR_GSI),
             Err(vm_allocator::Error::ResourceNotAvailable)
         );
         // But allocating all of them at once should work
         assert_eq!(
-            allocator.allocate_gsi(MAX_IRQS),
+            allocator.allocate_gsi(NR_GSI),
             Ok((arch::GSI_BASE..=arch::GSI_MAX).collect::<Vec<_>>())
         );
         // And now we ran out of GSIs
@@ -200,7 +264,7 @@ mod tests {
         assert_eq!(allocator.allocate_gsi(1), Ok(vec![arch::GSI_BASE]));
         // We can't allocate MAX_IRQS any more
         assert_eq!(
-            allocator.allocate_gsi(MAX_IRQS),
+            allocator.allocate_gsi(NR_GSI),
             Err(vm_allocator::Error::ResourceNotAvailable)
         );
         // We can allocate another one and it should be the second available
@@ -221,10 +285,14 @@ mod tests {
     #[test]
     fn test_save_restore() {
         let mut allocator0 = ResourceAllocator::new();
+        let irq_0 = allocator0.allocate_irq(1).unwrap()[0];
+        assert_eq!(irq_0, IRQ_BASE);
         let gsi_0 = allocator0.allocate_gsi(1).unwrap()[0];
         assert_eq!(gsi_0, GSI_BASE);
 
         let mut allocator1 = clone_allocator(&allocator0);
+        let irq_1 = allocator1.allocate_irq(1).unwrap()[0];
+        assert_eq!(irq_1, IRQ_BASE + 1);
         let gsi_1 = allocator1.allocate_gsi(1).unwrap()[0];
         assert_eq!(gsi_1, GSI_BASE + 1);
         let mmio32_mem = allocator1
@@ -251,6 +319,8 @@ mod tests {
             .allocate_system_memory(0x42, 1, AllocPolicy::ExactMatch(system_mem))
             .unwrap_err();
 
+        let irq_2 = allocator2.allocate_irq(1).unwrap()[0];
+        assert_eq!(irq_2, IRQ_BASE + 2);
         let gsi_2 = allocator2.allocate_gsi(1).unwrap()[0];
         assert_eq!(gsi_2, GSI_BASE + 2);
         let mmio32_mem = allocator1
diff --git a/tests/integration_tests/functional/test_max_devices.py b/tests/integration_tests/functional/test_max_devices.py
