fix: Correct GSI numbering for aarch64 to resolve device limit issues

On aarch64, Linux expects device tree interrupts (GSIs) to start at 0,
as it internally adds an offset of 32 for SPIs. Firecracker previously
defined IRQ_BASE as 32, unintentionally causing a double offset and
limiting the maximum number of attachable devices to 64.

This commit introduces new constants, GSI_BASE and GSI_MAX, properly
adjusted for aarch64, with GSIs starting at 1 (since GSI 0 is disallowed
by device_manager/mmio.rs). These changes ensure interrupts are numbered
correctly, resolving the issue and allowing an aarch64 VM to handle
up to 95 devices.

Additional adjustments are made to tests to reflect this correction.

Signed-off-by: Sheng-Wei (Way) Chen <[email protected]>

Author: Swchexit

src/vmm/src/arch/aarch64/layout.rs

@@ -80,5 +80,15 @@ pub const IRQ_MAX: u32 = 128;
 /// First usable interrupt on aarch64.
 pub const IRQ_BASE: u32 = 32;
 
+// The Linux kernel automatically shifts the GSI by 32 if it is an SPI,
+// allowing us to start numbering from 0 instead of 32.
+// But device_manager/mmio.rs states that 0 is not allowed for GSIs
+// So we start with 1 instead.
+/// The first usable GSI on aarch64.
+pub const GSI_BASE: u32 = 1;
+
+/// The maximum usable GSI on aarch64.
+pub const GSI_MAX: u32 = IRQ_MAX - IRQ_BASE - 1;
+
 /// Below this address will reside the GIC, above this address will reside the MMIO devices.
 pub const MAPPED_IO_START: u64 = 1 << 30; // 1 GB

src/vmm/src/arch/mod.rs

@@ -22,8 +22,8 @@ pub use aarch64::vm::{ArchVm, ArchVmError, VmState};
 pub use aarch64::{
     ConfigurationError, MMIO_MEM_SIZE, MMIO_MEM_START, arch_memory_regions,
     configure_system_for_boot, get_kernel_start, initrd_load_addr, layout::CMDLINE_MAX_SIZE,
-    layout::IRQ_BASE, layout::IRQ_MAX, layout::SYSTEM_MEM_SIZE, layout::SYSTEM_MEM_START,
-    load_kernel,
+    layout::GSI_BASE, layout::GSI_MAX, layout::IRQ_BASE, layout::IRQ_MAX, layout::SYSTEM_MEM_SIZE,
+    layout::SYSTEM_MEM_START, load_kernel,
 };
 
 /// Module for x86_64 related functionality.
@@ -41,8 +41,9 @@ pub use x86_64::vm::{ArchVm, ArchVmError, VmState};
 pub use crate::arch::x86_64::{
     ConfigurationError, MMIO_MEM_SIZE, MMIO_MEM_START, arch_memory_regions,
     configure_system_for_boot, get_kernel_start, initrd_load_addr, layout::APIC_ADDR,
-    layout::CMDLINE_MAX_SIZE, layout::IOAPIC_ADDR, layout::IRQ_BASE, layout::IRQ_MAX,
-    layout::SYSTEM_MEM_SIZE, layout::SYSTEM_MEM_START, load_kernel,
+    layout::CMDLINE_MAX_SIZE, layout::GSI_BASE, layout::GSI_MAX, layout::IOAPIC_ADDR,
+    layout::IRQ_BASE, layout::IRQ_MAX, layout::SYSTEM_MEM_SIZE, layout::SYSTEM_MEM_START,
+    load_kernel,
 };
 
 /// Types of devices that can get attached to this platform.

src/vmm/src/arch/x86_64/layout.rs

@@ -24,6 +24,12 @@ pub const IRQ_BASE: u32 = 5;
 /// Last usable IRQ ID for virtio device interrupts on x86_64.
 pub const IRQ_MAX: u32 = 23;
 
+/// The first usable GSI on x86_64 is the same as the first usable IRQ ID.
+pub const GSI_BASE: u32 = IRQ_BASE;
+
+/// The maximum usable GSI on x86_64 is the same as the last usable IRQ ID.
+pub const GSI_MAX: u32 = IRQ_MAX;
+
 /// Address for the TSS setup.
 pub const KVM_TSS_ADDRESS: u64 = 0xfffb_d000;
 

src/vmm/src/device_manager/mmio.rs

@@ -834,7 +834,7 @@ mod tests {
         let device_info = device_manager
             .allocate_mmio_resources(&mut resource_allocator, 1)
             .unwrap();
-        assert_eq!(device_info.irq.unwrap().get(), crate::arch::IRQ_BASE);
+        assert_eq!(device_info.irq.unwrap().get(), crate::arch::GSI_BASE);
     }
 
     #[test]

src/vmm/src/device_manager/resources.rs

@@ -27,7 +27,7 @@ impl ResourceAllocator {
     /// Create a new resource allocator for Firecracker devices
     pub fn new() -> Result<Self, vm_allocator::Error> {
         Ok(Self {
-            gsi_allocator: IdAllocator::new(arch::IRQ_BASE, arch::IRQ_MAX)?,
+            gsi_allocator: IdAllocator::new(arch::GSI_BASE, arch::GSI_MAX)?,
             mmio_memory: AddressAllocator::new(arch::MMIO_MEM_START, arch::MMIO_MEM_SIZE)?,
             system_memory: AddressAllocator::new(arch::SYSTEM_MEM_START, arch::SYSTEM_MEM_SIZE)?,
         })
@@ -102,7 +102,7 @@ mod tests {
     use super::ResourceAllocator;
     use crate::arch;
 
-    const MAX_IRQS: u32 = arch::IRQ_MAX - arch::IRQ_BASE + 1;
+    const MAX_IRQS: u32 = arch::GSI_MAX - arch::GSI_BASE + 1;
 
     #[test]
     fn test_allocate_gsi() {
@@ -117,7 +117,7 @@ mod tests {
         // But allocating all of them at once should work
         assert_eq!(
             allocator.allocate_gsi(MAX_IRQS),
-            Ok((arch::IRQ_BASE..=arch::IRQ_MAX).collect::<Vec<_>>())
+            Ok((arch::GSI_BASE..=arch::GSI_MAX).collect::<Vec<_>>())
         );
         // And now we ran out of GSIs
         assert_eq!(
@@ -129,16 +129,16 @@ mod tests {
 
         let mut allocator = ResourceAllocator::new().unwrap();
         // We should be able to allocate 1 GSI
-        assert_eq!(allocator.allocate_gsi(1), Ok(vec![arch::IRQ_BASE]));
+        assert_eq!(allocator.allocate_gsi(1), Ok(vec![arch::GSI_BASE]));
         // We can't allocate MAX_IRQS any more
         assert_eq!(
             allocator.allocate_gsi(MAX_IRQS),
             Err(vm_allocator::Error::ResourceNotAvailable)
         );
         // We can allocate another one and it should be the second available
-        assert_eq!(allocator.allocate_gsi(1), Ok(vec![arch::IRQ_BASE + 1]));
+        assert_eq!(allocator.allocate_gsi(1), Ok(vec![arch::GSI_BASE + 1]));
         // Let's allocate the rest in a loop
-        for i in arch::IRQ_BASE + 2..=arch::IRQ_MAX {
+        for i in arch::GSI_BASE + 2..=arch::GSI_MAX {
             assert_eq!(allocator.allocate_gsi(1), Ok(vec![i]));
         }
     }