arm: support MSI-X on ARM

Add support for ITS device which provides support for MSI interrupts on
ARM architecture. This is currently supported only on systems with GICv3
interrupt controller.

In order to make saving/restore of ITS state work properly, we need to
change the order in which we restore redistributor register GICR_CTLR.
We need to make sure that this register is restored last. Otherwise,
restoring GICR_PROPBASER doesn't have any effect and ITS depends on it
in order to save/restore ITS tables to/from guest memory.

Signed-off-by: Babis Chalios <[email protected]>

Author: bchalios

src/vmm/src/arch/aarch64/fdt.rs

@@ -28,6 +28,8 @@ use crate::vstate::memory::{Address, GuestMemory, GuestMemoryMmap};
 const GIC_PHANDLE: u32 = 1;
 // This is a value for uniquely identifying the FDT node containing the clock definition.
 const CLOCK_PHANDLE: u32 = 2;
+// This is a value for uniquely identifying the FDT node declaring the MSI controller.
+const MSI_PHANDLE: u32 = 3;
 // You may be wondering why this big value?
 // This phandle is used to uniquely identify the FDT nodes containing cache information. Each cpu
 // can have a variable number of caches, some of these caches may be shared with other cpus.
@@ -302,6 +304,16 @@ fn create_gic_node(fdt: &mut FdtWriter, gic_device: &GICDevice) -> Result<(), Fd
     ];
 
     fdt.property_array_u32("interrupts", &gic_intr)?;
+
+    if let Some(msi_properties) = gic_device.msi_properties() {
+        let msic_node = fdt.begin_node("msic")?;
+        fdt.property_string("compatible", "arm,gic-v3-its")?;
+        fdt.property_null("msi-controller")?;
+        fdt.property_u32("phandle", MSI_PHANDLE)?;
+        fdt.property_array_u64("reg", msi_properties)?;
+        fdt.end_node(msic_node)?;
+    }
+
     fdt.end_node(interrupt)?;
 
     Ok(())
@@ -471,6 +483,21 @@ fn create_pci_nodes(fdt: &mut FdtWriter, pci_devices: &PciDevices) -> Result<(),
         (MEM_64BIT_DEVICES_SIZE >> 32) as u32, // Range size
         ((MEM_64BIT_DEVICES_SIZE & 0xffff_ffff) >> 32) as u32,
     ];
+
+    // See kernel document Documentation/devicetree/bindings/pci/pci-msi.txt
+    let msi_map = [
+        // rid-base: A single cell describing the first RID matched by the entry.
+        0x0,
+        // msi-controller: A single phandle to an MSI controller.
+        MSI_PHANDLE,
+        // msi-base: An msi-specifier describing the msi-specifier produced for the
+        // first RID matched by the entry.
+        segment.id as u32,
+        // length: A single cell describing how many consecutive RIDs are matched
+        // following the rid-base.
+        0x100,
+    ];
+
     let pci_node = fdt.begin_node(&pci_node_name)?;
 
     fdt.property_string("compatible", "pci-host-ecam-generic")?;
@@ -491,6 +518,9 @@ fn create_pci_nodes(fdt: &mut FdtWriter, pci_devices: &PciDevices) -> Result<(),
     fdt.property_null("interrupt-map")?;
     fdt.property_null("interrupt-map-mask")?;
     fdt.property_null("dma-coherent")?;
+    fdt.property_array_u32("msi-map", &msi_map)?;
+    fdt.property_u32("msi-parent", MSI_PHANDLE)?;
+
     Ok(fdt.end_node(pci_node)?)
 }
 

src/vmm/src/arch/aarch64/gic/gicv2/mod.rs

@@ -68,7 +68,9 @@ impl GICv2 {
                 GICv2::get_cpu_addr(),
                 GICv2::get_cpu_size(),
             ],
+            msi_properties: None,
             vcpu_count,
+            its_device: None,
         })
     }
 
@@ -82,7 +84,7 @@ impl GICv2 {
 
     pub fn init_device_attributes(gic_device: &Self) -> Result<(), GicError> {
         // Setting up the distributor attribute.
-        // We are placing the GIC below 1GB so we need to substract the size of the distributor.
+        // We are placing the GIC below 1GB so we need to subtract the size of the distributor.
         Self::set_device_attribute(
             gic_device.device_fd(),
             kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR,

src/vmm/src/arch/aarch64/gic/gicv2/regs/mod.rs

@@ -22,6 +22,7 @@ pub fn save_state(fd: &DeviceFd, mpidrs: &[u64]) -> Result<GicState, GicError> {
     Ok(GicState {
         dist: dist_regs::get_dist_regs(fd)?,
         gic_vcpu_states: vcpu_states,
+        ..Default::default()
     })
 }
 

src/vmm/src/arch/aarch64/gic/gicv3/mod.rs

@@ -1,7 +1,7 @@
 // Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-mod regs;
+pub mod regs;
 
 use kvm_ioctls::{DeviceFd, VmFd};
 
@@ -18,12 +18,19 @@ impl std::ops::Deref for GICv3 {
     }
 }
 
+impl std::ops::DerefMut for GICv3 {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+
 impl GICv3 {
     // Unfortunately bindgen omits defines that are based on other defines.
     // See arch/arm64/include/uapi/asm/kvm.h file from the linux kernel.
     const SZ_64K: u64 = 0x0001_0000;
     const KVM_VGIC_V3_DIST_SIZE: u64 = GICv3::SZ_64K;
     const KVM_VGIC_V3_REDIST_SIZE: u64 = (2 * GICv3::SZ_64K);
+    const GIC_V3_ITS_SIZE: u64 = 0x2_0000;
 
     // Device trees specific constants
     const ARCH_GIC_V3_MAINT_IRQ: u32 = 9;
@@ -48,6 +55,16 @@ impl GICv3 {
         vcpu_count * GICv3::KVM_VGIC_V3_REDIST_SIZE
     }
 
+    /// Get the MSI address
+    fn get_msi_address(vcpu_count: u64) -> u64 {
+        Self::get_redists_addr(vcpu_count) - GICv3::GIC_V3_ITS_SIZE
+    }
+
+    /// Get the MSI size
+    const fn get_msi_size() -> u64 {
+        GICv3::GIC_V3_ITS_SIZE
+    }
+
     pub const VERSION: u32 = kvm_bindings::kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_V3;
 
     pub fn fdt_compatibility(&self) -> &str {
@@ -59,30 +76,43 @@ impl GICv3 {
     }
 
     /// Create the GIC device object
-    pub fn create_device(fd: DeviceFd, vcpu_count: u64) -> Self {
-        GICv3(super::GIC {
-            fd,
+    pub fn create_device(vm: &VmFd, vcpu_count: u64) -> Result<Self, GicError> {
+        // Create the GIC device
+        let mut gic_device = kvm_bindings::kvm_create_device {
+            type_: Self::VERSION,
+            fd: 0,
+            flags: 0,
+        };
+
+        let gic_fd = vm
+            .create_device(&mut gic_device)
+            .map_err(GicError::CreateGIC)?;
+
+        Ok(GICv3(super::GIC {
+            fd: gic_fd,
             properties: [
                 GICv3::get_dist_addr(),
                 GICv3::get_dist_size(),
                 GICv3::get_redists_addr(vcpu_count),
                 GICv3::get_redists_size(vcpu_count),
             ],
+            msi_properties: Some([GICv3::get_msi_address(vcpu_count), GICv3::get_msi_size()]),
             vcpu_count,
-        })
+            its_device: None,
+        }))
     }
 
     pub fn save_device(&self, mpidrs: &[u64]) -> Result<GicState, GicError> {
-        regs::save_state(&self.fd, mpidrs)
+        regs::save_state(&self.fd, self.its_device.as_ref().unwrap(), mpidrs)
     }
 
     pub fn restore_device(&self, mpidrs: &[u64], state: &GicState) -> Result<(), GicError> {
-        regs::restore_state(&self.fd, mpidrs, state)
+        regs::restore_state(&self.fd, self.its_device.as_ref().unwrap(), mpidrs, state)
     }
 
     pub fn init_device_attributes(gic_device: &Self) -> Result<(), GicError> {
         // Setting up the distributor attribute.
-        // We are placing the GIC below 1GB so we need to substract the size of the distributor.
+        // We are placing the GIC below 1GB so we need to subtract the size of the distributor.
         Self::set_device_attribute(
             gic_device.device_fd(),
             kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR,
@@ -104,25 +134,45 @@ impl GICv3 {
         Ok(())
     }
 
-    /// Initialize a GIC device
-    pub fn init_device(vm: &VmFd) -> Result<DeviceFd, GicError> {
-        let mut gic_device = kvm_bindings::kvm_create_device {
-            type_: Self::VERSION,
+    fn init_its(vm: &VmFd, gic_device: &mut Self) -> Result<(), GicError> {
+        // ITS part attributes
+        let mut its_device = kvm_bindings::kvm_create_device {
+            type_: kvm_bindings::kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_ITS,
             fd: 0,
             flags: 0,
         };
 
-        vm.create_device(&mut gic_device)
-            .map_err(GicError::CreateGIC)
+        let its_fd = vm
+            .create_device(&mut its_device)
+            .map_err(GicError::CreateGIC)?;
+
+        // Setting up the ITS attributes
+        Self::set_device_attribute(
+            &its_fd,
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR,
+            u64::from(kvm_bindings::KVM_VGIC_ITS_ADDR_TYPE),
+            &Self::get_msi_address(gic_device.vcpu_count()) as *const u64 as u64,
+            0,
+        )?;
+
+        Self::set_device_attribute(
+            &its_fd,
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_CTRL,
+            u64::from(kvm_bindings::KVM_DEV_ARM_VGIC_CTRL_INIT),
+            0,
+            0,
+        )?;
+
+        gic_device.its_device = Some(its_fd);
+        Ok(())
     }
 
     /// Method to initialize the GIC device
     pub fn create(vm: &VmFd, vcpu_count: u64) -> Result<Self, GicError> {
-        let vgic_fd = Self::init_device(vm)?;
-
-        let device = Self::create_device(vgic_fd, vcpu_count);
+        let mut device = Self::create_device(vm, vcpu_count)?;
 
         Self::init_device_attributes(&device)?;
+        Self::init_its(vm, &mut device)?;
 
         Self::finalize_device(&device)?;
 
@@ -184,14 +234,14 @@ impl GICv3 {
 /// RDIST pending tables into guest RAM.
 ///
 /// The tables get flushed to guest RAM whenever the VM gets stopped.
-fn save_pending_tables(fd: &DeviceFd) -> Result<(), GicError> {
+fn save_pending_tables(gic_device: &DeviceFd) -> Result<(), GicError> {
     let init_gic_attr = kvm_bindings::kvm_device_attr {
         group: kvm_bindings::KVM_DEV_ARM_VGIC_GRP_CTRL,
         attr: u64::from(kvm_bindings::KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES),
         addr: 0,
         flags: 0,
     };
-    fd.set_device_attr(&init_gic_attr).map_err(|err| {
+    gic_device.set_device_attr(&init_gic_attr).map_err(|err| {
         GicError::DeviceAttribute(err, true, kvm_bindings::KVM_DEV_ARM_VGIC_GRP_CTRL)
     })
 }

src/vmm/src/arch/aarch64/gic/gicv3/regs/its_regs.rs

@@ -0,0 +1,135 @@
+// Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use kvm_bindings::{
+    KVM_DEV_ARM_ITS_RESTORE_TABLES, KVM_DEV_ARM_ITS_SAVE_TABLES, KVM_DEV_ARM_VGIC_GRP_CTRL,
+    KVM_DEV_ARM_VGIC_GRP_ITS_REGS,
+};
+use kvm_ioctls::DeviceFd;
+use serde::{Deserialize, Serialize};
+
+use crate::arch::aarch64::gic::GicError;
+
+// ITS registers that we want to preserve across snapshots
+const GITS_CTLR: u32 = 0x0000;
+const GITS_IIDR: u32 = 0x0004;
+const GITS_CBASER: u32 = 0x0080;
+const GITS_CWRITER: u32 = 0x0088;
+const GITS_CREADR: u32 = 0x0090;
+const GITS_BASER: u32 = 0x0100;
+
+fn set_device_attribute(
+    its_device: &DeviceFd,
+    group: u32,
+    attr: u32,
+    val: u64,
+) -> Result<(), GicError> {
+    let gicv3_its_attr = kvm_bindings::kvm_device_attr {
+        group,
+        attr: attr as u64,
+        addr: &val as *const u64 as u64,
+        flags: 0,
+    };
+
+    its_device
+        .set_device_attr(&gicv3_its_attr)
+        .map_err(|err| GicError::DeviceAttribute(err, true, group))
+}
+
+fn get_device_attribute(its_device: &DeviceFd, group: u32, attr: u32) -> Result<u64, GicError> {
+    let mut val = 0;
+
+    let mut gicv3_its_attr = kvm_bindings::kvm_device_attr {
+        group,
+        attr: attr as u64,
+        addr: &mut val as *mut u64 as u64,
+        flags: 0,
+    };
+
+    // SAFETY: gicv3_its_attr.addr is safe to write to.
+    unsafe { its_device.get_device_attr(&mut gicv3_its_attr) }
+        .map_err(|err| GicError::DeviceAttribute(err, false, group))?;
+
+    Ok(val)
+}
+
+fn its_read_register(its_fd: &DeviceFd, attr: u32) -> Result<u64, GicError> {
+    get_device_attribute(its_fd, KVM_DEV_ARM_VGIC_GRP_ITS_REGS, attr)
+}
+
+fn its_set_register(its_fd: &DeviceFd, attr: u32, val: u64) -> Result<(), GicError> {
+    set_device_attribute(its_fd, KVM_DEV_ARM_VGIC_GRP_ITS_REGS, attr, val)
+}
+
+pub fn its_save_tables(its_fd: &DeviceFd) -> Result<(), GicError> {
+    set_device_attribute(
+        its_fd,
+        KVM_DEV_ARM_VGIC_GRP_CTRL,
+        KVM_DEV_ARM_ITS_SAVE_TABLES,
+        0,
+    )
+}
+
+pub fn its_restore_tables(its_fd: &DeviceFd) -> Result<(), GicError> {
+    set_device_attribute(
+        its_fd,
+        KVM_DEV_ARM_VGIC_GRP_CTRL,
+        KVM_DEV_ARM_ITS_RESTORE_TABLES,
+        0,
+    )
+}
+
+/// ITS registers that we save/restore during snapshot
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct ItsRegisterState {
+    iidr: u64,
+    cbaser: u64,
+    creadr: u64,
+    cwriter: u64,
+    baser: [u64; 8],
+    ctlr: u64,
+}
+
+impl ItsRegisterState {
+    /// Save ITS state
+    pub fn save(its_fd: &DeviceFd) -> Result<Self, GicError> {
+        let mut state = ItsRegisterState::default();
+
+        for i in 0..8 {
+            state.baser[i as usize] = its_read_register(its_fd, GITS_BASER + i * 8)?;
+        }
+        state.ctlr = its_read_register(its_fd, GITS_CTLR)?;
+        state.cbaser = its_read_register(its_fd, GITS_CBASER)?;
+        state.creadr = its_read_register(its_fd, GITS_CREADR)?;
+        state.cwriter = its_read_register(its_fd, GITS_CWRITER)?;
+        state.iidr = its_read_register(its_fd, GITS_IIDR)?;
+
+        Ok(state)
+    }
+
+    /// Restore ITS state
+    ///
+    /// We need to restore ITS registers in a very specific order for things to work. Take a look
+    /// at:
+    /// https://elixir.bootlin.com/linux/v6.1.141/source/Documentation/virt/kvm/devices/arm-vgic-its.rst#L60
+    /// and
+    /// https://elixir.bootlin.com/linux/v6.1.141/source/Documentation/virt/kvm/devices/arm-vgic-its.rst#L123
+    ///
+    /// for more details, but TL;DR is:
+    ///
+    /// We need to restore GITS_CBASER, GITS_CREADER, GITS_CWRITER, GITS_BASER and GITS_IIDR
+    /// registers before restoring ITS tables from guest memory. We also need to set GITS_CTLR
+    /// last.
+    pub fn restore(&self, its_fd: &DeviceFd) -> Result<(), GicError> {
+        its_set_register(its_fd, GITS_IIDR, self.iidr)?;
+        its_set_register(its_fd, GITS_CBASER, self.cbaser)?;
+        its_set_register(its_fd, GITS_CREADR, self.creadr)?;
+        its_set_register(its_fd, GITS_CWRITER, self.cwriter)?;
+        for i in 0..8 {
+            its_set_register(its_fd, GITS_BASER + i * 8, self.baser[i as usize])?;
+        }
+        // We need to restore saved ITS tables before restoring GITS_CTLR
+        its_restore_tables(its_fd)?;
+        its_set_register(its_fd, GITS_CTLR, self.ctlr)
+    }
+}