arm: support MSI-X on ARM

Expose MSI properties for GIC via the FDT.

Signed-off-by: Babis Chalios <[email protected]>

Author: bchalios

src/vmm/src/arch/aarch64/fdt.rs

@@ -22,12 +22,15 @@ use crate::device_manager::mmio::MMIODeviceInfo;
 use crate::device_manager::pci_mngr::PciDevices;
 use crate::devices::acpi::vmgenid::{VMGENID_MEM_SIZE, VmGenId};
 use crate::initrd::InitrdConfig;
+use crate::logger::info;
 use crate::vstate::memory::{Address, GuestMemory, GuestMemoryMmap};
 
 // This is a value for uniquely identifying the FDT node declaring the interrupt controller.
 const GIC_PHANDLE: u32 = 1;
 // This is a value for uniquely identifying the FDT node containing the clock definition.
 const CLOCK_PHANDLE: u32 = 2;
+// This is a value for uniquely identifying the FDT node declaring the MSI controller.
+const MSI_PHANDLE: u32 = 3;
 // You may be wondering why this big value?
 // This phandle is used to uniquely identify the FDT nodes containing cache information. Each cpu
 // can have a variable number of caches, some of these caches may be shared with other cpus.
@@ -302,6 +305,17 @@ fn create_gic_node(fdt: &mut FdtWriter, gic_device: &GICDevice) -> Result<(), Fd
     ];
 
     fdt.property_array_u32("interrupts", &gic_intr)?;
+
+    if let Some(msi_properties) = gic_device.msi_properties() {
+        info!("msi_properties: {msi_properties:#?}");
+        let msic_node = fdt.begin_node("msic")?;
+        fdt.property_string("compatible", "arm,gic-v3-its")?;
+        fdt.property_null("msi-controller")?;
+        fdt.property_u32("phandle", MSI_PHANDLE)?;
+        fdt.property_array_u64("reg", msi_properties)?;
+        fdt.end_node(msic_node)?;
+    }
+
     fdt.end_node(interrupt)?;
 
     Ok(())
@@ -471,6 +485,21 @@ fn create_pci_nodes(fdt: &mut FdtWriter, pci_devices: &PciDevices) -> Result<(),
         (MEM_64BIT_DEVICES_SIZE >> 32) as u32, // Range size
         ((MEM_64BIT_DEVICES_SIZE & 0xffff_ffff) >> 32) as u32,
     ];
+
+    // See kernel document Documentation/devicetree/bindings/pci/pci-msi.txt
+    let msi_map = [
+        // rid-base: A single cell describing the first RID matched by the entry.
+        0x0,
+        // msi-controller: A single phandle to an MSI controller.
+        MSI_PHANDLE,
+        // msi-base: An msi-specifier describing the msi-specifier produced for the
+        // first RID matched by the entry.
+        segment.id as u32,
+        // length: A single cell describing how many consecutive RIDs are matched
+        // following the rid-base.
+        0x100,
+    ];
+
     let pci_node = fdt.begin_node(&pci_node_name)?;
 
     fdt.property_string("compatible", "pci-host-ecam-generic")?;
@@ -491,6 +520,9 @@ fn create_pci_nodes(fdt: &mut FdtWriter, pci_devices: &PciDevices) -> Result<(),
     fdt.property_null("interrupt-map")?;
     fdt.property_null("interrupt-map-mask")?;
     fdt.property_null("dma-coherent")?;
+    fdt.property_array_u32("msi-map", &msi_map)?;
+    fdt.property_u32("msi-parent", MSI_PHANDLE)?;
+
     Ok(fdt.end_node(pci_node)?)
 }
 

src/vmm/src/arch/aarch64/gic/gicv2/mod.rs

@@ -68,7 +68,9 @@ impl GICv2 {
                 GICv2::get_cpu_addr(),
                 GICv2::get_cpu_size(),
             ],
+            msi_properties: None,
             vcpu_count,
+            its_device: None,
         })
     }
 

src/vmm/src/arch/aarch64/gic/gicv3/mod.rs

@@ -24,6 +24,7 @@ impl GICv3 {
     const SZ_64K: u64 = 0x0001_0000;
     const KVM_VGIC_V3_DIST_SIZE: u64 = GICv3::SZ_64K;
     const KVM_VGIC_V3_REDIST_SIZE: u64 = (2 * GICv3::SZ_64K);
+    const GIC_V3_ITS_SIZE: u64 = 0x2_0000;
 
     // Device trees specific constants
     const ARCH_GIC_V3_MAINT_IRQ: u32 = 9;
@@ -48,6 +49,16 @@ impl GICv3 {
         vcpu_count * GICv3::KVM_VGIC_V3_REDIST_SIZE
     }
 
+    /// Get the MSI address
+    fn get_msi_address(vcpu_count: u64) -> u64 {
+        Self::get_redists_addr(vcpu_count) - GICv3::GIC_V3_ITS_SIZE
+    }
+
+    /// Get the MSI size
+    const fn get_msi_size() -> u64 {
+        GICv3::GIC_V3_ITS_SIZE
+    }
+
     pub const VERSION: u32 = kvm_bindings::kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_V3;
 
     pub fn fdt_compatibility(&self) -> &str {
@@ -59,17 +70,41 @@ impl GICv3 {
     }
 
     /// Create the GIC device object
-    pub fn create_device(fd: DeviceFd, vcpu_count: u64) -> Self {
-        GICv3(super::GIC {
-            fd,
+    pub fn create_device(vm: &VmFd, vcpu_count: u64) -> Result<Self, GicError> {
+        // Create the GIC device
+        let mut gic_device = kvm_bindings::kvm_create_device {
+            type_: Self::VERSION,
+            fd: 0,
+            flags: 0,
+        };
+
+        let gic_fd = vm
+            .create_device(&mut gic_device)
+            .map_err(GicError::CreateGIC)?;
+
+        // ITS part attributes
+        let mut its_device = kvm_bindings::kvm_create_device {
+            type_: kvm_bindings::kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_ITS,
+            fd: 0,
+            flags: 0,
+        };
+
+        let its_fd = vm
+            .create_device(&mut its_device)
+            .map_err(GicError::CreateGIC)?;
+
+        Ok(GICv3(super::GIC {
+            fd: gic_fd,
             properties: [
                 GICv3::get_dist_addr(),
                 GICv3::get_dist_size(),
                 GICv3::get_redists_addr(vcpu_count),
                 GICv3::get_redists_size(vcpu_count),
             ],
+            msi_properties: Some([GICv3::get_msi_address(vcpu_count), GICv3::get_msi_size()]),
             vcpu_count,
-        })
+            its_device: Some(its_fd),
+        }))
     }
 
     pub fn save_device(&self, mpidrs: &[u64]) -> Result<GicState, GicError> {
@@ -82,7 +117,7 @@ impl GICv3 {
 
     pub fn init_device_attributes(gic_device: &Self) -> Result<(), GicError> {
         // Setting up the distributor attribute.
-        // We are placing the GIC below 1GB so we need to substract the size of the distributor.
+        // We are placing the GIC below 1GB so we need to subtract the size of the distributor.
         Self::set_device_attribute(
             gic_device.device_fd(),
             kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR,
@@ -101,26 +136,29 @@ impl GICv3 {
             0,
         )?;
 
-        Ok(())
-    }
+        // Setting up the ITS attributes
+        Self::set_device_attribute(
+            gic_device.its_device.as_ref().unwrap(),
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR,
+            u64::from(kvm_bindings::KVM_VGIC_ITS_ADDR_TYPE),
+            Self::get_msi_address(gic_device.vcpu_count()) as *const u64 as u64,
+            0,
+        )?;
 
-    /// Initialize a GIC device
-    pub fn init_device(vm: &VmFd) -> Result<DeviceFd, GicError> {
-        let mut gic_device = kvm_bindings::kvm_create_device {
-            type_: Self::VERSION,
-            fd: 0,
-            flags: 0,
-        };
+        Self::set_device_attribute(
+            gic_device.its_device.as_ref().unwrap(),
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_CTRL,
+            u64::from(kvm_bindings::KVM_DEV_ARM_VGIC_CTRL_INIT),
+            0,
+            0,
+        )?;
 
-        vm.create_device(&mut gic_device)
-            .map_err(GicError::CreateGIC)
+        Ok(())
     }
 
     /// Method to initialize the GIC device
     pub fn create(vm: &VmFd, vcpu_count: u64) -> Result<Self, GicError> {
-        let vgic_fd = Self::init_device(vm)?;
-
-        let device = Self::create_device(vgic_fd, vcpu_count);
+        let device = Self::create_device(vm, vcpu_count)?;
 
         Self::init_device_attributes(&device)?;
 

src/vmm/src/arch/aarch64/gic/mod.rs

@@ -21,8 +21,14 @@ pub struct GIC {
     /// GIC device properties, to be used for setting up the fdt entry
     properties: [u64; 4],
 
+    /// MSI properties of the GIC device
+    msi_properties: Option<[u64; 2]>,
+
     /// Number of CPUs handled by the device
     vcpu_count: u64,
+
+    /// ITS device
+    its_device: Option<DeviceFd>,
 }
 impl GIC {
     /// Returns the file descriptor of the GIC device
@@ -88,6 +94,14 @@ impl GICDevice {
         }
     }
 
+    /// Returns an array with MSI properties if GIC supports it
+    pub fn msi_properties(&self) -> Option<&[u64; 2]> {
+        match self {
+            Self::V2(x) => x.msi_properties.as_ref(),
+            Self::V3(x) => x.msi_properties.as_ref(),
+        }
+    }
+
     /// Returns the number of vCPUs this GIC handles
     pub fn vcpu_count(&self) -> u64 {
         match self {