arm: support MSI-X on ARM

Expose MSI properties for GIC via the FDT.

Signed-off-by: Babis Chalios <[email protected]>

Author: bchalios

src/vmm/src/arch/aarch64/fdt.rs

@@ -28,6 +28,8 @@ use crate::vstate::memory::{Address, GuestMemory, GuestMemoryMmap};
 const GIC_PHANDLE: u32 = 1;
 // This is a value for uniquely identifying the FDT node containing the clock definition.
 const CLOCK_PHANDLE: u32 = 2;
+// This is a value for uniquely identifying the FDT node declaring the MSI controller.
+const MSI_PHANDLE: u32 = 3;
 // You may be wondering why this big value?
 // This phandle is used to uniquely identify the FDT nodes containing cache information. Each cpu
 // can have a variable number of caches, some of these caches may be shared with other cpus.
@@ -302,6 +304,16 @@ fn create_gic_node(fdt: &mut FdtWriter, gic_device: &GICDevice) -> Result<(), Fd
     ];
 
     fdt.property_array_u32("interrupts", &gic_intr)?;
+
+    if let Some(msi_properties) = gic_device.msi_properties() {
+        let msic_node = fdt.begin_node("msic")?;
+        fdt.property_string("compatible", "arm,gic-v3-its")?;
+        fdt.property_null("msi-controller")?;
+        fdt.property_u32("phandle", MSI_PHANDLE)?;
+        fdt.property_array_u64("reg", msi_properties)?;
+        fdt.end_node(msic_node)?;
+    }
+
     fdt.end_node(interrupt)?;
 
     Ok(())
@@ -471,6 +483,21 @@ fn create_pci_nodes(fdt: &mut FdtWriter, pci_devices: &PciDevices) -> Result<(),
         (MEM_64BIT_DEVICES_SIZE >> 32) as u32, // Range size
         ((MEM_64BIT_DEVICES_SIZE & 0xffff_ffff) >> 32) as u32,
     ];
+
+    // See kernel document Documentation/devicetree/bindings/pci/pci-msi.txt
+    let msi_map = [
+        // rid-base: A single cell describing the first RID matched by the entry.
+        0x0,
+        // msi-controller: A single phandle to an MSI controller.
+        MSI_PHANDLE,
+        // msi-base: An msi-specifier describing the msi-specifier produced for the
+        // first RID matched by the entry.
+        segment.id as u32,
+        // length: A single cell describing how many consecutive RIDs are matched
+        // following the rid-base.
+        0x100,
+    ];
+
     let pci_node = fdt.begin_node(&pci_node_name)?;
 
     fdt.property_string("compatible", "pci-host-ecam-generic")?;
@@ -491,6 +518,9 @@ fn create_pci_nodes(fdt: &mut FdtWriter, pci_devices: &PciDevices) -> Result<(),
     fdt.property_null("interrupt-map")?;
     fdt.property_null("interrupt-map-mask")?;
     fdt.property_null("dma-coherent")?;
+    fdt.property_array_u32("msi-map", &msi_map)?;
+    fdt.property_u32("msi-parent", MSI_PHANDLE)?;
+
     Ok(fdt.end_node(pci_node)?)
 }
 

src/vmm/src/arch/aarch64/gic/gicv2/mod.rs

@@ -68,6 +68,7 @@ impl GICv2 {
                 GICv2::get_cpu_addr(),
                 GICv2::get_cpu_size(),
             ],
+            msi_properties: None,
             vcpu_count,
         })
     }

src/vmm/src/arch/aarch64/gic/gicv3/mod.rs

@@ -24,6 +24,7 @@ impl GICv3 {
     const SZ_64K: u64 = 0x0001_0000;
     const KVM_VGIC_V3_DIST_SIZE: u64 = GICv3::SZ_64K;
     const KVM_VGIC_V3_REDIST_SIZE: u64 = (2 * GICv3::SZ_64K);
+    const GIC_V3_ITS_SIZE: u64 = 0x2_0000;
 
     // Device trees specific constants
     const ARCH_GIC_V3_MAINT_IRQ: u32 = 9;
@@ -68,6 +69,10 @@ impl GICv3 {
                 GICv3::get_redists_addr(vcpu_count),
                 GICv3::get_redists_size(vcpu_count),
             ],
+            msi_properties: Some([
+                Self::get_redists_addr(vcpu_count - GICv3::GIC_V3_ITS_SIZE),
+                GICv3::GIC_V3_ITS_SIZE,
+            ]),
             vcpu_count,
         })
     }

src/vmm/src/arch/aarch64/gic/mod.rs

@@ -21,6 +21,9 @@ pub struct GIC {
     /// GIC device properties, to be used for setting up the fdt entry
     properties: [u64; 4],
 
+    /// MSI properties of the GIC device
+    msi_properties: Option<[u64; 2]>,
+
     /// Number of CPUs handled by the device
     vcpu_count: u64,
 }
@@ -88,6 +91,14 @@ impl GICDevice {
         }
     }
 
+    /// Returns an array with MSI properties if GIC supports it
+    pub fn msi_properties(&self) -> Option<&[u64; 2]> {
+        match self {
+            Self::V2(x) => x.msi_properties.as_ref(),
+            Self::V3(x) => x.msi_properties.as_ref(),
+        }
+    }
+
     /// Returns the number of vCPUs this GIC handles
     pub fn vcpu_count(&self) -> u64 {
         match self {