vmgenid: add support for ARM systems

Expose the VMGenID device via Device Tree bindings. We add a node in the
Device Tree that passes to the guest the address of the generation ID
and the interrupt we are using to notify it about generation changes,
aka snapshot events.

Signed-off-by: Sudan Landge <[email protected]>
Signed-off-by: Babis Chalios <[email protected]>

Author: Sudan Landge

src/vmm/src/arch/aarch64/fdt.rs

@@ -16,6 +16,7 @@ use super::super::{DeviceType, InitrdConfig};
 use super::cache_info::{read_cache_config, CacheEntry};
 use super::get_fdt_addr;
 use super::gic::GICDevice;
+use crate::devices::acpi::vmgenid::{VmGenId, VMGENID_MEM_SIZE};
 use crate::vstate::memory::{Address, Bytes, GuestAddress, GuestMemory, GuestMemoryMmap};
 
 // This is a value for uniquely identifying the FDT node declaring the interrupt controller.
@@ -25,7 +26,7 @@ const CLOCK_PHANDLE: u32 = 2;
 // You may be wondering why this big value?
 // This phandle is used to uniquely identify the FDT nodes containing cache information. Each cpu
 // can have a variable number of caches, some of these caches may be shared with other cpus.
-// So, we start the indexing of the phandles used from a really big number and then substract from
+// So, we start the indexing of the phandles used from a really big number and then subtract from
 // it as we need more and more phandle for each cache representation.
 const LAST_CACHE_PHANDLE: u32 = 4000;
 // Read the documentation specified when appending the root node to the FDT.
@@ -70,6 +71,7 @@ pub fn create_fdt<T: DeviceInfoForFDT + Clone + Debug, S: std::hash::BuildHasher
     cmdline: CString,
     device_info: &HashMap<(DeviceType, String), T, S>,
     gic_device: &GICDevice,
+    vmgenid: &Option<VmGenId>,
     initrd: &Option<InitrdConfig>,
 ) -> Result<Vec<u8>, FdtError> {
     // Allocate stuff necessary for storing the blob.
@@ -97,6 +99,7 @@ pub fn create_fdt<T: DeviceInfoForFDT + Clone + Debug, S: std::hash::BuildHasher
     create_clock_node(&mut fdt_writer)?;
     create_psci_node(&mut fdt_writer)?;
     create_devices_node(&mut fdt_writer, device_info)?;
+    create_vmgenid_node(&mut fdt_writer, vmgenid)?;
 
     // End Header node.
     fdt_writer.end_node(root)?;
@@ -219,12 +222,26 @@ fn create_cpu_nodes(fdt: &mut FdtWriter, vcpu_mpidr: &[u64]) -> Result<(), FdtEr
 }
 
 fn create_memory_node(fdt: &mut FdtWriter, guest_mem: &GuestMemoryMmap) -> Result<(), FdtError> {
-    let mem_size = guest_mem.last_addr().raw_value() - super::layout::DRAM_MEM_START + 1;
     // See https://github.com/torvalds/linux/blob/master/Documentation/devicetree/booting-without-of.txt#L960
     // for an explanation of this.
-    let mem_reg_prop = &[super::layout::DRAM_MEM_START, mem_size];
 
-    let mem = fdt.begin_node("memory")?;
+    // On ARM we reserve some memory so that it can be utilized for devices like VMGenID to send
+    // data to kernel drivers. The range of this memory is:
+    //
+    // [layout::DRAM_MEM_START, layout::DRAM_MEM_START + layout::SYSTEM_MEM_SIZE)
+    //
+    // The reason we do this is that Linux does not allow remapping system memory. However, without
+    // remap, kernel drivers cannot get virtual addresses to read data from device memory. Leaving
+    // this memory region out allows Linux kernel modules to remap and thus read this region.
+    let mem_size = guest_mem.last_addr().raw_value()
+        - super::layout::DRAM_MEM_START
+        - super::layout::SYSTEM_MEM_SIZE
+        + 1;
+    let mem_reg_prop = &[
+        super::layout::DRAM_MEM_START + super::layout::SYSTEM_MEM_SIZE,
+        mem_size,
+    ];
+    let mem = fdt.begin_node("memory@ram")?;
     fdt.property_string("device_type", "memory")?;
     fdt.property_array_u64("reg", mem_reg_prop)?;
     fdt.end_node(mem)?;
@@ -258,6 +275,20 @@ fn create_chosen_node(
     Ok(())
 }
 
+fn create_vmgenid_node(fdt: &mut FdtWriter, vmgenid: &Option<VmGenId>) -> Result<(), FdtError> {
+    if let Some(vmgenid_info) = vmgenid {
+        let vmgenid = fdt.begin_node("vmgenid")?;
+        fdt.property_string("compatible", "microsoft,vmgenid")?;
+        fdt.property_array_u64("reg", &[vmgenid_info.guest_address.0, VMGENID_MEM_SIZE])?;
+        fdt.property_array_u32(
+            "interrupts",
+            &[GIC_FDT_IRQ_TYPE_SPI, vmgenid_info.gsi, IRQ_TYPE_EDGE_RISING],
+        )?;
+        fdt.end_node(vmgenid)?;
+    }
+    Ok(())
+}
+
 fn create_gic_node(fdt: &mut FdtWriter, gic_device: &GICDevice) -> Result<(), FdtError> {
     let interrupt = fdt.begin_node("intc")?;
     fdt.property_string("compatible", gic_device.fdt_compatibility())?;
@@ -428,6 +459,7 @@ mod tests {
     use super::*;
     use crate::arch::aarch64::gic::create_gic;
     use crate::arch::aarch64::layout;
+    use crate::device_manager::resources::ResourceAllocator;
     use crate::utilities::test_utils::arch_mem;
 
     const LEN: u64 = 4096;
@@ -492,6 +524,27 @@ mod tests {
             &dev_info,
             &gic,
             &None,
+            &None,
+        )
+        .unwrap();
+    }
+
+    #[test]
+    fn test_create_fdt_with_vmgenid() {
+        let mem = arch_mem(layout::FDT_MAX_SIZE + 0x1000);
+        let mut resource_allocator = ResourceAllocator::new().unwrap();
+        let vmgenid = VmGenId::new(&mem, &mut resource_allocator).unwrap();
+        let kvm = Kvm::new().unwrap();
+        let vm = kvm.create_vm().unwrap();
+        let gic = create_gic(&vm, 1, None).unwrap();
+        create_fdt(
+            &mem,
+            vec![0],
+            CString::new("console=tty0").unwrap(),
+            &HashMap::<(DeviceType, std::string::String), MMIODeviceInfo>::new(),
+            &gic,
+            &Some(vmgenid),
+            &None,
         )
         .unwrap();
     }
@@ -516,6 +569,7 @@ mod tests {
             &HashMap::<(DeviceType, std::string::String), MMIODeviceInfo>::new(),
             &gic,
             &None,
+            &None,
         )
         .unwrap();
 
@@ -576,6 +630,7 @@ mod tests {
             CString::new("console=tty0").unwrap(),
             &HashMap::<(DeviceType, std::string::String), MMIODeviceInfo>::new(),
             &gic,
+            &None,
             &Some(initrd),
         )
         .unwrap();

src/vmm/src/arch/aarch64/layout.rs

@@ -54,6 +54,14 @@ pub const DRAM_MEM_START: u64 = 0x8000_0000; // 2 GB.
 /// The maximum RAM size.
 pub const DRAM_MEM_MAX_SIZE: usize = 0x00FF_8000_0000; // 1024 - 2 = 1022G.
 
+/// Start of RAM on 64 bit ARM.
+pub const SYSTEM_MEM_START: u64 = DRAM_MEM_START;
+
+/// This is used by ACPI device manager for acpi tables or devices like vmgenid
+/// In reality, 2MBs is an overkill, but immediately after this we write the kernel
+/// image, which needs to be 2MB aligned.
+pub const SYSTEM_MEM_SIZE: u64 = 0x20_0000;
+
 /// Kernel command line maximum size.
 /// As per `arch/arm64/include/uapi/asm/setup.h`.
 pub const CMDLINE_MAX_SIZE: usize = 2048;

src/vmm/src/arch/aarch64/mod.rs

@@ -20,6 +20,7 @@ use std::fmt::Debug;
 pub use self::fdt::DeviceInfoForFDT;
 use self::gic::GICDevice;
 use crate::arch::DeviceType;
+use crate::devices::acpi::vmgenid::VmGenId;
 use crate::vstate::memory::{Address, GuestAddress, GuestMemory, GuestMemoryMmap};
 
 /// Errors thrown while configuring aarch64 system.
@@ -60,6 +61,7 @@ pub fn configure_system<T: DeviceInfoForFDT + Clone + Debug, S: std::hash::Build
     vcpu_mpidr: Vec<u64>,
     device_info: &HashMap<(DeviceType, String), T, S>,
     gic_device: &GICDevice,
+    vmgenid: &Option<VmGenId>,
     initrd: &Option<super::InitrdConfig>,
 ) -> Result<(), ConfigurationError> {
     fdt::create_fdt(
@@ -68,14 +70,15 @@ pub fn configure_system<T: DeviceInfoForFDT + Clone + Debug, S: std::hash::Build
         cmdline_cstring,
         device_info,
         gic_device,
+        vmgenid,
         initrd,
     )?;
     Ok(())
 }
 
 /// Returns the memory address where the kernel could be loaded.
 pub fn get_kernel_start() -> u64 {
-    layout::DRAM_MEM_START
+    layout::SYSTEM_MEM_START + layout::SYSTEM_MEM_SIZE
 }
 
 /// Returns the memory address where the initrd could be loaded.

src/vmm/src/arch/mod.rs

@@ -12,8 +12,8 @@ pub mod aarch64;
 #[cfg(target_arch = "aarch64")]
 pub use aarch64::{
     arch_memory_regions, configure_system, get_kernel_start, initrd_load_addr,
-    layout::CMDLINE_MAX_SIZE, layout::IRQ_BASE, layout::IRQ_MAX, ConfigurationError, MMIO_MEM_SIZE,
-    MMIO_MEM_START,
+    layout::CMDLINE_MAX_SIZE, layout::IRQ_BASE, layout::IRQ_MAX, layout::SYSTEM_MEM_SIZE,
+    layout::SYSTEM_MEM_START, ConfigurationError, MMIO_MEM_SIZE, MMIO_MEM_START,
 };
 
 /// Module for x86_64 related functionality.

src/vmm/src/builder.rs

@@ -36,18 +36,14 @@ use crate::cpu_config::templates::{
     CpuConfiguration, CustomCpuTemplate, GetCpuTemplate, GetCpuTemplateError, GuestConfigError,
     KvmCapability,
 };
-#[cfg(target_arch = "x86_64")]
 use crate::device_manager::acpi::ACPIDeviceManager;
 #[cfg(target_arch = "x86_64")]
 use crate::device_manager::legacy::PortIODeviceManager;
 use crate::device_manager::mmio::MMIODeviceManager;
-use crate::device_manager::persist::MMIODevManagerConstructorArgs;
-#[cfg(target_arch = "x86_64")]
 use crate::device_manager::persist::{
-    ACPIDeviceManagerConstructorArgs, ACPIDeviceManagerRestoreError,
+    ACPIDeviceManagerConstructorArgs, ACPIDeviceManagerRestoreError, MMIODevManagerConstructorArgs,
 };
 use crate::device_manager::resources::ResourceAllocator;
-#[cfg(target_arch = "x86_64")]
 use crate::devices::acpi::vmgenid::{VmGenId, VmGenIdError};
 use crate::devices::legacy::serial::SerialOut;
 #[cfg(target_arch = "aarch64")]
@@ -79,7 +75,6 @@ pub enum StartMicrovmError {
     /// Unable to attach block device to Vmm: {0}
     AttachBlockDevice(io::Error),
     /// Unable to attach the VMGenID device: {0}
-    #[cfg(target_arch = "x86_64")]
     AttachVmgenidDevice(kvm_ioctls::Error),
     /// System configuration error: {0}
     ConfigureSystem(crate::arch::ConfigurationError),
@@ -93,7 +88,6 @@ pub enum StartMicrovmError {
     #[cfg(target_arch = "x86_64")]
     CreateLegacyDevice(device_manager::legacy::LegacyDeviceError),
     /// Error creating VMGenID device: {0}
-    #[cfg(target_arch = "x86_64")]
     CreateVMGenID(VmGenIdError),
     /// Invalid Memory Configuration: {0}
     GuestMemory(crate::vstate::memory::MemoryError),
@@ -175,7 +169,6 @@ fn create_vmm_and_vcpus(
     let mmio_device_manager = MMIODeviceManager::new();
 
     // Instantiate ACPI device manager.
-    #[cfg(target_arch = "x86_64")]
     let acpi_device_manager = ACPIDeviceManager::new();
 
     // For x86_64 we need to create the interrupt controller before calling `KVM_CREATE_VCPUS`
@@ -233,7 +226,6 @@ fn create_vmm_and_vcpus(
         mmio_device_manager,
         #[cfg(target_arch = "x86_64")]
         pio_device_manager,
-        #[cfg(target_arch = "x86_64")]
         acpi_device_manager,
     };
 
@@ -347,7 +339,6 @@ pub fn build_microvm_for_boot(
     #[cfg(target_arch = "aarch64")]
     attach_legacy_devices_aarch64(event_manager, &mut vmm, &mut boot_cmdline).map_err(Internal)?;
 
-    #[cfg(target_arch = "x86_64")]
     attach_vmgenid_device(&mut vmm)?;
 
     configure_system_for_boot(
@@ -449,7 +440,6 @@ pub enum BuildMicrovmFromSnapshotError {
     /// Failed to apply VMM secccomp filter: {0}
     SeccompFiltersInternal(#[from] seccompiler::InstallationError),
     /// Failed to restore ACPI device manager: {0}
-    #[cfg(target_arch = "x86_64")]
     ACPIDeviManager(#[from] ACPIDeviceManagerRestoreError),
     /// VMGenID update failed: {0}
     VMGenIDUpdate(std::io::Error),
@@ -532,7 +522,6 @@ pub fn build_microvm_from_snapshot(
             .map_err(MicrovmStateError::RestoreDevices)?;
     vmm.emulate_serial_init()?;
 
-    #[cfg(target_arch = "x86_64")]
     {
         let acpi_ctor_args = ACPIDeviceManagerConstructorArgs {
             mem: &guest_memory,
@@ -859,6 +848,7 @@ pub fn configure_system_for_boot(
             vcpu_mpidr,
             vmm.mmio_device_manager.get_device_info(),
             vmm.vm.get_irqchip(),
+            &vmm.acpi_device_manager.vmgenid,
             initrd,
         )
         .map_err(ConfigureSystem)?;
@@ -908,7 +898,6 @@ pub(crate) fn attach_boot_timer_device(
     Ok(())
 }
 
-#[cfg(target_arch = "x86_64")]
 fn attach_vmgenid_device(vmm: &mut Vmm) -> Result<(), StartMicrovmError> {
     let vmgenid = VmGenId::new(&vmm.guest_memory, &mut vmm.resource_allocator)
         .map_err(StartMicrovmError::CreateVMGenID)?;
@@ -1117,7 +1106,6 @@ pub mod tests {
         let mut vm = Vm::new(vec![]).unwrap();
         vm.memory_init(&guest_memory, false).unwrap();
         let mmio_device_manager = MMIODeviceManager::new();
-        #[cfg(target_arch = "x86_64")]
         let acpi_device_manager = ACPIDeviceManager::new();
         #[cfg(target_arch = "x86_64")]
         let pio_device_manager = PortIODeviceManager::new(
@@ -1158,7 +1146,6 @@ pub mod tests {
             mmio_device_manager,
             #[cfg(target_arch = "x86_64")]
             pio_device_manager,
-            #[cfg(target_arch = "x86_64")]
             acpi_device_manager,
         }
     }

src/vmm/src/device_manager/mod.rs

@@ -6,7 +6,6 @@
 // found in the THIRD-PARTY file.
 
 /// ACPI device manager.
-#[cfg(target_arch = "x86_64")]
 pub mod acpi;
 /// Legacy Device Manager.
 pub mod legacy;

src/vmm/src/device_manager/persist.rs

@@ -12,13 +12,11 @@ use log::{error, warn};
 use serde::{Deserialize, Serialize};
 use vm_allocator::AllocPolicy;
 
-#[cfg(target_arch = "x86_64")]
 use super::acpi::ACPIDeviceManager;
 use super::mmio::*;
 use super::resources::ResourceAllocator;
 #[cfg(target_arch = "aarch64")]
 use crate::arch::DeviceType;
-#[cfg(target_arch = "x86_64")]
 use crate::devices::acpi::vmgenid::{VMGenIDState, VMGenIdConstructorArgs, VmGenId, VmGenIdError};
 use crate::devices::virtio::balloon::persist::{BalloonConstructorArgs, BalloonState};
 use crate::devices::virtio::balloon::{Balloon, BalloonError};
@@ -230,20 +228,17 @@ impl fmt::Debug for MMIODevManagerConstructorArgs<'_> {
     }
 }
 
-#[cfg(target_arch = "x86_64")]
 #[derive(Default, Debug, Clone, Serialize, Deserialize)]
 pub struct ACPIDeviceManagerState {
     vmgenid: Option<VMGenIDState>,
 }
 
-#[cfg(target_arch = "x86_64")]
 pub struct ACPIDeviceManagerConstructorArgs<'a> {
     pub mem: &'a GuestMemoryMmap,
     pub resource_allocator: &'a mut ResourceAllocator,
     pub vm: &'a VmFd,
 }
 
-#[cfg(target_arch = "x86_64")]
 #[derive(Debug, thiserror::Error, displaydoc::Display)]
 pub enum ACPIDeviceManagerRestoreError {
     /// Could not register device: {0}
@@ -252,7 +247,6 @@ pub enum ACPIDeviceManagerRestoreError {
     VMGenID(#[from] VmGenIdError),
 }
 
-#[cfg(target_arch = "x86_64")]
 impl<'a> Persist<'a> for ACPIDeviceManager {
     type State = ACPIDeviceManagerState;
     type ConstructorArgs = ACPIDeviceManagerConstructorArgs<'a>;

src/vmm/src/device_manager/resources.rs

@@ -20,7 +20,6 @@ pub struct ResourceAllocator {
     // Allocator for memory in the MMIO address space
     mmio_memory: AddressAllocator,
     // Memory allocator for system data
-    #[cfg(target_arch = "x86_64")]
     system_memory: AddressAllocator,
 }
 
@@ -30,7 +29,6 @@ impl ResourceAllocator {
         Ok(Self {
             gsi_allocator: IdAllocator::new(arch::IRQ_BASE, arch::IRQ_MAX)?,
             mmio_memory: AddressAllocator::new(arch::MMIO_MEM_START, arch::MMIO_MEM_SIZE)?,
-            #[cfg(target_arch = "x86_64")]
             system_memory: AddressAllocator::new(arch::SYSTEM_MEM_START, arch::SYSTEM_MEM_SIZE)?,
         })
     }
@@ -86,7 +84,6 @@ impl ResourceAllocator {
     /// * `size` - The size in bytes of the memory to allocate
     /// * `alignment` - The alignment of the address of the first byte
     /// * `policy` - A [`vm_allocator::AllocPolicy`] variant for determining the allocation policy
-    #[cfg(target_arch = "x86_64")]
     pub fn allocate_system_memory(
         &mut self,
         size: u64,

src/vmm/src/devices/mod.rs

@@ -9,7 +9,6 @@
 
 use std::io;
 
-#[cfg(target_arch = "x86_64")]
 pub mod acpi;
 pub mod bus;
 pub mod legacy;