arch: define 64-bit capable MMIO memory regions

PCIe distinguishes MMIO regions between 32bit and 64bit, caring for
devices that can't deal with 64-bit addresses. This commit defines the
appropriate regions for both x86 and aarch64 architectures, extends the
resource allocator to handle allocations for both of these regions and
adjusts the logic that calculates the memory regions for the
architecture.

Signed-off-by: Babis Chalios <[email protected]>

Author: bchalios

src/vmm/src/arch/aarch64/layout.rs

@@ -82,3 +82,46 @@ pub const IRQ_BASE: u32 = 32;
 
 /// Below this address will reside the GIC, above this address will reside the MMIO devices.
 pub const MAPPED_IO_START: u64 = 1 << 30; // 1 GB
+
+/// The start of the memory area reserved for MMIO 32-bit accesses.
+pub const MMIO32_MEM_START: u64 = MAPPED_IO_START;
+/// The size of the memory area reserved for MMIO 32-bit accesses (1GiB).
+pub const MMIO32_MEM_SIZE: u64 = DRAM_MEM_START - MMIO32_MEM_START;
+
+// The rest of the MMIO address space (256 MiB) we dedicate to PCIe for memory-mapped access to
+// configuration.
+/// Size of MMIO region for PCIe configuration accesses.
+pub const PCI_MMCONFIG_SIZE: u64 = 256 << 20;
+/// Start of MMIO region for PCIe configuration accesses.
+pub const PCI_MMCONFIG_START: u64 = DRAM_MEM_START - PCI_MMCONFIG_SIZE;
+/// MMIO space per PCIe segment
+pub const PCI_MMIO_CONFIG_SIZE_PER_SEGMENT: u64 = 4096 * 256;
+
+// We reserve 768 MiB for devices at the beginning of the MMIO region. This includes space both for
+// pure MMIO and PCIe devices.
+
+/// Memory region start for boot device.
+pub const BOOT_DEVICE_MEM_START: u64 = MMIO32_MEM_START;
+/// Memory region start for RTC device.
+pub const RTC_MEM_START: u64 = BOOT_DEVICE_MEM_START + 0x1000;
+/// Memory region start for Serial device.
+pub const SERIAL_MEM_START: u64 = RTC_MEM_START + 0x1000;
+
+/// Beginning of memory region for device MMIO 32-bit accesses
+pub const MEM_32BIT_DEVICES_START: u64 = SERIAL_MEM_START + 0x1000;
+/// Size of memory region for device MMIO 32-bit accesses
+pub const MEM_32BIT_DEVICES_SIZE: u64 = PCI_MMCONFIG_START - MEM_32BIT_DEVICES_START;
+
+// 64-bits region for MMIO accesses
+/// The start of the memory area reserved for MMIO 64-bit accesses.
+pub const MMIO64_MEM_START: u64 = 256 << 30;
+/// The size of the memory area reserved for MMIO 64-bit accesses.
+pub const MMIO64_MEM_SIZE: u64 = 256 << 30;
+
+// At the moment, all of this region goes to devices
+/// Beginning of memory region for device MMIO 64-bit accesses
+pub const MEM_64BIT_DEVICES_START: u64 = MMIO64_MEM_START;
+/// Size of memory region for device MMIO 32-bit accesses
+pub const MEM_64BIT_DEVICES_SIZE: u64 = MMIO64_MEM_SIZE;
+/// First address past the 64-bit MMIO gap
+pub const FIRST_ADDR_PAST_64BITS_MMIO: u64 = MMIO64_MEM_START + MMIO64_MEM_SIZE;

src/vmm/src/arch/aarch64/mod.rs

@@ -24,11 +24,11 @@ use linux_loader::loader::pe::PE as Loader;
 use linux_loader::loader::{Cmdline, KernelLoader};
 use vm_memory::GuestMemoryError;
 
-use crate::arch::{BootProtocol, EntryPoint};
+use crate::arch::{BootProtocol, EntryPoint, arch_memory_regions_with_gap};
 use crate::cpu_config::aarch64::{CpuConfiguration, CpuConfigurationError};
 use crate::cpu_config::templates::CustomCpuTemplate;
 use crate::initrd::InitrdConfig;
-use crate::utils::{align_up, usize_to_u64};
+use crate::utils::{align_up, u64_to_usize, usize_to_u64};
 use crate::vmm_config::machine_config::MachineConfig;
 use crate::vstate::memory::{Address, Bytes, GuestAddress, GuestMemory, GuestMemoryMmap};
 use crate::vstate::vcpu::KvmVcpuError;
@@ -51,42 +51,34 @@ pub enum ConfigurationError {
     VcpuConfigure(#[from] KvmVcpuError),
 }
 
-/// The start of the memory area reserved for MMIO devices.
-pub const MMIO_MEM_START: u64 = layout::MAPPED_IO_START;
-/// The size of the memory area reserved for MMIO devices.
-pub const MMIO_MEM_SIZE: u64 = layout::DRAM_MEM_START - layout::MAPPED_IO_START; //>> 1GB
-
 /// Returns a Vec of the valid memory addresses for aarch64.
 /// See [`layout`](layout) module for a drawing of the specific memory model for this platform.
-///
-/// The `offset` parameter specified the offset from [`layout::DRAM_MEM_START`].
-pub fn arch_memory_regions(offset: usize, size: usize) -> Vec<(GuestAddress, usize)> {
+pub fn arch_memory_regions(size: usize) -> Vec<(GuestAddress, usize)> {
     assert!(size > 0, "Attempt to allocate guest memory of length 0");
-    assert!(
-        offset.checked_add(size).is_some(),
-        "Attempt to allocate guest memory such that the address space would wrap around"
-    );
-    assert!(
-        offset < layout::DRAM_MEM_MAX_SIZE,
-        "offset outside allowed DRAM range"
-    );
 
-    let dram_size = min(size, layout::DRAM_MEM_MAX_SIZE - offset);
+    let dram_size = min(size, layout::DRAM_MEM_MAX_SIZE);
 
     if dram_size != size {
         logger::warn!(
-            "Requested offset/memory size {}/{} exceeds architectural maximum (1022GiB). Size has \
-             been truncated to {}",
-            offset,
+            "Requested memory size {} exceeds architectural maximum (1022GiB). Size has been \
+             truncated to {}",
             size,
             dram_size
         );
     }
 
-    vec![(
-        GuestAddress(layout::DRAM_MEM_START + offset as u64),
+    let mut regions = vec![];
+    if let Some((offset, remaining)) = arch_memory_regions_with_gap(
+        &mut regions,
+        u64_to_usize(layout::DRAM_MEM_START),
         dram_size,
-    )]
+        u64_to_usize(layout::MMIO64_MEM_START),
+        u64_to_usize(layout::MMIO64_MEM_SIZE),
+    ) {
+        regions.push((GuestAddress(offset as u64), remaining));
+    }
+
+    regions
 }
 
 /// Configures the system for booting Linux.
@@ -211,73 +203,109 @@ pub fn load_kernel(
 
 #[cfg(kani)]
 mod verification {
-    use vm_memory::GuestAddress;
-
-    use crate::arch::aarch64::layout;
+    use crate::arch::aarch64::layout::{
+        DRAM_MEM_MAX_SIZE, DRAM_MEM_START, FIRST_ADDR_PAST_64BITS_MMIO, MMIO64_MEM_START,
+    };
     use crate::arch::arch_memory_regions;
 
     #[kani::proof]
     #[kani::unwind(3)]
     fn verify_arch_memory_regions() {
-        let offset: u64 = kani::any::<u64>();
-        let len: u64 = kani::any::<u64>();
-
+        let len: usize = kani::any::<usize>();
         kani::assume(len > 0);
-        kani::assume(offset.checked_add(len).is_some());
-        kani::assume(offset < layout::DRAM_MEM_MAX_SIZE as u64);
 
-        let regions = arch_memory_regions(offset as usize, len as usize);
+        let regions = arch_memory_regions(len);
 
-        // No MMIO gap on ARM
-        assert_eq!(regions.len(), 1);
+        for region in &regions {
+            println!(
+                "region: [{:x}:{:x})",
+                region.0.0,
+                region.0.0 + region.1 as u64
+            );
+        }
 
-        let (GuestAddress(start), actual_len) = regions[0];
-        let actual_len = actual_len as u64;
+        // On Arm we have one MMIO gap that might fall within addressable ranges,
+        // so we can get either 1 or 2 regions.
+        assert!(regions.len() >= 1);
+        assert!(regions.len() <= 2);
 
-        assert_eq!(start, layout::DRAM_MEM_START + offset);
-        assert!(actual_len <= layout::DRAM_MEM_MAX_SIZE as u64);
+        // The total length of all regions cannot exceed DRAM_MEM_MAX_SIZE
+        let actual_len = regions.iter().map(|&(_, len)| len).sum::<usize>();
+        assert!(actual_len <= DRAM_MEM_MAX_SIZE);
+        // The total length is smaller or equal to the length we asked
         assert!(actual_len <= len);
+        // If it's smaller, it's because we asked more than the the maximum possible.
+        if (actual_len) < len {
+            assert!(len > DRAM_MEM_MAX_SIZE);
+        }
 
-        if actual_len < len {
-            assert_eq!(
-                start + actual_len,
-                layout::DRAM_MEM_START + layout::DRAM_MEM_MAX_SIZE as u64
-            );
-            assert!(offset + len >= layout::DRAM_MEM_MAX_SIZE as u64);
+        // No region overlaps the 64-bit MMIO gap
+        assert!(
+            regions
+                .iter()
+                .all(|&(start, len)| start.0 >= FIRST_ADDR_PAST_64BITS_MMIO
+                    || start.0 + len as u64 <= MMIO64_MEM_START)
+        );
+
+        // All regions start after our DRAM_MEM_START
+        assert!(regions.iter().all(|&(start, _)| start.0 >= DRAM_MEM_START));
+
+        // All regions have non-zero length
+        assert!(regions.iter().all(|&(_, len)| len > 0));
+
+        // If there's two regions, they perfectly snuggle up the 64bit MMIO gap
+        if regions.len() == 2 {
+            kani::cover!();
+
+            // The very first address should be DRAM_MEM_START
+            assert_eq!(regions[0].0.0, DRAM_MEM_START);
+            // The first region ends at the beginning of the 64 bits gap.
+            assert_eq!(regions[0].0.0 + regions[0].1 as u64, MMIO64_MEM_START);
+            // The second region starts exactly after the 64 bits gap.
+            assert_eq!(regions[1].0.0, FIRST_ADDR_PAST_64BITS_MMIO);
         }
     }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::arch::aarch64::layout::{
+        DRAM_MEM_MAX_SIZE, DRAM_MEM_START, FDT_MAX_SIZE, FIRST_ADDR_PAST_64BITS_MMIO,
+        MMIO64_MEM_START,
+    };
     use crate::test_utils::arch_mem;
 
     #[test]
     fn test_regions_lt_1024gb() {
-        let regions = arch_memory_regions(0, 1usize << 29);
+        let regions = arch_memory_regions(1usize << 29);
         assert_eq!(1, regions.len());
-        assert_eq!(GuestAddress(super::layout::DRAM_MEM_START), regions[0].0);
+        assert_eq!(GuestAddress(DRAM_MEM_START), regions[0].0);
         assert_eq!(1usize << 29, regions[0].1);
     }
 
     #[test]
     fn test_regions_gt_1024gb() {
-        let regions = arch_memory_regions(0, 1usize << 41);
-        assert_eq!(1, regions.len());
-        assert_eq!(GuestAddress(super::layout::DRAM_MEM_START), regions[0].0);
-        assert_eq!(super::layout::DRAM_MEM_MAX_SIZE, regions[0].1);
+        let regions = arch_memory_regions(1usize << 41);
+        assert_eq!(2, regions.len());
+        assert_eq!(GuestAddress(DRAM_MEM_START), regions[0].0);
+        assert_eq!(MMIO64_MEM_START - DRAM_MEM_START, regions[0].1 as u64);
+        assert_eq!(GuestAddress(FIRST_ADDR_PAST_64BITS_MMIO), regions[1].0);
+        assert_eq!(
+            DRAM_MEM_MAX_SIZE as u64 - MMIO64_MEM_START + DRAM_MEM_START,
+            regions[1].1 as u64
+        );
     }
 
     #[test]
     fn test_get_fdt_addr() {
-        let mem = arch_mem(layout::FDT_MAX_SIZE - 0x1000);
-        assert_eq!(get_fdt_addr(&mem), layout::DRAM_MEM_START);
+        let mem = arch_mem(FDT_MAX_SIZE - 0x1000);
+        assert_eq!(get_fdt_addr(&mem), DRAM_MEM_START);
 
-        let mem = arch_mem(layout::FDT_MAX_SIZE);
-        assert_eq!(get_fdt_addr(&mem), layout::DRAM_MEM_START);
+        let mem = arch_mem(FDT_MAX_SIZE);
+        assert_eq!(get_fdt_addr(&mem), DRAM_MEM_START);
 
-        let mem = arch_mem(layout::FDT_MAX_SIZE + 0x1000);
-        assert_eq!(get_fdt_addr(&mem), 0x1000 + layout::DRAM_MEM_START);
+        let mem = arch_mem(FDT_MAX_SIZE + 0x1000);
+        assert_eq!(get_fdt_addr(&mem), 0x1000 + DRAM_MEM_START);
     }
 }

src/vmm/src/arch/mod.rs

@@ -20,10 +20,13 @@ pub use aarch64::vcpu::*;
 pub use aarch64::vm::{ArchVm, ArchVmError, VmState};
 #[cfg(target_arch = "aarch64")]
 pub use aarch64::{
-    ConfigurationError, MMIO_MEM_SIZE, MMIO_MEM_START, arch_memory_regions,
-    configure_system_for_boot, get_kernel_start, initrd_load_addr, layout::CMDLINE_MAX_SIZE,
-    layout::IRQ_BASE, layout::IRQ_MAX, layout::SYSTEM_MEM_SIZE, layout::SYSTEM_MEM_START,
-    load_kernel,
+    ConfigurationError, arch_memory_regions, configure_system_for_boot, get_kernel_start,
+    initrd_load_addr, layout::BOOT_DEVICE_MEM_START, layout::CMDLINE_MAX_SIZE, layout::IRQ_BASE,
+    layout::IRQ_MAX, layout::MEM_32BIT_DEVICES_SIZE, layout::MEM_32BIT_DEVICES_START,
+    layout::MEM_64BIT_DEVICES_SIZE, layout::MEM_64BIT_DEVICES_START, layout::MMIO32_MEM_SIZE,
+    layout::MMIO32_MEM_START, layout::PCI_MMCONFIG_SIZE, layout::PCI_MMCONFIG_START,
+    layout::PCI_MMIO_CONFIG_SIZE_PER_SEGMENT, layout::RTC_MEM_START, layout::SERIAL_MEM_START,
+    layout::SYSTEM_MEM_SIZE, layout::SYSTEM_MEM_START, load_kernel,
 };
 
 /// Module for x86_64 related functionality.
@@ -39,9 +42,11 @@ pub use x86_64::vm::{ArchVm, ArchVmError, VmState};
 
 #[cfg(target_arch = "x86_64")]
 pub use crate::arch::x86_64::{
-    ConfigurationError, MMIO_MEM_SIZE, MMIO_MEM_START, arch_memory_regions,
-    configure_system_for_boot, get_kernel_start, initrd_load_addr, layout::APIC_ADDR,
-    layout::CMDLINE_MAX_SIZE, layout::IOAPIC_ADDR, layout::IRQ_BASE, layout::IRQ_MAX,
+    ConfigurationError, arch_memory_regions, configure_system_for_boot, get_kernel_start,
+    initrd_load_addr, layout::APIC_ADDR, layout::BOOT_DEVICE_MEM_START, layout::CMDLINE_MAX_SIZE,
+    layout::IOAPIC_ADDR, layout::IRQ_BASE, layout::IRQ_MAX, layout::MEM_32BIT_DEVICES_SIZE,
+    layout::MEM_32BIT_DEVICES_START, layout::MEM_64BIT_DEVICES_SIZE,
+    layout::MEM_64BIT_DEVICES_START, layout::MMIO32_MEM_SIZE, layout::MMIO32_MEM_START,
     layout::SYSTEM_MEM_SIZE, layout::SYSTEM_MEM_START, load_kernel,
 };
 
@@ -114,3 +119,32 @@ pub struct EntryPoint {
     /// Specifies which boot protocol to use
     pub protocol: BootProtocol,
 }
+
+/// Adds in [`regions`] the valid memory regions suitable for RAM taking into account a gap in the
+/// available address space and returns the remaining region (if any) past this gap
+fn arch_memory_regions_with_gap(
+    regions: &mut Vec<(GuestAddress, usize)>,
+    offset: usize,
+    size: usize,
+    gap_start: usize,
+    gap_size: usize,
+) -> Option<(usize, usize)> {
+    // 0-sized gaps don't really make sense. We should never receive such a gap.
+    assert!(gap_size > 0);
+
+    let first_addr_past_gap = gap_start + gap_size;
+    match (size + offset).checked_sub(gap_start) {
+        // case0: region fits all before gap
+        None | Some(0) => {
+            regions.push((GuestAddress(offset as u64), size));
+            None
+        }
+        // case1: region starts before the gap and goes past it
+        Some(remaining) if offset < gap_start => {
+            regions.push((GuestAddress(offset as u64), gap_start - offset));
+            Some((first_addr_past_gap, remaining))
+        }
+        // case2: region starts past the gap
+        Some(_) => Some((first_addr_past_gap.max(offset), size)),
+    }
+}

src/vmm/src/arch/x86_64/layout.rs

@@ -7,6 +7,8 @@
 
 //! Magic addresses externally used to lay out x86_64 VMs.
 
+use crate::utils::mib_to_bytes;
+
 /// Initial stack for the boot CPU.
 pub const BOOT_STACK_POINTER: u64 = 0x8ff0;
 
@@ -77,3 +79,45 @@ pub const SYSTEM_MEM_START: u64 = 0x9fc00;
 /// 257KiB is more than we need, however we reserve this space for potential future use of
 /// ACPI features (new tables and/or devices).
 pub const SYSTEM_MEM_SIZE: u64 = RSDP_ADDR - SYSTEM_MEM_START;
+
+/// First address that cannot be addressed using 32 bit anymore.
+pub const FIRST_ADDR_PAST_32BITS: u64 = 1 << 32;
+
+/// The size of the memory area reserved for MMIO 32-bit accesses.
+pub const MMIO32_MEM_SIZE: u64 = mib_to_bytes(1024) as u64;
+/// The start of the memory area reserved for MMIO 32-bit accesses.
+pub const MMIO32_MEM_START: u64 = FIRST_ADDR_PAST_32BITS - MMIO32_MEM_SIZE;
+
+// We dedicate the last 256 MiB of the 32-bit MMIO address space PCIe for memory-mapped access to
+// configuration.
+/// Size of MMIO region for PCIe configuration accesses.
+pub const PCI_MMCONFIG_SIZE: u64 = 256 << 20;
+/// Start of MMIO region for PCIe configuration accesses.
+pub const PCI_MMCONFIG_START: u64 = FIRST_ADDR_PAST_32BITS - PCI_MMCONFIG_SIZE;
+/// MMIO space per PCIe segment
+pub const PCI_MMIO_CONFIG_SIZE_PER_SEGMENT: u64 = 4096 * 256;
+
+// We reserve 768 MiB for devices at the beginning of the MMIO region. This includes space both for
+// pure MMIO and PCIe devices.
+
+/// Memory region start for boot device.
+pub const BOOT_DEVICE_MEM_START: u64 = MMIO32_MEM_START;
+
+/// Beginning of memory region for device MMIO 32-bit accesses
+pub const MEM_32BIT_DEVICES_START: u64 = BOOT_DEVICE_MEM_START + 0x1000;
+/// Size of memory region for device MMIO 32-bit accesses
+pub const MEM_32BIT_DEVICES_SIZE: u64 = PCI_MMCONFIG_START - MEM_32BIT_DEVICES_START;
+
+// 64-bits region for MMIO accesses
+/// The start of the memory area reserved for MMIO 64-bit accesses.
+pub const MMIO64_MEM_START: u64 = 256 << 30;
+/// The size of the memory area reserved for MMIO 64-bit accesses.
+pub const MMIO64_MEM_SIZE: u64 = 256 << 30;
+
+// At the moment, all of this region goes to devices
+/// Beginning of memory region for device MMIO 64-bit accesses
+pub const MEM_64BIT_DEVICES_START: u64 = MMIO64_MEM_START;
+/// Size of memory region for device MMIO 32-bit accesses
+pub const MEM_64BIT_DEVICES_SIZE: u64 = MMIO64_MEM_SIZE;
+/// First address past the 64-bit MMIO gap
+pub const FIRST_ADDR_PAST_64BITS_MMIO: u64 = MMIO64_MEM_START + MMIO64_MEM_SIZE;