arch: define 64-bit capable MMIO memory regions

PCIe distinguishes MMIO regions between 32bit and 64bit, caring for
devices that can't deal with 64-bit addresses. This commit defines the
appropriate regions for both x86 and aarch64 architectures, extends the
resource allocator to handle allocations for both of these regions and
adjusts the logic that calculates the memory regions for the
architecture.

Also, un-do the change that added an `offset` argument
`arch_memory_regions` function. We won't be using this for "secret
hiding" so it just made the logic (especially for kani proofs) too
convoluted.

Signed-off-by: Babis Chalios <[email protected]>

Author: bchalios

src/vmm/src/arch/aarch64/gic/gicv2/mod.rs

@@ -30,7 +30,7 @@ impl GICv2 {
 
     /// Get the address of the GICv2 distributor.
     const fn get_dist_addr() -> u64 {
-        super::layout::MAPPED_IO_START - GICv2::KVM_VGIC_V2_DIST_SIZE
+        super::layout::MMIO32_MEM_START - GICv2::KVM_VGIC_V2_DIST_SIZE
     }
 
     /// Get the size of the GIC_v2 distributor.

src/vmm/src/arch/aarch64/gic/gicv3/mod.rs

@@ -30,7 +30,7 @@ impl GICv3 {
 
     /// Get the address of the GIC distributor.
     fn get_dist_addr() -> u64 {
-        super::layout::MAPPED_IO_START - GICv3::KVM_VGIC_V3_DIST_SIZE
+        super::layout::MMIO32_MEM_START - GICv3::KVM_VGIC_V3_DIST_SIZE
     }
 
     /// Get the size of the GIC distributor.

src/vmm/src/arch/aarch64/layout.rs

@@ -4,51 +4,53 @@
 //      ==== Address map in use in ARM development systems today ====
 //
 //              - 32-bit -              - 36-bit -          - 40-bit -
-// 1024GB    +                   +                      +-------------------+     <- 40-bit
+// 1024GB   +                   +                       +-------------------+     <- 40-bit
 //          |                                           | DRAM              |
 //          ~                   ~                       ~                   ~
 //          |                                           |                   |
 //          |                                           |                   |
 //          |                                           |                   |
 //          |                                           |                   |
-// 544GB     +                   +                       +-------------------+
+// 544GB    +                   +                       +-------------------+
 //          |                                           | Hole or DRAM      |
 //          |                                           |                   |
-// 512GB     +                   +                       +-------------------+
+// 512GB    +                   +                       +-------------------+
 //          |                                           |       Mapped      |
 //          |                                           |       I/O         |
 //          ~                   ~                       ~                   ~
 //          |                                           |                   |
-// 256GB     +                   +                       +-------------------+
+// 256GB    +                   +                       +-------------------+
 //          |                                           |       Reserved    |
 //          ~                   ~                       ~                   ~
 //          |                                           |                   |
-// 64GB      +                   +-----------------------+-------------------+   <- 36-bit
+// 64GB     +                   +-----------------------+-------------------+   <- 36-bit
 //          |                   |                   DRAM                    |
 //          ~                   ~                   ~                       ~
 //          |                   |                                           |
 //          |                   |                                           |
-// 34GB      +                   +-----------------------+-------------------+
+// 34GB     +                   +-----------------------+-------------------+
 //          |                   |                  Hole or DRAM             |
-// 32GB      +                   +-----------------------+-------------------+
+// 32GB     +                   +-----------------------+-------------------+
 //          |                   |                   Mapped I/O              |
 //          ~                   ~                       ~                   ~
 //          |                   |                                           |
-// 16GB      +                   +-----------------------+-------------------+
+// 16GB     +                   +-----------------------+-------------------+
 //          |                   |                   Reserved                |
 //          ~                   ~                       ~                   ~
-// 4GB       +-------------------+-----------------------+-------------------+   <- 32-bit
+// 4GB      +-------------------+-----------------------+-------------------+   <- 32-bit
 //          |           2GB of DRAM                                         |
 //          |                                                               |
-// 2GB       +-------------------+-----------------------+-------------------+
+// 2GB      +-------------------+-----------------------+-------------------+
 //          |                           Mapped I/O                          |
-// 1GB       +-------------------+-----------------------+-------------------+
+// 1GB      +-------------------+-----------------------+-------------------+
 //          |                          ROM & RAM & I/O                      |
-// 0GB       +-------------------+-----------------------+-------------------+   0
+// 0GB      +-------------------+-----------------------+-------------------+   0
 //              - 32-bit -              - 36-bit -              - 40-bit -
 //
 // Taken from (http://infocenter.arm.com/help/topic/com.arm.doc.den0001c/DEN0001C_principles_of_arm_memory_maps.pdf).
 
+use crate::device_manager::mmio::MMIO_LEN;
+
 /// Start of RAM on 64 bit ARM.
 pub const DRAM_MEM_START: u64 = 0x8000_0000; // 2 GB.
 /// The maximum RAM size.
@@ -80,5 +82,46 @@ pub const IRQ_MAX: u32 = 128;
 /// First usable interrupt on aarch64.
 pub const IRQ_BASE: u32 = 32;
 
+/// The start of the memory area reserved for MMIO 32-bit accesses.
 /// Below this address will reside the GIC, above this address will reside the MMIO devices.
-pub const MAPPED_IO_START: u64 = 1 << 30; // 1 GB
+pub const MMIO32_MEM_START: u64 = 1 << 30; // 1GiB
+/// The size of the memory area reserved for MMIO 32-bit accesses (1GiB).
+pub const MMIO32_MEM_SIZE: u64 = DRAM_MEM_START - MMIO32_MEM_START;
+
+// The rest of the MMIO address space (256 MiB) we dedicate to PCIe for memory-mapped access to
+// configuration.
+/// Size of MMIO region for PCIe configuration accesses.
+pub const PCI_MMCONFIG_SIZE: u64 = 256 << 20;
+/// Start of MMIO region for PCIe configuration accesses.
+pub const PCI_MMCONFIG_START: u64 = DRAM_MEM_START - PCI_MMCONFIG_SIZE;
+/// MMIO space per PCIe segment
+pub const PCI_MMIO_CONFIG_SIZE_PER_SEGMENT: u64 = 4096 * 256;
+
+// We reserve 768 MiB for devices at the beginning of the MMIO region. This includes space both for
+// pure MMIO and PCIe devices.
+
+/// Memory region start for boot device.
+pub const BOOT_DEVICE_MEM_START: u64 = MMIO32_MEM_START;
+/// Memory region start for RTC device.
+pub const RTC_MEM_START: u64 = BOOT_DEVICE_MEM_START + MMIO_LEN;
+/// Memory region start for Serial device.
+pub const SERIAL_MEM_START: u64 = RTC_MEM_START + MMIO_LEN;
+
+/// Beginning of memory region for device MMIO 32-bit accesses
+pub const MEM_32BIT_DEVICES_START: u64 = SERIAL_MEM_START + MMIO_LEN;
+/// Size of memory region for device MMIO 32-bit accesses
+pub const MEM_32BIT_DEVICES_SIZE: u64 = PCI_MMCONFIG_START - MEM_32BIT_DEVICES_START;
+
+// 64-bits region for MMIO accesses
+/// The start of the memory area reserved for MMIO 64-bit accesses.
+pub const MMIO64_MEM_START: u64 = 256 << 30;
+/// The size of the memory area reserved for MMIO 64-bit accesses.
+pub const MMIO64_MEM_SIZE: u64 = 256 << 30;
+
+// At the moment, all of this region goes to devices
+/// Beginning of memory region for device MMIO 64-bit accesses
+pub const MEM_64BIT_DEVICES_START: u64 = MMIO64_MEM_START;
+/// Size of memory region for device MMIO 32-bit accesses
+pub const MEM_64BIT_DEVICES_SIZE: u64 = MMIO64_MEM_SIZE;
+/// First address past the 64-bit MMIO gap
+pub const FIRST_ADDR_PAST_64BITS_MMIO: u64 = MMIO64_MEM_START + MMIO64_MEM_SIZE;

src/vmm/src/arch/aarch64/mod.rs

@@ -24,11 +24,11 @@ use linux_loader::loader::pe::PE as Loader;
 use linux_loader::loader::{Cmdline, KernelLoader};
 use vm_memory::GuestMemoryError;
 
-use crate::arch::{BootProtocol, EntryPoint};
+use crate::arch::{BootProtocol, EntryPoint, arch_memory_regions_with_gap};
 use crate::cpu_config::aarch64::{CpuConfiguration, CpuConfigurationError};
 use crate::cpu_config::templates::CustomCpuTemplate;
 use crate::initrd::InitrdConfig;
-use crate::utils::{align_up, usize_to_u64};
+use crate::utils::{align_up, u64_to_usize, usize_to_u64};
 use crate::vmm_config::machine_config::MachineConfig;
 use crate::vstate::memory::{Address, Bytes, GuestAddress, GuestMemory, GuestMemoryMmap};
 use crate::vstate::vcpu::KvmVcpuError;
@@ -51,42 +51,34 @@ pub enum ConfigurationError {
     VcpuConfigure(#[from] KvmVcpuError),
 }
 
-/// The start of the memory area reserved for MMIO devices.
-pub const MMIO_MEM_START: u64 = layout::MAPPED_IO_START;
-/// The size of the memory area reserved for MMIO devices.
-pub const MMIO_MEM_SIZE: u64 = layout::DRAM_MEM_START - layout::MAPPED_IO_START; //>> 1GB
-
 /// Returns a Vec of the valid memory addresses for aarch64.
 /// See [`layout`](layout) module for a drawing of the specific memory model for this platform.
-///
-/// The `offset` parameter specified the offset from [`layout::DRAM_MEM_START`].
-pub fn arch_memory_regions(offset: usize, size: usize) -> Vec<(GuestAddress, usize)> {
+pub fn arch_memory_regions(size: usize) -> Vec<(GuestAddress, usize)> {
     assert!(size > 0, "Attempt to allocate guest memory of length 0");
-    assert!(
-        offset.checked_add(size).is_some(),
-        "Attempt to allocate guest memory such that the address space would wrap around"
-    );
-    assert!(
-        offset < layout::DRAM_MEM_MAX_SIZE,
-        "offset outside allowed DRAM range"
-    );
 
-    let dram_size = min(size, layout::DRAM_MEM_MAX_SIZE - offset);
+    let dram_size = min(size, layout::DRAM_MEM_MAX_SIZE);
 
     if dram_size != size {
         logger::warn!(
-            "Requested offset/memory size {}/{} exceeds architectural maximum (1022GiB). Size has \
-             been truncated to {}",
-            offset,
+            "Requested memory size {} exceeds architectural maximum (1022GiB). Size has been \
+             truncated to {}",
             size,
             dram_size
         );
     }
 
-    vec![(
-        GuestAddress(layout::DRAM_MEM_START + offset as u64),
+    let mut regions = vec![];
+    if let Some((offset, remaining)) = arch_memory_regions_with_gap(
+        &mut regions,
+        u64_to_usize(layout::DRAM_MEM_START),
         dram_size,
-    )]
+        u64_to_usize(layout::MMIO64_MEM_START),
+        u64_to_usize(layout::MMIO64_MEM_SIZE),
+    ) {
+        regions.push((GuestAddress(offset as u64), remaining));
+    }
+
+    regions
 }
 
 /// Configures the system for booting Linux.
@@ -211,73 +203,109 @@ pub fn load_kernel(
 
 #[cfg(kani)]
 mod verification {
-    use vm_memory::GuestAddress;
-
-    use crate::arch::aarch64::layout;
+    use crate::arch::aarch64::layout::{
+        DRAM_MEM_MAX_SIZE, DRAM_MEM_START, FIRST_ADDR_PAST_64BITS_MMIO, MMIO64_MEM_START,
+    };
     use crate::arch::arch_memory_regions;
 
     #[kani::proof]
     #[kani::unwind(3)]
     fn verify_arch_memory_regions() {
-        let offset: u64 = kani::any::<u64>();
-        let len: u64 = kani::any::<u64>();
-
+        let len: usize = kani::any::<usize>();
         kani::assume(len > 0);
-        kani::assume(offset.checked_add(len).is_some());
-        kani::assume(offset < layout::DRAM_MEM_MAX_SIZE as u64);
 
-        let regions = arch_memory_regions(offset as usize, len as usize);
+        let regions = arch_memory_regions(len);
 
-        // No MMIO gap on ARM
-        assert_eq!(regions.len(), 1);
+        for region in &regions {
+            println!(
+                "region: [{:x}:{:x})",
+                region.0.0,
+                region.0.0 + region.1 as u64
+            );
+        }
 
-        let (GuestAddress(start), actual_len) = regions[0];
-        let actual_len = actual_len as u64;
+        // On Arm we have one MMIO gap that might fall within addressable ranges,
+        // so we can get either 1 or 2 regions.
+        assert!(regions.len() >= 1);
+        assert!(regions.len() <= 2);
 
-        assert_eq!(start, layout::DRAM_MEM_START + offset);
-        assert!(actual_len <= layout::DRAM_MEM_MAX_SIZE as u64);
+        // The total length of all regions cannot exceed DRAM_MEM_MAX_SIZE
+        let actual_len = regions.iter().map(|&(_, len)| len).sum::<usize>();
+        assert!(actual_len <= DRAM_MEM_MAX_SIZE);
+        // The total length is smaller or equal to the length we asked
         assert!(actual_len <= len);
+        // If it's smaller, it's because we asked more than the the maximum possible.
+        if (actual_len) < len {
+            assert!(len > DRAM_MEM_MAX_SIZE);
+        }
 
-        if actual_len < len {
-            assert_eq!(
-                start + actual_len,
-                layout::DRAM_MEM_START + layout::DRAM_MEM_MAX_SIZE as u64
-            );
-            assert!(offset + len >= layout::DRAM_MEM_MAX_SIZE as u64);
+        // No region overlaps the 64-bit MMIO gap
+        assert!(
+            regions
+                .iter()
+                .all(|&(start, len)| start.0 >= FIRST_ADDR_PAST_64BITS_MMIO
+                    || start.0 + len as u64 <= MMIO64_MEM_START)
+        );
+
+        // All regions start after our DRAM_MEM_START
+        assert!(regions.iter().all(|&(start, _)| start.0 >= DRAM_MEM_START));
+
+        // All regions have non-zero length
+        assert!(regions.iter().all(|&(_, len)| len > 0));
+
+        // If there's two regions, they perfectly snuggle up the 64bit MMIO gap
+        if regions.len() == 2 {
+            kani::cover!();
+
+            // The very first address should be DRAM_MEM_START
+            assert_eq!(regions[0].0.0, DRAM_MEM_START);
+            // The first region ends at the beginning of the 64 bits gap.
+            assert_eq!(regions[0].0.0 + regions[0].1 as u64, MMIO64_MEM_START);
+            // The second region starts exactly after the 64 bits gap.
+            assert_eq!(regions[1].0.0, FIRST_ADDR_PAST_64BITS_MMIO);
         }
     }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::arch::aarch64::layout::{
+        DRAM_MEM_MAX_SIZE, DRAM_MEM_START, FDT_MAX_SIZE, FIRST_ADDR_PAST_64BITS_MMIO,
+        MMIO64_MEM_START,
+    };
     use crate::test_utils::arch_mem;
 
     #[test]
     fn test_regions_lt_1024gb() {
-        let regions = arch_memory_regions(0, 1usize << 29);
+        let regions = arch_memory_regions(1usize << 29);
         assert_eq!(1, regions.len());
-        assert_eq!(GuestAddress(super::layout::DRAM_MEM_START), regions[0].0);
+        assert_eq!(GuestAddress(DRAM_MEM_START), regions[0].0);
         assert_eq!(1usize << 29, regions[0].1);
     }
 
     #[test]
     fn test_regions_gt_1024gb() {
-        let regions = arch_memory_regions(0, 1usize << 41);
-        assert_eq!(1, regions.len());
-        assert_eq!(GuestAddress(super::layout::DRAM_MEM_START), regions[0].0);
-        assert_eq!(super::layout::DRAM_MEM_MAX_SIZE, regions[0].1);
+        let regions = arch_memory_regions(1usize << 41);
+        assert_eq!(2, regions.len());
+        assert_eq!(GuestAddress(DRAM_MEM_START), regions[0].0);
+        assert_eq!(MMIO64_MEM_START - DRAM_MEM_START, regions[0].1 as u64);
+        assert_eq!(GuestAddress(FIRST_ADDR_PAST_64BITS_MMIO), regions[1].0);
+        assert_eq!(
+            DRAM_MEM_MAX_SIZE as u64 - MMIO64_MEM_START + DRAM_MEM_START,
+            regions[1].1 as u64
+        );
     }
 
     #[test]
     fn test_get_fdt_addr() {
-        let mem = arch_mem(layout::FDT_MAX_SIZE - 0x1000);
-        assert_eq!(get_fdt_addr(&mem), layout::DRAM_MEM_START);
+        let mem = arch_mem(FDT_MAX_SIZE - 0x1000);
+        assert_eq!(get_fdt_addr(&mem), DRAM_MEM_START);
 
-        let mem = arch_mem(layout::FDT_MAX_SIZE);
-        assert_eq!(get_fdt_addr(&mem), layout::DRAM_MEM_START);
+        let mem = arch_mem(FDT_MAX_SIZE);
+        assert_eq!(get_fdt_addr(&mem), DRAM_MEM_START);
 
-        let mem = arch_mem(layout::FDT_MAX_SIZE + 0x1000);
-        assert_eq!(get_fdt_addr(&mem), 0x1000 + layout::DRAM_MEM_START);
+        let mem = arch_mem(FDT_MAX_SIZE + 0x1000);
+        assert_eq!(get_fdt_addr(&mem), 0x1000 + DRAM_MEM_START);
     }
 }