Add option to force guests to use swiotlb in dedicated memory region for all I/O

Cargo.toml

@@ -21,6 +21,7 @@ exit = "warn"
 tests_outside_test_module = "warn"
 assertions_on_result_states = "warn"
 error_impl_error = "warn"
+needless-update = "allow"
 
 [profile.dev]
 panic = "abort"

resources/guest_configs/ci.config

@@ -5,6 +5,7 @@ CONFIG_SQUASHFS_ZSTD=y
 # aarch64 only TBD split into a separate file
 CONFIG_DEVMEM=y
 # CONFIG_ARM64_ERRATUM_3194386 is not set
+CONFIG_DMA_RESTRICTED_POOL=y
 # Needed for CTRL+ALT+DEL support
 CONFIG_SERIO=y
 CONFIG_SERIO_I8042=y

src/firecracker/src/api_server/request/machine_configuration.rs

@@ -119,6 +119,7 @@ mod tests {
             let expected_config = MachineConfigUpdate {
                 vcpu_count: Some(8),
                 mem_size_mib: Some(1024),
+                mem_config: Some(Default::default()),
                 smt: Some(false),
                 cpu_template: None,
                 track_dirty_pages: Some(false),
@@ -140,6 +141,7 @@ mod tests {
         let expected_config = MachineConfigUpdate {
             vcpu_count: Some(8),
             mem_size_mib: Some(1024),
+            mem_config: Some(Default::default()),
             smt: Some(false),
             cpu_template: Some(StaticCpuTemplate::None),
             track_dirty_pages: Some(false),
@@ -161,6 +163,7 @@ mod tests {
         let expected_config = MachineConfigUpdate {
             vcpu_count: Some(8),
             mem_size_mib: Some(1024),
+            mem_config: Some(Default::default()),
             smt: Some(false),
             cpu_template: None,
             track_dirty_pages: Some(true),
@@ -186,6 +189,7 @@ mod tests {
             let expected_config = MachineConfigUpdate {
                 vcpu_count: Some(8),
                 mem_size_mib: Some(1024),
+                mem_config: Some(Default::default()),
                 smt: Some(false),
                 cpu_template: Some(StaticCpuTemplate::T2),
                 track_dirty_pages: Some(true),
@@ -213,6 +217,7 @@ mod tests {
         let expected_config = MachineConfigUpdate {
             vcpu_count: Some(8),
             mem_size_mib: Some(1024),
+            mem_config: Some(Default::default()),
             smt: Some(true),
             cpu_template: None,
             track_dirty_pages: Some(true),

src/firecracker/swagger/firecracker.yaml

@@ -1044,6 +1044,8 @@ definitions:
       mem_size_mib:
         type: integer
         description: Memory size of VM
+      mem_config:
+        $ref: "#/definitions/MemoryConfig"
       track_dirty_pages:
         type: boolean
         description:
@@ -1064,6 +1066,17 @@ definitions:
           - 2M
         description: Which huge pages configuration (if any) should be used to back guest memory.
 
+  MemoryConfig:
+    type: object
+    description:
+      Describes special configuration options for guest memory, such as swiotlb
+    properties:
+      initial_swiotlb_size:
+        type: integer
+        description:
+          Aarch64 only. If non-zero, causes Firecracker to allocate a specific memory area of this size (in MiB)
+          which the guest will use as swiotlb region for all I/O.
+
   MemoryBackend:
     type: object
     required:

src/vmm/src/arch/aarch64/fdt.rs

@@ -10,20 +10,23 @@
 use std::fmt::Debug;
 
 use vm_fdt::{Error as VmFdtError, FdtWriter, FdtWriterNode};
-use vm_memory::GuestMemoryError;
+use vm_memory::{GuestMemoryError, GuestMemoryRegion};
 
 use super::super::DeviceType;
 use super::cache_info::{CacheEntry, read_cache_config};
 use super::gic::GICDevice;
 use crate::device_manager::mmio::MMIODeviceInfo;
 use crate::devices::acpi::vmgenid::{VMGENID_MEM_SIZE, VmGenId};
 use crate::initrd::InitrdConfig;
-use crate::vstate::memory::{Address, GuestMemory, GuestMemoryMmap};
+use crate::vstate::memory::{Address, GuestMemory, GuestMemoryMmap, KvmRegion};
 
 // This is a value for uniquely identifying the FDT node declaring the interrupt controller.
 const GIC_PHANDLE: u32 = 1;
 // This is a value for uniquely identifying the FDT node containing the clock definition.
 const CLOCK_PHANDLE: u32 = 2;
+/// Unique identifier for the /reserved-memory node describing the memory region the guest
+/// is allowed to use for swiotlb
+const IO_MEM_PHANDLE: u32 = 3;
 // You may be wondering why this big value?
 // This phandle is used to uniquely identify the FDT nodes containing cache information. Each cpu
 // can have a variable number of caches, some of these caches may be shared with other cpus.
@@ -56,8 +59,10 @@
 }
 
 /// Creates the flattened device tree for this aarch64 microVM.
+#[allow(clippy::too_many_arguments)]
 pub fn create_fdt(
     guest_mem: &GuestMemoryMmap,
+    swiotlb_region: Option<&KvmRegion>,
     vcpu_mpidr: Vec<u64>,
     cmdline: CString,
     device_info: &HashMap<(DeviceType, String), MMIODeviceInfo>,
@@ -83,7 +88,7 @@
     // containing description of the interrupt controller for this VM.
     fdt_writer.property_u32("interrupt-parent", GIC_PHANDLE)?;
     create_cpu_nodes(&mut fdt_writer, &vcpu_mpidr)?;
-    create_memory_node(&mut fdt_writer, guest_mem)?;
+    create_memory_node(&mut fdt_writer, guest_mem, swiotlb_region)?;
     create_chosen_node(&mut fdt_writer, cmdline, initrd)?;
     create_gic_node(&mut fdt_writer, gic_device)?;
     create_timer_node(&mut fdt_writer)?;
@@ -208,7 +213,11 @@
     Ok(())
 }
 
-fn create_memory_node(fdt: &mut FdtWriter, guest_mem: &GuestMemoryMmap) -> Result<(), FdtError> {
+fn create_memory_node(
+    fdt: &mut FdtWriter,
+    guest_mem: &GuestMemoryMmap,
+    swiotlb_region: Option<&KvmRegion>,
+) -> Result<(), FdtError> {
     // See https://github.com/torvalds/linux/blob/master/Documentation/devicetree/booting-without-of.txt#L960
     // for an explanation of this.
 
@@ -220,9 +229,13 @@
     // The reason we do this is that Linux does not allow remapping system memory. However, without
     // remap, kernel drivers cannot get virtual addresses to read data from device memory. Leaving
     // this memory region out allows Linux kernel modules to remap and thus read this region.
+    //
+    // The generic memory description must include the range that we later declare as a reserved
+    // carve out.
     let mem_size = guest_mem.last_addr().raw_value()
         - super::layout::DRAM_MEM_START
         - super::layout::SYSTEM_MEM_SIZE
+        + swiotlb_region.map(|r| r.len()).unwrap_or(0)
         + 1;
     let mem_reg_prop = &[
         super::layout::DRAM_MEM_START + super::layout::SYSTEM_MEM_SIZE,
@@ -233,6 +246,26 @@
     fdt.property_array_u64("reg", mem_reg_prop)?;
     fdt.end_node(mem)?;
 
+    if let Some(region) = swiotlb_region {
+        // See https://github.com/devicetree-org/dt-schema/blob/dd3e3dce83607661f2831a8fac9112fae5ebe6cd/dtschema/schemas/reserved-memory/reserved-memory.yaml#L4
+        let rmem = fdt.begin_node("reserved-memory")?;
+        fdt.property_u32("#address-cells", ADDRESS_CELLS)?;
+        fdt.property_u32("#size-cells", SIZE_CELLS)?;
+        fdt.property_null("ranges")?;
+        let dma = fdt.begin_node("bouncy_boi")?;
+        fdt.property_string("compatible", "restricted-dma-pool")?;
+        fdt.property_phandle(IO_MEM_PHANDLE)?;
+        fdt.property_array_u64("reg", &[region.start_addr().0, region.len()])?;
+        fdt.end_node(dma)?;
+        fdt.end_node(rmem)?;
+
+        log::info!(
+            "Placed swiotlb region at [{}, {})",
+            region.start_addr().0,
+            region.start_addr().0 + region.len()
+        );
+    }
+
     Ok(())
 }
 
@@ -358,6 +391,9 @@
 
     fdt.property_string("compatible", "virtio,mmio")?;
     fdt.property_array_u64("reg", &[dev_info.addr, dev_info.len])?;
+    // Force all virtio device to place their vrings and buffer into the dedicated I/O memory region
+    // that is backed by traditional memory (e.g. not secret-free).
+    fdt.property_u32("memory-region", IO_MEM_PHANDLE)?;
     fdt.property_array_u32(
         "interrupts",
         &[
@@ -499,6 +535,7 @@
         let gic = create_gic(&vm, 1, None).unwrap();
         create_fdt(
             &mem,
+            None,
             vec![0],
             CString::new("console=tty0").unwrap(),
             &dev_info,
@@ -519,6 +556,7 @@
         let gic = create_gic(&vm, 1, None).unwrap();
         create_fdt(
             &mem,
+            None,
             vec![0],
             CString::new("console=tty0").unwrap(),
             &HashMap::<(DeviceType, std::string::String), MMIODeviceInfo>::new(),
@@ -544,6 +582,7 @@
 
         let current_dtb_bytes = create_fdt(
             &mem,
+            None,
             vec![0],
             CString::new("console=tty0").unwrap(),
             &HashMap::<(DeviceType, std::string::String), MMIODeviceInfo>::new(),
@@ -606,6 +645,7 @@
 
         let current_dtb_bytes = create_fdt(
             &mem,
+            None,
             vec![0],
             CString::new("console=tty0").unwrap(),
             &HashMap::<(DeviceType, std::string::String), MMIODeviceInfo>::new(),

src/vmm/src/arch/aarch64/mod.rs

@@ -89,6 +89,15 @@
     )]
 }
 
+/// The minimal offset that needs to be passed to `arch_memory_region` to ensure
+/// the function returned a single contiguous memory region placed after
+/// all gaps in guest physical address space.
+///
+/// There are no architectural gaps in the physical address space on aarch64, so this is 0
+pub fn offset_after_last_gap() -> usize {
+    0
+}
+
 /// Configures the system for booting Linux.
 pub fn configure_system_for_boot(
     vmm: &mut Vmm,
@@ -130,8 +139,16 @@
         .as_cstring()
         .expect("Cannot create cstring from cmdline string");
 
+    let swiotlb_region = match vmm.vm.swiotlb_regions().num_regions() {
+        0 | 1 => vmm.vm.swiotlb_regions().iter().next(),
+        _ => panic!(
+            "Firecracker tried to configure more than one swiotlb region. This is a logic bug."
+        ),
+    };
+
     let fdt = fdt::create_fdt(
         vmm.vm.guest_memory(),
+        swiotlb_region,
         vcpu_mpidr,
         cmdline,
         vmm.mmio_device_manager.get_device_info(),

src/vmm/src/arch/aarch64/vm.rs

@@ -70,6 +70,7 @@ impl ArchVm {
     pub fn save_state(&self, mpidrs: &[u64]) -> Result<VmState, ArchVmError> {
         Ok(VmState {
             memory: self.common.guest_memory.describe(),
+            io_memory: self.common.swiotlb_regions.describe(),
             gic: self
                 .get_irqchip()
                 .save_device(mpidrs)
@@ -96,6 +97,8 @@ impl ArchVm {
 pub struct VmState {
     /// Guest memory state
     pub memory: GuestMemoryState,
+    /// io memory state
+    pub io_memory: GuestMemoryState,
     /// GIC state.
     pub gic: GicState,
 }

src/vmm/src/arch/mod.rs

@@ -23,7 +23,7 @@ pub use aarch64::{
     ConfigurationError, MMIO_MEM_SIZE, MMIO_MEM_START, arch_memory_regions,
     configure_system_for_boot, get_kernel_start, initrd_load_addr, layout::CMDLINE_MAX_SIZE,
     layout::IRQ_BASE, layout::IRQ_MAX, layout::SYSTEM_MEM_SIZE, layout::SYSTEM_MEM_START,
-    load_kernel,
+    load_kernel, offset_after_last_gap,
 };
 
 /// Module for x86_64 related functionality.
@@ -42,7 +42,7 @@ pub use crate::arch::x86_64::{
     ConfigurationError, MMIO_MEM_SIZE, MMIO_MEM_START, arch_memory_regions,
     configure_system_for_boot, get_kernel_start, initrd_load_addr, layout::APIC_ADDR,
     layout::CMDLINE_MAX_SIZE, layout::IOAPIC_ADDR, layout::IRQ_BASE, layout::IRQ_MAX,
-    layout::SYSTEM_MEM_SIZE, layout::SYSTEM_MEM_START, load_kernel,
+    layout::SYSTEM_MEM_SIZE, layout::SYSTEM_MEM_START, load_kernel, offset_after_last_gap,
 };
 
 /// Types of devices that can get attached to this platform.

src/vmm/src/arch/x86_64/mod.rs

@@ -140,6 +140,15 @@
     }
 }
 
+/// The minimal offset that needs to be passed to `arch_memory_region` to ensure
+/// the function returned a single contiguous memory region placed after
+/// all gaps in guest physical address space.
+///
+/// On x86_64, this is the offset needed to place memory after the MMIO gap
+pub fn offset_after_last_gap() -> usize {
+    u64_to_usize(FIRST_ADDR_PAST_32BITS)
+}
+
 /// Returns the memory address where the kernel could be loaded.
 pub fn get_kernel_start() -> u64 {
     layout::HIMEM_START

src/vmm/src/arch/x86_64/vm.rs

@@ -187,6 +187,7 @@ impl ArchVm {
 
         Ok(VmState {
             memory: self.common.guest_memory.describe(),
+            io_memory: self.common.swiotlb_regions.describe(),
             pitstate,
             clock,
             pic_master,
@@ -211,6 +212,8 @@ impl ArchVm {
 pub struct VmState {
     /// guest memory state
     pub memory: GuestMemoryState,
+    /// io memory state
+    pub io_memory: GuestMemoryState,
     pitstate: kvm_pit_state2,
     clock: kvm_clock_data,
     // TODO: rename this field to adopt inclusive language once Linux updates it, too.