Use guest_memfd to back non swiotlb regions

roypat · roypat · commit 73f627a6d1d6 · 2025-04-07T13:31:38.000+01:00
If the `secret_free` field of the memory_config is set to true in the
/machine-config endpoint, back all non-swiotlb regions using
guest_memfd. For our setup, this means both setting the
guest_memfd[_offset] fields in kvm_user_memory_region2, as well as
mmaping the guest memory and reflecting this VMA back into the memslot's
userspace_addr (which is how kvm internal accesses to guest memory will
work for these guest_memfd regions, such as mmio emulation on x86).

Signed-off-by: Patrick Roy &lt;roypat@amazon.co.uk&gt;
diff --git a/src/vmm/benches/memory_access.rs b/src/vmm/benches/memory_access.rs
@@ -9,7 +9,7 @@ fn bench_single_page_fault(c: &mut Criterion, configuration: VmResources) {
     c.bench_function("page_fault", |b| {
         b.iter_batched(
             || {
-                let memory = configuration.allocate_guest_memory().unwrap();
+                let memory = configuration.allocate_guest_memory(None).unwrap();
                 // Get a pointer to the first memory region (cannot do `.get_slice(GuestAddress(0),
                 // 1)`, because on ARM64 guest memory does not start at physical
                 // address 0).
diff --git a/src/vmm/src/builder.rs b/src/vmm/src/builder.rs
@@ -65,7 +65,7 @@ use crate::vmm_config::snapshot::{MemBackendConfig, MemBackendType};
 use crate::vstate::kvm::Kvm;
 use crate::vstate::memory::Bounce;
 use crate::vstate::vcpu::{Vcpu, VcpuError};
-use crate::vstate::vm::Vm;
+use crate::vstate::vm::{KVM_GMEM_NO_DIRECT_MAP, Vm};
 use crate::{EventManager, Vmm, VmmError, device_manager};
 
 /// Errors associated with starting the instance.
@@ -217,14 +217,6 @@ pub fn build_microvm_for_boot(
         .as_ref()
         .ok_or(MissingKernelConfig)?;
 
-    let guest_memory = vm_resources
-        .allocate_guest_memory()
-        .map_err(StartMicrovmError::GuestMemory)?;
-
-    let swiotlb = vm_resources
-        .allocate_swiotlb_region()
-        .map_err(StartMicrovmError::GuestMemory)?;
-
     // Clone the command-line so that a failed boot doesn't pollute the original.
     #[allow(unused_mut)]
     let mut boot_cmdline = boot_config.cmdline.clone();
@@ -234,16 +226,37 @@ pub fn build_microvm_for_boot(
         .cpu_template
         .get_cpu_template()?;
 
+    let secret_free = vm_resources.machine_config.mem_config.secret_free;
+
     let (mut vmm, mut vcpus) = create_vmm_and_vcpus(
         instance_info,
         event_manager,
         vm_resources.machine_config.vcpu_count,
         cpu_template.kvm_capabilities.clone(),
     )?;
 
-    vmm.vm
-        .register_memory_regions(guest_memory)
-        .map_err(VmmError::Vm)?;
+    let guest_memfd = match secret_free {
+        true => Some(
+            vmm.vm
+                .create_guest_memfd(vm_resources.memory_size(), KVM_GMEM_NO_DIRECT_MAP)
+                .map_err(VmmError::Vm)?,
+        ),
+        false => None,
+    };
+
+    let guest_memory = vm_resources
+        .allocate_guest_memory(guest_memfd)
+        .map_err(StartMicrovmError::GuestMemory)?;
+
+    let swiotlb = vm_resources
+        .allocate_swiotlb_region()
+        .map_err(StartMicrovmError::GuestMemory)?;
+
+    for region in guest_memory {
+        vmm.vm
+            .register_memory_region(region, secret_free)
+            .map_err(VmmError::Vm)?;
+    }
 
     if let Some(swiotlb) = swiotlb {
         vmm.vm
@@ -252,10 +265,7 @@ pub fn build_microvm_for_boot(
     }
 
     let entry_point = load_kernel(
-        Bounce::new(
-            &boot_config.kernel_file,
-            vm_resources.machine_config.mem_config.secret_free,
-        ),
+        Bounce::new(&boot_config.kernel_file, secret_free),
         vmm.vm.guest_memory(),
     )?;
     let initrd = match &boot_config.initrd_file {
@@ -267,10 +277,7 @@ pub fn build_microvm_for_boot(
 
             Some(InitrdConfig::from_reader(
                 vmm.vm.guest_memory(),
-                Bounce::new(
-                    initrd_file,
-                    vm_resources.machine_config.mem_config.secret_free,
-                ),
+                Bounce::new(initrd_file, secret_free),
                 u64_to_usize(size),
             )?)
         }
@@ -493,8 +500,9 @@ pub fn build_microvm_from_snapshot(
         guest_memory.iter().map(|r| r.len()).sum(),
     )?;
 
+    // TODO: sort out gmem support for snapshot restore
     vmm.vm
-        .register_memory_regions(guest_memory)
+        .register_memory_regions(guest_memory, false)
         .map_err(VmmError::Vm)
         .map_err(StartMicrovmError::Internal)?;
 
diff --git a/src/vmm/src/device_manager/mmio.rs b/src/vmm/src/device_manager/mmio.rs
@@ -656,7 +656,7 @@ mod tests {
         let guest_mem = multi_region_mem_raw(&[(start_addr1, 0x1000), (start_addr2, 0x1000)]);
         let kvm = Kvm::new(vec![]).expect("Cannot create Kvm");
         let mut vm = Vm::new(&kvm).unwrap();
-        vm.register_memory_regions(guest_mem).unwrap();
+        vm.register_memory_regions(guest_mem, false).unwrap();
         let mut device_manager = MMIODeviceManager::new();
         let mut resource_allocator = ResourceAllocator::new().unwrap();
 
@@ -687,7 +687,7 @@ mod tests {
         let guest_mem = multi_region_mem_raw(&[(start_addr1, 0x1000), (start_addr2, 0x1000)]);
         let kvm = Kvm::new(vec![]).expect("Cannot create Kvm");
         let mut vm = Vm::new(&kvm).unwrap();
-        vm.register_memory_regions(guest_mem).unwrap();
+        vm.register_memory_regions(guest_mem, false).unwrap();
         let mut device_manager = MMIODeviceManager::new();
         let mut resource_allocator = ResourceAllocator::new().unwrap();
 
@@ -743,7 +743,7 @@ mod tests {
         let guest_mem = multi_region_mem_raw(&[(start_addr1, 0x1000), (start_addr2, 0x1000)]);
         let kvm = Kvm::new(vec![]).expect("Cannot create Kvm");
         let mut vm = Vm::new(&kvm).unwrap();
-        vm.register_memory_regions(guest_mem).unwrap();
+        vm.register_memory_regions(guest_mem, false).unwrap();
 
         #[cfg(target_arch = "x86_64")]
         vm.setup_irqchip().unwrap();
diff --git a/src/vmm/src/persist.rs b/src/vmm/src/persist.rs
@@ -330,7 +330,7 @@ pub fn restore_memory(
             }
 
             let mem_file = File::open(path)?;
-            memory::snapshot_file(mem_file, state.regions(), track_dirty, offset)?
+            memory::file_private(mem_file, state.regions(), track_dirty, offset)?
         }
         None => memory::anonymous(state.regions(), track_dirty, huge_pages)?,
     };
diff --git a/src/vmm/src/resources.rs b/src/vmm/src/resources.rs
@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use std::convert::From;
+use std::fs::File;
 use std::path::PathBuf;
 use std::sync::{Arc, Mutex, MutexGuard};
 
@@ -30,7 +31,7 @@ use crate::vmm_config::mmds::{MmdsConfig, MmdsConfigError};
 use crate::vmm_config::net::*;
 use crate::vmm_config::vsock::*;
 use crate::vstate::memory;
-use crate::vstate::memory::{GuestRegionMmap, MemoryError};
+use crate::vstate::memory::{GuestRegionMmap, MemoryError, create_memfd};
 
 /// Errors encountered when configuring microVM resources.
 #[derive(Debug, thiserror::Error, displaydoc::Display)]
@@ -492,28 +493,45 @@ impl VmResources {
         offset: usize,
         size: usize,
         vhost_accessible: bool,
+        file: Option<File>,
     ) -> Result<Vec<GuestRegionMmap>, MemoryError> {
-        let regions = crate::arch::arch_memory_regions(offset, size);
-        if vhost_accessible {
-            memory::memfd_backed(
-                regions.as_ref(),
+        let regions = crate::arch::arch_memory_regions(offset, size).into_iter();
+        match file {
+            Some(file) => memory::file_shared(
+                file,
+                regions,
                 self.machine_config.track_dirty_pages,
                 self.machine_config.huge_pages,
-            )
-        } else {
-            memory::anonymous(
-                regions.into_iter(),
-                self.machine_config.track_dirty_pages,
-                self.machine_config.huge_pages,
-            )
+            ),
+            None => {
+                if vhost_accessible {
+                    let memfd = create_memfd(size as u64, self.machine_config.huge_pages.into())?
+                        .into_file();
+                    memory::file_shared(
+                        memfd,
+                        regions,
+                        self.machine_config.track_dirty_pages,
+                        self.machine_config.huge_pages,
+                    )
+                } else {
+                    memory::anonymous(
+                        regions.into_iter(),
+                        self.machine_config.track_dirty_pages,
+                        self.machine_config.huge_pages,
+                    )
+                }
+            }
         }
     }
 
     /// Allocates guest memory in a configuration most appropriate for these [`VmResources`].
     ///
     /// If vhost-user-blk devices are in use, allocates memfd-backed shared memory, otherwise
     /// prefers anonymous memory for performance reasons.
-    pub fn allocate_guest_memory(&self) -> Result<Vec<GuestRegionMmap>, MemoryError> {
+    pub fn allocate_guest_memory(
+        &self,
+        guest_memfd: Option<File>,
+    ) -> Result<Vec<GuestRegionMmap>, MemoryError> {
         // Page faults are more expensive for shared memory mapping, including  memfd.
         // For this reason, we only back guest memory with a memfd
         // if a vhost-user-blk device is configured in the VM, otherwise we fall back to
@@ -530,6 +548,7 @@ impl VmResources {
             0,
             self.memory_size(),
             self.vhost_user_devices_used() && !self.swiotlb_used(),
+            guest_memfd,
         )
     }
 
@@ -542,8 +561,12 @@ impl VmResources {
         let start = self.memory_size();
         let start = start.max(crate::arch::bytes_before_last_gap());
 
-        let mut mem =
-            self.allocate_memory(start, self.swiotlb_size(), self.vhost_user_devices_used())?;
+        let mut mem = self.allocate_memory(
+            start,
+            self.swiotlb_size(),
+            self.vhost_user_devices_used(),
+            None,
+        )?;
 
         assert_eq!(mem.len(), 1);
 
@@ -1614,7 +1637,7 @@ mod tests {
             ..Default::default()
         };
 
-        let normal_mem = resources.allocate_guest_memory().unwrap();
+        let normal_mem = resources.allocate_guest_memory(None).unwrap();
         assert_eq!(
             normal_mem.iter().map(|r| r.len()).sum::<usize>(),
             mib_to_bytes(16)
diff --git a/src/vmm/src/vstate/memory.rs b/src/vmm/src/vstate/memory.rs
@@ -12,7 +12,7 @@ use std::os::fd::AsRawFd;
 use std::ptr::null_mut;
 use std::sync::Arc;
 
-use kvm_bindings::{KVM_MEM_LOG_DIRTY_PAGES, kvm_userspace_memory_region2};
+use kvm_bindings::{KVM_MEM_GUEST_MEMFD, KVM_MEM_LOG_DIRTY_PAGES, kvm_userspace_memory_region2};
 use serde::{Deserialize, Serialize};
 pub use vm_memory::bitmap::{AtomicBitmap, BS, Bitmap, BitmapSlice};
 pub use vm_memory::mmap::MmapRegionBuilder;
@@ -138,14 +138,16 @@ impl KvmRegion {
     pub(crate) fn from_mmap_region(
         region: GuestRegionMmap,
         slot: u32,
-        guest_memfd: Option<&FileOffset>,
+        guest_memfd: Option<FileOffset>,
     ) -> Self {
         let region = ManuallyDrop::new(region);
-        let flags = if region.bitmap().is_some() {
-            KVM_MEM_LOG_DIRTY_PAGES
-        } else {
-            0
-        };
+        let mut flags = 0;
+        if region.bitmap().is_some() {
+            flags |= KVM_MEM_LOG_DIRTY_PAGES;
+        }
+        if guest_memfd.is_some() {
+            flags |= KVM_MEM_GUEST_MEMFD;
+        }
 
         #[allow(clippy::cast_sign_loss)]
         let (guest_memfd, guest_memfd_offset) = guest_memfd
@@ -301,18 +303,16 @@ pub fn create(
 }
 
 /// Creates a GuestMemoryMmap with `size` in MiB backed by a memfd.
-pub fn memfd_backed(
-    regions: &[(GuestAddress, usize)],
+pub fn file_shared(
+    file: File,
+    regions: impl Iterator<Item = (GuestAddress, usize)>,
     track_dirty_pages: bool,
     huge_pages: HugePageConfig,
 ) -> Result<Vec<GuestRegionMmap>, MemoryError> {
-    let size = regions.iter().map(|&(_, size)| size as u64).sum();
-    let memfd_file = create_memfd(size, huge_pages.into())?.into_file();
-
     create(
-        regions.iter().copied(),
+        regions,
         libc::MAP_SHARED | huge_pages.mmap_flags(),
-        Some(memfd_file),
+        Some(file),
         track_dirty_pages,
         0,
     )
@@ -335,7 +335,7 @@ pub fn anonymous(
 
 /// Creates a GuestMemoryMmap given a `file` containing the data
 /// and a `state` containing mapping information.
-pub fn snapshot_file(
+pub fn file_private(
     file: File,
     regions: impl Iterator<Item = (GuestAddress, usize)>,
     track_dirty_pages: bool,
@@ -528,7 +528,8 @@ impl GuestMemoryExtension for GuestMemoryMmap {
     }
 }
 
-fn create_memfd(
+/// Creates a memfd of the given size and huge pages configuration
+pub fn create_memfd(
     mem_size: u64,
     hugetlb_size: Option<memfd::HugetlbSize>,
 ) -> Result<memfd::Memfd, MemoryError> {
@@ -781,7 +782,7 @@ mod tests {
         guest_memory.dump(&mut memory_file).unwrap();
 
         let restored_guest_memory =
-            kvmify(snapshot_file(memory_file, memory_state.regions(), false, 0).unwrap());
+            kvmify(file_private(memory_file, memory_state.regions(), false, 0).unwrap());
 
         // Check that the region contents are the same.
         let mut restored_region = vec![0u8; page_size * 2];
@@ -839,7 +840,7 @@ mod tests {
 
         // We can restore from this because this is the first dirty dump.
         let restored_guest_memory =
-            kvmify(snapshot_file(file, memory_state.regions(), false, 0).unwrap());
+            kvmify(file_private(file, memory_state.regions(), false, 0).unwrap());
 
         // Check that the region contents are the same.
         let mut restored_region = vec![0u8; region_size];
diff --git a/src/vmm/src/vstate/vm.rs b/src/vmm/src/vstate/vm.rs

Original file line number	Diff line number	Diff line change
`@@ -330,7 +330,7 @@ pub fn restore_memory(`
`330`	`330`	`}`
`331`	`331`
`332`	`332`	`let mem_file = File::open(path)?;`
`333`		`- memory::snapshot_file(mem_file, state.regions(), track_dirty, offset)?`
	`333`	`+ memory::file_private(mem_file, state.regions(), track_dirty, offset)?`
`334`	`334`	`}`
`335`	`335`	`None => memory::anonymous(state.regions(), track_dirty, huge_pages)?,`
`336`	`336`	`};`