add concept of "io memory" to struct Vm

Add a special `GuestMemoryMmap` to `struct Vm` that has the semantics
that if its not empty, it represents the swiotlb area in the guest.

Have all virtio devices use io memory by default. This means that in
case of swiotlb regions being set up, virtio devices cannot access
generic memory, so if the guest decides to (incorrectly) place I/O
buffers outside of swiotlb we'll get a somewhat meaningful error of
"invalid guest address" (or similar), instead of random things failing
further down the line in case generic memory isn't accessible by
firecracker/the host kernel for some reason (e.g. if its guest_memfd
backed in the future).

Signed-off-by: Patrick Roy <[email protected]>

Author: roypat

src/vmm/src/builder.rs

@@ -870,7 +870,7 @@ fn attach_virtio_device<T: 'static + VirtioDevice + MutEventSubscriber + Debug>(
     event_manager.add_subscriber(device.clone());
 
     // The device mutex mustn't be locked here otherwise it will deadlock.
-    let device = MmioTransport::new(vmm.guest_memory().clone(), device, is_vhost_user);
+    let device = MmioTransport::new(vmm.vm.io_memory().clone(), device, is_vhost_user);
     vmm.mmio_device_manager
         .register_mmio_virtio_for_boot(
             vmm.vm.fd(),

src/vmm/src/vstate/vm/mod.rs

@@ -33,6 +33,7 @@ struct VmCommon {
     fd: VmFd,
     max_memslots: usize,
     guest_memory: GuestMemoryMmap,
+    io_memory: GuestMemoryMmap,
 }
 
 /// Errors associated with the wrappers over KVM ioctls.
@@ -99,6 +100,7 @@ impl Vm {
             fd,
             max_memslots: kvm.max_nr_memslots(),
             guest_memory: GuestMemoryMmap::default(),
+            io_memory: GuestMemoryMmap::default(),
         })
     }
 
@@ -136,11 +138,41 @@ impl Vm {
 
     /// Register a new memory region to this [`Vm`].
     pub fn register_memory_region(&mut self, region: GuestRegionMmap) -> Result<(), VmError> {
+        let arcd_region = Arc::new(region);
+        let new_guest_memory = self
+            .common
+            .guest_memory
+            .insert_region(Arc::clone(&arcd_region))?;
+
+        self.register_kvm_region(arcd_region.as_ref())?;
+
+        self.common.guest_memory = new_guest_memory;
+        Ok(())
+    }
+
+    /// Registers a new io memory region to this [`Vm`].
+    pub fn register_io_region(&mut self, region: GuestRegionMmap) -> Result<(), VmError> {
+        let arcd_region = Arc::new(region);
+        let new_io_memory = self
+            .common
+            .io_memory
+            .insert_region(Arc::clone(&arcd_region))?;
+
+        self.register_kvm_region(arcd_region.as_ref())?;
+
+        self.common.io_memory = new_io_memory;
+        Ok(())
+    }
+
+    fn register_kvm_region(&mut self, region: &GuestRegionMmap) -> Result<(), VmError> {
         let next_slot = self
             .guest_memory()
             .num_regions()
+            .checked_add(self.io_memory().num_regions())
+            .ok_or(VmError::NotEnoughMemorySlots)?
             .try_into()
             .map_err(|_| VmError::NotEnoughMemorySlots)?;
+
         if next_slot as usize + 1 >= self.common.max_memslots {
             return Err(VmError::NotEnoughMemorySlots);
         }
@@ -159,17 +191,13 @@ impl Vm {
             flags,
         };
 
-        let new_guest_memory = self.common.guest_memory.insert_region(Arc::new(region))?;
-
         // SAFETY: Safe because the fd is a valid KVM file descriptor.
         unsafe {
             self.fd()
                 .set_user_memory_region(memory_region)
                 .map_err(VmError::SetUserMemoryRegion)?;
         }
 
-        self.common.guest_memory = new_guest_memory;
-
         Ok(())
     }
 
@@ -182,6 +210,17 @@ impl Vm {
     pub fn guest_memory(&self) -> &GuestMemoryMmap {
         &self.common.guest_memory
     }
+
+    /// Returns a reference to the [`GuestMemoryMmap`] that I/O devices attached to this [`Vm`]
+    /// have access to. If no I/O regions were registered, return the same as [`Vm::guest_memory`],
+    /// otherwise returns the [`GuestMemoryMmap`] describing a specific swiotlb region.
+    pub fn io_memory(&self) -> &GuestMemoryMmap {
+        if self.common.io_memory.num_regions() > 0 {
+            &self.common.io_memory
+        } else {
+            &self.common.guest_memory
+        }
+    }
 }
 
 #[cfg(test)]