feat(vmm): add userfault channels to vm and vcpus

These will be used to communicate vCPU faults between vCPUs and the VM
if secret freedom is enabled.

Signed-off-by: Nikita Kalyazin <[email protected]>

Author: kalyazin

src/vmm/src/builder.rs

@@ -162,7 +162,7 @@ fn create_vmm_and_vcpus(
     // Instantiate ACPI device manager.
     let acpi_device_manager = ACPIDeviceManager::new();
 
-    let (vcpus, vcpus_exit_evt) = vm.create_vcpus(vcpu_count)?;
+    let (vcpus, vcpus_exit_evt, userfault_channels) = vm.create_vcpus(vcpu_count, secret_free)?;
 
     #[cfg(target_arch = "x86_64")]
     let pio_device_manager = {
@@ -192,6 +192,7 @@ fn create_vmm_and_vcpus(
         vm,
         uffd: None,
         uffd_socket: None,
+        userfault_channels,
         vcpus_handles: Vec::new(),
         vcpus_exit_evt,
         resource_allocator,
@@ -1034,7 +1035,7 @@ pub(crate) mod tests {
         )
         .unwrap();
 
-        let (_, vcpus_exit_evt) = vm.create_vcpus(1).unwrap();
+        let (_, vcpus_exit_evt, _) = vm.create_vcpus(1, false).unwrap();
 
         Vmm {
             events_observer: Some(std::io::stdin()),
@@ -1044,6 +1045,7 @@ pub(crate) mod tests {
             vm,
             uffd: None,
             uffd_socket: None,
+            userfault_channels: None,
             vcpus_handles: Vec::new(),
             vcpus_exit_evt,
             resource_allocator: ResourceAllocator::new().unwrap(),

src/vmm/src/lib.rs

@@ -154,6 +154,7 @@ use crate::vmm_config::instance_info::{InstanceInfo, VmState};
 use crate::vstate::memory::{GuestMemory, GuestMemoryMmap, GuestMemoryRegion};
 use crate::vstate::vcpu::VcpuState;
 pub use crate::vstate::vcpu::{Vcpu, VcpuConfig, VcpuEvent, VcpuHandle, VcpuResponse};
+use crate::vstate::vm::UserfaultChannel;
 pub use crate::vstate::vm::Vm;
 
 /// Shorthand type for the EventManager flavour used by Firecracker.
@@ -313,6 +314,8 @@ pub struct Vmm {
     uffd: Option<Uffd>,
     // Used for userfault communication with the UFFD handler when secret freedom is enabled
     uffd_socket: Option<UnixStream>,
+    // Used for userfault communication with vCPUs when secret freedom is enabled
+    userfault_channels: Option<Vec<UserfaultChannel>>,
     vcpus_handles: Vec<VcpuHandle>,
     // Used by Vcpus and devices to initiate teardown; Vmm should never write here.
     vcpus_exit_evt: EventFd,

src/vmm/src/vstate/vcpu.rs

@@ -31,7 +31,7 @@ use crate::logger::{IncMetric, METRICS};
 use crate::seccomp::{BpfProgram, BpfProgramRef};
 use crate::utils::signal::{Killable, register_signal_handler, sigrtmin};
 use crate::utils::sm::StateMachine;
-use crate::vstate::vm::Vm;
+use crate::vstate::vm::{UserfaultChannel, Vm};
 
 /// Signal number (SIGRTMIN) used to kick Vcpus.
 pub const VCPU_RTSIG_OFFSET: i32 = 0;
@@ -109,6 +109,8 @@ pub struct Vcpu {
     response_receiver: Option<Receiver<VcpuResponse>>,
     /// The transmitting end of the responses channel owned by the vcpu side.
     response_sender: Sender<VcpuResponse>,
+    /// Channel for communicating userfaults with the VMM thread
+    userfault_channel: Option<UserfaultChannel>,
 }
 
 impl Vcpu {
@@ -201,7 +203,13 @@ impl Vcpu {
     /// * `index` - Represents the 0-based CPU index between [0, max vcpus).
     /// * `vm` - The vm to which this vcpu will get attached.
     /// * `exit_evt` - An `EventFd` that will be written into when this vcpu exits.
-    pub fn new(index: u8, vm: &Vm, exit_evt: EventFd) -> Result<Self, VcpuError> {
+    /// * `userfault_channel` - An optional userfault channel for handling page faults.
+    pub fn new(
+        index: u8,
+        vm: &Vm,
+        exit_evt: EventFd,
+        userfault_channel: Option<UserfaultChannel>,
+    ) -> Result<Self, VcpuError> {
         let (event_sender, event_receiver) = channel();
         let (response_sender, response_receiver) = channel();
         let kvm_vcpu = KvmVcpu::new(index, vm).unwrap();
@@ -215,6 +223,7 @@ impl Vcpu {
             #[cfg(feature = "gdb")]
             gdb_event: None,
             kvm_vcpu,
+            userfault_channel,
         })
     }
 
@@ -922,7 +931,7 @@ pub(crate) mod tests {
     pub(crate) fn setup_vcpu(mem_size: usize) -> (Kvm, Vm, Vcpu) {
         let (kvm, mut vm) = setup_vm_with_memory(mem_size);
 
-        let (mut vcpus, _) = vm.create_vcpus(1).unwrap();
+        let (mut vcpus, _, _) = vm.create_vcpus(1, false).unwrap();
         let mut vcpu = vcpus.remove(0);
 
         #[cfg(target_arch = "aarch64")]

src/vmm/src/vstate/vm.rs

@@ -12,6 +12,7 @@ use std::os::fd::{AsFd, AsRawFd, FromRawFd};
 use std::path::Path;
 use std::sync::Arc;
 
+use bincode::{Decode, Encode};
 use kvm_bindings::{
     KVM_MEM_GUEST_MEMFD, KVM_MEM_LOG_DIRTY_PAGES, KVM_MEMORY_ATTRIBUTE_PRIVATE, KVMIO,
     kvm_create_guest_memfd, kvm_memory_attributes, kvm_userspace_memory_region,
@@ -36,6 +37,47 @@ use crate::{DirtyBitmap, Vcpu, mem_size_mib};
 
 pub(crate) const KVM_GMEM_NO_DIRECT_MAP: u64 = 1;
 
+/// KVM userfault information
+#[derive(Copy, Clone, Decode, Default, Eq, PartialEq, Debug, Encode)]
+pub struct UserfaultData {
+    /// Flags
+    pub flags: u64,
+    /// Guest physical address
+    pub gpa: u64,
+    /// Size
+    pub size: u64,
+}
+
+/// KVM userfault channel
+#[derive(Debug)]
+pub struct UserfaultChannel {
+    /// Sender
+    pub sender: File,
+    /// Receiver
+    pub receiver: File,
+}
+
+fn pipe2(flags: libc::c_int) -> std::io::Result<(File, File)> {
+    let mut fds = [0, 0];
+
+    // SAFETY: pipe2() is safe to call with a valid mutable pointer to an array of 2 integers
+    // The fds array is stack-allocated and lives for the entire unsafe block.
+    let res = unsafe { libc::pipe2(fds.as_mut_ptr(), flags) };
+
+    if res == 0 {
+        Ok((
+            // SAFETY: fds[0] contains a valid file descriptor for the read end of the pipe
+            // We only convert successful pipe2() calls, and each fd is used exactly once.
+            unsafe { File::from_raw_fd(fds[0]) },
+            // SAFETY: fds[1] contains a valid file descriptor for the write end of the pipe
+            // We only convert successful pipe2() calls, and each fd is used exactly once.
+            unsafe { File::from_raw_fd(fds[1]) },
+        ))
+    } else {
+        Err(std::io::Error::last_os_error())
+    }
+}
+
 /// Architecture independent parts of a VM.
 #[derive(Debug)]
 pub struct VmCommon {
@@ -60,6 +102,8 @@ pub enum VmError {
     Arch(#[from] ArchVmError),
     /// Error during eventfd operations: {0}
     EventFd(std::io::Error),
+    /// Failed to create a userfault channel: {0}
+    UserfaultChannel(std::io::Error),
     /// Failed to create vcpu: {0}
     CreateVcpu(VcpuError),
     /// The number of configured slots is bigger than the maximum reported by KVM
@@ -92,6 +136,8 @@ struct kvm_userspace_memory_region2 {
     pad2: [u64; 13],
 }
 
+type VcpuCreationResult = Result<(Vec<Vcpu>, EventFd, Option<Vec<UserfaultChannel>>), VmError>;
+
 /// Contains Vm functions that are usable across CPU architectures
 impl Vm {
     /// Create a KVM VM
@@ -154,24 +200,65 @@ impl Vm {
         })
     }
 
+    fn create_userfault_channels(
+        &self,
+        secret_free: bool,
+    ) -> Result<(Option<UserfaultChannel>, Option<UserfaultChannel>), std::io::Error> {
+        if secret_free {
+            let (receiver_vcpu_to_vm, sender_vcpu_to_vm) = pipe2(libc::O_NONBLOCK)?;
+            let (receiver_vm_to_vcpu, sender_vm_to_vcpu) = pipe2(0)?;
+            Ok((
+                Some(UserfaultChannel {
+                    sender: sender_vcpu_to_vm,
+                    receiver: receiver_vm_to_vcpu,
+                }),
+                Some(UserfaultChannel {
+                    sender: sender_vm_to_vcpu,
+                    receiver: receiver_vcpu_to_vm,
+                }),
+            ))
+        } else {
+            Ok((None, None))
+        }
+    }
+
     /// Creates the specified number of [`Vcpu`]s.
     ///
     /// The returned [`EventFd`] is written to whenever any of the vcpus exit.
-    pub fn create_vcpus(&mut self, vcpu_count: u8) -> Result<(Vec<Vcpu>, EventFd), VmError> {
+    pub fn create_vcpus(&mut self, vcpu_count: u8, secret_free: bool) -> VcpuCreationResult {
         self.arch_pre_create_vcpus(vcpu_count)?;
 
         let exit_evt = EventFd::new(libc::EFD_NONBLOCK).map_err(VmError::EventFd)?;
 
         let mut vcpus = Vec::with_capacity(vcpu_count as usize);
+        let mut userfault_channels = Vec::with_capacity(vcpu_count as usize);
         for cpu_idx in 0..vcpu_count {
             let exit_evt = exit_evt.try_clone().map_err(VmError::EventFd)?;
-            let vcpu = Vcpu::new(cpu_idx, self, exit_evt).map_err(VmError::CreateVcpu)?;
+
+            let (vcpu_channel, vmm_channel) = self
+                .create_userfault_channels(secret_free)
+                .map_err(VmError::UserfaultChannel)?;
+
+            let vcpu =
+                Vcpu::new(cpu_idx, self, exit_evt, vcpu_channel).map_err(VmError::CreateVcpu)?;
             vcpus.push(vcpu);
+
+            if secret_free {
+                userfault_channels.push(vmm_channel.unwrap());
+            }
         }
 
         self.arch_post_create_vcpus(vcpu_count)?;
 
-        Ok((vcpus, exit_evt))
+        Ok((
+            vcpus,
+            exit_evt,
+            if secret_free {
+                Some(userfault_channels)
+            } else {
+                None
+            },
+        ))
     }
 
     /// Create a guest_memfd of the specified size
@@ -605,7 +692,7 @@ pub(crate) mod tests {
         let vcpu_count = 2;
         let (_, mut vm) = setup_vm_with_memory(mib_to_bytes(128));
 
-        let (vcpu_vec, _) = vm.create_vcpus(vcpu_count).unwrap();
+        let (vcpu_vec, _, _) = vm.create_vcpus(vcpu_count, false).unwrap();
 
         assert_eq!(vcpu_vec.len(), vcpu_count as usize);
     }