feat(vmm): implement secret-free fault handling protocol

It contains two parts:
 - external: between the VMM thread and the UFFD handler
 - internal: between vCPUs and the VMM thread

An outline of the workflow:
 - When a vCPU fault occurs, vCPU exits to userspace
 - The vCPU thread sends sends the exit syndrome in the vCPU to VMM
   channel and writes to the eventfd
 - The VMM thread forwards the syndrome to the UFFD handler via the UDS
   socket
 - The UFFD handler populates the page, clears the corresponding bit in
   the userfault bitmap and sends a reply to Firecracker
 - The VMM thread receives the reply and updates a vCPU condvar to
   notify the vCPU that the fault has been resolved
 - The vCPU resumes execution

Note that as a result of this change, an ability to exit the VM
gracefully is lost (at least on x86).  In the existing implementation,
the VMM thread initiated an exit if an event was read from the eventfd,
but no VcpuResponse::Exited responses were read for unknown reason.
Since the exit_evt eventfd is now also used by vCPUs to notify the VMM
thread of the VM exits caused by pagefaults, this situation (an eventfd
event, but response in the channel) can occur also because we have read
all VcpuResponse::Userfault in response to the previous eventfd event.

Signed-off-by: Nikita Kalyazin <[email protected]>

Author: kalyazin

src/vmm/src/builder.rs

@@ -4,8 +4,9 @@
 //! Enables pre-boot setup, instantiation and booting of a Firecracker VMM.
 
 use std::fmt::Debug;
+use std::fs::File;
 use std::io::{self, Write};
-use std::os::fd::AsFd;
+use std::os::fd::{AsFd, AsRawFd};
 use std::os::unix::fs::MetadataExt;
 #[cfg(feature = "gdb")]
 use std::sync::mpsc;
@@ -162,7 +163,7 @@ fn create_vmm_and_vcpus(
     // Instantiate ACPI device manager.
     let acpi_device_manager = ACPIDeviceManager::new();
 
-    let (vcpus, vcpus_exit_evt) = vm.create_vcpus(vcpu_count)?;
+    let (vcpus, vcpus_exit_evt) = vm.create_vcpus(vcpu_count, secret_free)?;
 
     #[cfg(target_arch = "x86_64")]
     let pio_device_manager = {
@@ -482,6 +483,41 @@ pub enum BuildMicrovmFromSnapshotError {
     UserfaultBitmapMemfd(#[from] crate::vstate::memory::MemoryError),
 }
 
+fn memfd_to_slice(memfd: &Option<File>) -> Option<&[u8]> {
+    if let Some(bitmap_file) = memfd {
+        let len = u64_to_usize(
+            bitmap_file
+                .metadata()
+                .expect("Failed to get metadata")
+                .len(),
+        );
+
+        // SAFETY: the arguments to mmap cannot cause any memory unsafety in the rust sense
+        let bitmap_addr = unsafe {
+            libc::mmap(
+                std::ptr::null_mut(),
+                len,
+                libc::PROT_WRITE,
+                libc::MAP_SHARED,
+                bitmap_file.as_raw_fd(),
+                0,
+            )
+        };
+
+        if bitmap_addr == libc::MAP_FAILED {
+            panic!(
+                "Failed to mmap userfault bitmap file: {}",
+                std::io::Error::last_os_error()
+            );
+        }
+
+        // SAFETY: `bitmap_addr` is a valid memory address returned by `mmap`.
+        Some(unsafe { std::slice::from_raw_parts(bitmap_addr as *const u8, len) })
+    } else {
+        None
+    }
+}
+
 /// Builds and starts a microVM based on the provided MicrovmState.
 ///
 /// An `Arc` reference of the built `Vmm` is also plugged in the `EventManager`, while another
@@ -527,7 +563,7 @@ pub fn build_microvm_from_snapshot(
     };
 
     let userfault_bitmap_memfd = if secret_free {
-        let bitmap_size = vm_resources.memory_size() / host_page_size();
+        let bitmap_size = vm_resources.memory_size() / host_page_size() / u8::BITS as usize;
         let bitmap_file = create_memfd(bitmap_size as u64, None)?;
 
         // Set all bits so a fault on any page will cause a VM exit
@@ -580,8 +616,10 @@ pub fn build_microvm_from_snapshot(
         }
     };
 
+    let userfault_bitmap = memfd_to_slice(&userfault_bitmap_memfd);
+
     vmm.vm
-        .register_memory_regions(guest_memory, userfault_bitmap_memfd.as_ref())
+        .register_memory_regions(guest_memory, userfault_bitmap)
         .map_err(VmmError::Vm)
         .map_err(StartMicrovmError::Internal)?;
     vmm.uffd = uffd;
@@ -1045,7 +1083,7 @@ pub(crate) mod tests {
         )
         .unwrap();
 
-        let (_, vcpus_exit_evt) = vm.create_vcpus(1).unwrap();
+        let (_, vcpus_exit_evt) = vm.create_vcpus(1, false).unwrap();
 
         Vmm {
             events_observer: Some(std::io::stdin()),

src/vmm/src/lib.rs

@@ -115,7 +115,8 @@ pub mod vstate;
 pub mod initrd;
 
 use std::collections::HashMap;
-use std::io;
+use std::io::{self, Read, Write};
+use std::os::fd::RawFd;
 use std::os::unix::io::AsRawFd;
 use std::os::unix::net::UnixStream;
 use std::sync::mpsc::RecvTimeoutError;
@@ -128,6 +129,7 @@ use devices::acpi::vmgenid::VmGenIdError;
 use event_manager::{EventManager as BaseEventManager, EventOps, Events, MutEventSubscriber};
 use seccomp::BpfProgram;
 use userfaultfd::Uffd;
+use vm_memory::GuestAddress;
 use vmm_sys_util::epoll::EventSet;
 use vmm_sys_util::eventfd::EventFd;
 use vmm_sys_util::terminal::Terminal;
@@ -147,13 +149,16 @@ use crate::devices::virtio::block::device::Block;
 use crate::devices::virtio::net::Net;
 use crate::devices::virtio::{TYPE_BALLOON, TYPE_BLOCK, TYPE_NET};
 use crate::logger::{METRICS, MetricsError, error, info, warn};
-use crate::persist::{MicrovmState, MicrovmStateError, VmInfo};
+use crate::persist::{FaultReply, FaultRequest, MicrovmState, MicrovmStateError, VmInfo};
 use crate::rate_limiter::BucketUpdate;
 use crate::snapshot::Persist;
 use crate::vmm_config::instance_info::{InstanceInfo, VmState};
-use crate::vstate::memory::{GuestMemory, GuestMemoryMmap, GuestMemoryRegion};
+use crate::vstate::memory::{
+    GuestMemory, GuestMemoryExtension, GuestMemoryMmap, GuestMemoryRegion,
+};
 use crate::vstate::vcpu::VcpuState;
 pub use crate::vstate::vcpu::{Vcpu, VcpuConfig, VcpuEvent, VcpuHandle, VcpuResponse};
+use crate::vstate::vm::UserfaultData;
 pub use crate::vstate::vm::Vm;
 
 /// Shorthand type for the EventManager flavour used by Firecracker.
@@ -800,6 +805,111 @@ impl Vmm {
         self.shutdown_exit_code = Some(exit_code);
     }
 
+    fn process_vcpu_userfault(&mut self, vcpu: usize, userfault_data: UserfaultData) {
+        let offset = self
+            .vm
+            .guest_memory()
+            .gpa_to_offset(GuestAddress(userfault_data.gpa))
+            .expect("Failed to convert GPA to offset");
+
+        let fault_request = FaultRequest {
+            vcpu: vcpu.try_into().expect("Invalid vCPU index"),
+            offset,
+            flags: userfault_data.flags,
+            token: None,
+        };
+        let fault_request_json =
+            serde_json::to_string(&fault_request).expect("Failed to serialize fault request");
+
+        let written = self
+            .uffd_socket
+            .as_ref()
+            .expect("Uffd socket is not set")
+            .write(fault_request_json.as_bytes())
+            .expect("Failed to write to uffd socket");
+
+        if written != fault_request_json.len() {
+            panic!(
+                "Failed to write the entire fault request to the uffd socket: expected {}, \
+                 written {}",
+                fault_request_json.len(),
+                written
+            );
+        }
+    }
+
+    fn active_event_in_uffd_socket(&self, source: RawFd, event_set: EventSet) -> bool {
+        if let Some(uffd_socket) = &self.uffd_socket {
+            uffd_socket.as_raw_fd() == source && event_set == EventSet::IN
+        } else {
+            false
+        }
+    }
+
+    fn process_uffd_socket(&mut self) {
+        const BUFFER_SIZE: usize = 4096;
+
+        let stream = self.uffd_socket.as_mut().expect("Uffd socket is not set");
+
+        let mut buffer = [0u8; BUFFER_SIZE];
+        let mut current_pos = 0;
+
+        loop {
+            if current_pos < BUFFER_SIZE {
+                match stream.read(&mut buffer[current_pos..]) {
+                    Ok(0) => break,
+                    Ok(n) => current_pos += n,
+                    Err(e) if e.kind() == io::ErrorKind::WouldBlock => {
+                        if current_pos == 0 {
+                            break;
+                        }
+                    }
+                    Err(e) => panic!("Read error: {}", e),
+                }
+            }
+
+            let mut parser = serde_json::Deserializer::from_slice(&buffer[..current_pos])
+                .into_iter::<FaultReply>();
+            let mut total_consumed = 0;
+            let mut needs_more = false;
+
+            while let Some(result) = parser.next() {
+                match result {
+                    Ok(fault_reply) => {
+                        let vcpu = fault_reply.vcpu.expect("vCPU must be set");
+
+                        self.vcpus_handles
+                            .get(vcpu as usize)
+                            .expect("Invalid vcpu index")
+                            .send_userfault_resolved();
+
+                        total_consumed = parser.byte_offset();
+                    }
+                    Err(e) if e.is_eof() => {
+                        needs_more = true;
+                        break;
+                    }
+                    Err(e) => {
+                        println!(
+                            "Buffer content: {:?}",
+                            std::str::from_utf8(&buffer[..current_pos])
+                        );
+                        panic!("Invalid JSON: {}", e);
+                    }
+                }
+            }
+
+            if total_consumed > 0 {
+                buffer.copy_within(total_consumed..current_pos, 0);
+                current_pos -= total_consumed;
+            }
+
+            if needs_more {
+                continue;
+            }
+        }
+    }
+
     /// Gets a reference to kvm-ioctls Vm
     #[cfg(feature = "gdb")]
     pub fn vm(&self) -> &Vm {
@@ -882,38 +992,55 @@ impl MutEventSubscriber for Vmm {
         let event_set = event.event_set();
 
         if source == self.vcpus_exit_evt.as_raw_fd() && event_set == EventSet::IN {
-            // Exit event handling should never do anything more than call 'self.stop()'.
             let _ = self.vcpus_exit_evt.read();
 
-            let exit_code = 'exit_code: {
-                // Query each vcpu for their exit_code.
-                for handle in &self.vcpus_handles {
-                    // Drain all vcpu responses that are pending from this vcpu until we find an
-                    // exit status.
-                    for response in handle.response_receiver().try_iter() {
-                        if let VcpuResponse::Exited(status) = response {
-                            // It could be that some vcpus exited successfully while others
-                            // errored out. Thus make sure that error exits from one vcpu always
-                            // takes precedence over "ok" exits
+            let mut pending_userfaults = Vec::with_capacity(self.vcpus_handles.len());
+            let mut should_exit = false;
+            let mut final_exit_code = FcExitCode::Ok;
+
+            // First pass: collect all responses and determine exit status
+            for (index, handle) in self.vcpus_handles.iter().enumerate() {
+                for response in handle.response_receiver().try_iter() {
+                    match response {
+                        VcpuResponse::Exited(status) => {
+                            should_exit = true;
                             if status != FcExitCode::Ok {
-                                break 'exit_code status;
+                                final_exit_code = status;
                             }
                         }
+                        VcpuResponse::Userfault(userfault_data) => {
+                            pending_userfaults.push((index, userfault_data));
+                        }
+                        _ => panic!("Unexpected response from vcpu: {:?}", response),
                     }
                 }
+            }
 
-                // No CPUs exited with error status code, report "Ok"
-                FcExitCode::Ok
-            };
-            self.stop(exit_code);
-        } else {
-            error!("Spurious EventManager event for handler: Vmm");
+            // Process any pending userfaults
+            for (index, userfault_data) in pending_userfaults {
+                self.process_vcpu_userfault(index, userfault_data);
+            }
+
+            // Stop if we received an exit event
+            if should_exit {
+                self.stop(final_exit_code);
+            }
+        }
+
+        if self.active_event_in_uffd_socket(source, event_set) {
+            self.process_uffd_socket();
         }
     }
 
     fn init(&mut self, ops: &mut EventOps) {
         if let Err(err) = ops.add(Events::new(&self.vcpus_exit_evt, EventSet::IN)) {
             error!("Failed to register vmm exit event: {}", err);
         }
+
+        if let Some(uffd_socket) = self.uffd_socket.as_ref() {
+            if let Err(err) = ops.add(Events::new(uffd_socket, EventSet::IN)) {
+                panic!("Failed to register UFFD socket: {}", err);
+            }
+        }
     }
 }

src/vmm/src/persist.rs

@@ -649,6 +649,10 @@ fn send_uffd_handshake(
     let backend_mappings = serde_json::to_string(backend_mappings).unwrap();
 
     let socket = UnixStream::connect(mem_uds_path)?;
+    socket
+        .set_nonblocking(true)
+        .expect("Cannot set non-blocking");
+
     socket.send_with_fds(
         &[backend_mappings.as_bytes()],
         // In the happy case we can close the fd since the other process has it open and is