vmm: Coalesce around pm_sender

Signed-off-by: Tyler Fanelli <[email protected]>

Author: tylerfanelli

Cargo.lock

@@ -1601,7 +1601,7 @@ pub extern "C" fn krun_start_enter(ctx_id: u32) -> i32 {
             })
             .unwrap();
     }
-    
+
     #[cfg(feature = "tee")]
     let guest_mem = _vmm.lock().unwrap().guest_memory().clone();
 

src/libkrun/src/lib.rs

@@ -3,8 +3,6 @@
 
 //! Enables pre-boot setup, instantiation and booting of a Firecracker VMM.
 
-#[cfg(feature = "tee")]
-use crate::vstate::MemProperties;
 #[cfg(target_os = "macos")]
 use crossbeam_channel::unbounded;
 
@@ -715,8 +713,6 @@ pub fn build_microvm(
             &guest_memory,
             payload_config.entry_addr,
             &exit_evt,
-            #[cfg(feature = "tee")]
-            io_sender,
         )
         .map_err(StartMicrovmError::Internal)?;
 
@@ -778,8 +774,6 @@ pub fn build_microvm(
         mmio_device_manager,
         #[cfg(target_arch = "x86_64")]
         pio_device_manager,
-        #[cfg(feature = "tee")]
-        guest_memfd_vec: guest_memfd,
     };
 
     #[cfg(not(feature = "tee"))]
@@ -1494,16 +1488,13 @@ fn create_vcpus_aarch64(
     guest_mem: &GuestMemoryMmap,
     entry_addr: GuestAddress,
     exit_evt: &EventFd,
-    #[cfg(feature = "tee")] sender_io: Sender<MemProperties>,
 ) -> super::Result<Vec<Vcpu>> {
     let mut vcpus = Vec::with_capacity(vcpu_config.vcpu_count as usize);
     for cpu_index in 0..vcpu_config.vcpu_count {
         let mut vcpu = Vcpu::new_aarch64(
             cpu_index,
             vm.fd(),
             exit_evt.try_clone().map_err(Error::EventFd)?,
-            #[cfg(feature = "tee")]
-            sender_io.clone(),
         )
         .map_err(Error::Vcpu)?;
 

src/vmm/src/builder.rs

@@ -207,9 +207,6 @@ pub struct Vmm {
     mmio_device_manager: MMIODeviceManager,
     #[cfg(target_arch = "x86_64")]
     pio_device_manager: PortIODeviceManager,
-
-    #[cfg(feature = "tee")]
-    pub guest_memfd_vec: Vec<RawFd>,
 }
 
 impl Vmm {

src/vmm/src/lib.rs

@@ -13,12 +13,6 @@ use std::io;
 
 use std::os::unix::io::RawFd;
 
-#[cfg(feature = "tee")]
-use kvm_ioctls::VcpuExit::Unsupported;
-
-use std::sync::Arc;
-use std::sync::Mutex;
-
 use std::result;
 use std::sync::atomic::{fence, Ordering};
 #[cfg(not(test))]
@@ -45,18 +39,14 @@ use kvm_bindings::{
     KVM_CLOCK_TSC_STABLE, KVM_IRQCHIP_IOAPIC, KVM_IRQCHIP_PIC_MASTER, KVM_IRQCHIP_PIC_SLAVE,
     KVM_MAX_CPUID_ENTRIES,
 };
-#[cfg(feature = "tee")]
 use kvm_bindings::{
-    kvm_create_guest_memfd, kvm_memory_attributes, kvm_userspace_memory_region2, KVM_API_VERSION,
-    KVM_EXIT_MEMORY_FAULT, KVM_MEMORY_ATTRIBUTE_PRIVATE, KVM_MEMORY_EXIT_FLAG_PRIVATE,
-    KVM_MEM_GUEST_MEMFD,
+    kvm_create_guest_memfd, kvm_memory_attributes, kvm_userspace_memory_region,
+    kvm_userspace_memory_region2, KVM_API_VERSION, KVM_MEMORY_ATTRIBUTE_PRIVATE,
+    KVM_MEM_GUEST_MEMFD, KVM_SYSTEM_EVENT_RESET, KVM_SYSTEM_EVENT_SHUTDOWN,
 };
-#[cfg(not(feature = "tee"))]
-use kvm_bindings::{kvm_userspace_memory_region, KVM_API_VERSION};
-use kvm_bindings::{
-    kvm_userspace_memory_region, KVM_API_VERSION, KVM_SYSTEM_EVENT_RESET, KVM_SYSTEM_EVENT_SHUTDOWN,
-};
-use kvm_ioctls::*;
+#[cfg(feature = "tee")]
+use kvm_bindings::{kvm_enable_cap, KVM_CAP_EXIT_HYPERCALL, KVM_MEMORY_EXIT_FLAG_PRIVATE};
+use kvm_ioctls::{Cap::*, *};
 use utils::eventfd::EventFd;
 use utils::signal::{register_signal_handler, sigrtmin, Killable};
 use utils::sm::StateMachine;
@@ -781,13 +771,6 @@ pub struct VcpuConfig {
     pub cpu_template: Option<CpuFeaturesTemplate>,
 }
 
-#[cfg(feature = "tee")]
-pub struct MemProperties {
-    pub addr: u64,
-    pub size: u64,
-    pub attributes: u32,
-}
-
 // Using this for easier explicit type-casting to help IDEs interpret the code.
 type VcpuCell = Cell<Option<*mut Vcpu>>;
 
@@ -810,10 +793,6 @@ pub struct Vcpu {
     #[cfg(target_arch = "aarch64")]
     mpidr: u64,
 
-    // The transmitting end of the events channel which will be given to the vcpu side
-    #[cfg(feature = "tee")]
-    sender_io: Sender<MemProperties>,
-
     // The receiving end of events channel owned by the vcpu side.
     event_receiver: Receiver<VcpuEvent>,
     // The transmitting end of the events channel which will be given to the handler.
@@ -961,12 +940,7 @@ impl Vcpu {
     /// * `exit_evt` - An `EventFd` that will be written into when this vcpu exits.
     /// * `create_ts` - A timestamp used by the vcpu to calculate its lifetime.
     #[cfg(target_arch = "aarch64")]
-    pub fn new_aarch64(
-        id: u8,
-        vm_fd: &VmFd,
-        exit_evt: EventFd,
-        #[cfg(feature = "tee")] sender_io: Sender<MemProperties>,
-    ) -> Result<Self> {
+    pub fn new_aarch64(id: u8, vm_fd: &VmFd, exit_evt: EventFd) -> Result<Self> {
         let kvm_vcpu = vm_fd.create_vcpu(id as u64).map_err(Error::VcpuFd)?;
         let (event_sender, event_receiver) = unbounded();
         let (response_sender, response_receiver) = unbounded();
@@ -981,8 +955,6 @@ impl Vcpu {
             event_sender: Some(event_sender),
             response_receiver: Some(response_receiver),
             response_sender,
-            #[cfg(feature = "tee")]
-            sender_io,
         })
     }
 
@@ -1271,6 +1243,17 @@ impl Vcpu {
                     self.io_bus.write(0, u64::from(addr), data);
                     Ok(VcpuEmulation::Handled)
                 }
+                #[cfg(feature = "tee")]
+                VcpuExit::MemoryFault { gpa, size, flags } => {
+                    let private = (flags & (KVM_MEMORY_EXIT_FLAG_PRIVATE as u64)) != 0;
+
+                    let mem_properties = MemoryProperties { gpa, size, private };
+
+                    self.pm_sender.0.send(mem_properties).unwrap();
+                    let _ = self.pm_sender.1.read().unwrap();
+
+                    Ok(VcpuEmulation::Handled)
+                }
                 VcpuExit::MmioRead(addr, data) => {
                     if let Some(ref mmio_bus) = self.mmio_bus {
                         mmio_bus.read(0, addr, data);
@@ -1291,38 +1274,6 @@ impl Vcpu {
                     info!("Received KVM_EXIT_SHUTDOWN signal");
                     Ok(VcpuEmulation::Stopped)
                 }
-                #[cfg(feature = "tee")]
-                VcpuExit::MemoryFault { flags, gpa, size } => {
-                    if flags & !KVM_MEMORY_EXIT_FLAG_PRIVATE as u64 != 0 {
-                        error!("KVM_EXIT_MEMORY_FAULT: Unknown flag {}", flags);
-                        Err(Error::VcpuUnhandledKvmExit)
-                    } else {
-                        // from private to shared
-                        let mut attr = 0;
-                        // from shared to private
-                        if flags & KVM_MEMORY_EXIT_FLAG_PRIVATE as u64
-                            == KVM_MEMORY_EXIT_FLAG_PRIVATE as u64
-                        {
-                            attr = KVM_MEMORY_ATTRIBUTE_PRIVATE;
-                        };
-
-                        let _ = self.sender_io.try_send(MemProperties {
-                            addr: gpa,
-                            size,
-                            attributes: attr,
-                        });
-                        Ok(VcpuEmulation::Handled)
-                    }
-                }
-                // Documentation specifices that when KVM exists with KVM_EXIT_MEMORY_FAULT,
-                // userspace should assume kvm_run.exit_reason is stale/undefined for error numbers
-                // different than EFAULT or EHWPOISON
-                #[cfg(feature = "tee")]
-                Unsupported(KVM_EXIT_MEMORY_FAULT) => Ok(VcpuEmulation::Handled),
-                VcpuExit::InternalError => {
-                    error!("Received KVM_EXIT_INTERNAL_ERROR signal");
-                    Err(Error::VcpuUnhandledKvmExit)
-                }
                 // Documentation specifies that below kvm exits are considered
                 // errors.
                 VcpuExit::FailEntry(reason, vcpu) => {