vmm/linux/tee: Handle KVM_EXIT_HYPERCALL exits

SEV-SNP guests use KVM_EXIT_HYPERCALL exits to signal to the hypervisor
it would like some pages set to private or shared.

Implements a handler that manages guest memory and can set regions to
private or shared. vCPUs can send "memory properties" messages
to the handler indicating:

- Guest GPA
- Size of memory region
- Whether the region should be set to private or shared

The handler will read these messages and configure the memory regions
accordingly.

Signed-off-by: Tyler Fanelli <[email protected]>

Author: tylerfanelli

Cargo.lock

@@ -31,6 +31,10 @@ vmm = { path = "../vmm" }
 [target.'cfg(target_os = "macos")'.dependencies]
 hvf = { path = "../hvf" }
 
+[target.'cfg(target_os = "linux")'.dependencies]
+kvm-bindings = ">=0.11"
+vm-memory = ">=0.13"
+
 [lib]
 name = "krun"
 crate-type = ["cdylib"]

src/libkrun/Cargo.toml

@@ -5,6 +5,8 @@ use std::collections::hash_map::Entry;
 use std::collections::HashMap;
 use std::convert::TryInto;
 use std::env;
+#[cfg(feature = "tee")]
+use std::ffi::c_void;
 use std::ffi::CStr;
 #[cfg(target_os = "linux")]
 use std::ffi::CString;
@@ -20,7 +22,7 @@ use std::sync::atomic::{AtomicI32, Ordering};
 use std::sync::LazyLock;
 use std::sync::Mutex;
 
-#[cfg(target_os = "macos")]
+#[cfg(any(target_os = "macos", feature = "tee"))]
 use crossbeam_channel::unbounded;
 #[cfg(feature = "blk")]
 use devices::virtio::block::ImageType;
@@ -54,6 +56,17 @@ use vmm::vmm_config::machine_config::VmConfig;
 use vmm::vmm_config::net::NetworkInterfaceConfig;
 use vmm::vmm_config::vsock::VsockDeviceConfig;
 
+#[cfg(feature = "tee")]
+use kvm_bindings::{kvm_memory_attributes, KVM_MEMORY_ATTRIBUTE_PRIVATE};
+
+#[cfg(feature = "tee")]
+use vm_memory::{guest_memory::GuestMemory, GuestAddress, GuestMemoryRegion, MemoryRegionAddress};
+
+#[cfg(feature = "tee")]
+use libc::{
+    fallocate, madvise, EFD_SEMAPHORE, FALLOC_FL_KEEP_SIZE, FALLOC_FL_PUNCH_HOLE, MADV_DONTNEED,
+};
+
 // Value returned on success. We use libc's errors otherwise.
 const KRUN_SUCCESS: i32 = 0;
 // Maximum number of arguments/environment variables we allow
@@ -1471,12 +1484,25 @@ pub extern "C" fn krun_start_enter(ctx_id: u32) -> i32 {
     #[cfg(target_os = "macos")]
     let (sender, receiver) = unbounded();
 
+    #[cfg(feature = "tee")]
+    let (pm_sender, pm_receiver) = unbounded();
+    #[cfg(feature = "tee")]
+    let pm_efd =
+        EventFd::new(EFD_SEMAPHORE).expect("unable to create TEE memory properties eventfd");
+
     let _vmm = match vmm::builder::build_microvm(
         &ctx_cfg.vmr,
         &mut event_manager,
         ctx_cfg.shutdown_efd,
         #[cfg(target_os = "macos")]
         sender,
+        #[cfg(feature = "tee")]
+        (
+            pm_sender,
+            pm_efd
+                .try_clone()
+                .expect("unable to clone TEE memory properties eventfd"),
+        ),
     ) {
         Ok(vmm) => vmm,
         Err(e) => {
@@ -1485,7 +1511,7 @@ pub extern "C" fn krun_start_enter(ctx_id: u32) -> i32 {
         }
     };
 
-    #[cfg(target_os = "macos")]
+    #[cfg(any(target_os = "macos", feature = "tee"))]
     let mapper_vmm = _vmm.clone();
 
     #[cfg(target_os = "macos")]
@@ -1508,6 +1534,93 @@ pub extern "C" fn krun_start_enter(ctx_id: u32) -> i32 {
             .unwrap();
     }
 
+    #[cfg(feature = "tee")]
+    let guest_mem = _vmm.lock().unwrap().guest_memory().clone();
+
+    #[cfg(feature = "tee")]
+    std::thread::Builder::new()
+        .name("TEE memory properties worker".into())
+        .spawn(move || loop {
+            match pm_receiver.recv() {
+                Err(e) => error!("Error in pm receiver: {:?}", e),
+                Ok(m) => {
+                    let (guest_memfd, region_start) = mapper_vmm
+                        .lock()
+                        .unwrap()
+                        .kvm_vm()
+                        .guest_memfd_get(m.gpa)
+                        .unwrap_or_else(|| panic!("unable to find KVM guest_memfd for memory region corresponding to GPA 0x{:x}", m.gpa));
+
+                    let attributes: u64 = if m.private {
+                        KVM_MEMORY_ATTRIBUTE_PRIVATE as u64
+                    } else {
+                        0
+                    };
+
+                    let attr = kvm_memory_attributes {
+                        address: m.gpa,
+                        size: m.size,
+                        attributes,
+                        flags: 0,
+                    };
+
+                    mapper_vmm
+                        .lock()
+                        .unwrap()
+                        .kvm_vm()
+                        .fd()
+                        .set_memory_attributes(attr)
+                        .unwrap_or_else(|_| panic!("unable to set memory attributes for memory region corresponding to guest address 0x{:x}", m.gpa));
+
+                    let region = guest_mem.find_region(GuestAddress(m.gpa));
+                    if region.is_none() {
+                        error!("{}", format!("guest memory region corresponding to GPA 0x{:x} not found", m.gpa));
+                        pm_efd.write(1).unwrap();
+                        continue;
+                    }
+
+                    let offset = m.gpa - region_start;
+
+                    if m.private {
+                        let region_addr = MemoryRegionAddress(offset);
+
+                        let host_startaddr = region
+                            .unwrap()
+                            .get_host_address(region_addr)
+                            .expect("host address corresponding to memory region address 0x{:x} not found");
+
+                        let ret = unsafe {
+                            madvise(
+                                host_startaddr as *mut c_void,
+                                m.size.try_into().unwrap(),
+                                MADV_DONTNEED,
+                            )
+                        };
+
+                        if ret < 0 {
+                            error!("{}", format!("unable to advise kernel that memory region corresponding to GPA 0x{:x} will likely not be needed (madvise)", m.gpa));
+                        }
+                    } else {
+                        let ret = unsafe {
+                            fallocate(
+                                guest_memfd,
+                                FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
+                                offset as i64,
+                                m.size as i64,
+                            )
+                        };
+
+                        if ret < 0 {
+                            error!("{}", format!("unable to allocate space in guest_memfd for shared memory (fallocate)"));
+                        }
+                    }
+
+                    pm_efd.write(1).unwrap();
+                }
+            }
+        })
+        .unwrap();
+
     loop {
         match event_manager.run() {
             Ok(_) => {}

src/libkrun/src/lib.rs

@@ -21,6 +21,7 @@ libc = ">=0.2.39"
 linux-loader = { version = "0.13.0", features = ["bzimage", "elf", "pe"] }
 log = "0.4.0"
 vm-memory = { version = ">=0.13", features = ["backend-mmap"] }
+rangemap = "1.5.1"
 
 arch = { path = "../arch" }
 devices = { path = "../devices" }

src/vmm/Cargo.toml

@@ -4,7 +4,11 @@
 //! Enables pre-boot setup, instantiation and booting of a Firecracker VMM.
 
 #[cfg(target_os = "macos")]
-use crossbeam_channel::{unbounded, Sender};
+use crossbeam_channel::unbounded;
+
+#[cfg(any(target_os = "macos", feature = "tee"))]
+use crossbeam_channel::Sender;
+
 use kernel::cmdline::Cmdline;
 #[cfg(target_os = "macos")]
 use std::collections::HashMap;
@@ -21,6 +25,8 @@ use super::{Error, Vmm};
 #[cfg(target_arch = "x86_64")]
 use crate::device_manager::legacy::PortIODeviceManager;
 use crate::device_manager::mmio::MMIODeviceManager;
+#[cfg(feature = "tee")]
+use crate::linux::vstate::MemoryProperties;
 use crate::resources::VmResources;
 use crate::vmm_config::external_kernel::{ExternalKernel, KernelFormat};
 #[cfg(all(target_os = "linux", target_arch = "aarch64"))]
@@ -507,6 +513,7 @@ pub fn build_microvm(
     event_manager: &mut EventManager,
     _shutdown_efd: Option<EventFd>,
     #[cfg(target_os = "macos")] _map_sender: Sender<MemoryMapping>,
+    #[cfg(feature = "tee")] pm_sender: (Sender<MemoryProperties>, EventFd),
 ) -> std::result::Result<Arc<Mutex<Vmm>>, StartMicrovmError> {
     let payload = choose_payload(vm_resources)?;
 
@@ -672,6 +679,8 @@ pub fn build_microvm(
             payload_config.entry_addr,
             &pio_device_manager.io_bus,
             &exit_evt,
+            #[cfg(feature = "tee")]
+            pm_sender,
         )
         .map_err(StartMicrovmError::Internal)?;
     }
@@ -1422,6 +1431,7 @@ fn create_vcpus_x86_64(
     entry_addr: GuestAddress,
     io_bus: &devices::Bus,
     exit_evt: &EventFd,
+    #[cfg(feature = "tee")] pm_sender: (Sender<MemoryProperties>, EventFd),
 ) -> super::Result<Vec<Vcpu>> {
     let mut vcpus = Vec::with_capacity(vcpu_config.vcpu_count as usize);
     for cpu_index in 0..vcpu_config.vcpu_count {
@@ -1432,6 +1442,8 @@ fn create_vcpus_x86_64(
             vm.supported_msrs().clone(),
             io_bus.clone(),
             exit_evt.try_clone().map_err(Error::EventFd)?,
+            #[cfg(feature = "tee")]
+            (pm_sender.0.clone(), pm_sender.1.try_clone().unwrap()),
         )
         .map_err(Error::Vcpu)?;