mshv works

Signed-off-by: Ludvig Liljenberg <[email protected]>

Author: ludfjig

src/hyperlight_host/src/hypervisor/hyperv_linux.rs

@@ -25,6 +25,8 @@ extern crate mshv_bindings3 as mshv_bindings;
 extern crate mshv_ioctls3 as mshv_ioctls;
 
 use std::fmt::{Debug, Formatter};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::Arc;
 
 use log::{error, LevelFilter};
 #[cfg(mshv2)]
@@ -56,8 +58,9 @@ use super::gdb::{DebugCommChannel, DebugMsg, DebugResponse, GuestDebug, MshvDebu
 use super::handlers::DbgMemAccessHandlerWrapper;
 use super::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper};
 use super::{
-    Hypervisor, VirtualCPU, CR0_AM, CR0_ET, CR0_MP, CR0_NE, CR0_PE, CR0_PG, CR0_WP, CR4_OSFXSR,
-    CR4_OSXMMEXCPT, CR4_PAE, EFER_LMA, EFER_LME, EFER_NX, EFER_SCE,
+    Hypervisor, InterruptHandle, LinuxInterruptHandle, VirtualCPU, CR0_AM, CR0_ET, CR0_MP, CR0_NE,
+    CR0_PE, CR0_PG, CR0_WP, CR4_OSFXSR, CR4_OSXMMEXCPT, CR4_PAE, EFER_LMA, EFER_LME, EFER_NX,
+    EFER_SCE,
 };
 use crate::hypervisor::HyperlightExit;
 use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
@@ -292,6 +295,7 @@ pub(crate) struct HypervLinuxDriver {
     entrypoint: u64,
     mem_regions: Vec<MemoryRegion>,
     orig_rsp: GuestPtr,
+    interrupt_handle: Arc<LinuxInterruptHandle>,
 
     #[cfg(gdb)]
     debug: Option<MshvDebug>,
@@ -389,6 +393,11 @@ impl HypervLinuxDriver {
             mem_regions,
             entrypoint: entrypoint_ptr.absolute()?,
             orig_rsp: rsp_ptr,
+            interrupt_handle: Arc::new(LinuxInterruptHandle {
+                running: AtomicBool::new(false),
+                tid: AtomicU64::new(unsafe { libc::pthread_self() }),
+                dropped: AtomicBool::new(false),
+            }),
 
             #[cfg(gdb)]
             debug,
@@ -572,13 +581,38 @@ impl Hypervisor for HypervLinuxDriver {
         #[cfg(gdb)]
         const EXCEPTION_INTERCEPT: hv_message_type = hv_message_type_HVMSG_X64_EXCEPTION_INTERCEPT;
 
+        self.interrupt_handle
+            .tid
+            .store(unsafe { libc::pthread_self() as u64 }, Ordering::Relaxed);
+        // Note: if a `InterruptHandle::kill()` signal is delivered to this thread **here**
+        // - before we've set the running to true,
+        // Then the signal does not have any effect, because the signal handler is a no-op.
+        self.interrupt_handle.running.store(true, Ordering::Relaxed);
+        // Note: if a `InterruptHandle::kill()` signal is delivered to this thread **here**
+        // - after we've set the running to true,
+        // - before we've called `VcpuFd::run()`
+        // Then the individual signal is lost, because the signal is only processed after we've left userspace.
+        // However, for this reason, we keep sending the signal again and again until we see that the atomic `running` is set to false.
         #[cfg(mshv2)]
         let run_result = {
             let hv_message: hv_message = Default::default();
-            &self.vcpu_fd.run(hv_message)
+            self.vcpu_fd.run(hv_message)
         };
         #[cfg(mshv3)]
-        let run_result = &self.vcpu_fd.run();
+        let run_result = self.vcpu_fd.run();
+        // Note: if a `InterruptHandle::kill()` signal is delivered to this thread **here**
+        // - after we've called `VcpuFd::run()`
+        // - before we've set the running to false
+        // Then this is fine because the call to `VcpuFd::run()` is already finished,
+        // the signal handler itself is a no-op, and the signals will stop being sent
+        // once we've set the `running` to false.
+        self.interrupt_handle
+            .running
+            .store(false, Ordering::Relaxed);
+        // Note: if a `InterruptHandle::kill()` signal is delivered to this thread **here**
+        // - after we've set the running to true,
+        // Then the signal does not have any effect, because the signal handler is a no-op.
+        // This is fine since we are already done with the `VcpuFd::run()` call.
 
         let result = match run_result {
             Ok(m) => match m.header.message_type {
@@ -673,6 +707,10 @@ impl Hypervisor for HypervLinuxDriver {
         self as &mut dyn Hypervisor
     }
 
+    fn interrupt_handle(&self) -> Arc<dyn InterruptHandle> {
+        self.interrupt_handle.clone()
+    }
+
     #[cfg(crashdump)]
     fn get_memory_regions(&self) -> &[MemoryRegion] {
         &self.mem_regions
@@ -722,10 +760,6 @@ impl Hypervisor for HypervLinuxDriver {
 
         Ok(())
     }
-
-    fn interrupt_handle(&self) -> std::sync::Arc<dyn super::InterruptHandle> {
-        todo!()
-    }
 }
 
 impl Drop for HypervLinuxDriver {

src/hyperlight_host/src/hypervisor/kvm.rs

@@ -34,9 +34,9 @@ use super::gdb::{DebugCommChannel, DebugMsg, DebugResponse, GuestDebug, KvmDebug
 use super::handlers::DbgMemAccessHandlerWrapper;
 use super::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper};
 use super::{
-    HyperlightExit, Hypervisor, InterruptHandle, VirtualCPU, CR0_AM, CR0_ET, CR0_MP, CR0_NE,
-    CR0_PE, CR0_PG, CR0_WP, CR4_OSFXSR, CR4_OSXMMEXCPT, CR4_PAE, EFER_LMA, EFER_LME, EFER_NX,
-    EFER_SCE,
+    HyperlightExit, Hypervisor, InterruptHandle, LinuxInterruptHandle, VirtualCPU, CR0_AM, CR0_ET,
+    CR0_MP, CR0_NE, CR0_PE, CR0_PG, CR0_WP, CR4_OSFXSR, CR4_OSXMMEXCPT, CR4_PAE, EFER_LMA,
+    EFER_LME, EFER_NX, EFER_SCE,
 };
 use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
 use crate::mem::ptr::{GuestPtr, RawPtr};
@@ -283,24 +283,14 @@ pub(crate) struct KVMDriver {
     entrypoint: u64,
     orig_rsp: GuestPtr,
     mem_regions: Vec<MemoryRegion>,
-    interrupt_handle: Arc<KvmInterruptHandle>,
+    interrupt_handle: Arc<LinuxInterruptHandle>,
 
     #[cfg(gdb)]
     debug: Option<KvmDebug>,
     #[cfg(gdb)]
     gdb_conn: Option<DebugCommChannel<DebugResponse, DebugMsg>>,
 }
 
-#[derive(Debug)]
-pub(super) struct KvmInterruptHandle {
-    /// True when the vcpu is currently running and blocking the thread
-    running: AtomicBool,
-    /// The thread id on which the vcpu was most recently run on or is currently running on
-    tid: AtomicU64,
-    /// Whether the corresponding vm is dropped
-    dropped: AtomicBool,
-}
-
 impl KVMDriver {
     /// Create a new instance of a `KVMDriver`, with only control registers
     /// set. Standard registers will not be set, and `initialise` must
@@ -358,7 +348,7 @@ impl KVMDriver {
             entrypoint,
             orig_rsp: rsp_gp,
             mem_regions,
-            interrupt_handle: Arc::new(KvmInterruptHandle {
+            interrupt_handle: Arc::new(LinuxInterruptHandle {
                 running: AtomicBool::new(false),
                 tid: AtomicU64::new(unsafe { libc::pthread_self() }),
                 dropped: AtomicBool::new(false),
@@ -681,23 +671,3 @@ impl Drop for KVMDriver {
         self.interrupt_handle.dropped.store(true, Ordering::Relaxed);
     }
 }
-
-impl InterruptHandle for KvmInterruptHandle {
-    fn kill(&self) -> bool {
-        let sigrtmin = libc::SIGRTMIN();
-        let mut sent_signal = false;
-
-        while self.running.load(Ordering::Relaxed) {
-            sent_signal = true;
-            unsafe {
-                libc::pthread_kill(self.tid.load(Ordering::Relaxed) as _, sigrtmin);
-            }
-            std::thread::sleep(std::time::Duration::from_micros(50));
-        }
-
-        sent_signal
-    }
-    fn dropped(&self) -> bool {
-        self.dropped.load(Ordering::Relaxed)
-    }
-}

src/hyperlight_host/src/hypervisor/mod.rs

@@ -36,7 +36,7 @@ pub(crate) mod hyperv_windows;
 
 /// GDB debugging support
 #[cfg(gdb)]
-mod gdb;
+pub(crate) mod gdb;
 
 #[cfg(kvm)]
 /// Functionality to manipulate KVM-based virtual machines
@@ -59,6 +59,7 @@ pub(crate) mod crashdump;
 
 use std::fmt::Debug;
 use std::str::FromStr;
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::sync::{Arc, Mutex};
 
 #[cfg(gdb)]
@@ -335,3 +336,33 @@ pub trait InterruptHandle: Send + Sync {
     /// Returns true iff the corresponding sandbox has been dropped
     fn dropped(&self) -> bool;
 }
+
+#[derive(Debug)]
+pub(super) struct LinuxInterruptHandle {
+    /// True when the vcpu is currently running and blocking the thread
+    running: AtomicBool,
+    /// The thread id on which the vcpu was most recently run on or is currently running on
+    tid: AtomicU64,
+    /// Whether the corresponding vm is dropped
+    dropped: AtomicBool,
+}
+
+impl InterruptHandle for LinuxInterruptHandle {
+    fn kill(&self) -> bool {
+        let sigrtmin = libc::SIGRTMIN();
+        let mut sent_signal = false;
+
+        while self.running.load(Ordering::Relaxed) {
+            sent_signal = true;
+            unsafe {
+                libc::pthread_kill(self.tid.load(Ordering::Relaxed) as _, sigrtmin);
+            }
+            std::thread::sleep(std::time::Duration::from_micros(50));
+        }
+
+        sent_signal
+    }
+    fn dropped(&self) -> bool {
+        self.dropped.load(Ordering::Relaxed)
+    }
+}

src/hyperlight_host/src/sandbox/initialized_multi_use.rs

@@ -26,6 +26,8 @@ use super::host_funcs::FunctionRegistry;
 use super::{MemMgrWrapper, WrapperGetter};
 use crate::func::call_ctx::MultiUseGuestCallContext;
 use crate::func::guest_err::check_for_guest_error;
+#[cfg(gdb)]
+use crate::hypervisor::handlers::DbgMemAccessHandlerWrapper;
 use crate::hypervisor::handlers::{MemAccessHandlerCaller, OutBHandlerCaller};
 use crate::hypervisor::{Hypervisor, InterruptHandle};
 use crate::mem::ptr::RawPtr;
@@ -51,6 +53,8 @@ pub struct MultiUseSandbox {
     out_hdl: Arc<Mutex<dyn OutBHandlerCaller>>,
     mem_hdl: Arc<Mutex<dyn MemAccessHandlerCaller>>,
     dispatch_ptr: RawPtr,
+    #[cfg(gdb)]
+    dbg_mem_access_fn: DbgMemAccessHandlerWrapper,
 }
 
 impl MultiUseSandbox {
@@ -67,6 +71,7 @@ impl MultiUseSandbox {
         out_hdl: Arc<Mutex<dyn OutBHandlerCaller>>,
         mem_hdl: Arc<Mutex<dyn MemAccessHandlerCaller>>,
         dispatch_ptr: RawPtr,
+        #[cfg(gdb)] dbg_mem_access_fn: DbgMemAccessHandlerWrapper,
     ) -> MultiUseSandbox {
         Self {
             _host_funcs: host_funcs,
@@ -75,6 +80,8 @@ impl MultiUseSandbox {
             out_hdl,
             mem_hdl,
             dispatch_ptr,
+            #[cfg(gdb)]
+            dbg_mem_access_fn,
         }
     }
 
@@ -194,6 +201,7 @@ impl MultiUseSandbox {
             self.dispatch_ptr.clone(),
             self.out_hdl.clone(),
             self.mem_hdl.clone(),
+            self.dbg_mem_access_fn.clone(),
         )?;
 
         self.check_stack_guard()?;

src/hyperlight_host/src/sandbox/uninitialized_evolve.rs

@@ -67,8 +67,13 @@ where
     ) -> Result<ResSandbox>,
 {
     let (hshm, mut gshm) = u_sbox.mgr.build();
-    let mut vm = set_up_hypervisor_partition(&mut gshm)?;
+    let mut vm = set_up_hypervisor_partition(
+        &mut gshm,
+        #[cfg(gdb)]
+        &u_sbox.debug_info,
+    )?;
     let outb_hdl = outb_handler_wrapper(hshm.clone(), u_sbox.host_funcs.clone());
+
     let seed = {
         let mut rng = rand::rng();
         rng.random::<u64>()
@@ -94,7 +99,7 @@ where
         mem_access_hdl.clone(),
         u_sbox.max_guest_log_level,
         #[cfg(gdb)]
-        u_sbox.debug_info,
+        dbg_mem_access_hdl,
     )?;
 
     let dispatch_function_addr = hshm.as_ref().get_pointer_to_dispatch_function()?;
@@ -122,11 +127,13 @@ pub(super) fn evolve_impl_multi_use(u_sbox: UninitializedSandbox) -> Result<Mult
             }
             Ok(MultiUseSandbox::from_uninit(
                 hf,
-                hshm,
+                hshm.clone(),
                 vm,
                 out_hdl,
                 mem_hdl,
                 dispatch_ptr,
+                #[cfg(gdb)]
+                dbg_mem_access_handler_wrapper(hshm),
             ))
         },
     )
@@ -171,6 +178,8 @@ fn set_up_hypervisor_partition(
     // Create gdb thread if gdb is enabled and the configuration is provided
     #[cfg(gdb)]
     let gdb_conn = if let Some(DebugInfo { port }) = debug_info {
+        use crate::hypervisor::gdb::create_gdb_thread;
+
         let gdb_conn = create_gdb_thread(*port, unsafe { libc::pthread_self() });
 
         // in case the gdb thread creation fails, we still want to continue