add a way to read/write host shared mem when debugging

Signed-off-by: Doru Blânzeanu <[email protected]>

Author: dblnz

src/hyperlight_host/src/hypervisor/handlers.rs

@@ -102,3 +102,96 @@ impl MemAccessHandlerCaller for MemAccessHandler {
         func()
     }
 }
+
+/// The trait representing custom logic to handle the case when
+/// a Hypervisor's virtual CPU (vCPU) informs Hyperlight a debug memory access
+/// has been requested.
+pub trait DbgMemAccessHandlerCaller: Send {
+    /// Function that gets called when a read is requested.
+    fn read(&mut self, addr: usize, data: &mut [u8]) -> Result<()>;
+
+    /// Function that gets called when a write is requested.
+    fn write(&mut self, addr: usize, data: &[u8]) -> Result<()>;
+
+    /// Function that gets called for a request to get guest code offset.
+    fn get_code_offset(&mut self) -> Result<usize>;
+}
+
+/// A convenient type representing a common way `DbgMemAccessHandler` implementations
+/// are passed as parameters to functions
+///
+/// Note: This needs to be wrapped in a Mutex to be able to grab a mutable
+/// reference to the underlying data 
+pub type DbgMemAccessHandlerWrapper = Arc<Mutex<dyn DbgMemAccessHandlerCaller>>;
+
+pub(crate) type DbgReadMemAccessHandlerFunction =
+    Box<dyn FnMut(usize, &mut [u8]) -> Result<()> + Send>;
+pub(crate) type DbgWriteMemAccessHandlerFunction =
+    Box<dyn FnMut(usize, &[u8]) -> Result<()> + Send>;
+pub(crate) type DbgGetCodeAddrHandlerFunction = Box<dyn FnMut() -> Result<usize> + Send>;
+
+/// A `DbgMemAccessHandler` implementation using 
+/// (`DbgReadMemAccessHandlerFunction`, `DbgWriteMemAccessHandlerFunction `, `DbgGetCodeAddrHandlerFunction `).
+///
+/// Note: This handler must live for as long as its Sandbox or for
+/// static in the case of its C API usage.
+pub(crate) struct DbgMemAccessHandler (
+    Arc<Mutex<DbgReadMemAccessHandlerFunction>>,
+    Arc<Mutex<DbgWriteMemAccessHandlerFunction>>,
+    Arc<Mutex<DbgGetCodeAddrHandlerFunction>>,
+);
+
+impl
+    From<(
+        DbgReadMemAccessHandlerFunction,
+        DbgWriteMemAccessHandlerFunction,
+        DbgGetCodeAddrHandlerFunction,
+    )> for DbgMemAccessHandler
+{
+    #[instrument(skip_all, parent = Span::current(), level= "Trace")]
+    fn from(
+        (f1, f2, f3): (
+            DbgReadMemAccessHandlerFunction,
+            DbgWriteMemAccessHandlerFunction,
+            DbgGetCodeAddrHandlerFunction,
+        ),
+    ) -> Self {
+        Self(
+            Arc::new(Mutex::new(f1)),
+            Arc::new(Mutex::new(f2)),
+            Arc::new(Mutex::new(f3)),
+        )
+    }
+}
+
+impl DbgMemAccessHandlerCaller for DbgMemAccessHandler {
+    #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
+    fn read(&mut self, addr: usize, data: &mut [u8]) -> Result<()> {
+        let mut read_func = self
+            .0
+            .try_lock()
+            .map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))?;
+
+        read_func(addr, data)
+    }
+
+    #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
+    fn write(&mut self, addr: usize, data: &[u8]) -> Result<()> {
+        let mut write_func = self
+            .1
+            .try_lock()
+            .map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))?;
+
+        write_func(addr, data)
+    }
+
+    #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
+    fn get_code_offset(&mut self) -> Result<usize> {
+        let mut get_code_offset_func = self
+            .2
+            .try_lock()
+            .map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))?;
+
+        get_code_offset_func()
+    }
+}

src/hyperlight_host/src/hypervisor/hyperv_linux.rs

@@ -44,6 +44,8 @@ use mshv_ioctls::{Mshv, VcpuFd, VmFd};
 use tracing::{instrument, Span};
 
 use super::fpu::{FP_CONTROL_WORD_DEFAULT, FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT};
+#[cfg(gdb)]
+use super::handlers::DbgMemAccessHandlerWrapper;
 use super::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper};
 use super::{
     Hypervisor, VirtualCPU, CR0_AM, CR0_ET, CR0_MP, CR0_NE, CR0_PE, CR0_PG, CR0_WP, CR4_OSFXSR,
@@ -203,6 +205,7 @@ impl Hypervisor for HypervLinuxDriver {
         outb_hdl: OutBHandlerWrapper,
         mem_access_hdl: MemAccessHandlerWrapper,
         hv_handler: Option<HypervisorHandler>,
+        #[cfg(gdb)] dbg_mem_access_fn: DbgMemAccessHandlerWrapper,
     ) -> Result<()> {
         let regs = StandardRegisters {
             rip: self.entrypoint,
@@ -224,6 +227,8 @@ impl Hypervisor for HypervLinuxDriver {
             hv_handler,
             outb_hdl,
             mem_access_hdl,
+            #[cfg(gdb)]
+            dbg_mem_access_fn,
         )?;
 
         // reset RSP to what it was before initialise
@@ -242,6 +247,7 @@ impl Hypervisor for HypervLinuxDriver {
         outb_handle_fn: OutBHandlerWrapper,
         mem_access_fn: MemAccessHandlerWrapper,
         hv_handler: Option<HypervisorHandler>,
+        #[cfg(gdb)] dbg_mem_access_fn: DbgMemAccessHandlerWrapper,
     ) -> Result<()> {
         // Reset general purpose registers except RSP, then set RIP
         let rsp_before = self.vcpu_fd.get_regs()?.rsp;
@@ -268,6 +274,8 @@ impl Hypervisor for HypervLinuxDriver {
             hv_handler,
             outb_handle_fn,
             mem_access_fn,
+            #[cfg(gdb)]
+            dbg_mem_access_fn,
         )?;
 
         // reset RSP to what it was before function call

src/hyperlight_host/src/hypervisor/hyperv_windows.rs

@@ -28,6 +28,8 @@ use windows::Win32::System::Hypervisor::{
 };
 
 use super::fpu::{FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT};
+#[cfg(gdb)]
+use super::handlers::DbgMemAccessHandlerWrapper;
 use super::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper};
 use super::surrogate_process::SurrogateProcess;
 use super::surrogate_process_manager::*;
@@ -305,6 +307,7 @@ impl Hypervisor for HypervWindowsDriver {
         outb_hdl: OutBHandlerWrapper,
         mem_access_hdl: MemAccessHandlerWrapper,
         hv_handler: Option<HypervisorHandler>,
+        #[cfg(gdb)] dbg_mem_access_hdl: DbgMemAccessHandlerWrapper,
     ) -> Result<()> {
         let regs = WHvGeneralRegisters {
             rip: self.entrypoint,
@@ -326,6 +329,8 @@ impl Hypervisor for HypervWindowsDriver {
             hv_handler,
             outb_hdl,
             mem_access_hdl,
+            #[cfg(gdb)]
+            dbg_mem_access_hdl,
         )?;
 
         // reset RSP to what it was before initialise
@@ -344,6 +349,7 @@ impl Hypervisor for HypervWindowsDriver {
         outb_hdl: OutBHandlerWrapper,
         mem_access_hdl: MemAccessHandlerWrapper,
         hv_handler: Option<HypervisorHandler>,
+        #[cfg(gdb)] dbg_mem_access_hdl: DbgMemAccessHandlerWrapper,
     ) -> Result<()> {
         // Reset general purpose registers except RSP, then set RIP
         let rsp_before = self.processor.get_regs()?.rsp;
@@ -368,6 +374,7 @@ impl Hypervisor for HypervWindowsDriver {
             hv_handler,
             outb_hdl,
             mem_access_hdl,
+            dbg_mem_access_hdl,
         )?;
 
         // reset RSP to what it was before function call

src/hyperlight_host/src/hypervisor/hypervisor_handler.rs

@@ -37,6 +37,8 @@ use windows::Win32::System::Hypervisor::{WHvCancelRunVirtualProcessor, WHV_PARTI
 
 #[cfg(feature = "function_call_metrics")]
 use crate::histogram_vec_observe;
+#[cfg(gdb)]
+use crate::hypervisor::handlers::DbgMemAccessHandlerWrapper;
 use crate::hypervisor::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper};
 #[cfg(target_os = "windows")]
 use crate::hypervisor::wrappers::HandleWrapper;
@@ -187,6 +189,8 @@ pub(crate) struct HvHandlerConfig {
     pub(crate) outb_handler: OutBHandlerWrapper,
     pub(crate) mem_access_handler: MemAccessHandlerWrapper,
     pub(crate) max_wait_for_cancellation: Duration,
+    #[cfg(gdb)]
+    pub(crate) dbg_mem_access_handler: DbgMemAccessHandlerWrapper,
 }
 
 impl HypervisorHandler {
@@ -351,6 +355,8 @@ impl HypervisorHandler {
                                     configuration.outb_handler.clone(),
                                     configuration.mem_access_handler.clone(),
                                     Some(hv_handler_clone.clone()),
+                                    #[cfg(gdb)]
+                                    configuration.dbg_mem_access_handler.clone(),
                                 );
                                 drop(mem_lock_guard);
                                 drop(evar_lock_guard);
@@ -436,6 +442,8 @@ impl HypervisorHandler {
                                             configuration.outb_handler.clone(),
                                             configuration.mem_access_handler.clone(),
                                             Some(hv_handler_clone.clone()),
+                                            #[cfg(gdb)]
+                                            configuration.dbg_mem_access_handler.clone(),
                                         );
                                         histogram_vec_observe!(
                                             &GuestFunctionCallDurationMicroseconds,
@@ -451,6 +459,8 @@ impl HypervisorHandler {
                                         configuration.outb_handler.clone(),
                                         configuration.mem_access_handler.clone(),
                                         Some(hv_handler_clone.clone()),
+                                        #[cfg(gdb)]
+                                        configuration.dbg_mem_access_handler.clone(),
                                     )
                                 };
                                 drop(mem_lock_guard);

src/hyperlight_host/src/hypervisor/inprocess.rs

@@ -17,6 +17,8 @@ limitations under the License.
 use std::fmt::Debug;
 use std::os::raw::c_void;
 
+#[cfg(gdb)]
+use super::handlers::DbgMemAccessHandlerWrapper;
 use super::{HyperlightExit, Hypervisor};
 #[cfg(crashdump)]
 use crate::mem::memory_region::MemoryRegion;
@@ -73,6 +75,7 @@ impl<'a> Hypervisor for InprocessDriver<'a> {
         _outb_handle_fn: super::handlers::OutBHandlerWrapper,
         _mem_access_fn: super::handlers::MemAccessHandlerWrapper,
         _hv_handler: Option<super::hypervisor_handler::HypervisorHandler>,
+        #[cfg(gdb)] _dbg_mem_access_fn: DbgMemAccessHandlerWrapper,
     ) -> crate::Result<()> {
         let entrypoint_fn: extern "win64" fn(u64, u64, u64, u64) =
             unsafe { std::mem::transmute(self.args.entrypoint_raw as *const c_void) };
@@ -93,6 +96,7 @@ impl<'a> Hypervisor for InprocessDriver<'a> {
         _outb_handle_fn: super::handlers::OutBHandlerWrapper,
         _mem_access_fn: super::handlers::MemAccessHandlerWrapper,
         _hv_handler: Option<super::hypervisor_handler::HypervisorHandler>,
+        #[cfg(gdb)] _dbg_mem_access_fn: DbgMemAccessHandlerWrapper,
     ) -> crate::Result<()> {
         let ptr: u64 = dispatch_func_addr.into();
         let dispatch_func: extern "win64" fn() =

src/hyperlight_host/src/hypervisor/kvm.rs

@@ -16,6 +16,8 @@ limitations under the License.
 
 use std::convert::TryFrom;
 use std::fmt::Debug;
+#[cfg(gdb)]
+use std::sync::{Arc, Mutex};
 
 use kvm_bindings::{kvm_fpu, kvm_regs, kvm_userspace_memory_region, KVM_MEM_READONLY};
 use kvm_ioctls::Cap::UserMemory;
@@ -24,7 +26,9 @@ use tracing::{instrument, Span};
 
 use super::fpu::{FP_CONTROL_WORD_DEFAULT, FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT};
 #[cfg(gdb)]
-use super::gdb::{create_gdb_thread, DebugMsg, DebugResponse, DebugCommChannel, VcpuStopReason};
+use super::gdb::{create_gdb_thread, DebugCommChannel, DebugMsg, DebugResponse, VcpuStopReason};
+#[cfg(gdb)]
+use super::handlers::DbgMemAccessHandlerWrapper;
 use super::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper};
 use super::{
     HyperlightExit, Hypervisor, VirtualCPU, CR0_AM, CR0_ET, CR0_MP, CR0_NE, CR0_PE, CR0_PG, CR0_WP,
@@ -61,6 +65,7 @@ pub(crate) fn is_hypervisor_present() -> bool {
 
 #[cfg(gdb)]
 mod debug {
+    use std::sync::{Arc, Mutex};
     use kvm_bindings::{
         kvm_guest_debug, kvm_regs, KVM_GUESTDBG_ENABLE, KVM_GUESTDBG_SINGLESTEP,
         KVM_GUESTDBG_USE_HW_BP, KVM_GUESTDBG_USE_SW_BP,
@@ -69,6 +74,7 @@ mod debug {
 
     use super::KVMDriver;
     use crate::hypervisor::gdb::{DebugMsg, DebugResponse, VcpuStopReason, X86_64Regs};
+    use crate::hypervisor::handlers::DbgMemAccessHandlerCaller;
     use crate::{new_error, Result};
 
     /// KVM Debug struct
@@ -336,6 +342,7 @@ mod debug {
         pub fn process_dbg_request(
             &mut self,
             req: DebugMsg,
+            _dbg_mem_access_fn: Arc<Mutex<dyn DbgMemAccessHandlerCaller>>,
         ) -> Result<DebugResponse> {
             match req {
                 DebugMsg::AddHwBreakpoint(addr) => {
@@ -541,6 +548,7 @@ impl Hypervisor for KVMDriver {
         outb_hdl: OutBHandlerWrapper,
         mem_access_hdl: MemAccessHandlerWrapper,
         hv_handler: Option<HypervisorHandler>,
+        #[cfg(gdb)] dbg_mem_access_fn: DbgMemAccessHandlerWrapper,
     ) -> Result<()> {
         let regs = kvm_regs {
             rip: self.entrypoint,
@@ -561,6 +569,8 @@ impl Hypervisor for KVMDriver {
             hv_handler,
             outb_hdl,
             mem_access_hdl,
+            #[cfg(gdb)]
+            dbg_mem_access_fn,
         )?;
 
         // reset RSP to what it was before initialise
@@ -578,6 +588,7 @@ impl Hypervisor for KVMDriver {
         outb_handle_fn: OutBHandlerWrapper,
         mem_access_fn: MemAccessHandlerWrapper,
         hv_handler: Option<HypervisorHandler>,
+        #[cfg(gdb)] dbg_mem_access_fn: DbgMemAccessHandlerWrapper,
     ) -> Result<()> {
         // Reset general purpose registers except RSP, then set RIP
         let rsp_before = self.vcpu_fd.get_regs()?.rsp;
@@ -603,6 +614,8 @@ impl Hypervisor for KVMDriver {
             hv_handler,
             outb_handle_fn,
             mem_access_fn,
+            #[cfg(gdb)]
+            dbg_mem_access_fn,
         )?;
 
         // reset RSP to what it was before function call
@@ -714,6 +727,7 @@ impl Hypervisor for KVMDriver {
     #[cfg(gdb)]
     fn handle_debug(
         &mut self,
+        dbg_mem_access_fn: Arc<Mutex<dyn super::handlers::DbgMemAccessHandlerCaller>>,
         stop_reason: VcpuStopReason,
     ) -> Result<()> {
         self.send_dbg_msg(DebugResponse::VcpuStopped(stop_reason))
@@ -724,7 +738,7 @@ impl Hypervisor for KVMDriver {
             // Wait for a message from gdb
             let req = self.recv_dbg_msg()?;
 
-            let response = self.process_dbg_request(req)?;
+            let response = self.process_dbg_request(req, dbg_mem_access_fn.clone())?;
 
             // If the command was either step or continue, we need to run the vcpu
             let cont = matches!(
@@ -748,6 +762,8 @@ impl Hypervisor for KVMDriver {
 mod tests {
     use std::sync::{Arc, Mutex};
 
+    #[cfg(gdb)]
+    use crate::hypervisor::handlers::DbgMemAccessHandler;
     use crate::hypervisor::handlers::{MemAccessHandler, OutBHandler};
     use crate::hypervisor::tests::test_initialise;
     use crate::Result;
@@ -767,6 +783,28 @@ mod tests {
             let func: Box<dyn FnMut() -> Result<()> + Send> = Box::new(|| -> Result<()> { Ok(()) });
             Arc::new(Mutex::new(MemAccessHandler::from(func)))
         };
-        test_initialise(outb_handler, mem_access_handler).unwrap();
+        #[cfg(gdb)]
+        #[allow(clippy::type_complexity)]
+        let dbg_mem_access_handler = {
+            let read: Box<dyn FnMut(usize, &mut [u8]) -> Result<()> + Send> =
+                Box::new(|_: usize, _: &mut [u8]| -> Result<()> { Ok(()) });
+            let write: Box<dyn FnMut(usize, &[u8]) -> Result<()> + Send> =
+                Box::new(|_: usize, _: &[u8]| -> Result<()> { Ok(()) });
+            let get_code_offset: Box<dyn FnMut() -> Result<usize> + Send> =
+                Box::new(|| -> Result<usize> { Ok(0) });
+            Arc::new(Mutex::new(DbgMemAccessHandler::from((
+                read,
+                write,
+                get_code_offset,
+            ))))
+        };
+
+        test_initialise(
+            outb_handler,
+            mem_access_handler,
+            #[cfg(gdb)]
+            dbg_mem_access_handler,
+        )
+        .unwrap();
     }
 }