Gdb support for mshv guests

src/hyperlight_host/src/hypervisor/gdb/event_loop.rs

@@ -16,8 +16,9 @@ limitations under the License.
 
 use gdbstub::common::Signal;
 use gdbstub::conn::ConnectionExt;
-use gdbstub::stub::run_blocking::{self, WaitForStopReasonError};
-use gdbstub::stub::{BaseStopReason, DisconnectReason, GdbStub, SingleThreadStopReason};
+use gdbstub::stub::{
+    run_blocking, BaseStopReason, DisconnectReason, GdbStub, SingleThreadStopReason,
+};
 use libc::{pthread_kill, SIGRTMIN};
 
 use super::x86_64_target::HyperlightSandboxTarget;
@@ -57,13 +58,10 @@ impl run_blocking::BlockingEventLoop for GdbBlockingEventLoop {
                             signal: Signal(SIGRTMIN() as u8),
                         },
                         VcpuStopReason::Unknown => {
-                            log::warn!("Unknown stop reason - resuming execution");
+                            log::warn!("Unknown stop reason received");
 
-                            target
-                                .resume_vcpu()
-                                .map_err(WaitForStopReasonError::Target)?;
-
-                            continue;
+                            // Marking as a SwBreak so the gdb inspect where/why it stopped
+                            BaseStopReason::SwBreak(())
                         }
                     };
 

src/hyperlight_host/src/hypervisor/gdb/kvm_debug.rs

@@ -17,14 +17,20 @@ limitations under the License.
 use std::collections::HashMap;
 
 use kvm_bindings::{
-    kvm_guest_debug, kvm_regs, KVM_GUESTDBG_ENABLE, KVM_GUESTDBG_SINGLESTEP,
+    kvm_debug_exit_arch, kvm_guest_debug, kvm_regs, KVM_GUESTDBG_ENABLE, KVM_GUESTDBG_SINGLESTEP,
     KVM_GUESTDBG_USE_HW_BP, KVM_GUESTDBG_USE_SW_BP,
 };
 use kvm_ioctls::VcpuFd;
 
-use super::{GuestDebug, VcpuStopReason, X86_64Regs, MAX_NO_OF_HW_BP, SW_BP_SIZE};
+use super::{
+    GuestDebug, VcpuStopReason, X86_64Regs, DR6_BS_FLAG_MASK, DR6_HW_BP_FLAGS_MASK,
+    MAX_NO_OF_HW_BP, SW_BP_SIZE,
+};
 use crate::{new_error, HyperlightError, Result};
 
+/// Exception id for SW breakpoint
+const SW_BP_ID: u32 = 3;
+
 /// KVM Debug struct
 /// This struct is used to abstract the internal details of the kvm
 /// guest debugging settings
@@ -109,20 +115,25 @@ impl KvmDebug {
     pub(crate) fn get_stop_reason(
         &mut self,
         vcpu_fd: &VcpuFd,
+        debug_exit: kvm_debug_exit_arch,
         entrypoint: u64,
     ) -> Result<VcpuStopReason> {
-        if self.single_step {
+        // If the BS flag in DR6 register is set, it means a single step
+        // instruction triggered the exit
+        // Check page 19-4 Vol. 3B of Intel 64 and IA-32
+        // Architectures Software Developer's Manual
+        if debug_exit.dr6 & DR6_BS_FLAG_MASK != 0 && self.single_step {
             return Ok(VcpuStopReason::DoneStep);
         }
 
         let ip = self.get_instruction_pointer(vcpu_fd)?;
         let gpa = self.translate_gva(vcpu_fd, ip)?;
 
-        if self.sw_breakpoints.contains_key(&gpa) {
-            return Ok(VcpuStopReason::SwBp);
-        }
-
-        if self.hw_breakpoints.contains(&gpa) {
+        // If any of the B0-B3 flags in DR6 register is set, it means a
+        // hardware breakpoint triggered the exit
+        // Check page 19-4 Vol. 3B of Intel 64 and IA-32
+        // Architectures Software Developer's Manual
+        if DR6_HW_BP_FLAGS_MASK & debug_exit.dr6 != 0 && self.hw_breakpoints.contains(&gpa) {
             // In case the hw breakpoint is the entry point, remove it to
             // avoid hanging here as gdb does not remove breakpoints it
             // has not set.
@@ -133,6 +144,24 @@ impl KvmDebug {
             return Ok(VcpuStopReason::HwBp);
         }
 
+        // If the exception ID matches #BP (3) - it means a software breakpoint
+        // caused the exit
+        if SW_BP_ID == debug_exit.exception && self.sw_breakpoints.contains_key(&gpa) {
+            return Ok(VcpuStopReason::SwBp);
+        }
+
+        // Log an error and provide internal debugging info for fixing
+        log::error!(
+            r"The vCPU exited because of an unknown reason:
+            kvm_debug_exit_arch: {:?}
+            single_step: {:?}
+            hw_breakpoints: {:?}
+            sw_breakpoints: {:?}",
+            debug_exit,
+            self.single_step,
+            self.hw_breakpoints,
+            self.sw_breakpoints,
+        );
         Ok(VcpuStopReason::Unknown)
     }
 }

src/hyperlight_host/src/hypervisor/gdb/mod.rs

@@ -50,6 +50,17 @@ const SW_BP: [u8; SW_BP_SIZE] = [SW_BP_OP];
 /// Maximum number of supported hardware breakpoints
 const MAX_NO_OF_HW_BP: usize = 4;
 
+/// Check page 19-4 Vol. 3B of Intel 64 and IA-32
+/// Architectures Software Developer's Manual
+/// Bit position of BS flag in DR6 debug register
+const DR6_BS_FLAG_POS: usize = 14;
+/// Bit mask of BS flag in DR6 debug register
+const DR6_BS_FLAG_MASK: u64 = 1 << DR6_BS_FLAG_POS;
+/// Bit position of HW breakpoints status in DR6 debug register
+const DR6_HW_BP_FLAGS_POS: usize = 0;
+/// Bit mask of HW breakpoints status in DR6 debug register
+const DR6_HW_BP_FLAGS_MASK: u64 = 0x0F << DR6_HW_BP_FLAGS_POS;
+
 #[derive(Debug, Error)]
 pub(crate) enum GdbTargetError {
     #[error("Error encountered while binding to address and port")]

src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs

@@ -70,11 +70,6 @@ impl HyperlightSandboxTarget {
         self.hyp_conn.recv()
     }
 
-    /// Non-Blocking check for a response over the communication channel
-    pub(crate) fn try_recv(&self) -> Result<DebugResponse, TryRecvError> {
-        self.hyp_conn.try_recv()
-    }
-
     /// Sends an event to the Hypervisor that tells it to resume vCPU execution
     /// Note: The method waits for a confirmation message
     pub(crate) fn resume_vcpu(&mut self) -> Result<(), GdbTargetError> {
@@ -89,6 +84,11 @@ impl HyperlightSandboxTarget {
         }
     }
 
+    /// Non-Blocking check for a response over the communication channel
+    pub(crate) fn try_recv(&self) -> Result<DebugResponse, TryRecvError> {
+        self.hyp_conn.try_recv()
+    }
+
     /// Sends an event to the Hypervisor that tells it to disable debugging
     /// and continue executing
     /// Note: The method waits for a confirmation message

src/hyperlight_host/src/hypervisor/kvm.rs

@@ -67,6 +67,8 @@ pub(crate) fn is_hypervisor_present() -> bool {
 mod debug {
     use std::sync::{Arc, Mutex};
 
+    use kvm_bindings::kvm_debug_exit_arch;
+
     use super::KVMDriver;
     use crate::hypervisor::gdb::{
         DebugMsg, DebugResponse, GuestDebug, KvmDebug, VcpuStopReason, X86_64Regs,
@@ -87,13 +89,16 @@ mod debug {
         }
 
         /// Get the reason the vCPU has stopped
-        pub(crate) fn get_stop_reason(&mut self) -> Result<VcpuStopReason> {
+        pub(crate) fn get_stop_reason(
+            &mut self,
+            debug_exit: kvm_debug_exit_arch,
+        ) -> Result<VcpuStopReason> {
             let debug = self
                 .debug
                 .as_mut()
                 .ok_or_else(|| new_error!("Debug is not enabled"))?;
 
-            debug.get_stop_reason(&self.vcpu_fd, self.entrypoint)
+            debug.get_stop_reason(&self.vcpu_fd, debug_exit, self.entrypoint)
         }
 
         pub(crate) fn process_dbg_request(
@@ -469,7 +474,8 @@ impl Hypervisor for KVMDriver {
                 }
             }
             #[cfg(gdb)]
-            Ok(VcpuExit::Debug(_)) => match self.get_stop_reason() {
+            // KVM provides architecture specific information about the vCPU state when exiting
+            Ok(VcpuExit::Debug(debug_exit)) => match self.get_stop_reason(debug_exit) {
                 Ok(reason) => HyperlightExit::Debug(reason),
                 Err(e) => {
                     log_then_return!("Error getting stop reason: {:?}", e);