feat: Enable gdb debugging on x86

Enabling GDB support for debugging the guest kernel. This allows us to
connect a gdb server to firecracker and debug the guest.

Signed-off-by: Jack Thomson <[email protected]>

Author: JackThomson2

Cargo.lock

@@ -49,6 +49,7 @@ serde_json = "1.0.128"
 
 [features]
 tracing = ["log-instrument", "seccompiler/tracing", "utils/tracing", "vmm/tracing"]
+gdb = ["vmm/gdb"]
 
 [lints]
 workspace = true

src/firecracker/Cargo.toml

@@ -11,6 +11,7 @@ bench = false
 [dependencies]
 acpi_tables = { path = "../acpi-tables" } 
 aes-gcm =  { version = "0.10.1", default-features = false, features = ["aes"] }
+arrayvec = { version = "0.7.6", optional = true }
 aws-lc-rs = { version = "1.10.0", features = ["bindgen"] }
 base64 = "0.22.1"
 bincode = "1.2.1"
@@ -19,6 +20,8 @@ crc64 = "2.0.0"
 derive_more = { version = "1.0.0", default-features = false, features = ["from", "display"] }
 displaydoc = "0.2.5"
 event-manager = "0.4.0"
+gdbstub = { version = "0.7.2", optional = true }
+gdbstub_arch = { version = "0.3.0", optional = true }
 kvm-bindings = { version = "0.9.1", features = ["fam-wrappers", "serde"] }
 kvm-ioctls = "0.18.0"
 lazy_static = "1.5.0"
@@ -55,7 +58,9 @@ itertools = "0.13.0"
 proptest = { version = "1.5.0", default-features = false, features = ["std"] }
 
 [features]
+default = []
 tracing = ["log-instrument"]
+gdb = ["arrayvec", "gdbstub", "gdbstub_arch"]
 
 [[bench]]
 name = "cpu_templates"

src/vmm/Cargo.toml

@@ -7,6 +7,8 @@
 use std::convert::TryFrom;
 use std::fmt::Debug;
 use std::io::{self, Seek, SeekFrom};
+#[cfg(feature = "gdb")]
+use std::sync::mpsc;
 use std::sync::{Arc, Mutex};
 
 use event_manager::{MutEventSubscriber, SubscriberOps};
@@ -26,6 +28,9 @@ use vm_superio::Rtc;
 use vm_superio::Serial;
 use vmm_sys_util::eventfd::EventFd;
 
+#[cfg(all(feature = "gdb", target_arch = "aarch64"))]
+compile_error!("GDB feature not supported on ARM");
+
 #[cfg(target_arch = "x86_64")]
 use crate::acpi;
 use crate::arch::InitrdConfig;
@@ -56,6 +61,8 @@ use crate::devices::virtio::net::Net;
 use crate::devices::virtio::rng::Entropy;
 use crate::devices::virtio::vsock::{Vsock, VsockUnixBackend};
 use crate::devices::BusDevice;
+#[cfg(feature = "gdb")]
+use crate::gdb;
 use crate::logger::{debug, error};
 use crate::persist::{MicrovmState, MicrovmStateError};
 use crate::resources::VmResources;
@@ -128,6 +135,12 @@ pub enum StartMicrovmError {
     /// Error configuring ACPI: {0}
     #[cfg(target_arch = "x86_64")]
     Acpi(#[from] crate::acpi::AcpiError),
+    /// Error starting GDB debug session
+    #[cfg(feature = "gdb")]
+    GdbServer(gdb::target::GdbTargetError),
+    /// Error cloning Vcpu fds
+    #[cfg(feature = "gdb")]
+    VcpuFdCloneError(#[from] crate::vstate::vcpu::CopyKvmFdError),
 }
 
 /// It's convenient to automatically convert `linux_loader::cmdline::Error`s
@@ -274,6 +287,18 @@ pub fn build_microvm_for_boot(
         cpu_template.kvm_capabilities.clone(),
     )?;
 
+    #[cfg(feature = "gdb")]
+    let (gdb_tx, gdb_rx) = mpsc::channel();
+    #[cfg(feature = "gdb")]
+    vcpus
+        .iter_mut()
+        .for_each(|vcpu| vcpu.attach_debug_info(gdb_tx.clone()));
+    #[cfg(feature = "gdb")]
+    let vcpu_fds = vcpus
+        .iter()
+        .map(|vcpu| vcpu.copy_kvm_vcpu_fd(vmm.vm()))
+        .collect::<Result<Vec<_>, _>>()?;
+
     // The boot timer device needs to be the first device attached in order
     // to maintain the same MMIO address referenced in the documentation
     // and tests.
@@ -321,16 +346,28 @@ pub fn build_microvm_for_boot(
         boot_cmdline,
     )?;
 
+    let vmm = Arc::new(Mutex::new(vmm));
+
+    #[cfg(feature = "gdb")]
+    if let Some(gdb_socket_addr) = &vm_resources.gdb_socket_addr {
+        gdb::gdb_thread(vmm.clone(), vcpu_fds, gdb_rx, entry_addr, gdb_socket_addr)
+            .map_err(GdbServer)?;
+    } else {
+        debug!("No GDB socket provided not starting gdb server.");
+    }
+
     // Move vcpus to their own threads and start their state machine in the 'Paused' state.
-    vmm.start_vcpus(
-        vcpus,
-        seccomp_filters
-            .get("vcpu")
-            .ok_or_else(|| MissingSeccompFilters("vcpu".to_string()))?
-            .clone(),
-    )
-    .map_err(VmmError::VcpuStart)
-    .map_err(Internal)?;
+    vmm.lock()
+        .unwrap()
+        .start_vcpus(
+            vcpus,
+            seccomp_filters
+                .get("vcpu")
+                .ok_or_else(|| MissingSeccompFilters("vcpu".to_string()))?
+                .clone(),
+        )
+        .map_err(VmmError::VcpuStart)
+        .map_err(Internal)?;
 
     // Load seccomp filters for the VMM thread.
     // Execution panics if filters cannot be loaded, use --no-seccomp if skipping filters
@@ -344,7 +381,6 @@ pub fn build_microvm_for_boot(
     .map_err(VmmError::SeccompFilters)
     .map_err(Internal)?;
 
-    let vmm = Arc::new(Mutex::new(vmm));
     event_manager.add_subscriber(vmm.clone());
 
     Ok(vmm)

src/vmm/src/builder.rs

@@ -0,0 +1,62 @@
+// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use gdbstub_arch::aarch64::reg::AArch64CoreRegs as CoreRegs;
+use kvm_ioctls::VcpuFd;
+use vm_memory::GuestAddress;
+
+use crate::gdb::target::GdbTargetError;
+
+/// Configures the number of bytes required for a software breakpoint
+pub const SW_BP_SIZE: usize = 1;
+
+/// The bytes stored for a software breakpoint
+pub const SW_BP: [u8; SW_BP_SIZE] = [0];
+
+/// Gets the RIP value for a Vcpu
+pub fn get_instruction_pointer(_vcpu_fd: &VcpuFd) -> Result<u64, GdbTargetError> {
+    unimplemented!()
+}
+
+/// Translates a virtual address according to the vCPU's current address translation mode.
+pub fn translate_gva(_vcpu_fd: &VcpuFd, _gva: u64) -> Result<u64, GdbTargetError> {
+    unimplemented!()
+}
+
+/// Configures the kvm guest debug regs to register the hardware breakpoints
+fn set_kvm_debug(
+    _control: u32,
+    _vcpu_fd: &VcpuFd,
+    _addrs: &[GuestAddress],
+) -> Result<(), GdbTargetError> {
+    unimplemented!()
+}
+
+/// Configures the Vcpu for debugging and sets the hardware breakpoints on the Vcpu
+pub fn vcpu_set_debug(
+    _vcpu_fd: &VcpuFd,
+    _addrs: &[GuestAddress],
+    _step: bool,
+) -> Result<(), GdbTargetError> {
+    unimplemented!()
+}
+
+/// Injects a BP back into the guest kernel for it to handle, this is particularly useful for the
+/// kernels selftesting which can happen during boot.
+pub fn vcpu_inject_bp(
+    _vcpu_fd: &VcpuFd,
+    _addrs: &[GuestAddress],
+    _step: bool,
+) -> Result<(), GdbTargetError> {
+    unimplemented!()
+}
+
+/// Reads the registers for the Vcpu
+pub fn read_registers(_vcpu_fd: &VcpuFd, _regs: &mut CoreRegs) -> Result<(), GdbTargetError> {
+    unimplemented!()
+}
+
+/// Writes to the registers for the Vcpu
+pub fn write_registers(_vcpu_fd: &VcpuFd, _regs: &CoreRegs) -> Result<(), GdbTargetError> {
+    unimplemented!()
+}

src/vmm/src/gdb/arch/aarch64.rs

@@ -0,0 +1,13 @@
+// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#[cfg(target_arch = "aarch64")]
+mod aarch64;
+#[cfg(target_arch = "aarch64")]
+pub use aarch64::*;
+
+#[cfg(target_arch = "x86_64")]
+mod x86;
+#[cfg(target_arch = "x86_64")]
+pub use x86::*;