diff --git a/Cargo.lock b/Cargo.lock index 1733e9ac749..45324e6a5c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -111,6 +111,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "autocfg" version = "1.4.0" @@ -627,6 +633,30 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "gdbstub" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbcc892208d6998fb57e7c3e05883def66f8130924bba066beb0cfe71566a9f6" +dependencies = [ + "bitflags 2.6.0", + "cfg-if", + "log", + "managed", + "num-traits", + "paste", +] + +[[package]] +name = "gdbstub_arch" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e3b1357bd3203fc09a6601327ae0ab38865d14231d0b65d3143f5762cc7977d" +dependencies = [ + "gdbstub", + "num-traits", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -888,6 +918,12 @@ dependencies = [ "syn", ] +[[package]] +name = "managed" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca88d725a0a943b096803bd34e73a4437208b6077654cc4ecb2947a5f91618d" + [[package]] name = "memchr" version = "2.7.4" @@ -1534,6 +1570,7 @@ version = "0.1.0" dependencies = [ "acpi_tables", "aes-gcm", + "arrayvec", "aws-lc-rs", "base64", "bincode", @@ -1544,6 +1581,8 @@ dependencies = [ "device_tree", "displaydoc", "event-manager", + "gdbstub", + "gdbstub_arch", "itertools 0.13.0", "kvm-bindings", "kvm-ioctls", diff --git a/docs/gdb-debugging.md b/docs/gdb-debugging.md new file mode 100644 index 00000000000..6e96983a309 --- /dev/null +++ b/docs/gdb-debugging.md @@ -0,0 +1,110 @@ +# GDB Debugging with Firecracker + +Firecracker supports debugging the guest kernel via GDB remote serial protocol. +This allows us to connect GDB to the firecracker process and step through debug +the guest kernel. Currently only debugging on x86 is supported. + +The GDB feature requires Firecracker to be booted with a config file. + +## Prerequisites + +Firstly, to enable GDB debugging we need to compile Firecracker with the `debug` +feature enabled, this will enable the necessary components for the debugging +process. + +To build firecracker with the `gdb` feature enabled we run: + +```bash +cargo build --features "gdb" +``` + +Secondly, we need to compile a kernel with specific features enabled for +debugging to work. The key config options to enable are: + +``` +CONFIG_FRAME_POINTER=y +CONFIG_KGDB=y +CONFIG_KGDB_SERIAL_CONSOLE=y +CONFIG_DEBUG_INFO=y +``` + +For GDB debugging the `gdb-socket` option should be set in your config file. In +this example we set it to `/tmp/gdb.socket` + +``` +{ + ... + "gdb-socket": "/tmp/gdb.socket" + ... +} +``` + +## Starting Firecracker with GDB + +With all the prerequisites in place you can now start firecracker ready to +connect to GDB. When you start the firecracker binary now you'll notice it'll be +blocked waiting for the GDB connection. This is done to allow us to set +breakpoints before the boot process begins. + +With Firecracker running and waiting for GDB we are now able to start GDB and +connect to Firecracker. You may need to set the permissions of your GDB socket +E.g. `/tmp/gdb.socket` to `0666` before connecting. + +An example of the steps taken to start GDB, load the symbols and connect to +Firecracker: + +1. Start the GDB process, you can attach the symbols by appending the kernel + blob, for example here `vmlinux` + + ```bash + gdb vmlinux + ``` + +1. When GDB has started set the target remote to `/tmp/gdb.socket` to connect to + Firecracker + + ```bash + (gdb) target remote /tmp/gdb.socket + ``` + +With these steps completed you'll now see GDB has stopped at the entry point +ready for us to start inserting breakpoints and debugging. + +## Notes + +### Software Breakpoints not working on start + +When at the initial paused state you'll notice software breakpoints won't work +and only hardware breakpoints will until memory virtualisation is enabled. To +circumvent this one solution is to set a hardware breakpoint at `start_kernel` +and continue. Once you've hit the `start_kernel` set the regular breakpoints as +you would do normally. E.g. + +```bash +> hbreak start_kernel +> c +``` + +### Pausing Firecracker while it's running + +While Firecracker is running you can pause vcpu 1 by pressing `Ctrl+C` which +will stop the vcpu and allow you to set breakpoints or inspect the current +location. + +### Halting execution of GDB and Firecracker + +To end the debugging session and shut down Firecracker you can run the `exit` +command in the GDB session which will terminate both. + +## Known limitations + +- The multi-core scheduler can in some cases cause issues with GDB, this can be + mitigated by setting these kernel config values: + + ``` + CONFIG_SCHED_MC=y + CONFIG_SCHED_MC_PRIO=y + ``` + +- Currently we support a limited subset of cpu registers for get and set + operations, if more are required feel free to contribute. diff --git a/src/firecracker/Cargo.toml b/src/firecracker/Cargo.toml index e3e5bca6ded..0d678afc28e 100644 --- a/src/firecracker/Cargo.toml +++ b/src/firecracker/Cargo.toml @@ -49,6 +49,7 @@ serde_json = "1.0.128" [features] tracing = ["log-instrument", "seccompiler/tracing", "utils/tracing", "vmm/tracing"] +gdb = ["vmm/gdb"] [lints] workspace = true diff --git a/src/vmm/Cargo.toml b/src/vmm/Cargo.toml index 88a421136ad..7be2200b8b2 100644 --- a/src/vmm/Cargo.toml +++ b/src/vmm/Cargo.toml @@ -11,6 +11,7 @@ bench = false [dependencies] acpi_tables = { path = "../acpi-tables" } aes-gcm = { version = "0.10.1", default-features = false, features = ["aes"] } +arrayvec = { version = "0.7.6", optional = true } aws-lc-rs = { version = "1.10.0", features = ["bindgen"] } base64 = "0.22.1" bincode = "1.2.1" @@ -19,6 +20,8 @@ crc64 = "2.0.0" derive_more = { version = "1.0.0", default-features = false, features = ["from", "display"] } displaydoc = "0.2.5" event-manager = "0.4.0" +gdbstub = { version = "0.7.2", optional = true } +gdbstub_arch = { version = "0.3.0", optional = true } kvm-bindings = { version = "0.9.1", features = ["fam-wrappers", "serde"] } kvm-ioctls = "0.18.0" lazy_static = "1.5.0" @@ -55,7 +58,9 @@ itertools = "0.13.0" proptest = { version = "1.5.0", default-features = false, features = ["std"] } [features] +default = [] tracing = ["log-instrument"] +gdb = ["arrayvec", "gdbstub", "gdbstub_arch"] [[bench]] name = "cpu_templates" diff --git a/src/vmm/src/builder.rs b/src/vmm/src/builder.rs index 6bbfec6fabb..760c22a27b5 100644 --- a/src/vmm/src/builder.rs +++ b/src/vmm/src/builder.rs @@ -7,6 +7,8 @@ use std::convert::TryFrom; use std::fmt::Debug; use std::io::{self, Seek, SeekFrom}; +#[cfg(feature = "gdb")] +use std::sync::mpsc; use std::sync::{Arc, Mutex}; use event_manager::{MutEventSubscriber, SubscriberOps}; @@ -26,6 +28,9 @@ use vm_superio::Rtc; use vm_superio::Serial; use vmm_sys_util::eventfd::EventFd; +#[cfg(all(feature = "gdb", target_arch = "aarch64"))] +compile_error!("GDB feature not supported on ARM"); + #[cfg(target_arch = "x86_64")] use crate::acpi; use crate::arch::InitrdConfig; @@ -56,6 +61,8 @@ use crate::devices::virtio::net::Net; use crate::devices::virtio::rng::Entropy; use crate::devices::virtio::vsock::{Vsock, VsockUnixBackend}; use crate::devices::BusDevice; +#[cfg(feature = "gdb")] +use crate::gdb; use crate::logger::{debug, error}; use crate::persist::{MicrovmState, MicrovmStateError}; use crate::resources::VmResources; @@ -128,6 +135,12 @@ pub enum StartMicrovmError { /// Error configuring ACPI: {0} #[cfg(target_arch = "x86_64")] Acpi(#[from] crate::acpi::AcpiError), + /// Error starting GDB debug session + #[cfg(feature = "gdb")] + GdbServer(gdb::target::GdbTargetError), + /// Error cloning Vcpu fds + #[cfg(feature = "gdb")] + VcpuFdCloneError(#[from] crate::vstate::vcpu::CopyKvmFdError), } /// It's convenient to automatically convert `linux_loader::cmdline::Error`s @@ -274,6 +287,18 @@ pub fn build_microvm_for_boot( cpu_template.kvm_capabilities.clone(), )?; + #[cfg(feature = "gdb")] + let (gdb_tx, gdb_rx) = mpsc::channel(); + #[cfg(feature = "gdb")] + vcpus + .iter_mut() + .for_each(|vcpu| vcpu.attach_debug_info(gdb_tx.clone())); + #[cfg(feature = "gdb")] + let vcpu_fds = vcpus + .iter() + .map(|vcpu| vcpu.copy_kvm_vcpu_fd(vmm.vm())) + .collect::, _>>()?; + // The boot timer device needs to be the first device attached in order // to maintain the same MMIO address referenced in the documentation // and tests. @@ -321,16 +346,28 @@ pub fn build_microvm_for_boot( boot_cmdline, )?; + let vmm = Arc::new(Mutex::new(vmm)); + + #[cfg(feature = "gdb")] + if let Some(gdb_socket_addr) = &vm_resources.gdb_socket_addr { + gdb::gdb_thread(vmm.clone(), vcpu_fds, gdb_rx, entry_addr, gdb_socket_addr) + .map_err(GdbServer)?; + } else { + debug!("No GDB socket provided not starting gdb server."); + } + // Move vcpus to their own threads and start their state machine in the 'Paused' state. - vmm.start_vcpus( - vcpus, - seccomp_filters - .get("vcpu") - .ok_or_else(|| MissingSeccompFilters("vcpu".to_string()))? - .clone(), - ) - .map_err(VmmError::VcpuStart) - .map_err(Internal)?; + vmm.lock() + .unwrap() + .start_vcpus( + vcpus, + seccomp_filters + .get("vcpu") + .ok_or_else(|| MissingSeccompFilters("vcpu".to_string()))? + .clone(), + ) + .map_err(VmmError::VcpuStart) + .map_err(Internal)?; // Load seccomp filters for the VMM thread. // Execution panics if filters cannot be loaded, use --no-seccomp if skipping filters @@ -344,7 +381,6 @@ pub fn build_microvm_for_boot( .map_err(VmmError::SeccompFilters) .map_err(Internal)?; - let vmm = Arc::new(Mutex::new(vmm)); event_manager.add_subscriber(vmm.clone()); Ok(vmm) diff --git a/src/vmm/src/gdb/arch/aarch64.rs b/src/vmm/src/gdb/arch/aarch64.rs new file mode 100644 index 00000000000..d6e667f9fcb --- /dev/null +++ b/src/vmm/src/gdb/arch/aarch64.rs @@ -0,0 +1,62 @@ +// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use gdbstub_arch::aarch64::reg::AArch64CoreRegs as CoreRegs; +use kvm_ioctls::VcpuFd; +use vm_memory::GuestAddress; + +use crate::gdb::target::GdbTargetError; + +/// Configures the number of bytes required for a software breakpoint +pub const SW_BP_SIZE: usize = 1; + +/// The bytes stored for a software breakpoint +pub const SW_BP: [u8; SW_BP_SIZE] = [0]; + +/// Gets the RIP value for a Vcpu +pub fn get_instruction_pointer(_vcpu_fd: &VcpuFd) -> Result { + unimplemented!() +} + +/// Translates a virtual address according to the vCPU's current address translation mode. +pub fn translate_gva(_vcpu_fd: &VcpuFd, _gva: u64) -> Result { + unimplemented!() +} + +/// Configures the kvm guest debug regs to register the hardware breakpoints +fn set_kvm_debug( + _control: u32, + _vcpu_fd: &VcpuFd, + _addrs: &[GuestAddress], +) -> Result<(), GdbTargetError> { + unimplemented!() +} + +/// Configures the Vcpu for debugging and sets the hardware breakpoints on the Vcpu +pub fn vcpu_set_debug( + _vcpu_fd: &VcpuFd, + _addrs: &[GuestAddress], + _step: bool, +) -> Result<(), GdbTargetError> { + unimplemented!() +} + +/// Injects a BP back into the guest kernel for it to handle, this is particularly useful for the +/// kernels selftesting which can happen during boot. +pub fn vcpu_inject_bp( + _vcpu_fd: &VcpuFd, + _addrs: &[GuestAddress], + _step: bool, +) -> Result<(), GdbTargetError> { + unimplemented!() +} + +/// Reads the registers for the Vcpu +pub fn read_registers(_vcpu_fd: &VcpuFd, _regs: &mut CoreRegs) -> Result<(), GdbTargetError> { + unimplemented!() +} + +/// Writes to the registers for the Vcpu +pub fn write_registers(_vcpu_fd: &VcpuFd, _regs: &CoreRegs) -> Result<(), GdbTargetError> { + unimplemented!() +} diff --git a/src/vmm/src/gdb/arch/mod.rs b/src/vmm/src/gdb/arch/mod.rs new file mode 100644 index 00000000000..424b6626ba1 --- /dev/null +++ b/src/vmm/src/gdb/arch/mod.rs @@ -0,0 +1,13 @@ +// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// + +#[cfg(target_arch = "aarch64")] +mod aarch64; +#[cfg(target_arch = "aarch64")] +pub use aarch64::*; + +#[cfg(target_arch = "x86_64")] +mod x86; +#[cfg(target_arch = "x86_64")] +pub use x86::*; diff --git a/src/vmm/src/gdb/arch/x86.rs b/src/vmm/src/gdb/arch/x86.rs new file mode 100644 index 00000000000..6671b83443f --- /dev/null +++ b/src/vmm/src/gdb/arch/x86.rs @@ -0,0 +1,160 @@ +// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use gdbstub_arch::x86::reg::X86_64CoreRegs as CoreRegs; +use kvm_bindings::*; +use kvm_ioctls::VcpuFd; +use vm_memory::GuestAddress; + +use crate::gdb::target::GdbTargetError; +use crate::logger::error; + +/// Sets the 9th (Global Exact Breakpoint enable) and the 10th (always 1) bits for the DR7 debug +/// control register +const X86_GLOBAL_DEBUG_ENABLE: u64 = 0b11 << 9; + +/// Op code to trigger a software breakpoint in x86 +const X86_SW_BP_OP: u8 = 0xCC; + +/// Configures the number of bytes required for a software breakpoint +pub const SW_BP_SIZE: usize = 1; + +/// The bytes stored for an x86 software breakpoint +pub const SW_BP: [u8; SW_BP_SIZE] = [X86_SW_BP_OP]; + +/// Gets the RIP value for a Vcpu +pub fn get_instruction_pointer(vcpu_fd: &VcpuFd) -> Result { + let regs = vcpu_fd.get_regs()?; + + Ok(regs.rip) +} + +/// Translates a virtual address according to the vCPU's current address translation mode. +pub fn translate_gva(vcpu_fd: &VcpuFd, gva: u64) -> Result { + let tr = vcpu_fd.translate_gva(gva)?; + + if tr.valid == 0 { + return Err(GdbTargetError::KvmGvaTranslateError); + } + + Ok(tr.physical_address) +} + +/// Configures the kvm guest debug regs to register the hardware breakpoints, the `arch.debugreg` +/// attribute is used to store the location of the hardware breakpoints, with the 8th slot being +/// used as a bitfield to track which registers are enabled and setting the +/// `X86_GLOBAL_DEBUG_ENABLE` flags. Further reading on the DR7 register can be found here: +/// https://en.wikipedia.org/wiki/X86_debug_register#DR7_-_Debug_control +fn set_kvm_debug( + control: u32, + vcpu_fd: &VcpuFd, + addrs: &[GuestAddress], +) -> Result<(), GdbTargetError> { + let mut dbg = kvm_guest_debug { + control, + ..Default::default() + }; + + dbg.arch.debugreg[7] = X86_GLOBAL_DEBUG_ENABLE; + + for (i, addr) in addrs.iter().enumerate() { + dbg.arch.debugreg[i] = addr.0; + // Set global breakpoint enable flag for the specific breakpoint number by setting the bit + dbg.arch.debugreg[7] |= 2 << (i * 2); + } + + vcpu_fd.set_guest_debug(&dbg)?; + + Ok(()) +} + +/// Configures the Vcpu for debugging and sets the hardware breakpoints on the Vcpu +pub fn vcpu_set_debug( + vcpu_fd: &VcpuFd, + addrs: &[GuestAddress], + step: bool, +) -> Result<(), GdbTargetError> { + let mut control = KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW_BP | KVM_GUESTDBG_USE_SW_BP; + if step { + control |= KVM_GUESTDBG_SINGLESTEP; + } + + set_kvm_debug(control, vcpu_fd, addrs) +} + +/// Injects a BP back into the guest kernel for it to handle, this is particularly useful for the +/// kernels selftesting which can happen during boot. +pub fn vcpu_inject_bp( + vcpu_fd: &VcpuFd, + addrs: &[GuestAddress], + step: bool, +) -> Result<(), GdbTargetError> { + let mut control = KVM_GUESTDBG_ENABLE + | KVM_GUESTDBG_USE_HW_BP + | KVM_GUESTDBG_USE_SW_BP + | KVM_GUESTDBG_INJECT_BP; + + if step { + control |= KVM_GUESTDBG_SINGLESTEP; + } + + set_kvm_debug(control, vcpu_fd, addrs) +} + +/// Reads the registers for the Vcpu +pub fn read_registers(vcpu_fd: &VcpuFd, regs: &mut CoreRegs) -> Result<(), GdbTargetError> { + let cpu_regs = vcpu_fd.get_regs()?; + + regs.regs[0] = cpu_regs.rax; + regs.regs[1] = cpu_regs.rbx; + regs.regs[2] = cpu_regs.rcx; + regs.regs[3] = cpu_regs.rdx; + regs.regs[4] = cpu_regs.rsi; + regs.regs[5] = cpu_regs.rdi; + regs.regs[6] = cpu_regs.rbp; + regs.regs[7] = cpu_regs.rsp; + + regs.regs[8] = cpu_regs.r8; + regs.regs[9] = cpu_regs.r9; + regs.regs[10] = cpu_regs.r10; + regs.regs[11] = cpu_regs.r11; + regs.regs[12] = cpu_regs.r12; + regs.regs[13] = cpu_regs.r13; + regs.regs[14] = cpu_regs.r14; + regs.regs[15] = cpu_regs.r15; + + regs.rip = cpu_regs.rip; + regs.eflags = u32::try_from(cpu_regs.rflags).map_err(|e| { + error!("Error {e:?} converting rflags to u32"); + GdbTargetError::RegFlagConversionError + })?; + + Ok(()) +} +/// Writes to the registers for the Vcpu +pub fn write_registers(vcpu_fd: &VcpuFd, regs: &CoreRegs) -> Result<(), GdbTargetError> { + let new_regs = kvm_regs { + rax: regs.regs[0], + rbx: regs.regs[1], + rcx: regs.regs[2], + rdx: regs.regs[3], + rsi: regs.regs[4], + rdi: regs.regs[5], + rbp: regs.regs[6], + rsp: regs.regs[7], + + r8: regs.regs[8], + r9: regs.regs[9], + r10: regs.regs[10], + r11: regs.regs[11], + r12: regs.regs[12], + r13: regs.regs[13], + r14: regs.regs[14], + r15: regs.regs[15], + + rip: regs.rip, + rflags: regs.eflags as u64, + }; + + Ok(vcpu_fd.set_regs(&new_regs)?) +} diff --git a/src/vmm/src/gdb/event_loop.rs b/src/vmm/src/gdb/event_loop.rs new file mode 100644 index 00000000000..ae4de0e64d7 --- /dev/null +++ b/src/vmm/src/gdb/event_loop.rs @@ -0,0 +1,159 @@ +// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::os::unix::net::UnixStream; +use std::sync::mpsc::Receiver; +use std::sync::mpsc::TryRecvError::Empty; +use std::sync::{Arc, Mutex}; + +use gdbstub::common::{Signal, Tid}; +use gdbstub::conn::{Connection, ConnectionExt}; +use gdbstub::stub::run_blocking::{self, WaitForStopReasonError}; +use gdbstub::stub::{DisconnectReason, GdbStub, MultiThreadStopReason}; +use gdbstub::target::Target; +use kvm_ioctls::VcpuFd; +use vm_memory::GuestAddress; + +use super::target::{vcpuid_to_tid, FirecrackerTarget, GdbTargetError}; +use crate::logger::{error, trace}; +use crate::Vmm; + +/// Starts the GDB event loop which acts as a proxy between the Vcpus and GDB +pub fn event_loop( + connection: UnixStream, + vmm: Arc>, + vcpu_fds: Vec, + gdb_event_receiver: Receiver, + entry_addr: GuestAddress, +) { + let target = FirecrackerTarget::new(vmm, vcpu_fds, gdb_event_receiver, entry_addr); + let connection: Box> = { Box::new(connection) }; + let debugger = GdbStub::new(connection); + + // We wait for the VM to reach the inital breakpoint we inserted before starting the event loop + target + .gdb_event + .recv() + .expect("Error getting initial gdb event"); + + gdb_event_loop_thread(debugger, target); +} + +struct GdbBlockingEventLoop {} + +impl run_blocking::BlockingEventLoop for GdbBlockingEventLoop { + type Target = FirecrackerTarget; + type Connection = Box>; + + type StopReason = MultiThreadStopReason; + + /// Poll for events from either Vcpu's or packets from the GDB connection + fn wait_for_stop_reason( + target: &mut FirecrackerTarget, + conn: &mut Self::Connection, + ) -> Result< + run_blocking::Event>, + run_blocking::WaitForStopReasonError< + ::Error, + ::Error, + >, + > { + loop { + match target.gdb_event.try_recv() { + Ok(cpu_id) => { + // The Vcpu reports it's id from raw_id so we straight convert here + let tid = Tid::new(cpu_id).expect("Error converting cpu id to Tid"); + // If notify paused returns false this means we were already debugging a single + // core, the target will track this for us to pick up later + target.set_paused_vcpu(tid); + trace!("Vcpu: {tid:?} paused from debug exit"); + + let stop_reason = target + .get_stop_reason(tid) + .map_err(WaitForStopReasonError::Target)?; + + let Some(stop_response) = stop_reason else { + // If we returned None this is a break which should be handled by + // the guest kernel (e.g. kernel int3 self testing) so we won't notify + // GDB and instead inject this back into the guest + target + .inject_bp_to_guest(tid) + .map_err(WaitForStopReasonError::Target)?; + target + .resume_vcpu(tid) + .map_err(WaitForStopReasonError::Target)?; + + trace!("Injected BP into guest early exit"); + continue; + }; + + trace!("Returned stop reason to gdb: {stop_response:?}"); + return Ok(run_blocking::Event::TargetStopped(stop_response)); + } + Err(Empty) => (), + Err(_) => { + return Err(WaitForStopReasonError::Target( + GdbTargetError::GdbQueueError, + )); + } + } + + if conn.peek().map(|b| b.is_some()).unwrap_or(false) { + let byte = conn + .read() + .map_err(run_blocking::WaitForStopReasonError::Connection)?; + return Ok(run_blocking::Event::IncomingData(byte)); + } + } + } + + /// Invoked when the GDB client sends a Ctrl-C interrupt. + fn on_interrupt( + target: &mut FirecrackerTarget, + ) -> Result>, ::Error> { + // notify the target that a ctrl-c interrupt has occurred. + let main_core = vcpuid_to_tid(0)?; + + target.pause_vcpu(main_core)?; + target.set_paused_vcpu(main_core); + + let exit_reason = MultiThreadStopReason::SignalWithThread { + tid: main_core, + signal: Signal::SIGINT, + }; + Ok(Some(exit_reason)) + } +} + +/// Runs while communication with GDB is in progress, after GDB disconnects we +/// shutdown firecracker +fn gdb_event_loop_thread( + debugger: GdbStub>>, + mut target: FirecrackerTarget, +) { + match debugger.run_blocking::(&mut target) { + Ok(disconnect_reason) => match disconnect_reason { + DisconnectReason::Disconnect => { + trace!("Client disconnected") + } + DisconnectReason::TargetExited(code) => { + trace!("Target exited with code {}", code) + } + DisconnectReason::TargetTerminated(sig) => { + trace!("Target terminated with signal {}", sig) + } + DisconnectReason::Kill => trace!("GDB sent a kill command"), + }, + Err(e) => { + if e.is_target_error() { + error!("target encountered a fatal error: {e:?}") + } else if e.is_connection_error() { + error!("connection error: {e:?}") + } else { + error!("gdbstub encountered a fatal error {e:?}") + } + } + } + + target.shutdown_vmm(); +} diff --git a/src/vmm/src/gdb/mod.rs b/src/vmm/src/gdb/mod.rs new file mode 100644 index 00000000000..9d209541425 --- /dev/null +++ b/src/vmm/src/gdb/mod.rs @@ -0,0 +1,66 @@ +// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +/// Arch specific implementations +mod arch; +/// Event loop for connection to GDB server +mod event_loop; +/// Target for gdb +pub mod target; + +use std::os::unix::net::UnixListener; +use std::path::Path; +use std::sync::mpsc::Receiver; +use std::sync::{Arc, Mutex}; + +use arch::vcpu_set_debug; +use event_loop::event_loop; +use kvm_ioctls::VcpuFd; +use target::GdbTargetError; +use vm_memory::GuestAddress; + +use crate::logger::trace; +use crate::Vmm; + +/// Kickstarts the GDB debugging process, it takes in the VMM object, a slice of +/// the paused Vcpu's, the GDB event queue which is used as a mechanism for the Vcpu's to notify +/// our GDB thread that they've been paused, then finally the entry address of the kernel. +/// +/// Firstly the function will start by configuring the Vcpus with KVM for debugging +/// +/// This will then create the GDB socket which will be used for communication to the GDB process. +/// After creating this, the function will block while waiting for GDB to connect. +/// +/// After the connection has been established the function will start a new thread for handling +/// communcation to the GDB server +pub fn gdb_thread( + vmm: Arc>, + vcpu_fds: Vec, + gdb_event_receiver: Receiver, + entry_addr: GuestAddress, + socket_addr: &str, +) -> Result<(), GdbTargetError> { + // We register a hw breakpoint at the entry point as GDB expects the application + // to be stopped as it connects. This also allows us to set breakpoints before kernel starts. + // This entry adddress is automatically used as it is not tracked inside the target state, so + // when resumed will be removed + vcpu_set_debug(&vcpu_fds[0], &[entry_addr], false)?; + + for vcpu_fd in &vcpu_fds[1..] { + vcpu_set_debug(vcpu_fd, &[], false)?; + } + + let path = Path::new(socket_addr); + let listener = UnixListener::bind(path).map_err(|_| GdbTargetError::ServerSocketError)?; + trace!("Waiting for GDB server connection on {}...", path.display()); + let (connection, _addr) = listener + .accept() + .map_err(|_| GdbTargetError::ServerSocketError)?; + + std::thread::Builder::new() + .name("gdb".into()) + .spawn(move || event_loop(connection, vmm, vcpu_fds, gdb_event_receiver, entry_addr)) + .map_err(|_| GdbTargetError::GdbThreadError)?; + + Ok(()) +} diff --git a/src/vmm/src/gdb/target.rs b/src/vmm/src/gdb/target.rs new file mode 100644 index 00000000000..c3234241379 --- /dev/null +++ b/src/vmm/src/gdb/target.rs @@ -0,0 +1,622 @@ +// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::HashMap; +use std::sync::mpsc::{Receiver, RecvError}; +use std::sync::{Arc, Mutex, PoisonError}; + +use arrayvec::ArrayVec; +use gdbstub::arch::Arch; +use gdbstub::common::{Signal, Tid}; +use gdbstub::stub::{BaseStopReason, MultiThreadStopReason}; +use gdbstub::target::ext::base::multithread::{ + MultiThreadBase, MultiThreadResume, MultiThreadResumeOps, MultiThreadSingleStep, + MultiThreadSingleStepOps, +}; +use gdbstub::target::ext::base::BaseOps; +use gdbstub::target::ext::breakpoints::{ + Breakpoints, BreakpointsOps, HwBreakpoint, HwBreakpointOps, SwBreakpoint, SwBreakpointOps, +}; +use gdbstub::target::ext::thread_extra_info::{ThreadExtraInfo, ThreadExtraInfoOps}; +use gdbstub::target::{Target, TargetError, TargetResult}; +#[cfg(target_arch = "aarch64")] +use gdbstub_arch::aarch64::reg::AArch64CoreRegs as CoreRegs; +#[cfg(target_arch = "aarch64")] +use gdbstub_arch::aarch64::AArch64 as GdbArch; +#[cfg(target_arch = "x86_64")] +use gdbstub_arch::x86::reg::X86_64CoreRegs as CoreRegs; +#[cfg(target_arch = "x86_64")] +use gdbstub_arch::x86::X86_64_SSE as GdbArch; +use kvm_ioctls::VcpuFd; +use vm_memory::{Bytes, GuestAddress}; + +use super::arch; +use crate::arch::PAGE_SIZE; +use crate::logger::{error, info}; +use crate::utils::u64_to_usize; +use crate::vstate::vcpu::VcpuSendEventError; +use crate::{FcExitCode, VcpuEvent, VcpuResponse, Vmm}; + +#[derive(Debug)] +/// Stores the current state of a Vcpu with a copy of the Vcpu file descriptor +struct VcpuState { + single_step: bool, + paused: bool, + vcpu_fd: VcpuFd, +} + +impl VcpuState { + /// Constructs a new instance of a VcpuState from a VcpuFd + fn from_vcpu_fd(vcpu_fd: VcpuFd) -> Self { + Self { + single_step: false, + paused: false, + vcpu_fd, + } + } + + /// Disables single stepping on the Vcpu state + fn reset_vcpu_state(&mut self) { + self.single_step = false; + } + + /// Updates the kvm debug flags set against the Vcpu with a check + fn update_kvm_debug(&self, hw_breakpoints: &[GuestAddress]) -> Result<(), GdbTargetError> { + if !self.paused { + info!("Attempted to update kvm debug on a non paused Vcpu"); + return Ok(()); + } + + arch::vcpu_set_debug(&self.vcpu_fd, hw_breakpoints, self.single_step) + } +} + +/// Errors from interactions between GDB and the VMM +#[derive(Debug, thiserror::Error, displaydoc::Display)] +pub enum GdbTargetError { + /// An error during a GDB request + GdbRequest, + /// An error with the queue between the target and the Vcpus + GdbQueueError, + /// The response from the Vcpu was not allowed + VcuRequestError, + /// No currently paused Vcpu error + NoPausedVcpu, + /// Error when setting Vcpu debug flags + VcpuKvmError, + /// Server socket Error + ServerSocketError, + /// Error with creating GDB thread + GdbThreadError, + /// VMM locking error + VmmLockError, + /// Vcpu send event error + VcpuSendEventError(#[from] VcpuSendEventError), + /// Recieve error from Vcpu channel + VcpuRecvError(#[from] RecvError), + /// TID Conversion error + TidConversionError, + /// KVM set guest debug error + KvmIoctlsError(#[from] kvm_ioctls::Error), + /// Gva no translation available + KvmGvaTranslateError, + /// Conversion error with cpu rflags + RegFlagConversionError, +} + +impl From for TargetError { + fn from(error: GdbTargetError) -> Self { + match error { + GdbTargetError::VmmLockError => TargetError::Fatal(GdbTargetError::VmmLockError), + _ => TargetError::NonFatal, + } + } +} + +impl From> for GdbTargetError { + fn from(_value: PoisonError) -> Self { + GdbTargetError::VmmLockError + } +} + +/// Debug Target for firecracker. +/// +/// This is used the manage the debug implementation and handle requests sent via GDB +#[derive(Debug)] +pub struct FirecrackerTarget { + /// A mutex around the VMM to allow communicataion to the Vcpus + vmm: Arc>, + /// Store the guest entry point + entry_addr: GuestAddress, + + /// Listener for events sent from the Vcpu + pub gdb_event: Receiver, + + /// Used to track the currently configured hardware breakpoints. + /// Limited to 4 in x86 see: + /// https://elixir.bootlin.com/linux/v6.1/source/arch/x86/include/asm/kvm_host.h#L210 + hw_breakpoints: ArrayVec, + /// Used to track the currently configured software breakpoints and store the op-code + /// which was swapped out + sw_breakpoints: HashMap<::Usize, [u8; arch::SW_BP_SIZE]>, + + /// Stores the current state of each Vcpu + vcpu_state: Vec, + + /// Stores the current paused thread id, GDB can inact commands without providing us a Tid to + /// run on and expects us to use the last paused thread. + paused_vcpu: Option, +} + +/// Convert the 1 indexed Tid to the 0 indexed Vcpuid +fn tid_to_vcpuid(tid: Tid) -> usize { + tid.get() - 1 +} + +/// Converts the inernal index of a Vcpu to +/// the Tid required by GDB +pub fn vcpuid_to_tid(cpu_id: usize) -> Result { + Tid::new(get_raw_tid(cpu_id)).ok_or(GdbTargetError::TidConversionError) +} + +/// Converts the inernal index of a Vcpu to +/// the 1 indexed value for GDB +pub fn get_raw_tid(cpu_id: usize) -> usize { + cpu_id + 1 +} + +impl FirecrackerTarget { + /// Creates a new Target for GDB stub. This is used as the layer between GDB and the VMM it + /// will handle requests from GDB and perform the appropriate actions, while also updating GDB + /// with the state of the VMM / Vcpu's as we hit debug events + pub fn new( + vmm: Arc>, + vcpu_fds: Vec, + gdb_event: Receiver, + entry_addr: GuestAddress, + ) -> Self { + let mut vcpu_state: Vec = + vcpu_fds.into_iter().map(VcpuState::from_vcpu_fd).collect(); + // By default vcpu 1 will be paused at the entry point + vcpu_state[0].paused = true; + + Self { + vmm, + entry_addr, + gdb_event, + // We only support 4 hw breakpoints on x86 this will need to be configurable on arm + hw_breakpoints: Default::default(), + sw_breakpoints: HashMap::new(), + vcpu_state, + + paused_vcpu: Tid::new(1), + } + } + + /// Retrieves the currently paused Vcpu id returns an error if there is no currently paused Vcpu + fn get_paused_vcpu_id(&self) -> Result { + self.paused_vcpu.ok_or(GdbTargetError::NoPausedVcpu) + } + + /// Retrieves the currently paused Vcpu state returns an error if there is no currently paused + /// Vcpu + fn get_paused_vcpu(&self) -> Result<&VcpuState, GdbTargetError> { + let vcpu_index = tid_to_vcpuid(self.get_paused_vcpu_id()?); + Ok(&self.vcpu_state[vcpu_index]) + } + + /// Updates state to reference the currently paused Vcpu and store that the cpu is currently + /// paused + pub fn set_paused_vcpu(&mut self, tid: Tid) { + self.vcpu_state[tid_to_vcpuid(tid)].paused = true; + self.paused_vcpu = Some(tid); + } + + /// Resumes execution of all paused Vcpus, update them with current kvm debug info + /// and resumes + fn resume_all_vcpus(&mut self) -> Result<(), GdbTargetError> { + self.vcpu_state + .iter() + .try_for_each(|state| state.update_kvm_debug(&self.hw_breakpoints))?; + + for cpu_id in 0..self.vcpu_state.len() { + let tid = vcpuid_to_tid(cpu_id)?; + self.resume_vcpu(tid)?; + } + + self.paused_vcpu = None; + + Ok(()) + } + + /// Resets all Vcpus to their base state + fn reset_all_vcpu_states(&mut self) { + for value in self.vcpu_state.iter_mut() { + value.reset_vcpu_state(); + } + } + + /// Shuts down the VMM + pub fn shutdown_vmm(&self) { + self.vmm + .lock() + .expect("error unlocking vmm") + .stop(FcExitCode::Ok) + } + + /// Pauses the requested Vcpu + pub fn pause_vcpu(&mut self, tid: Tid) -> Result<(), GdbTargetError> { + let vcpu_state = &mut self.vcpu_state[tid_to_vcpuid(tid)]; + + if vcpu_state.paused { + info!("Attempted to pause a vcpu already paused."); + // Pausing an already paused vcpu is not considered an error case from GDB + return Ok(()); + } + + let cpu_handle = &self.vmm.lock()?.vcpus_handles[tid_to_vcpuid(tid)]; + + cpu_handle.send_event(VcpuEvent::Pause)?; + let _ = cpu_handle.response_receiver().recv()?; + + vcpu_state.paused = true; + Ok(()) + } + + /// A helper function to allow the event loop to inject this breakpoint back into the Vcpu + pub fn inject_bp_to_guest(&mut self, tid: Tid) -> Result<(), GdbTargetError> { + let vcpu_state = &mut self.vcpu_state[tid_to_vcpuid(tid)]; + arch::vcpu_inject_bp(&vcpu_state.vcpu_fd, &self.hw_breakpoints, false) + } + + /// Resumes the Vcpu, will return early if the Vcpu is already running + pub fn resume_vcpu(&mut self, tid: Tid) -> Result<(), GdbTargetError> { + let vcpu_state = &mut self.vcpu_state[tid_to_vcpuid(tid)]; + + if !vcpu_state.paused { + info!("Attempted to resume a vcpu already running."); + // Resuming an already running Vcpu is not considered an error case from GDB + return Ok(()); + } + + let cpu_handle = &self.vmm.lock()?.vcpus_handles[tid_to_vcpuid(tid)]; + cpu_handle.send_event(VcpuEvent::Resume)?; + + let response = cpu_handle.response_receiver().recv()?; + if let VcpuResponse::NotAllowed(message) = response { + error!("Response resume : {message}"); + return Err(GdbTargetError::VcuRequestError); + } + + vcpu_state.paused = false; + Ok(()) + } + + /// Identifies why the specific core was paused to be returned to GDB if None is returned this + /// indicates to handle this internally and don't notify GDB + pub fn get_stop_reason( + &self, + tid: Tid, + ) -> Result>, GdbTargetError> { + let vcpu_state = &self.vcpu_state[tid_to_vcpuid(tid)]; + if vcpu_state.single_step { + return Ok(Some(MultiThreadStopReason::SignalWithThread { + tid, + signal: Signal::SIGTRAP, + })); + } + + let Ok(ip) = arch::get_instruction_pointer(&vcpu_state.vcpu_fd) else { + // If we error here we return an arbitrary Software Breakpoint, GDB will handle + // this gracefully + return Ok(Some(MultiThreadStopReason::SwBreak(tid))); + }; + + let gpa = arch::translate_gva(&vcpu_state.vcpu_fd, ip)?; + if self.sw_breakpoints.contains_key(&gpa) { + return Ok(Some(MultiThreadStopReason::SwBreak(tid))); + } + + if self.hw_breakpoints.contains(&GuestAddress(ip)) { + return Ok(Some(MultiThreadStopReason::HwBreak(tid))); + } + + if ip == self.entry_addr.0 { + return Ok(Some(MultiThreadStopReason::HwBreak(tid))); + } + + // This is not a breakpoint we've set, likely one set by the guest + Ok(None) + } +} + +impl Target for FirecrackerTarget { + type Error = GdbTargetError; + type Arch = GdbArch; + + #[inline(always)] + fn base_ops(&mut self) -> BaseOps { + BaseOps::MultiThread(self) + } + + #[inline(always)] + fn support_breakpoints(&mut self) -> Option> { + Some(self) + } + + /// We disable implicit sw breakpoints as we want to manage these internally so we can inject + /// breakpoints back into the guest if we didn't create them + #[inline(always)] + fn guard_rail_implicit_sw_breakpoints(&self) -> bool { + false + } +} + +impl MultiThreadBase for FirecrackerTarget { + /// Reads the registers for the Vcpu + fn read_registers(&mut self, regs: &mut CoreRegs, tid: Tid) -> TargetResult<(), Self> { + arch::read_registers(&self.vcpu_state[tid_to_vcpuid(tid)].vcpu_fd, regs)?; + + Ok(()) + } + + /// Writes to the registers for the Vcpu + fn write_registers(&mut self, regs: &CoreRegs, tid: Tid) -> TargetResult<(), Self> { + arch::write_registers(&self.vcpu_state[tid_to_vcpuid(tid)].vcpu_fd, regs)?; + + Ok(()) + } + + /// Writes data to a guest virtual address for the Vcpu + fn read_addrs( + &mut self, + mut gva: ::Usize, + mut data: &mut [u8], + tid: Tid, + ) -> TargetResult { + let data_len = data.len(); + let vcpu_state = &self.vcpu_state[tid_to_vcpuid(tid)]; + + while !data.is_empty() { + let gpa = arch::translate_gva(&vcpu_state.vcpu_fd, gva).map_err(|e| { + error!("Error {e:?} translating gva on read address: {gva:X}"); + })?; + + // Compute the amount space left in the page after the gpa + let read_len = std::cmp::min( + data.len(), + PAGE_SIZE - (u64_to_usize(gpa) & (PAGE_SIZE - 1)), + ); + + let vmm = &self.vmm.lock().map_err(|_| { + error!("Error locking vmm in read addr"); + TargetError::Fatal(GdbTargetError::VmmLockError) + })?; + vmm.guest_memory() + .read(&mut data[..read_len], GuestAddress(gpa as u64)) + .map_err(|e| { + error!("Error reading memory {e:?}"); + })?; + + data = &mut data[read_len..]; + gva += read_len as u64; + } + + Ok(data_len) + } + + /// Writes data at a guest virtual address for the Vcpu + fn write_addrs( + &mut self, + mut gva: ::Usize, + mut data: &[u8], + tid: Tid, + ) -> TargetResult<(), Self> { + let vcpu_state = &self.vcpu_state[tid_to_vcpuid(tid)]; + while !data.is_empty() { + let gpa = arch::translate_gva(&vcpu_state.vcpu_fd, gva).map_err(|e| { + error!("Error {e:?} translating gva on read address: {gva:X}"); + })?; + + // Compute the amount space left in the page after the gpa + let write_len = std::cmp::min( + data.len(), + PAGE_SIZE - (u64_to_usize(gpa) & (PAGE_SIZE - 1)), + ); + + let vmm = &self.vmm.lock().map_err(|_| { + error!("Error locking vmm in write addr"); + TargetError::Fatal(GdbTargetError::VmmLockError) + })?; + + vmm.guest_memory() + .write(&data[..write_len], GuestAddress(gpa)) + .map_err(|e| { + error!("Error {e:?} writing memory at {gpa:X}"); + })?; + + data = &data[write_len..]; + gva += write_len as u64; + } + + Ok(()) + } + + #[inline(always)] + /// Makes the callback provided with each Vcpu + /// GDB expects us to return all threads currently running with this command, for firecracker + /// this is all Vcpus + fn list_active_threads( + &mut self, + thread_is_active: &mut dyn FnMut(Tid), + ) -> Result<(), Self::Error> { + for id in 0..self.vcpu_state.len() { + thread_is_active(vcpuid_to_tid(id)?) + } + + Ok(()) + } + + #[inline(always)] + fn support_resume(&mut self) -> Option> { + Some(self) + } + + #[inline(always)] + fn support_thread_extra_info(&mut self) -> Option> { + Some(self) + } +} + +impl MultiThreadResume for FirecrackerTarget { + /// Disables single step on the Vcpu + fn set_resume_action_continue( + &mut self, + tid: Tid, + _signal: Option, + ) -> Result<(), Self::Error> { + self.vcpu_state[tid_to_vcpuid(tid)].single_step = false; + + Ok(()) + } + + /// Resumes the execution of all currently paused Vcpus + fn resume(&mut self) -> Result<(), Self::Error> { + self.resume_all_vcpus() + } + + /// Clears the state of all Vcpus setting it back to base config + fn clear_resume_actions(&mut self) -> Result<(), Self::Error> { + self.reset_all_vcpu_states(); + + Ok(()) + } + + #[inline(always)] + fn support_single_step(&mut self) -> Option> { + Some(self) + } +} + +impl MultiThreadSingleStep for FirecrackerTarget { + /// Enabled single step on the Vcpu + fn set_resume_action_step( + &mut self, + tid: Tid, + _signal: Option, + ) -> Result<(), Self::Error> { + self.vcpu_state[tid_to_vcpuid(tid)].single_step = true; + + Ok(()) + } +} + +impl Breakpoints for FirecrackerTarget { + #[inline(always)] + fn support_hw_breakpoint(&mut self) -> Option> { + Some(self) + } + + #[inline(always)] + fn support_sw_breakpoint(&mut self) -> Option> { + Some(self) + } +} + +impl HwBreakpoint for FirecrackerTarget { + /// Adds a hardware breakpoint The breakpoint addresses are + /// stored in state so we can track the reason for an exit. + fn add_hw_breakpoint( + &mut self, + gva: ::Usize, + _kind: ::BreakpointKind, + ) -> TargetResult { + let ga = GuestAddress(gva); + if self.hw_breakpoints.contains(&ga) { + return Ok(true); + } + + if self.hw_breakpoints.try_push(ga).is_err() { + return Ok(false); + } + + let state = self.get_paused_vcpu()?; + state.update_kvm_debug(&self.hw_breakpoints)?; + + Ok(true) + } + + /// Removes a hardware breakpoint. + fn remove_hw_breakpoint( + &mut self, + gva: ::Usize, + _kind: ::BreakpointKind, + ) -> TargetResult { + match self.hw_breakpoints.iter().position(|&b| b.0 == gva) { + None => return Ok(false), + Some(pos) => self.hw_breakpoints.remove(pos), + }; + + let state = self.get_paused_vcpu()?; + state.update_kvm_debug(&self.hw_breakpoints)?; + + Ok(true) + } +} + +impl SwBreakpoint for FirecrackerTarget { + /// Inserts a software breakpoint. + /// We initially translate the guest virtual address to a guest physical address and then check + /// if this is already present, if so we return early. Otherwise we store the opcode at the + /// specified guest physical address in our store and replace it with the `X86_SW_BP_OP` + fn add_sw_breakpoint( + &mut self, + addr: ::Usize, + _kind: ::BreakpointKind, + ) -> TargetResult { + let gpa = arch::translate_gva(&self.get_paused_vcpu()?.vcpu_fd, addr)?; + + if self.sw_breakpoints.contains_key(&gpa) { + return Ok(true); + } + + let paused_vcpu_id = self.get_paused_vcpu_id()?; + + let mut saved_register = [0; arch::SW_BP_SIZE]; + self.read_addrs(addr, &mut saved_register, paused_vcpu_id)?; + self.sw_breakpoints.insert(gpa, saved_register); + + self.write_addrs(addr, &arch::SW_BP, paused_vcpu_id)?; + Ok(true) + } + + /// Removes a software breakpoint. + /// We firstly translate the guest virtual address to a guest physical address, we then check if + /// the resulting gpa is in our store, if so we load the stored opcode and write this back + fn remove_sw_breakpoint( + &mut self, + addr: ::Usize, + _kind: ::BreakpointKind, + ) -> TargetResult { + let gpa = arch::translate_gva(&self.get_paused_vcpu()?.vcpu_fd, addr)?; + + if let Some(removed) = self.sw_breakpoints.remove(&gpa) { + self.write_addrs(addr, &removed, self.get_paused_vcpu_id()?)?; + return Ok(true); + } + + Ok(false) + } +} + +impl ThreadExtraInfo for FirecrackerTarget { + /// Allows us to configure the formatting of the thread information, we just return the ID of + /// the Vcpu + fn thread_extra_info(&self, tid: Tid, buf: &mut [u8]) -> Result { + let info = format!("Vcpu ID: {}", tid_to_vcpuid(tid)); + let size = buf.len().min(info.len()); + + buf[..size].copy_from_slice(&info.as_bytes()[..size]); + Ok(size) + } +} diff --git a/src/vmm/src/lib.rs b/src/vmm/src/lib.rs index 94dfcfbf409..c80f004e789 100644 --- a/src/vmm/src/lib.rs +++ b/src/vmm/src/lib.rs @@ -83,6 +83,9 @@ pub(crate) mod device_manager; pub mod devices; /// minimalist HTTP/TCP/IPv4 stack named DUMBO pub mod dumbo; +/// Support for GDB debugging the guest +#[cfg(feature = "gdb")] +pub mod gdb; /// Logger pub mod logger; /// microVM Metadata Service MMDS @@ -846,6 +849,12 @@ impl Vmm { // Break the main event loop, propagating the Vmm exit-code. self.shutdown_exit_code = Some(exit_code); } + + /// Gets a reference to kvm-ioctls Vm + #[cfg(feature = "gdb")] + pub fn vm(&self) -> &Vm { + &self.vm + } } /// Process the content of the MPIDR_EL1 register in order to be able to pass it to KVM diff --git a/src/vmm/src/resources.rs b/src/vmm/src/resources.rs index a4d15641975..923225c6a8a 100644 --- a/src/vmm/src/resources.rs +++ b/src/vmm/src/resources.rs @@ -86,6 +86,9 @@ pub struct VmmConfig { vsock_device: Option, #[serde(rename = "entropy")] entropy_device: Option, + #[cfg(feature = "gdb")] + #[serde(rename = "gdb-socket")] + gdb_socket_addr: Option, } /// A data structure that encapsulates the device configurations @@ -114,6 +117,9 @@ pub struct VmResources { pub mmds_size_limit: usize, /// Whether or not to load boot timer device. pub boot_timer: bool, + #[cfg(feature = "gdb")] + /// Configures the location of the GDB socket + pub gdb_socket_addr: Option, } impl VmResources { @@ -136,6 +142,8 @@ impl VmResources { let mut resources: Self = Self { mmds_size_limit, + #[cfg(feature = "gdb")] + gdb_socket_addr: vmm_config.gdb_socket_addr, ..Default::default() }; if let Some(machine_config) = vmm_config.machine_config { @@ -521,6 +529,8 @@ impl From<&VmResources> for VmmConfig { net_devices: resources.net_builder.configs(), vsock_device: resources.vsock.config(), entropy_device: resources.entropy.config(), + #[cfg(feature = "gdb")] + gdb_socket_addr: resources.gdb_socket_addr.clone(), } } } @@ -630,6 +640,8 @@ mod tests { boot_timer: false, mmds_size_limit: HTTP_MAX_PAYLOAD_SIZE, entropy: Default::default(), + #[cfg(feature = "gdb")] + gdb_socket_addr: None, } } diff --git a/src/vmm/src/vstate/vcpu/mod.rs b/src/vmm/src/vstate/vcpu/mod.rs index 89b167bbd5b..43a0946931e 100644 --- a/src/vmm/src/vstate/vcpu/mod.rs +++ b/src/vmm/src/vstate/vcpu/mod.rs @@ -6,6 +6,8 @@ // found in the THIRD-PARTY file. use std::cell::Cell; +#[cfg(feature = "gdb")] +use std::os::fd::AsRawFd; use std::sync::atomic::{fence, Ordering}; use std::sync::mpsc::{channel, Receiver, Sender, TryRecvError}; use std::sync::{Arc, Barrier}; @@ -13,6 +15,8 @@ use std::{fmt, io, thread}; use kvm_bindings::{KVM_SYSTEM_EVENT_RESET, KVM_SYSTEM_EVENT_SHUTDOWN}; use kvm_ioctls::VcpuExit; +#[cfg(feature = "gdb")] +use kvm_ioctls::VcpuFd; use libc::{c_int, c_void, siginfo_t}; use log::{error, info, warn}; use seccompiler::{BpfProgram, BpfProgramRef}; @@ -20,6 +24,8 @@ use vmm_sys_util::errno; use vmm_sys_util::eventfd::EventFd; use crate::cpu_config::templates::{CpuConfiguration, GuestConfigError}; +#[cfg(feature = "gdb")] +use crate::gdb::target::{get_raw_tid, GdbTargetError}; use crate::logger::{IncMetric, METRICS}; use crate::utils::signal::{register_signal_handler, sigrtmin, Killable}; use crate::utils::sm::StateMachine; @@ -60,6 +66,9 @@ pub enum VcpuError { VcpuTlsInit, /// Vcpu not present in TLS VcpuTlsNotPresent, + /// Error with gdb request sent + #[cfg(feature = "gdb")] + GdbRequest(GdbTargetError), } /// Encapsulates configuration parameters for the guest vCPUS. @@ -81,6 +90,17 @@ type VcpuCell = Cell>; #[error("Failed to spawn vCPU thread: {0}")] pub struct StartThreadedError(std::io::Error); +/// Error type for [`Vcpu::copy_kvm_vcpu_fd`]. +#[cfg(feature = "gdb")] +#[derive(Debug, thiserror::Error)] +#[error("Failed to clone kvm Vcpu fd: {0}")] +pub enum CopyKvmFdError { + /// Error with libc dup of kvm Vcpu fd + DupError(#[from] std::io::Error), + /// Error creating the Vcpu from the duplicated Vcpu fd + CreateVcpuError(#[from] kvm_ioctls::Error), +} + /// A wrapper around creating and using a vcpu. #[derive(Debug)] pub struct Vcpu { @@ -89,6 +109,9 @@ pub struct Vcpu { /// File descriptor for vcpu to trigger exit event on vmm. exit_evt: EventFd, + /// Debugger emitter for gdb events + #[cfg(feature = "gdb")] + gdb_event: Option>, /// The receiving end of events channel owned by the vcpu side. event_receiver: Receiver, /// The transmitting end of the events channel which will be given to the handler. @@ -200,6 +223,8 @@ impl Vcpu { event_sender: Some(event_sender), response_receiver: Some(response_receiver), response_sender, + #[cfg(feature = "gdb")] + gdb_event: None, kvm_vcpu, }) } @@ -209,6 +234,24 @@ impl Vcpu { self.kvm_vcpu.peripherals.mmio_bus = Some(mmio_bus); } + /// Attaches the fields required for debugging + #[cfg(feature = "gdb")] + pub fn attach_debug_info(&mut self, gdb_event: Sender) { + self.gdb_event = Some(gdb_event); + } + + /// Obtains a copy of the VcpuFd + #[cfg(feature = "gdb")] + pub fn copy_kvm_vcpu_fd(&self, vm: &Vm) -> Result { + // SAFETY: We own this fd so it is considered safe to clone + let r = unsafe { libc::dup(self.kvm_vcpu.fd.as_raw_fd()) }; + if r < 0 { + return Err(std::io::Error::last_os_error().into()); + } + // SAFETY: We assert this is a valid fd by checking the result from the dup + unsafe { Ok(vm.fd().create_vcpu_from_rawfd(r)?) } + } + /// Moves the vcpu to its own thread and constructs a VcpuHandle. /// The handle can be used to control the remote vcpu. pub fn start_threaded( @@ -271,6 +314,11 @@ impl Vcpu { // - the other vCPUs won't ever exit out of `KVM_RUN`, but they won't consume CPU. // So we pause vCPU0 and send a signal to the emulation thread to stop the VMM. Ok(VcpuEmulation::Stopped) => return self.exit(FcExitCode::Ok), + // If the emulation requests a pause lets do this + #[cfg(feature = "gdb")] + Ok(VcpuEmulation::Paused) => { + return StateMachine::next(Self::paused); + } // Emulation errors lead to vCPU exit. Err(_) => return self.exit(FcExitCode::GenericError), } @@ -448,6 +496,16 @@ impl Vcpu { // Notify that this KVM_RUN was interrupted. Ok(VcpuEmulation::Interrupted) } + #[cfg(feature = "gdb")] + Ok(VcpuExit::Debug(_)) => { + if let Some(gdb_event) = &self.gdb_event { + gdb_event + .send(get_raw_tid(self.kvm_vcpu.index.into())) + .expect("Unable to notify gdb event"); + } + + Ok(VcpuEmulation::Paused) + } emulation_result => handle_kvm_exit(&mut self.kvm_vcpu.peripherals, emulation_result), } } @@ -688,6 +746,9 @@ pub enum VcpuEmulation { Interrupted, /// Stopped. Stopped, + /// Pause request + #[cfg(feature = "gdb")] + Paused, } #[cfg(test)] diff --git a/tests/integration_tests/build/test_gdb.py b/tests/integration_tests/build/test_gdb.py new file mode 100644 index 00000000000..541807dd5c1 --- /dev/null +++ b/tests/integration_tests/build/test_gdb.py @@ -0,0 +1,19 @@ +# Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A test that ensures that firecracker builds with GDB feature enabled at integration time.""" + +import platform + +import pytest + +import host_tools.cargo_build as host + +MACHINE = platform.machine() +TARGET = "{}-unknown-linux-musl".format(MACHINE) + + +@pytest.mark.skipif(MACHINE != "x86_64", reason="GDB runs only on x86_64.") +def test_gdb_compiles(): + """Checks that Firecracker compiles with GDB enabled""" + + host.cargo("build", f"--features gdb --target {TARGET}")