diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index 66f0a0ff8..e5215e91d 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -95,6 +95,7 @@ jobs: # make sure certain cargo features compile cargo check -p hyperlight-host --features crashdump cargo check -p hyperlight-host --features print_debug + cargo check -p hyperlight-host --features gdb # without any driver (shouldn't compile) just test-rust-feature-compilation-fail ${{ matrix.config }} @@ -114,6 +115,13 @@ jobs: RUST_LOG: debug run: just run-rust-examples-linux ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}} + - name: Run Rust Gdb tests - linux + if: runner.os == 'Linux' && matrix.hypervisor == 'kvm' + env: + CARGO_TERM_COLOR: always + RUST_LOG: debug + run: just test-rust-gdb-debugging ${{ matrix.config }} + ### Benchmarks ### - name: Install github-cli (Linux mariner) if: runner.os == 'Linux' && matrix.hypervisor == 'mshv' diff --git a/Cargo.lock b/Cargo.lock index 559ed652c..2c8ed847e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -805,6 +805,30 @@ dependencies = [ "slab", ] +[[package]] +name = "gdbstub" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31c683a9f13de31432e6097131d5f385898c7f0635c0f392b9d0fa165063c8ac" +dependencies = [ + "bitflags 2.8.0", + "cfg-if", + "log", + "managed", + "num-traits", + "paste", +] + +[[package]] +name = "gdbstub_arch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "328a9e9425db13770d0d11de6332a608854266e44c53d12776be7b4aa427e3de" +dependencies = [ + "gdbstub", + "num-traits", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1106,6 +1130,8 @@ dependencies = [ "env_logger", "envy", "flatbuffers", + "gdbstub", + "gdbstub_arch", "goblin", "hyperlight-common", "hyperlight-testing", @@ -1574,6 +1600,12 @@ dependencies = [ "libc", ] +[[package]] +name = "managed" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca88d725a0a943b096803bd34e73a4437208b6077654cc4ecb2947a5f91618d" + [[package]] name = "matchers" version = "0.1.0" diff --git a/Justfile b/Justfile index 56045d0b8..d5e73f312 100644 --- a/Justfile +++ b/Justfile @@ -110,6 +110,11 @@ test-rust-feature-compilation-fail target=default-target: @# the following should fail on linux because one of kvm, mshv, or mshv3 feature must be specified, which is why the exit code is inverted with an !. {{ if os() == "linux" { "! cargo check -p hyperlight-host --no-default-features 2> /dev/null"} else { "" } }} +# Test rust gdb debugging +test-rust-gdb-debugging target=default-target: (build-rust target) + {{ set-trace-env-vars }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --example guest-debugging --features gdb + {{ set-trace-env-vars }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --features gdb -- test_gdb + test target=default-target: (test-rust target) # RUST LINTING diff --git a/docs/README.md b/docs/README.md index 1b1572dda..fff51c662 100644 --- a/docs/README.md +++ b/docs/README.md @@ -34,6 +34,7 @@ This project is composed internally of several internal components, depicted in * [Security guidance for developers](./security-guidance-for-developers.md) * [Paging Development Notes](./paging-development-notes.md) +* [How to debug a Hyperlight guest](./how-to-debug-a-hyperlight-guest.md) * [How to use Flatbuffers in Hyperlight](./how-to-use-flatbuffers.md) * [How to make a Hyperlight release](./how-to-make-releases.md) * [Getting Hyperlight Metrics, Logs, and Traces](./hyperlight-metrics-logs-and-traces.md) diff --git a/docs/how-to-debug-a-hyperlight-guest.md b/docs/how-to-debug-a-hyperlight-guest.md new file mode 100644 index 000000000..843fc7ae8 --- /dev/null +++ b/docs/how-to-debug-a-hyperlight-guest.md @@ -0,0 +1,193 @@ +# How to debug a Hyperlight **KVM** guest using gdb + +Hyperlight supports gdb debugging of a **KVM** guest running inside a Hyperlight sandbox. +When Hyperlight is compiled with the `gdb` feature enabled, a Hyperlight KVM sandbox can be configured +to start listening for a gdb connection. + +## Supported features + +The Hyperlight `gdb` feature enables **KVM** guest debugging: + - an entry point breakpoint is automatically set for the guest to stop + - add and remove HW breakpoints (maximum 4 set breakpoints at a time) + - add and remove SW breakpoints + - read and write registers + - read and write addresses + - step/continue + - get code offset from target + +## Expected behavior + +Below is a list describing some cases of expected behavior from a gdb debug +session of a guest binary running inside a KVM Hyperlight sandbox. + +- when the `gdb` feature is enabled and a SandboxConfiguration is provided a + debug port, the created sandbox will wait for a gdb client to connect on the + configured port +- when the gdb client attaches, the guest vCPU is expected to be stopped at the + entry point +- if a gdb client disconnects unexpectedly, the debug session will be closed and + the guest will continue executing disregarding any prior breakpoints +- if multiple sandbox instances are created, each instance will have its own + gdb thread listening on the configured port +- if two sandbox instances are created with the same debug port, the second + instance logs an error and the gdb thread will not be created, but the sandbox + will continue to run without gdb debugging + +## Example + +### Sandbox configuration + +The `guest-debugging` example in Hyperlight demonstrates how to configure a Hyperlight +sandbox to listen for a gdb client on a specific port. + +### CLI Gdb configuration + +One can use a gdb config file to provide the symbols and desired configuration. + +The below contents of the `.gdbinit` file can be used to provide a basic configuration +to gdb startup. + +```gdb +# Path to symbols +file path/to/symbols.elf +# The port on which Hyperlight listens for a connection +target remote :8080 +set disassembly-flavor intel +set disassemble-next-line on +enable pretty-printer +layout src +``` +One can find more information about the `.gdbinit` file at [gdbinit(5)](https://www.man7.org/linux/man-pages/man5/gdbinit.5.html). + +### End to end example + +Using the example mentioned at [Sandbox configuration](#sandbox-configuration) +one can run the below commands to debug the guest binary: + +```bash +# Terminal 1 +$ cargo run --example guest-debugging --features gdb +``` + +```bash +# Terminal 2 +$ cat .gdbinit +file src/tests/rust_guests/bin/debug/simpleguest +target remote :8080 +set disassembly-flavor intel +set disassemble-next-line on +enable pretty-printer +layout src + +$ gdb +``` + +### Using VSCode to debug a Hyperlight guest + +To replicate the above behavior using VSCode follow the below steps: +- install the `gdb` package on the host machine +- install the `C/C++` extension in VSCode to add debugging capabilities +- create a `.vscode/launch.json` file in the project directory with the below content: + ```json + { + "version": "0.2.0", + "configurations": [ + { + "name": "GDB", + "type": "cppdbg", + "request": "launch", + "program": "${workspaceFolder}/src/tests/rust_guests/bin/debug/simpleguest", + "args": [], + "stopAtEntry": true, + "hardwareBreakpoints": {"require": false, "limit": 4}, + "cwd": "${workspaceFolder}", + "environment": [], + "externalConsole": false, + "MIMode": "gdb", + "miDebuggerPath": "/usr/bin/gdb", + "miDebuggerServerAddress": "localhost:8080", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + }, + { + "description": "Set Disassembly Flavor to Intel", + "text": "-gdb-set disassembly-flavor intel", + "ignoreFailures": true + } + ] + } + ] + } + ``` +- in `Run and Debug` tab, select the `GDB` configuration and click on the `Run` + button to start the debugging session. + The gdb client will connect to the Hyperlight sandbox and the guest vCPU will + stop at the entry point. + + +## How it works + +The gdb feature is designed to work like a Request - Response protocol between +a thread that accepts commands from a gdb client and the hypervisor handler over +a communication channel. + +All the functionality is implemented on the hypervisor side so it has access to +the shared memory and the vCPU. + +The gdb thread uses the `gdbstub` crate to handle the communication with the gdb client. +When the gdb client requests one of the supported features mentioned above, a request +is sent over the communication channel to the hypervisor handler for the sandbox +to resolve. + +Below is a sequence diagram that shows the interaction between the entities +involved in the gdb debugging of a Hyperlight guest running inside a KVM sandbox. + +``` + ┌───────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Hyperlight Sandbox │ + USER │ │ +┌────────────┐ │ ┌──────────────┐ ┌───────────────────────────┐ ┌────────┐ │ +│ gdb client │ │ │ gdb thread │ │ hypervisor handler thread │ │ vCPU │ │ +└────────────┘ │ └──────────────┘ └───────────────────────────┘ └────────┘ │ + | │ | create_gdb_thread | | │ + | │ |◄─────────────────────────────────────────┌─┐ vcpu stopped ┌─┐ │ + | attach │ ┌─┐ │ │◄──────────────────────────────┴─┘ │ + ┌─┐───────────────────────┼────────►│ │ │ │ entrypoint breakpoint | │ + │ │ attach response │ │ │ │ │ | │ + │ │◄──────────────────────┼─────────│ │ │ │ | │ + │ │ │ │ │ │ │ | │ + │ │ add_breakpoint │ │ │ │ │ | │ + │ │───────────────────────┼────────►│ │ add_breakpoint │ │ | │ + │ │ │ │ │────────────────────────────────────────►│ │ add_breakpoint | │ + │ │ │ │ │ │ │────┐ | │ + │ │ │ │ │ │ │ │ | │ + │ │ │ │ │ │ │◄───┘ | │ + │ │ │ │ │ add_breakpoint response │ │ | │ + │ │ add_breakpoint response │ │◄────────────────────────────────────────│ │ | │ + │ │◄──────────────────────┬─────────│ │ │ │ | │ + │ │ continue │ │ │ │ │ | │ + │ │───────────────────────┼────────►│ │ continue │ │ | │ + │ │ │ │ │────────────────────────────────────────►│ │ resume vcpu | │ + │ │ │ │ │ │ │──────────────────────────────►┌─┐ │ + │ │ │ │ │ │ │ │ │ │ + │ │ │ │ │ │ │ │ │ │ + │ │ │ │ │ │ │ │ │ │ + │ │ │ │ │ │ │ │ │ │ + │ │ │ │ │ │ │ vcpu stopped │ │ │ + │ │ │ │ │ notify vcpu stop reason │ │◄──────────────────────────────┴─┘ │ + │ │ notify vcpu stop reason │ │◄────────────────────────────────────────│ │ | │ + │ │◄──────────────────────┬─────────│ │ │ │ | │ + │ │ continue until end │ │ │ │ │ | │ + │ │───────────────────────┼────────►│ │ continue │ │ resume vcpu | │ + │ │ │ │ │────────────────────────────────────────►│ │──────────────────────────────►┌─┐ │ + │ │ │ │ │ │ │ │ │ │ + │ │ │ │ │ comm channel disconnected │ │ vcpu halted │ │ │ + │ │ target finished exec│ │ │◄────────────────────────────────────────┤ │◄──────────────────────────────┴─┘ │ + │ │◄──────────────────────┼─────────┴─┘ target finished exec └─┘ | │ + │ │ │ | | | │ + └─┘ │ | | | │ + | └───────────────────────────────────────────────────────────────────────────────────────────────┘ +``` diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index 692389d04..8440c2c13 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -71,6 +71,8 @@ sha256 = "1.4.0" windows-version = "0.1" [target.'cfg(unix)'.dependencies] +gdbstub = { version = "0.7.3", optional = true } +gdbstub_arch = { version = "0.3.1", optional = true } seccompiler = { version = "0.4.0", optional = true } kvm-bindings = { version = "0.11", features = ["fam-wrappers"], optional = true } kvm-ioctls = { version = "0.20", optional = true } @@ -128,6 +130,8 @@ kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"] mshv2 = ["dep:mshv-bindings2", "dep:mshv-ioctls2"] mshv3 = ["dep:mshv-bindings3", "dep:mshv-ioctls3"] inprocess = [] +# This enables easy debug in the guest +gdb = ["dep:gdbstub", "dep:gdbstub_arch"] [[bench]] name = "benchmarks" diff --git a/src/hyperlight_host/build.rs b/src/hyperlight_host/build.rs index 7600f6470..a8370725d 100644 --- a/src/hyperlight_host/build.rs +++ b/src/hyperlight_host/build.rs @@ -89,6 +89,7 @@ fn main() -> Result<()> { // Essentially the kvm and mshv features are ignored on windows as long as you use #[cfg(kvm)] and not #[cfg(feature = "kvm")]. // You should never use #[cfg(feature = "kvm")] or #[cfg(feature = "mshv")] in the codebase. cfg_aliases::cfg_aliases! { + gdb: { all(feature = "gdb", debug_assertions, feature = "kvm", target_os = "linux") }, kvm: { all(feature = "kvm", target_os = "linux") }, mshv: { all(any(feature = "mshv2", feature = "mshv3"), target_os = "linux") }, // inprocess feature is aliased with debug_assertions to make it only available in debug-builds. diff --git a/src/hyperlight_host/examples/guest-debugging/main.rs b/src/hyperlight_host/examples/guest-debugging/main.rs new file mode 100644 index 000000000..da3010b9c --- /dev/null +++ b/src/hyperlight_host/examples/guest-debugging/main.rs @@ -0,0 +1,245 @@ +/* +Copyright 2024 The Hyperlight Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +use std::sync::{Arc, Mutex}; +use std::thread; + +use hyperlight_common::flatbuffer_wrappers::function_types::{ParameterValue, ReturnType}; +use hyperlight_host::func::HostFunction0; +#[cfg(gdb)] +use hyperlight_host::sandbox::config::DebugInfo; +use hyperlight_host::sandbox::SandboxConfiguration; +use hyperlight_host::sandbox_state::sandbox::EvolvableSandbox; +use hyperlight_host::sandbox_state::transition::Noop; +use hyperlight_host::{MultiUseSandbox, UninitializedSandbox}; + +/// Build a sandbox configuration that enables GDB debugging when the `gdb` feature is enabled. +fn get_sandbox_cfg() -> Option { + #[cfg(gdb)] + { + let mut cfg = SandboxConfiguration::default(); + let debug_info = DebugInfo { port: 8080 }; + cfg.set_guest_debug_info(debug_info); + + Some(cfg) + } + + #[cfg(not(gdb))] + None +} + +fn main() -> hyperlight_host::Result<()> { + let cfg = get_sandbox_cfg(); + + // Create an uninitialized sandbox with a guest binary + let mut uninitialized_sandbox = UninitializedSandbox::new( + hyperlight_host::GuestBinary::FilePath( + hyperlight_testing::simple_guest_as_string().unwrap(), + ), + cfg, // sandbox configuration + None, // default run options + None, // default host print function + )?; + + // Register a host functions + fn sleep_5_secs() -> hyperlight_host::Result<()> { + thread::sleep(std::time::Duration::from_secs(5)); + Ok(()) + } + + let host_function = Arc::new(Mutex::new(sleep_5_secs)); + + host_function.register(&mut uninitialized_sandbox, "Sleep5Secs")?; + // Note: This function is unused, it's just here for demonstration purposes + + // Initialize sandbox to be able to call host functions + let mut multi_use_sandbox: MultiUseSandbox = uninitialized_sandbox.evolve(Noop::default())?; + + // Call guest function + let message = "Hello, World! I am executing inside of a VM :)\n".to_string(); + let result = multi_use_sandbox.call_guest_function_by_name( + "PrintOutput", // function must be defined in the guest binary + ReturnType::Int, + Some(vec![ParameterValue::String(message.clone())]), + ); + + assert!(result.is_ok()); + + Ok(()) +} + +#[cfg(gdb)] +#[cfg(test)] +mod tests { + use std::fs::File; + use std::io; + use std::process::{Command, Stdio}; + use std::time::Duration; + + use hyperlight_host::{new_error, Result}; + use io::{BufReader, BufWriter, Read, Write}; + + use super::*; + + fn write_cmds_file(cmd_file_path: &str, out_file_path: &str) -> io::Result<()> { + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").expect("Failed to get manifest dir"); + let file = File::create(cmd_file_path)?; + let mut writer = BufWriter::new(file); + + // write from string to file + writer.write_all( + format!( + "file {manifest_dir}/../tests/rust_guests/bin/debug/simpleguest + target remote :8080 + + set pagination off + set logging file {out_file_path} + set logging enabled on + + break hyperlight_main + commands + echo \"Stopped at hyperlight_main breakpoint\\n\" + backtrace + continue + end + + continue + + set logging enabled off + quit + " + ) + .as_bytes(), + )?; + + writer.flush() + } + + fn run_guest_and_gdb(cmd_file_path: &str, out_file_path: &str) -> Result<()> { + // write gdb commands to file + + write_cmds_file(&cmd_file_path, &out_file_path) + .expect("Failed to write gdb commands to file"); + + let mut guest_child = Command::new("cargo") + .arg("run") + .arg("--example") + .arg("guest-debugging") + .arg("--features") + .arg("gdb") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .map_err(|e| new_error!("Failed to start guest process: {}", e))?; + + // wait 3 seconds for the gdb to connect + thread::sleep(Duration::from_secs(3)); + + let mut gdb = Command::new("rust-gdb") + .arg("--nw") + .arg("-x") + .arg(cmd_file_path) + .spawn() + .map_err(|e| new_error!("Failed to start gdb process: {}", e))?; + + // wait 3 seconds for the gdb to connect + thread::sleep(Duration::from_secs(10)); + + // check if the guest process has finished + match guest_child.try_wait() { + Ok(Some(status)) => { + if !status.success() { + Err(new_error!( + "Guest process exited with non-zero status: {}", + status + ))?; + } + } + Ok(None) => { + guest_child + .kill() + .map_err(|e| new_error!("Failed to kill child process: {}", e))?; + } + Err(e) => { + Err(new_error!("error attempting to wait guest: {}", e))?; + } + } + + // check if the gdb process has finished + match gdb.try_wait() { + Ok(Some(status)) => { + if !status.success() { + Err(new_error!( + "Gdb process exited with non-zero status: {}", + status + ))?; + } + } + Ok(None) => { + gdb.kill() + .map_err(|e| new_error!("Failed to kill guest process: {}", e))?; + } + Err(e) => { + Err(new_error!("error attempting to wait gdb: {}", e))?; + } + } + + check_output(&out_file_path) + } + + fn check_output(out_file_path: &str) -> Result<()> { + let results = File::open(out_file_path) + .map_err(|e| new_error!("Failed to open gdb.output file: {}", e))?; + let mut reader = BufReader::new(results); + let mut contents = String::new(); + reader.read_to_string(&mut contents).unwrap(); + + if contents.contains("Stopped at hyperlight_main breakpoint") { + Ok(()) + } else { + Err(new_error!( + "Failed to find expected output in gdb.output file" + )) + } + } + + fn cleanup(out_file_path: &str, cmd_file_path: &str) -> Result<()> { + let res1 = std::fs::remove_file(out_file_path) + .map_err(|e| new_error!("Failed to remove gdb.output file: {}", e)); + let res2 = std::fs::remove_file(cmd_file_path) + .map_err(|e| new_error!("Failed to remove gdb-commands.txt file: {}", e)); + + res1?; + res2?; + + Ok(()) + } + + #[test] + fn test_gdb_end_to_end() { + let out_dir = std::env::var("OUT_DIR").expect("Failed to get out dir"); + let out_file_path = format!("{out_dir}/gdb.output"); + let cmd_file_path = format!("{out_dir}/gdb-commands.txt"); + + let result = run_guest_and_gdb(&cmd_file_path, &out_file_path); + + // cleanup + let cleanup_result = cleanup(&out_file_path, &cmd_file_path); + assert!(cleanup_result.is_ok(), "{}", cleanup_result.unwrap_err()); + // check if the test passed - done at the end to ensure cleanup is done + assert!(result.is_ok(), "{}", result.unwrap_err()); + } +} diff --git a/src/hyperlight_host/src/error.rs b/src/hyperlight_host/src/error.rs index e846ee12e..76fdb8d05 100644 --- a/src/hyperlight_host/src/error.rs +++ b/src/hyperlight_host/src/error.rs @@ -275,6 +275,11 @@ pub enum HyperlightError { #[error("SystemTimeError {0:?}")] SystemTimeError(#[from] SystemTimeError), + /// Error occurred when translating guest address + #[error("An error occurred when translating guest address: {0:?}")] + #[cfg(gdb)] + TranslateGuestAddress(u64), + /// Error occurred converting a slice to an array #[error("TryFromSliceError {0:?}")] TryFromSliceError(#[from] TryFromSliceError), diff --git a/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs b/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs new file mode 100644 index 000000000..ee009caa1 --- /dev/null +++ b/src/hyperlight_host/src/hypervisor/gdb/event_loop.rs @@ -0,0 +1,142 @@ +/* +Copyright 2024 The Hyperlight Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +use gdbstub::common::Signal; +use gdbstub::conn::ConnectionExt; +use gdbstub::stub::run_blocking::{self, WaitForStopReasonError}; +use gdbstub::stub::{BaseStopReason, DisconnectReason, GdbStub, SingleThreadStopReason}; +use libc::{pthread_kill, SIGRTMIN}; + +use super::x86_64_target::HyperlightSandboxTarget; +use super::{DebugResponse, GdbTargetError, VcpuStopReason}; + +pub struct GdbBlockingEventLoop; + +impl run_blocking::BlockingEventLoop for GdbBlockingEventLoop { + type Connection = Box>; + type StopReason = SingleThreadStopReason; + type Target = HyperlightSandboxTarget; + + fn wait_for_stop_reason( + target: &mut Self::Target, + conn: &mut Self::Connection, + ) -> Result< + run_blocking::Event, + run_blocking::WaitForStopReasonError< + ::Error, + ::Error, + >, + > { + loop { + match target.try_recv() { + Ok(DebugResponse::VcpuStopped(stop_reason)) => { + log::debug!("VcpuStopped with reason {:?}", stop_reason); + + // Resume execution if unknown reason for stop + let stop_response = match stop_reason { + VcpuStopReason::DoneStep => BaseStopReason::DoneStep, + VcpuStopReason::SwBp => BaseStopReason::SwBreak(()), + VcpuStopReason::HwBp => BaseStopReason::HwBreak(()), + // This is a consequence of the GDB client sending an interrupt signal + // to the target thread + VcpuStopReason::Interrupt => BaseStopReason::SignalWithThread { + tid: (), + signal: Signal(SIGRTMIN() as u8), + }, + VcpuStopReason::Unknown => { + log::warn!("Unknown stop reason - resuming execution"); + + target + .resume_vcpu() + .map_err(WaitForStopReasonError::Target)?; + + continue; + } + }; + + return Ok(run_blocking::Event::TargetStopped(stop_response)); + } + Ok(msg) => { + log::error!("Unexpected message received {:?}", msg); + } + Err(crossbeam_channel::TryRecvError::Empty) => (), + Err(crossbeam_channel::TryRecvError::Disconnected) => { + return Ok(run_blocking::Event::TargetStopped(BaseStopReason::Exited( + 0, + ))); + } + } + + // Check if there is any data to read from the connection + // If there is, return the data as an incoming data event + // Otherwise, continue waiting for a stop reason from the target + if conn.peek().map(|b| b.is_some()).unwrap_or(false) { + let byte = conn + .read() + .map_err(run_blocking::WaitForStopReasonError::Connection)?; + + return Ok(run_blocking::Event::IncomingData(byte)); + } + } + } + + /// Handle an interrupt from the GDB client. + /// This function is called when the GDB client sends an interrupt signal. + /// Passing `None` defers sending a stop reason to later (e.g. when the target stops). + fn on_interrupt( + target: &mut Self::Target, + ) -> Result, ::Error> { + log::info!("Received interrupt from GDB client - sending signal to target thread"); + + // Send a signal to the target thread to interrupt it + let ret = unsafe { pthread_kill(target.get_thread_id(), SIGRTMIN()) }; + + log::info!("pthread_kill returned {}", ret); + + if ret < 0 && ret != libc::ESRCH { + log::error!("Failed to send signal to target thread"); + return Err(GdbTargetError::SendSignalError); + } + + Ok(None) + } +} + +pub fn event_loop_thread( + debugger: GdbStub>>, + target: &mut HyperlightSandboxTarget, +) { + match debugger.run_blocking::(target) { + Ok(disconnect_reason) => match disconnect_reason { + DisconnectReason::Disconnect => { + log::info!("Gdb client disconnected"); + if let Err(e) = target.disable_debug() { + log::error!("Cannot disable debugging: {:?}", e); + } + } + DisconnectReason::TargetExited(_) => { + log::info!("Guest finalized execution and disconnected"); + } + DisconnectReason::TargetTerminated(sig) => { + log::info!("Gdb target terminated with signal {}", sig) + } + DisconnectReason::Kill => log::info!("Gdb sent a kill command"), + }, + Err(e) => { + log::error!("fatal error encountered: {e:?}"); + } + } +} diff --git a/src/hyperlight_host/src/hypervisor/gdb/mod.rs b/src/hyperlight_host/src/hypervisor/gdb/mod.rs new file mode 100644 index 000000000..b941040a3 --- /dev/null +++ b/src/hyperlight_host/src/hypervisor/gdb/mod.rs @@ -0,0 +1,240 @@ +/* +Copyright 2024 The Hyperlight Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +mod event_loop; +pub mod x86_64_target; + +use std::io::{self, ErrorKind}; +use std::net::TcpListener; +use std::thread; + +use crossbeam_channel::{Receiver, Sender, TryRecvError}; +use event_loop::event_loop_thread; +use gdbstub::conn::ConnectionExt; +use gdbstub::stub::GdbStub; +use gdbstub::target::TargetError; +use thiserror::Error; +use x86_64_target::HyperlightSandboxTarget; + +#[derive(Debug, Error)] +pub enum GdbTargetError { + #[error("Error encountered while binding to address and port")] + CannotBind, + #[error("Error encountered while listening for connections")] + ListenerError, + #[error("Error encountered when waiting to receive message")] + CannotReceiveMsg, + #[error("Error encountered when sending message")] + CannotSendMsg, + #[error("Error encountered when sending a signal to the hypervisor thread")] + SendSignalError, + #[error("Encountered an unexpected message over communication channel")] + UnexpectedMessage, + #[error("Unexpected error encountered")] + UnexpectedError, +} + +impl From for GdbTargetError { + fn from(err: io::Error) -> Self { + match err.kind() { + ErrorKind::AddrInUse => Self::CannotBind, + ErrorKind::AddrNotAvailable => Self::CannotBind, + ErrorKind::ConnectionReset + | ErrorKind::ConnectionAborted + | ErrorKind::ConnectionRefused => Self::ListenerError, + _ => Self::UnexpectedError, + } + } +} + +impl From for TargetError { + fn from(value: GdbTargetError) -> TargetError { + TargetError::Io(std::io::Error::other(value)) + } +} + +/// Struct that contains the x86_64 core registers +#[derive(Debug, Default)] +pub struct X86_64Regs { + pub rax: u64, + pub rbx: u64, + pub rcx: u64, + pub rdx: u64, + pub rsi: u64, + pub rdi: u64, + pub rbp: u64, + pub rsp: u64, + pub r8: u64, + pub r9: u64, + pub r10: u64, + pub r11: u64, + pub r12: u64, + pub r13: u64, + pub r14: u64, + pub r15: u64, + pub rip: u64, + pub rflags: u64, +} + +/// Defines the possible reasons for which a vCPU can be stopped when debugging +#[derive(Debug)] +pub enum VcpuStopReason { + DoneStep, + HwBp, + SwBp, + Interrupt, + Unknown, +} + +/// Enumerates the possible actions that a debugger can ask from a Hypervisor +#[derive(Debug)] +pub enum DebugMsg { + AddHwBreakpoint(u64), + AddSwBreakpoint(u64), + Continue, + DisableDebug, + GetCodeSectionOffset, + ReadAddr(u64, usize), + ReadRegisters, + RemoveHwBreakpoint(u64), + RemoveSwBreakpoint(u64), + Step, + WriteAddr(u64, Vec), + WriteRegisters(X86_64Regs), +} + +/// Enumerates the possible responses that a hypervisor can provide to a debugger +#[derive(Debug)] +pub enum DebugResponse { + AddHwBreakpoint(bool), + AddSwBreakpoint(bool), + Continue, + DisableDebug, + ErrorOccurred, + GetCodeSectionOffset(u64), + ReadAddr(Vec), + ReadRegisters(X86_64Regs), + RemoveHwBreakpoint(bool), + RemoveSwBreakpoint(bool), + Step, + VcpuStopped(VcpuStopReason), + WriteAddr, + WriteRegisters, +} + +/// Debug communication channel that is used for sending a request type and +/// receive a different response type +pub struct DebugCommChannel { + /// Transmit channel + tx: Sender, + /// Receive channel + rx: Receiver, +} + +impl DebugCommChannel { + pub fn unbounded() -> (DebugCommChannel, DebugCommChannel) { + let (hyp_tx, gdb_rx): (Sender, Receiver) = crossbeam_channel::unbounded(); + let (gdb_tx, hyp_rx): (Sender, Receiver) = crossbeam_channel::unbounded(); + + let gdb_conn = DebugCommChannel { + tx: gdb_tx, + rx: gdb_rx, + }; + + let hyp_conn = DebugCommChannel { + tx: hyp_tx, + rx: hyp_rx, + }; + + (gdb_conn, hyp_conn) + } + + /// Sends message over the transmit channel and expects a response + pub fn send(&self, msg: T) -> Result<(), GdbTargetError> { + self.tx.send(msg).map_err(|_| GdbTargetError::CannotSendMsg) + } + + /// Waits for a message over the receive channel + pub fn recv(&self) -> Result { + self.rx.recv().map_err(|_| GdbTargetError::CannotReceiveMsg) + } + + /// Checks whether there's a message waiting on the receive channel + pub fn try_recv(&self) -> Result { + self.rx.try_recv() + } +} + +/// Creates a thread that handles gdb protocol +pub fn create_gdb_thread( + port: u16, + thread_id: u64, +) -> Result, GdbTargetError> { + let (gdb_conn, hyp_conn) = DebugCommChannel::unbounded(); + let socket = format!("localhost:{}", port); + + log::info!("Listening on {:?}", socket); + let listener = TcpListener::bind(socket)?; + + log::info!("Starting GDB thread"); + let _handle = thread::Builder::new() + .name("GDB handler".to_string()) + .spawn(move || -> Result<(), GdbTargetError> { + log::info!("Waiting for GDB connection ... "); + let (conn, _) = listener.accept()?; + + let conn: Box> = Box::new(conn); + let debugger = GdbStub::new(conn); + + let mut target = HyperlightSandboxTarget::new(hyp_conn, thread_id); + + // Waits for vCPU to stop at entrypoint breakpoint + let res = target.recv()?; + if let DebugResponse::VcpuStopped(_) = res { + event_loop_thread(debugger, &mut target); + } + + Ok(()) + }); + + Ok(gdb_conn) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_gdb_debug_comm_channel() { + let (gdb_conn, hyp_conn) = DebugCommChannel::::unbounded(); + + let msg = DebugMsg::ReadRegisters; + let res = gdb_conn.send(msg); + assert!(res.is_ok()); + + let res = hyp_conn.recv(); + assert!(res.is_ok()); + + let res = gdb_conn.try_recv(); + assert!(res.is_err()); + + let res = hyp_conn.send(DebugResponse::ReadRegisters(X86_64Regs::default())); + assert!(res.is_ok()); + + let res = gdb_conn.recv(); + assert!(res.is_ok()); + } +} diff --git a/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs b/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs new file mode 100644 index 000000000..51bca3d19 --- /dev/null +++ b/src/hyperlight_host/src/hypervisor/gdb/x86_64_target.rs @@ -0,0 +1,450 @@ +/* +Copyright 2024 The Hyperlight Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +use crossbeam_channel::TryRecvError; +use gdbstub::arch::Arch; +use gdbstub::common::Signal; +use gdbstub::target::ext::base::singlethread::{ + SingleThreadBase, SingleThreadResume, SingleThreadResumeOps, SingleThreadSingleStep, + SingleThreadSingleStepOps, +}; +use gdbstub::target::ext::base::BaseOps; +use gdbstub::target::ext::breakpoints::{ + Breakpoints, BreakpointsOps, HwBreakpoint, HwBreakpointOps, SwBreakpoint, SwBreakpointOps, +}; +use gdbstub::target::ext::section_offsets::{Offsets, SectionOffsets}; +use gdbstub::target::{Target, TargetError, TargetResult}; +use gdbstub_arch::x86::X86_64_SSE as GdbTargetArch; + +use super::{DebugCommChannel, DebugMsg, DebugResponse, GdbTargetError, X86_64Regs}; + +/// Gdbstub target used by the gdbstub crate to provide GDB protocol implementation +pub struct HyperlightSandboxTarget { + /// Hypervisor communication channels + hyp_conn: DebugCommChannel, + /// Thread ID + thread_id: u64, +} + +impl HyperlightSandboxTarget { + pub fn new(hyp_conn: DebugCommChannel, thread_id: u64) -> Self { + HyperlightSandboxTarget { + hyp_conn, + thread_id, + } + } + + /// Sends a command over the communication channel and waits for response + fn send_command(&self, cmd: DebugMsg) -> Result { + self.send(cmd)?; + + // Wait for response + self.recv() + } + + /// Sends a command over the communication channel + fn send(&self, ev: DebugMsg) -> Result<(), GdbTargetError> { + self.hyp_conn.send(ev) + } + + /// Returns the thread ID + pub fn get_thread_id(&self) -> u64 { + self.thread_id + } + + /// Waits for a response over the communication channel + pub fn recv(&self) -> Result { + self.hyp_conn.recv() + } + + /// Non-Blocking check for a response over the communication channel + pub fn try_recv(&self) -> Result { + self.hyp_conn.try_recv() + } + + /// Sends an event to the Hypervisor that tells it to resume vCPU execution + /// Note: The method waits for a confirmation message + pub fn resume_vcpu(&mut self) -> Result<(), GdbTargetError> { + log::info!("Resume vCPU execution"); + + match self.send_command(DebugMsg::Continue)? { + DebugResponse::Continue => Ok(()), + msg => { + log::error!("Unexpected message received: {:?}", msg); + Err(GdbTargetError::UnexpectedMessage) + } + } + } + + /// Sends an event to the Hypervisor that tells it to disable debugging + /// and continue executing + /// Note: The method waits for a confirmation message + pub fn disable_debug(&mut self) -> Result<(), GdbTargetError> { + log::info!("Disable debugging and resume execution"); + + match self.send_command(DebugMsg::DisableDebug)? { + DebugResponse::DisableDebug => Ok(()), + DebugResponse::ErrorOccurred => { + log::error!("Error occurred"); + Err(GdbTargetError::UnexpectedError) + } + msg => { + log::error!("Unexpected message received: {:?}", msg); + Err(GdbTargetError::UnexpectedMessage) + } + } + } +} + +impl Target for HyperlightSandboxTarget { + type Arch = GdbTargetArch; + type Error = GdbTargetError; + + fn support_breakpoints(&mut self) -> Option> { + Some(self) + } + + #[inline(always)] + fn base_ops(&mut self) -> BaseOps { + BaseOps::SingleThread(self) + } + + fn support_section_offsets( + &mut self, + ) -> Option> { + Some(self) + } +} + +impl SingleThreadBase for HyperlightSandboxTarget { + fn read_addrs( + &mut self, + gva: ::Usize, + data: &mut [u8], + ) -> TargetResult { + log::debug!("Read addr: {:X} len: {:X}", gva, data.len()); + + match self.send_command(DebugMsg::ReadAddr(gva, data.len()))? { + DebugResponse::ReadAddr(v) => { + data.copy_from_slice(&v); + + Ok(v.len()) + } + DebugResponse::ErrorOccurred => { + log::error!("Error occurred"); + Err(TargetError::NonFatal) + } + msg => { + log::error!("Unexpected message received: {:?}", msg); + Err(TargetError::Fatal(GdbTargetError::UnexpectedMessage)) + } + } + } + + fn write_addrs( + &mut self, + gva: ::Usize, + data: &[u8], + ) -> TargetResult<(), Self> { + log::debug!("Write addr: {:X} len: {:X}", gva, data.len()); + let v = Vec::from(data); + + match self.send_command(DebugMsg::WriteAddr(gva, v))? { + DebugResponse::WriteAddr => Ok(()), + DebugResponse::ErrorOccurred => { + log::error!("Error occurred"); + Err(TargetError::NonFatal) + } + msg => { + log::error!("Unexpected message received: {:?}", msg); + Err(TargetError::Fatal(GdbTargetError::UnexpectedMessage)) + } + } + } + + fn read_registers( + &mut self, + regs: &mut ::Registers, + ) -> TargetResult<(), Self> { + log::debug!("Read regs"); + + match self.send_command(DebugMsg::ReadRegisters)? { + DebugResponse::ReadRegisters(read_regs) => { + regs.regs[0] = read_regs.rax; + regs.regs[1] = read_regs.rbp; + regs.regs[2] = read_regs.rcx; + regs.regs[3] = read_regs.rdx; + regs.regs[4] = read_regs.rsi; + regs.regs[5] = read_regs.rdi; + regs.regs[6] = read_regs.rbp; + regs.regs[7] = read_regs.rsp; + regs.regs[8] = read_regs.r8; + regs.regs[9] = read_regs.r9; + regs.regs[10] = read_regs.r10; + regs.regs[11] = read_regs.r11; + regs.regs[12] = read_regs.r12; + regs.regs[13] = read_regs.r13; + regs.regs[14] = read_regs.r14; + regs.regs[15] = read_regs.r15; + regs.rip = read_regs.rip; + regs.eflags = read_regs.rflags as u32; + + Ok(()) + } + DebugResponse::ErrorOccurred => { + log::error!("Error occurred"); + Err(TargetError::NonFatal) + } + + msg => { + log::error!("Unexpected message received: {:?}", msg); + Err(TargetError::Fatal(GdbTargetError::UnexpectedMessage)) + } + } + } + + fn write_registers( + &mut self, + regs: &::Registers, + ) -> TargetResult<(), Self> { + log::debug!("Write regs"); + + let regs = X86_64Regs { + rax: regs.regs[0], + rbx: regs.regs[1], + rcx: regs.regs[2], + rdx: regs.regs[3], + rsi: regs.regs[4], + rdi: regs.regs[5], + rbp: regs.regs[6], + rsp: regs.regs[7], + r8: regs.regs[8], + r9: regs.regs[9], + r10: regs.regs[10], + r11: regs.regs[11], + r12: regs.regs[12], + r13: regs.regs[13], + r14: regs.regs[14], + r15: regs.regs[15], + rip: regs.rip, + rflags: u64::from(regs.eflags), + }; + + match self.send_command(DebugMsg::WriteRegisters(regs))? { + DebugResponse::WriteRegisters => Ok(()), + DebugResponse::ErrorOccurred => { + log::error!("Error occurred"); + Err(TargetError::NonFatal) + } + msg => { + log::error!("Unexpected message received: {:?}", msg); + Err(TargetError::Fatal(GdbTargetError::UnexpectedMessage)) + } + } + } + + fn support_resume(&mut self) -> Option> { + Some(self) + } +} + +impl SectionOffsets for HyperlightSandboxTarget { + fn get_section_offsets(&mut self) -> Result::Usize>, Self::Error> { + log::debug!("Get section offsets"); + + match self.send_command(DebugMsg::GetCodeSectionOffset)? { + DebugResponse::GetCodeSectionOffset(text) => Ok(Offsets::Segments { + text_seg: text, + data_seg: None, + }), + DebugResponse::ErrorOccurred => { + log::error!("Error occurred"); + Err(GdbTargetError::UnexpectedError) + } + msg => { + log::error!("Unexpected message received: {:?}", msg); + Err(GdbTargetError::UnexpectedMessage) + } + } + } +} + +impl Breakpoints for HyperlightSandboxTarget { + fn support_hw_breakpoint(&mut self) -> Option> { + Some(self) + } + fn support_sw_breakpoint(&mut self) -> Option> { + Some(self) + } +} + +impl HwBreakpoint for HyperlightSandboxTarget { + fn add_hw_breakpoint( + &mut self, + addr: ::Usize, + _kind: ::BreakpointKind, + ) -> TargetResult { + log::debug!("Add hw breakpoint at address {:X}", addr); + + match self.send_command(DebugMsg::AddHwBreakpoint(addr))? { + DebugResponse::AddHwBreakpoint(rsp) => Ok(rsp), + DebugResponse::ErrorOccurred => { + log::error!("Error occurred"); + Err(TargetError::NonFatal) + } + msg => { + log::error!("Unexpected message received: {:?}", msg); + Err(TargetError::Fatal(GdbTargetError::UnexpectedMessage)) + } + } + } + + fn remove_hw_breakpoint( + &mut self, + addr: ::Usize, + _kind: ::BreakpointKind, + ) -> TargetResult { + log::debug!("Remove hw breakpoint at address {:X}", addr); + + match self.send_command(DebugMsg::RemoveHwBreakpoint(addr))? { + DebugResponse::RemoveHwBreakpoint(rsp) => Ok(rsp), + DebugResponse::ErrorOccurred => { + log::error!("Error occurred"); + Err(TargetError::NonFatal) + } + msg => { + log::error!("Unexpected message received: {:?}", msg); + Err(TargetError::Fatal(GdbTargetError::UnexpectedMessage)) + } + } + } +} + +impl SwBreakpoint for HyperlightSandboxTarget { + fn add_sw_breakpoint( + &mut self, + addr: ::Usize, + _kind: ::BreakpointKind, + ) -> TargetResult { + log::debug!("Add sw breakpoint at address {:X}", addr); + + match self.send_command(DebugMsg::AddSwBreakpoint(addr))? { + DebugResponse::AddSwBreakpoint(rsp) => Ok(rsp), + DebugResponse::ErrorOccurred => { + log::error!("Error occurred"); + Err(TargetError::NonFatal) + } + msg => { + log::error!("Unexpected message received: {:?}", msg); + Err(TargetError::Fatal(GdbTargetError::UnexpectedMessage)) + } + } + } + + fn remove_sw_breakpoint( + &mut self, + addr: ::Usize, + _kind: ::BreakpointKind, + ) -> TargetResult { + log::debug!("Remove sw breakpoint at address {:X}", addr); + + match self.send_command(DebugMsg::RemoveSwBreakpoint(addr))? { + DebugResponse::RemoveSwBreakpoint(rsp) => Ok(rsp), + DebugResponse::ErrorOccurred => { + log::error!("Error occurred"); + Err(TargetError::NonFatal) + } + msg => { + log::error!("Unexpected message received: {:?}", msg); + Err(TargetError::Fatal(GdbTargetError::UnexpectedMessage)) + } + } + } +} + +impl SingleThreadResume for HyperlightSandboxTarget { + /// Resumes the execution of the vCPU + /// Note: We do not handle signals passed to this method + fn resume(&mut self, _signal: Option) -> Result<(), Self::Error> { + log::debug!("Resume"); + self.resume_vcpu() + } + fn support_single_step(&mut self) -> Option> { + Some(self) + } +} + +impl SingleThreadSingleStep for HyperlightSandboxTarget { + /// Steps the vCPU execution by + /// Note: We do not handle signals passed to this method + fn step(&mut self, _signal: Option) -> Result<(), Self::Error> { + log::debug!("Step"); + match self.send_command(DebugMsg::Step)? { + DebugResponse::Step => Ok(()), + DebugResponse::ErrorOccurred => { + log::error!("Error occurred"); + Err(GdbTargetError::UnexpectedError) + } + msg => { + log::error!("Unexpected message received: {:?}", msg); + Err(GdbTargetError::UnexpectedMessage) + } + } + } +} + +#[cfg(test)] +mod tests { + use gdbstub_arch::x86::reg::X86_64CoreRegs; + + use super::*; + + #[test] + fn test_gdb_target() { + let (gdb_conn, hyp_conn) = DebugCommChannel::unbounded(); + + let mut target = HyperlightSandboxTarget::new(hyp_conn, 0); + + // Check response to read registers - send the response first to not be blocked + // by the recv call in the target + let msg = DebugResponse::ReadRegisters(X86_64Regs::default()); + let res = gdb_conn.send(msg); + assert!(res.is_ok()); + + let mut regs = X86_64CoreRegs::default(); + assert!( + target.read_registers(&mut regs).is_ok(), + "Failed to read registers" + ); + + // Check response to write registers + let msg = DebugResponse::WriteRegisters; + let res = gdb_conn.send(msg); + assert!(res.is_ok()); + assert!( + target.write_registers(®s).is_ok(), + "Failed to write registers" + ); + + // Check response when the channel is dropped + drop(gdb_conn); + assert!( + target.read_registers(&mut regs).is_err(), + "Succeeded to read registers when + expected to fail" + ); + } +} diff --git a/src/hyperlight_host/src/hypervisor/handlers.rs b/src/hyperlight_host/src/hypervisor/handlers.rs index b17fabfc1..3ffd64441 100644 --- a/src/hyperlight_host/src/hypervisor/handlers.rs +++ b/src/hyperlight_host/src/hypervisor/handlers.rs @@ -102,3 +102,25 @@ impl MemAccessHandlerCaller for MemAccessHandler { func() } } + +/// The trait representing custom logic to handle the case when +/// a Hypervisor's virtual CPU (vCPU) informs Hyperlight a debug memory access +/// has been requested. +#[cfg(gdb)] +pub trait DbgMemAccessHandlerCaller: Send { + /// Function that gets called when a read is requested. + fn read(&mut self, addr: usize, data: &mut [u8]) -> Result<()>; + + /// Function that gets called when a write is requested. + fn write(&mut self, addr: usize, data: &[u8]) -> Result<()>; + + /// Function that gets called for a request to get guest code offset. + fn get_code_offset(&mut self) -> Result; +} + +/// A convenient type representing an implementer of `DbgMemAccessHandlerCaller` +/// +/// Note: This needs to be wrapped in a Mutex to be able to grab a mutable +/// reference to the underlying data +#[cfg(gdb)] +pub type DbgMemAccessHandlerWrapper = Arc>; diff --git a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs index 33798d881..882b1c127 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs @@ -44,6 +44,8 @@ use mshv_ioctls::{Mshv, VcpuFd, VmFd}; use tracing::{instrument, Span}; use super::fpu::{FP_CONTROL_WORD_DEFAULT, FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT}; +#[cfg(gdb)] +use super::handlers::DbgMemAccessHandlerWrapper; use super::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper}; use super::{ Hypervisor, VirtualCPU, CR0_AM, CR0_ET, CR0_MP, CR0_NE, CR0_PE, CR0_PG, CR0_WP, CR4_OSFXSR, @@ -203,6 +205,7 @@ impl Hypervisor for HypervLinuxDriver { outb_hdl: OutBHandlerWrapper, mem_access_hdl: MemAccessHandlerWrapper, hv_handler: Option, + #[cfg(gdb)] dbg_mem_access_fn: DbgMemAccessHandlerWrapper, ) -> Result<()> { let regs = StandardRegisters { rip: self.entrypoint, @@ -224,6 +227,8 @@ impl Hypervisor for HypervLinuxDriver { hv_handler, outb_hdl, mem_access_hdl, + #[cfg(gdb)] + dbg_mem_access_fn, )?; // reset RSP to what it was before initialise @@ -242,6 +247,7 @@ impl Hypervisor for HypervLinuxDriver { outb_handle_fn: OutBHandlerWrapper, mem_access_fn: MemAccessHandlerWrapper, hv_handler: Option, + #[cfg(gdb)] dbg_mem_access_fn: DbgMemAccessHandlerWrapper, ) -> Result<()> { // Reset general purpose registers except RSP, then set RIP let rsp_before = self.vcpu_fd.get_regs()?.rsp; @@ -268,6 +274,8 @@ impl Hypervisor for HypervLinuxDriver { hv_handler, outb_handle_fn, mem_access_fn, + #[cfg(gdb)] + dbg_mem_access_fn, )?; // reset RSP to what it was before function call diff --git a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs index 067d0cb1a..acde5e8f8 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs @@ -28,6 +28,8 @@ use windows::Win32::System::Hypervisor::{ }; use super::fpu::{FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT}; +#[cfg(gdb)] +use super::handlers::DbgMemAccessHandlerWrapper; use super::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper}; use super::surrogate_process::SurrogateProcess; use super::surrogate_process_manager::*; @@ -305,6 +307,7 @@ impl Hypervisor for HypervWindowsDriver { outb_hdl: OutBHandlerWrapper, mem_access_hdl: MemAccessHandlerWrapper, hv_handler: Option, + #[cfg(gdb)] dbg_mem_access_hdl: DbgMemAccessHandlerWrapper, ) -> Result<()> { let regs = WHvGeneralRegisters { rip: self.entrypoint, @@ -326,6 +329,8 @@ impl Hypervisor for HypervWindowsDriver { hv_handler, outb_hdl, mem_access_hdl, + #[cfg(gdb)] + dbg_mem_access_hdl, )?; // reset RSP to what it was before initialise @@ -344,6 +349,7 @@ impl Hypervisor for HypervWindowsDriver { outb_hdl: OutBHandlerWrapper, mem_access_hdl: MemAccessHandlerWrapper, hv_handler: Option, + #[cfg(gdb)] dbg_mem_access_hdl: DbgMemAccessHandlerWrapper, ) -> Result<()> { // Reset general purpose registers except RSP, then set RIP let rsp_before = self.processor.get_regs()?.rsp; @@ -368,6 +374,8 @@ impl Hypervisor for HypervWindowsDriver { hv_handler, outb_hdl, mem_access_hdl, + #[cfg(gdb)] + dbg_mem_access_hdl, )?; // reset RSP to what it was before function call diff --git a/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs b/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs index a221570a9..a66ebc608 100644 --- a/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs +++ b/src/hyperlight_host/src/hypervisor/hypervisor_handler.rs @@ -35,8 +35,12 @@ use vmm_sys_util::signal::SIGRTMIN; #[cfg(target_os = "windows")] use windows::Win32::System::Hypervisor::{WHvCancelRunVirtualProcessor, WHV_PARTITION_HANDLE}; +#[cfg(gdb)] +use super::gdb::create_gdb_thread; #[cfg(feature = "function_call_metrics")] use crate::histogram_vec_observe; +#[cfg(gdb)] +use crate::hypervisor::handlers::DbgMemAccessHandlerWrapper; use crate::hypervisor::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper}; #[cfg(target_os = "windows")] use crate::hypervisor::wrappers::HandleWrapper; @@ -46,6 +50,8 @@ use crate::mem::mgr::SandboxMemoryManager; use crate::mem::ptr::{GuestPtr, RawPtr}; use crate::mem::ptr_offset::Offset; use crate::mem::shared_mem::{GuestSharedMemory, HostSharedMemory, SharedMemory}; +#[cfg(gdb)] +use crate::sandbox::config::DebugInfo; use crate::sandbox::hypervisor::{get_available_hypervisor, HypervisorType}; #[cfg(feature = "function_call_metrics")] use crate::sandbox::metrics::SandboxMetric::GuestFunctionCallDurationMicroseconds; @@ -185,6 +191,8 @@ pub(crate) struct HvHandlerConfig { pub(crate) outb_handler: OutBHandlerWrapper, pub(crate) mem_access_handler: MemAccessHandlerWrapper, pub(crate) max_wait_for_cancellation: Duration, + #[cfg(gdb)] + pub(crate) dbg_mem_access_handler: DbgMemAccessHandlerWrapper, } impl HypervisorHandler { @@ -232,6 +240,7 @@ impl HypervisorHandler { pub(crate) fn start_hypervisor_handler( &mut self, sandbox_memory_manager: SandboxMemoryManager, + #[cfg(gdb)] debug_info: Option, ) -> Result<()> { let configuration = self.configuration.clone(); #[cfg(target_os = "windows")] @@ -292,6 +301,8 @@ impl HypervisorHandler { hv = Some(set_up_hypervisor_partition( execution_variables.shm.try_lock().unwrap().deref_mut().as_mut().unwrap(), configuration.outb_handler.clone(), + #[cfg(gdb)] + &debug_info, )?); } let hv = hv.as_mut().unwrap(); @@ -346,6 +357,8 @@ impl HypervisorHandler { configuration.outb_handler.clone(), configuration.mem_access_handler.clone(), Some(hv_handler_clone.clone()), + #[cfg(gdb)] + configuration.dbg_mem_access_handler.clone(), ); drop(mem_lock_guard); drop(evar_lock_guard); @@ -431,6 +444,8 @@ impl HypervisorHandler { configuration.outb_handler.clone(), configuration.mem_access_handler.clone(), Some(hv_handler_clone.clone()), + #[cfg(gdb)] + configuration.dbg_mem_access_handler.clone(), ); histogram_vec_observe!( &GuestFunctionCallDurationMicroseconds, @@ -446,6 +461,8 @@ impl HypervisorHandler { configuration.outb_handler.clone(), configuration.mem_access_handler.clone(), Some(hv_handler_clone.clone()), + #[cfg(gdb)] + configuration.dbg_mem_access_handler.clone(), ) }; drop(mem_lock_guard); @@ -597,11 +614,19 @@ impl HypervisorHandler { /// and still have to receive after sorting that out without sending /// an extra message. pub(crate) fn try_receive_handler_msg(&self) -> Result<()> { - match self + // When gdb debugging is enabled, we don't want to timeout on receiving messages + // from the handler thread, as the thread may be paused by gdb. + // In this case, we will wait indefinitely for a message from the handler thread. + // Note: This applies to all the running sandboxes, not just the one being debugged. + #[cfg(gdb)] + let response = self.communication_channels.from_handler_rx.recv(); + #[cfg(not(gdb))] + let response = self .communication_channels .from_handler_rx - .recv_timeout(self.execution_variables.get_timeout()?) - { + .recv_timeout(self.execution_variables.get_timeout()?); + + match response { Ok(msg) => match msg { HandlerMsg::Error(e) => Err(e), HandlerMsg::FinishedHypervisorHandlerAction => Ok(()), @@ -823,6 +848,7 @@ fn set_up_hypervisor_partition( mgr: &mut SandboxMemoryManager, #[allow(unused_variables)] // parameter only used for in-process mode outb_handler: OutBHandlerWrapper, + #[cfg(gdb)] debug_info: &Option, ) -> Result> { let mem_size = u64::try_from(mgr.shared_mem.mem_size())?; let mut regions = mgr.layout.get_memory_regions(&mgr.shared_mem)?; @@ -882,6 +908,26 @@ fn set_up_hypervisor_partition( } } } else { + // Create gdb thread if gdb is enabled and the configuration is provided + // This is only done when the hypervisor is not in-process + #[cfg(gdb)] + let gdb_conn = if let Some(DebugInfo { port }) = debug_info { + let gdb_conn = create_gdb_thread(*port, unsafe { pthread_self() }); + + // in case the gdb thread creation fails, we still want to continue + // without gdb + match gdb_conn { + Ok(gdb_conn) => Some(gdb_conn), + Err(e) => { + log::error!("Could not create gdb connection: {:#}", e); + + None + } + } + } else { + None + }; + match *get_available_hypervisor() { #[cfg(mshv)] Some(HypervisorType::Mshv) => { @@ -901,6 +947,8 @@ fn set_up_hypervisor_partition( pml4_ptr.absolute()?, entrypoint_ptr.absolute()?, rsp_ptr.absolute()?, + #[cfg(gdb)] + gdb_conn, )?; Ok(Box::new(hv)) } diff --git a/src/hyperlight_host/src/hypervisor/inprocess.rs b/src/hyperlight_host/src/hypervisor/inprocess.rs index f7076a5b4..1fec3df54 100644 --- a/src/hyperlight_host/src/hypervisor/inprocess.rs +++ b/src/hyperlight_host/src/hypervisor/inprocess.rs @@ -17,6 +17,8 @@ limitations under the License. use std::fmt::Debug; use std::os::raw::c_void; +#[cfg(gdb)] +use super::handlers::DbgMemAccessHandlerWrapper; use super::{HyperlightExit, Hypervisor}; #[cfg(crashdump)] use crate::mem::memory_region::MemoryRegion; @@ -73,6 +75,7 @@ impl<'a> Hypervisor for InprocessDriver<'a> { _outb_handle_fn: super::handlers::OutBHandlerWrapper, _mem_access_fn: super::handlers::MemAccessHandlerWrapper, _hv_handler: Option, + #[cfg(gdb)] _dbg_mem_access_fn: DbgMemAccessHandlerWrapper, ) -> crate::Result<()> { let entrypoint_fn: extern "win64" fn(u64, u64, u64, u64) = unsafe { std::mem::transmute(self.args.entrypoint_raw as *const c_void) }; @@ -93,6 +96,7 @@ impl<'a> Hypervisor for InprocessDriver<'a> { _outb_handle_fn: super::handlers::OutBHandlerWrapper, _mem_access_fn: super::handlers::MemAccessHandlerWrapper, _hv_handler: Option, + #[cfg(gdb)] _dbg_mem_access_fn: DbgMemAccessHandlerWrapper, ) -> crate::Result<()> { let ptr: u64 = dispatch_func_addr.into(); let dispatch_func: extern "win64" fn() = diff --git a/src/hyperlight_host/src/hypervisor/kvm.rs b/src/hyperlight_host/src/hypervisor/kvm.rs index e21c6a35a..a09a23943 100644 --- a/src/hyperlight_host/src/hypervisor/kvm.rs +++ b/src/hyperlight_host/src/hypervisor/kvm.rs @@ -16,6 +16,8 @@ limitations under the License. use std::convert::TryFrom; use std::fmt::Debug; +#[cfg(gdb)] +use std::sync::{Arc, Mutex}; use kvm_bindings::{kvm_fpu, kvm_regs, kvm_userspace_memory_region, KVM_MEM_READONLY}; use kvm_ioctls::Cap::UserMemory; @@ -23,6 +25,10 @@ use kvm_ioctls::{Kvm, VcpuExit, VcpuFd, VmFd}; use tracing::{instrument, Span}; use super::fpu::{FP_CONTROL_WORD_DEFAULT, FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT}; +#[cfg(gdb)] +use super::gdb::{DebugCommChannel, DebugMsg, DebugResponse, VcpuStopReason}; +#[cfg(gdb)] +use super::handlers::DbgMemAccessHandlerWrapper; use super::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper}; use super::{ HyperlightExit, Hypervisor, VirtualCPU, CR0_AM, CR0_ET, CR0_MP, CR0_NE, CR0_PE, CR0_PG, CR0_WP, @@ -31,6 +37,8 @@ use super::{ use crate::hypervisor::hypervisor_handler::HypervisorHandler; use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags}; use crate::mem::ptr::{GuestPtr, RawPtr}; +#[cfg(gdb)] +use crate::HyperlightError; use crate::{log_then_return, new_error, Result}; /// Return `true` if the KVM API is available, version 12, and has UserMemory capability, or `false` otherwise @@ -55,6 +63,514 @@ pub(crate) fn is_hypervisor_present() -> bool { } } +#[cfg(gdb)] +mod debug { + use std::collections::HashMap; + use std::sync::{Arc, Mutex}; + + use hyperlight_common::mem::PAGE_SIZE; + use kvm_bindings::{ + kvm_guest_debug, kvm_regs, KVM_GUESTDBG_ENABLE, KVM_GUESTDBG_SINGLESTEP, + KVM_GUESTDBG_USE_HW_BP, KVM_GUESTDBG_USE_SW_BP, + }; + use kvm_ioctls::VcpuFd; + + use super::KVMDriver; + use crate::hypervisor::gdb::{DebugMsg, DebugResponse, VcpuStopReason, X86_64Regs}; + use crate::hypervisor::handlers::DbgMemAccessHandlerCaller; + use crate::mem::layout::SandboxMemoryLayout; + use crate::{new_error, HyperlightError, Result}; + + /// Software Breakpoint size in memory + pub const SW_BP_SIZE: usize = 1; + /// Software Breakpoint opcode + const SW_BP_OP: u8 = 0xCC; + /// Software Breakpoint written to memory + pub const SW_BP: [u8; SW_BP_SIZE] = [SW_BP_OP]; + + /// KVM Debug struct + /// This struct is used to abstract the internal details of the kvm + /// guest debugging settings + #[derive(Default)] + pub struct KvmDebug { + /// vCPU stepping state + single_step: bool, + + /// Array of addresses for HW breakpoints + hw_breakpoints: Vec, + /// Saves the bytes modified to enable SW breakpoints + sw_breakpoints: HashMap, + + /// Sent to KVM for enabling guest debug + pub dbg_cfg: kvm_guest_debug, + } + + impl KvmDebug { + const MAX_NO_OF_HW_BP: usize = 4; + + pub fn new() -> Self { + let dbg = kvm_guest_debug { + control: KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP, + ..Default::default() + }; + + Self { + single_step: false, + hw_breakpoints: vec![], + sw_breakpoints: HashMap::new(), + dbg_cfg: dbg, + } + } + + /// This method sets the kvm debugreg fields to enable breakpoints at + /// specific addresses + /// + /// The first 4 debug registers are used to set the addresses + /// The 4th and 5th debug registers are obsolete and not used + /// The 7th debug register is used to enable the breakpoints + /// For more information see: DEBUG REGISTERS chapter in the architecture + /// manual + fn set_debug_config(&mut self, vcpu_fd: &VcpuFd, step: bool) -> Result<()> { + let addrs = &self.hw_breakpoints; + + self.dbg_cfg.arch.debugreg = [0; 8]; + for (k, addr) in addrs.iter().enumerate() { + self.dbg_cfg.arch.debugreg[k] = *addr; + self.dbg_cfg.arch.debugreg[7] |= 1 << (k * 2); + } + + if !addrs.is_empty() { + self.dbg_cfg.control |= KVM_GUESTDBG_USE_HW_BP; + } else { + self.dbg_cfg.control &= !KVM_GUESTDBG_USE_HW_BP; + } + + if step { + self.dbg_cfg.control |= KVM_GUESTDBG_SINGLESTEP; + } else { + self.dbg_cfg.control &= !KVM_GUESTDBG_SINGLESTEP; + } + + log::debug!("Setting bp: {:?} cfg: {:?}", addrs, self.dbg_cfg); + vcpu_fd + .set_guest_debug(&self.dbg_cfg) + .map_err(|e| new_error!("Could not set guest debug: {:?}", e))?; + + self.single_step = step; + + Ok(()) + } + + /// Method that adds a breakpoint + fn add_breakpoint(&mut self, vcpu_fd: &VcpuFd, addr: u64) -> Result { + if self.hw_breakpoints.len() >= Self::MAX_NO_OF_HW_BP { + Ok(false) + } else if self.hw_breakpoints.contains(&addr) { + Ok(true) + } else { + self.hw_breakpoints.push(addr); + self.set_debug_config(vcpu_fd, self.single_step)?; + + Ok(true) + } + } + + /// Method that removes a breakpoint + fn remove_breakpoint(&mut self, vcpu_fd: &VcpuFd, addr: u64) -> Result { + if self.hw_breakpoints.contains(&addr) { + self.hw_breakpoints.retain(|&a| a != addr); + self.set_debug_config(vcpu_fd, self.single_step)?; + + Ok(true) + } else { + Ok(false) + } + } + } + + impl KVMDriver { + /// Resets the debug information to disable debugging + fn disable_debug(&mut self) -> Result<()> { + self.debug = Some(KvmDebug::default()); + + self.set_single_step(false) + } + + /// Returns the instruction pointer from the stopped vCPU + fn get_instruction_pointer(&self) -> Result { + let regs = self + .vcpu_fd + .get_regs() + .map_err(|e| new_error!("Could not retrieve registers from vCPU: {:?}", e))?; + + Ok(regs.rip) + } + + /// Sets or clears stepping for vCPU + fn set_single_step(&mut self, enable: bool) -> Result<()> { + let debug = self + .debug + .as_mut() + .ok_or_else(|| new_error!("Debug is not enabled"))?; + + debug.set_debug_config(&self.vcpu_fd, enable) + } + + /// Translates the guest address to physical address + fn translate_gva(&self, gva: u64) -> Result { + let tr = self + .vcpu_fd + .translate_gva(gva) + .map_err(|_| HyperlightError::TranslateGuestAddress(gva))?; + + if tr.valid == 0 { + Err(HyperlightError::TranslateGuestAddress(gva)) + } else { + Ok(tr.physical_address) + } + } + + fn read_addrs( + &mut self, + mut gva: u64, + mut data: &mut [u8], + dbg_mem_access_fn: Arc>, + ) -> Result<()> { + let data_len = data.len(); + log::debug!("Read addr: {:X} len: {:X}", gva, data_len); + + while !data.is_empty() { + let gpa = self.translate_gva(gva)?; + + let read_len = std::cmp::min( + data.len(), + (PAGE_SIZE - (gpa & (PAGE_SIZE - 1))).try_into().unwrap(), + ); + let offset = gpa as usize - SandboxMemoryLayout::BASE_ADDRESS; + + dbg_mem_access_fn + .try_lock() + .map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))? + .read(offset, &mut data[..read_len])?; + + data = &mut data[read_len..]; + gva += read_len as u64; + } + + Ok(()) + } + + fn write_addrs( + &mut self, + mut gva: u64, + mut data: &[u8], + dbg_mem_access_fn: Arc>, + ) -> Result<()> { + let data_len = data.len(); + log::debug!("Write addr: {:X} len: {:X}", gva, data_len); + + while !data.is_empty() { + let gpa = self.translate_gva(gva)?; + + let write_len = std::cmp::min( + data.len(), + (PAGE_SIZE - (gpa & (PAGE_SIZE - 1))).try_into().unwrap(), + ); + let offset = gpa as usize - SandboxMemoryLayout::BASE_ADDRESS; + + dbg_mem_access_fn + .try_lock() + .map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))? + .write(offset, data)?; + + data = &data[write_len..]; + gva += write_len as u64; + } + + Ok(()) + } + + fn read_regs(&self, regs: &mut X86_64Regs) -> Result<()> { + log::debug!("Read registers"); + let vcpu_regs = self + .vcpu_fd + .get_regs() + .map_err(|e| new_error!("Could not read guest registers: {:?}", e))?; + + regs.rax = vcpu_regs.rax; + regs.rbx = vcpu_regs.rbx; + regs.rcx = vcpu_regs.rcx; + regs.rdx = vcpu_regs.rdx; + regs.rsi = vcpu_regs.rsi; + regs.rdi = vcpu_regs.rdi; + regs.rbp = vcpu_regs.rbp; + regs.rsp = vcpu_regs.rsp; + regs.r8 = vcpu_regs.r8; + regs.r9 = vcpu_regs.r9; + regs.r10 = vcpu_regs.r10; + regs.r11 = vcpu_regs.r11; + regs.r12 = vcpu_regs.r12; + regs.r13 = vcpu_regs.r13; + regs.r14 = vcpu_regs.r14; + regs.r15 = vcpu_regs.r15; + + regs.rip = vcpu_regs.rip; + regs.rflags = vcpu_regs.rflags; + + Ok(()) + } + + fn write_regs(&self, regs: &X86_64Regs) -> Result<()> { + log::debug!("Write registers"); + let new_regs = kvm_regs { + rax: regs.rax, + rbx: regs.rbx, + rcx: regs.rcx, + rdx: regs.rdx, + rsi: regs.rsi, + rdi: regs.rdi, + rbp: regs.rbp, + rsp: regs.rsp, + r8: regs.r8, + r9: regs.r9, + r10: regs.r10, + r11: regs.r11, + r12: regs.r12, + r13: regs.r13, + r14: regs.r14, + r15: regs.r15, + + rip: regs.rip, + rflags: regs.rflags, + }; + + self.vcpu_fd + .set_regs(&new_regs) + .map_err(|e| new_error!("Could not write guest registers: {:?}", e)) + } + + fn add_hw_breakpoint(&mut self, addr: u64) -> Result { + let addr = self.translate_gva(addr)?; + + if let Some(debug) = self.debug.as_mut() { + debug.add_breakpoint(&self.vcpu_fd, addr) + } else { + Ok(false) + } + } + + fn remove_hw_breakpoint(&mut self, addr: u64) -> Result { + let addr = self.translate_gva(addr)?; + + if let Some(debug) = self.debug.as_mut() { + debug.remove_breakpoint(&self.vcpu_fd, addr) + } else { + Ok(false) + } + } + + fn add_sw_breakpoint( + &mut self, + addr: u64, + dbg_mem_access_fn: Arc>, + ) -> Result { + let addr = { + let debug = self + .debug + .as_ref() + .ok_or_else(|| new_error!("Debug is not enabled"))?; + let addr = self.translate_gva(addr)?; + if debug.sw_breakpoints.contains_key(&addr) { + return Ok(true); + } + + addr + }; + + let mut save_data = [0; SW_BP_SIZE]; + self.read_addrs(addr, &mut save_data[..], dbg_mem_access_fn.clone())?; + self.write_addrs(addr, &SW_BP, dbg_mem_access_fn)?; + + { + let debug = self + .debug + .as_mut() + .ok_or_else(|| new_error!("Debug is not enabled"))?; + debug.sw_breakpoints.insert(addr, save_data); + } + + Ok(true) + } + + fn remove_sw_breakpoint( + &mut self, + addr: u64, + dbg_mem_access_fn: Arc>, + ) -> Result { + let (ret, data) = { + let addr = self.translate_gva(addr)?; + let debug = self + .debug + .as_mut() + .ok_or_else(|| new_error!("Debug is not enabled"))?; + + if debug.sw_breakpoints.contains_key(&addr) { + let save_data = debug + .sw_breakpoints + .remove(&addr) + .ok_or_else(|| new_error!("Expected the hashmap to contain the address"))?; + + (true, Some(save_data)) + } else { + (false, None) + } + }; + + if ret { + self.write_addrs(addr, &data.unwrap(), dbg_mem_access_fn)?; + } + + Ok(ret) + } + + /// Gdb expects the target to be stopped when connected. + /// This method provides a way to set a breakpoint at the entry point + /// it does not keep this breakpoint set after the vCPU already stopped at the address + pub fn set_entrypoint_bp(&self) -> Result<()> { + if self.debug.is_some() { + log::debug!("Setting entrypoint bp {:X}", self.entrypoint); + let mut entrypoint_debug = KvmDebug::new(); + entrypoint_debug.add_breakpoint(&self.vcpu_fd, self.entrypoint)?; + + Ok(()) + } else { + Ok(()) + } + } + + /// Get the reason the vCPU has stopped + pub fn get_stop_reason(&self) -> Result { + let debug = self + .debug + .as_ref() + .ok_or_else(|| new_error!("Debug is not enabled"))?; + + if debug.single_step { + return Ok(VcpuStopReason::DoneStep); + } + + let ip = self.get_instruction_pointer()?; + let gpa = self.translate_gva(ip)?; + if debug.sw_breakpoints.contains_key(&gpa) { + return Ok(VcpuStopReason::SwBp); + } + + if debug.hw_breakpoints.contains(&gpa) { + return Ok(VcpuStopReason::HwBp); + } + + if ip == self.entrypoint { + return Ok(VcpuStopReason::HwBp); + } + + Ok(VcpuStopReason::Unknown) + } + + pub fn process_dbg_request( + &mut self, + req: DebugMsg, + dbg_mem_access_fn: Arc>, + ) -> Result { + match req { + DebugMsg::AddHwBreakpoint(addr) => self + .add_hw_breakpoint(addr) + .map(DebugResponse::AddHwBreakpoint), + DebugMsg::AddSwBreakpoint(addr) => self + .add_sw_breakpoint(addr, dbg_mem_access_fn) + .map(DebugResponse::AddSwBreakpoint), + DebugMsg::Continue => { + self.set_single_step(false)?; + Ok(DebugResponse::Continue) + } + DebugMsg::DisableDebug => { + self.disable_debug()?; + + Ok(DebugResponse::DisableDebug) + } + DebugMsg::GetCodeSectionOffset => { + let offset = dbg_mem_access_fn + .try_lock() + .map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))? + .get_code_offset()?; + + Ok(DebugResponse::GetCodeSectionOffset(offset as u64)) + } + DebugMsg::ReadAddr(addr, len) => { + let mut data = vec![0u8; len]; + + self.read_addrs(addr, &mut data, dbg_mem_access_fn)?; + + Ok(DebugResponse::ReadAddr(data)) + } + DebugMsg::ReadRegisters => { + let mut regs = X86_64Regs::default(); + + self.read_regs(&mut regs) + .map(|_| DebugResponse::ReadRegisters(regs)) + } + DebugMsg::RemoveHwBreakpoint(addr) => self + .remove_hw_breakpoint(addr) + .map(DebugResponse::RemoveHwBreakpoint), + DebugMsg::RemoveSwBreakpoint(addr) => self + .remove_sw_breakpoint(addr, dbg_mem_access_fn) + .map(DebugResponse::RemoveSwBreakpoint), + DebugMsg::Step => { + self.set_single_step(true)?; + Ok(DebugResponse::Step) + } + DebugMsg::WriteAddr(addr, data) => { + self.write_addrs(addr, &data, dbg_mem_access_fn)?; + + Ok(DebugResponse::WriteAddr) + } + DebugMsg::WriteRegisters(regs) => self + .write_regs(®s) + .map(|_| DebugResponse::WriteRegisters), + } + } + + pub fn recv_dbg_msg(&mut self) -> Result { + let gdb_conn = self + .gdb_conn + .as_mut() + .ok_or_else(|| new_error!("Debug is not enabled"))?; + + gdb_conn.recv().map_err(|e| { + new_error!( + "Got an error while waiting to receive a message from the gdb thread: {:?}", + e + ) + }) + } + + pub fn send_dbg_msg(&mut self, cmd: DebugResponse) -> Result<()> { + log::debug!("Sending {:?}", cmd); + + let gdb_conn = self + .gdb_conn + .as_mut() + .ok_or_else(|| new_error!("Debug is not enabled"))?; + + gdb_conn.send(cmd).map_err(|e| { + new_error!( + "Got an error while sending a response message to the gdb thread: {:?}", + e + ) + }) + } + } +} + /// A Hypervisor driver for KVM on Linux pub(super) struct KVMDriver { _kvm: Kvm, @@ -63,6 +579,11 @@ pub(super) struct KVMDriver { entrypoint: u64, orig_rsp: GuestPtr, mem_regions: Vec, + + #[cfg(gdb)] + debug: Option, + #[cfg(gdb)] + gdb_conn: Option>, } impl KVMDriver { @@ -75,6 +596,7 @@ impl KVMDriver { pml4_addr: u64, entrypoint: u64, rsp: u64, + #[cfg(gdb)] gdb_conn: Option>, ) -> Result { let kvm = Kvm::new()?; @@ -101,15 +623,33 @@ impl KVMDriver { let mut vcpu_fd = vm_fd.create_vcpu(0)?; Self::setup_initial_sregs(&mut vcpu_fd, pml4_addr)?; + #[cfg(gdb)] + let (debug, gdb_conn) = if let Some(gdb_conn) = gdb_conn { + (Some(debug::KvmDebug::new()), Some(gdb_conn)) + } else { + (None, None) + }; + let rsp_gp = GuestPtr::try_from(RawPtr::from(rsp))?; - Ok(Self { + + let ret = Self { _kvm: kvm, _vm_fd: vm_fd, vcpu_fd, entrypoint, orig_rsp: rsp_gp, mem_regions, - }) + + #[cfg(gdb)] + debug, + #[cfg(gdb)] + gdb_conn, + }; + + #[cfg(gdb)] + ret.set_entrypoint_bp()?; + + Ok(ret) } #[instrument(err(Debug), skip_all, parent = Span::current(), level = "Trace")] @@ -164,6 +704,7 @@ impl Hypervisor for KVMDriver { outb_hdl: OutBHandlerWrapper, mem_access_hdl: MemAccessHandlerWrapper, hv_handler: Option, + #[cfg(gdb)] dbg_mem_access_fn: DbgMemAccessHandlerWrapper, ) -> Result<()> { let regs = kvm_regs { rip: self.entrypoint, @@ -184,6 +725,8 @@ impl Hypervisor for KVMDriver { hv_handler, outb_hdl, mem_access_hdl, + #[cfg(gdb)] + dbg_mem_access_fn, )?; // reset RSP to what it was before initialise @@ -201,6 +744,7 @@ impl Hypervisor for KVMDriver { outb_handle_fn: OutBHandlerWrapper, mem_access_fn: MemAccessHandlerWrapper, hv_handler: Option, + #[cfg(gdb)] dbg_mem_access_fn: DbgMemAccessHandlerWrapper, ) -> Result<()> { // Reset general purpose registers except RSP, then set RIP let rsp_before = self.vcpu_fd.get_regs()?.rsp; @@ -226,6 +770,8 @@ impl Hypervisor for KVMDriver { hv_handler, outb_handle_fn, mem_access_fn, + #[cfg(gdb)] + dbg_mem_access_fn, )?; // reset RSP to what it was before function call @@ -301,8 +847,21 @@ impl Hypervisor for KVMDriver { None => HyperlightExit::Mmio(addr), } } + #[cfg(gdb)] + Ok(VcpuExit::Debug(_)) => match self.get_stop_reason() { + Ok(reason) => HyperlightExit::Debug(reason), + Err(e) => { + log_then_return!("Error getting stop reason: {:?}", e); + } + }, Err(e) => match e.errno() { + // In case of the gdb feature, the timeout is not enabled, this + // exit is because of a signal sent from the gdb thread to the + // hypervisor thread to cancel execution + #[cfg(gdb)] + libc::EINTR => HyperlightExit::Debug(VcpuStopReason::Interrupt), // we send a signal to the thread to cancel execution this results in EINTR being returned by KVM so we return Cancelled + #[cfg(not(gdb))] libc::EINTR => HyperlightExit::Cancelled(), libc::EAGAIN => HyperlightExit::Retry(), _ => { @@ -327,15 +886,78 @@ impl Hypervisor for KVMDriver { fn get_memory_regions(&self) -> &[MemoryRegion] { &self.mem_regions } + + #[cfg(gdb)] + fn handle_debug( + &mut self, + dbg_mem_access_fn: Arc>, + stop_reason: VcpuStopReason, + ) -> Result<()> { + self.send_dbg_msg(DebugResponse::VcpuStopped(stop_reason)) + .map_err(|e| new_error!("Couldn't signal vCPU stopped event to GDB thread: {:?}", e))?; + + loop { + log::debug!("Debug wait for event to resume vCPU"); + // Wait for a message from gdb + let req = self.recv_dbg_msg()?; + + let result = self.process_dbg_request(req, dbg_mem_access_fn.clone()); + + let response = match result { + Ok(response) => response, + // Treat non fatal errors separately so the guest doesn't fail + Err(HyperlightError::TranslateGuestAddress(_)) => DebugResponse::ErrorOccurred, + Err(e) => { + return Err(e); + } + }; + + // If the command was either step or continue, we need to run the vcpu + let cont = matches!( + response, + DebugResponse::Step | DebugResponse::Continue | DebugResponse::DisableDebug + ); + + self.send_dbg_msg(response) + .map_err(|e| new_error!("Couldn't send response to gdb: {:?}", e))?; + + if cont { + break; + } + } + + Ok(()) + } } + #[cfg(test)] mod tests { use std::sync::{Arc, Mutex}; + #[cfg(gdb)] + use crate::hypervisor::handlers::DbgMemAccessHandlerCaller; use crate::hypervisor::handlers::{MemAccessHandler, OutBHandler}; use crate::hypervisor::tests::test_initialise; use crate::Result; + #[cfg(gdb)] + struct DbgMemAccessHandler {} + + #[cfg(gdb)] + impl DbgMemAccessHandlerCaller for DbgMemAccessHandler { + fn read(&mut self, _offset: usize, _data: &mut [u8]) -> Result<()> { + Ok(()) + } + + fn write(&mut self, _offset: usize, _data: &[u8]) -> Result<()> { + Ok(()) + } + + fn get_code_offset(&mut self) -> Result { + Ok(0) + } + } + #[test] fn test_init() { if !super::is_hypervisor_present() { @@ -351,6 +973,15 @@ mod tests { let func: Box Result<()> + Send> = Box::new(|| -> Result<()> { Ok(()) }); Arc::new(Mutex::new(MemAccessHandler::from(func))) }; - test_initialise(outb_handler, mem_access_handler).unwrap(); + #[cfg(gdb)] + let dbg_mem_access_handler = Arc::new(Mutex::new(DbgMemAccessHandler {})); + + test_initialise( + outb_handler, + mem_access_handler, + #[cfg(gdb)] + dbg_mem_access_handler, + ) + .unwrap(); } } diff --git a/src/hyperlight_host/src/hypervisor/mod.rs b/src/hyperlight_host/src/hypervisor/mod.rs index c34d7eb1f..929aff0ab 100644 --- a/src/hyperlight_host/src/hypervisor/mod.rs +++ b/src/hyperlight_host/src/hypervisor/mod.rs @@ -34,6 +34,10 @@ pub mod hyperv_linux; pub(crate) mod hyperv_windows; pub(crate) mod hypervisor_handler; +/// GDB debugging support +#[cfg(gdb)] +mod gdb; + /// Driver for running in process instead of using hypervisor #[cfg(inprocess)] pub mod inprocess; @@ -61,6 +65,11 @@ pub(crate) mod crashdump; use std::fmt::Debug; use std::sync::{Arc, Mutex}; +#[cfg(gdb)] +use gdb::VcpuStopReason; + +#[cfg(gdb)] +use self::handlers::{DbgMemAccessHandlerCaller, DbgMemAccessHandlerWrapper}; use self::handlers::{ MemAccessHandlerCaller, MemAccessHandlerWrapper, OutBHandlerCaller, OutBHandlerWrapper, }; @@ -85,6 +94,9 @@ pub(crate) const EFER_NX: u64 = 1 << 11; /// These are the generic exit reasons that we can handle from a Hypervisor the Hypervisors run method is responsible for mapping from /// the hypervisor specific exit reasons to these generic ones pub enum HyperlightExit { + #[cfg(gdb)] + /// The vCPU has exited due to a debug event + Debug(VcpuStopReason), /// The vCPU has halted Halt(), /// The vCPU has issued a write to the given port with the given value @@ -118,6 +130,7 @@ pub(crate) trait Hypervisor: Debug + Sync + Send { outb_handle_fn: OutBHandlerWrapper, mem_access_fn: MemAccessHandlerWrapper, hv_handler: Option, + #[cfg(gdb)] dbg_mem_access_fn: DbgMemAccessHandlerWrapper, ) -> Result<()>; /// Dispatch a call from the host to the guest using the given pointer @@ -133,6 +146,7 @@ pub(crate) trait Hypervisor: Debug + Sync + Send { outb_handle_fn: OutBHandlerWrapper, mem_access_fn: MemAccessHandlerWrapper, hv_handler: Option, + #[cfg(gdb)] dbg_mem_access_fn: DbgMemAccessHandlerWrapper, ) -> Result<()>; /// Handle an IO exit from the internally stored vCPU. @@ -189,6 +203,16 @@ pub(crate) trait Hypervisor: Debug + Sync + Send { #[cfg(crashdump)] fn get_memory_regions(&self) -> &[MemoryRegion]; + + #[cfg(gdb)] + /// handles the cases when the vCPU stops due to a Debug event + fn handle_debug( + &mut self, + _dbg_mem_access_fn: Arc>, + _stop_reason: VcpuStopReason, + ) -> Result<()> { + unimplemented!() + } } /// A virtual CPU that can be run until an exit occurs @@ -202,9 +226,17 @@ impl VirtualCPU { hv_handler: Option, outb_handle_fn: Arc>, mem_access_fn: Arc>, + #[cfg(gdb)] dbg_mem_access_fn: Arc>, ) -> Result<()> { loop { match hv.run() { + #[cfg(gdb)] + Ok(HyperlightExit::Debug(stop_reason)) => { + if let Err(e) = hv.handle_debug(dbg_mem_access_fn.clone(), stop_reason) { + log_then_return!(e); + } + } + Ok(HyperlightExit::Halt()) => { break; } @@ -277,6 +309,8 @@ pub(crate) mod tests { use hyperlight_testing::dummy_guest_as_string; + #[cfg(gdb)] + use super::handlers::DbgMemAccessHandlerWrapper; use super::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper}; use crate::hypervisor::hypervisor_handler::{ HvHandlerConfig, HypervisorHandler, HypervisorHandlerAction, @@ -289,6 +323,7 @@ pub(crate) mod tests { pub(crate) fn test_initialise( outb_hdl: OutBHandlerWrapper, mem_access_hdl: MemAccessHandlerWrapper, + #[cfg(gdb)] dbg_mem_access_fn: DbgMemAccessHandlerWrapper, ) -> Result<()> { let filename = dummy_guest_as_string().map_err(|e| new_error!("{}", e))?; if !Path::new(&filename).exists() { @@ -306,6 +341,8 @@ pub(crate) mod tests { let hv_handler_config = HvHandlerConfig { outb_handler: outb_hdl, mem_access_handler: mem_access_hdl, + #[cfg(gdb)] + dbg_mem_access_handler: dbg_mem_access_fn, seed: 1234567890, page_size: 4096, peb_addr: RawPtr::from(0x230000), @@ -336,7 +373,11 @@ pub(crate) mod tests { // whether we can configure the shared memory region, load a binary // into it, and run the CPU to completion (e.g., a HLT interrupt) - hv_handler.start_hypervisor_handler(gshm)?; + hv_handler.start_hypervisor_handler( + gshm, + #[cfg(gdb)] + None, + )?; hv_handler.execute_hypervisor_handler_action(HypervisorHandlerAction::Initialise) } diff --git a/src/hyperlight_host/src/mem/layout.rs b/src/hyperlight_host/src/mem/layout.rs index 251032a25..3a13879c4 100644 --- a/src/hyperlight_host/src/mem/layout.rs +++ b/src/hyperlight_host/src/mem/layout.rs @@ -656,7 +656,7 @@ impl SandboxMemoryLayout { /// Get the guest address of the code section in the sandbox #[instrument(skip_all, parent = Span::current(), level= "Trace")] - pub(super) fn get_guest_code_address(&self) -> usize { + pub(crate) fn get_guest_code_address(&self) -> usize { Self::BASE_ADDRESS + self.guest_code_offset } diff --git a/src/hyperlight_host/src/mem/shared_mem.rs b/src/hyperlight_host/src/mem/shared_mem.rs index 0733f85dc..721704b41 100644 --- a/src/hyperlight_host/src/mem/shared_mem.rs +++ b/src/hyperlight_host/src/mem/shared_mem.rs @@ -840,7 +840,7 @@ impl HostSharedMemory { Ok(()) } - /// /Copy the contents of the sandbox at the specified offset into + /// Copy the contents of the sandbox at the specified offset into /// the slice pub fn copy_from_slice(&self, slice: &[u8], offset: usize) -> Result<()> { bounds_check!(offset, slice.len(), self.mem_size()); diff --git a/src/hyperlight_host/src/sandbox/config.rs b/src/hyperlight_host/src/sandbox/config.rs index c03528e31..e0bbaddbe 100644 --- a/src/hyperlight_host/src/sandbox/config.rs +++ b/src/hyperlight_host/src/sandbox/config.rs @@ -21,10 +21,21 @@ use tracing::{instrument, Span}; use crate::mem::exe::ExeInfo; +/// Used for passing debug configuration to a sandbox +#[cfg(gdb)] +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub struct DebugInfo { + /// Guest debug port + pub port: u16, +} + /// The complete set of configuration needed to create a Sandbox #[derive(Copy, Clone, Debug, Eq, PartialEq)] #[repr(C)] pub struct SandboxConfiguration { + /// Guest gdb debug port + #[cfg(gdb)] + guest_debug_info: Option, /// The maximum size of the guest error buffer. guest_error_buffer_size: usize, /// The size of the memory buffer that is made available for Guest Function @@ -153,6 +164,7 @@ impl SandboxConfiguration { max_initialization_time: Option, max_wait_for_cancellation: Option, guest_panic_context_buffer_size: usize, + #[cfg(gdb)] guest_debug_info: Option, ) -> Self { Self { input_data_size: max(input_data_size, Self::MIN_INPUT_SIZE), @@ -220,6 +232,8 @@ impl SandboxConfiguration { guest_panic_context_buffer_size, Self::MIN_GUEST_PANIC_CONTEXT_BUFFER_SIZE, ), + #[cfg(gdb)] + guest_debug_info, } } @@ -346,6 +360,13 @@ impl SandboxConfiguration { ); } + /// Sets the configuration for the guest debug + #[cfg(gdb)] + #[instrument(skip_all, parent = Span::current(), level= "Trace")] + pub fn set_guest_debug_info(&mut self, debug_info: DebugInfo) { + self.guest_debug_info = Some(debug_info); + } + #[instrument(skip_all, parent = Span::current(), level= "Trace")] pub(crate) fn get_guest_error_buffer_size(&self) -> usize { self.guest_error_buffer_size @@ -390,6 +411,12 @@ impl SandboxConfiguration { self.max_initialization_time } + #[cfg(gdb)] + #[instrument(skip_all, parent = Span::current(), level= "Trace")] + pub(crate) fn get_guest_debug_info(&self) -> Option { + self.guest_debug_info + } + #[instrument(skip_all, parent = Span::current(), level= "Trace")] fn stack_size_override_opt(&self) -> Option { (self.stack_size_override > 0).then_some(self.stack_size_override) @@ -438,6 +465,8 @@ impl Default for SandboxConfiguration { None, None, Self::DEFAULT_GUEST_PANIC_CONTEXT_BUFFER_SIZE, + #[cfg(gdb)] + None, ) } } @@ -480,6 +509,8 @@ mod tests { MAX_WAIT_FOR_CANCELLATION_OVERRIDE as u64, )), GUEST_PANIC_CONTEXT_BUFFER_SIZE_OVERRIDE, + #[cfg(gdb)] + None, ); let exe_infos = vec![ simple_guest_exe_info().unwrap(), @@ -543,6 +574,8 @@ mod tests { SandboxConfiguration::MIN_MAX_WAIT_FOR_CANCELLATION as u64 - 1, )), SandboxConfiguration::MIN_GUEST_PANIC_CONTEXT_BUFFER_SIZE - 1, + #[cfg(gdb)] + None, ); assert_eq!(SandboxConfiguration::MIN_INPUT_SIZE, cfg.input_data_size); assert_eq!(SandboxConfiguration::MIN_OUTPUT_SIZE, cfg.output_data_size); @@ -633,6 +666,8 @@ mod tests { use proptest::prelude::*; use super::SandboxConfiguration; + #[cfg(gdb)] + use crate::sandbox::config::DebugInfo; proptest! { #[test] @@ -711,6 +746,15 @@ mod tests { cfg.set_heap_size(size); prop_assert_eq!(size, cfg.heap_size_override); } + + #[test] + #[cfg(gdb)] + fn guest_debug_info(port in 9000..=u16::MAX) { + let mut cfg = SandboxConfiguration::default(); + let debug_info = DebugInfo { port }; + cfg.set_guest_debug_info(debug_info); + prop_assert_eq!(debug_info, *cfg.get_guest_debug_info().as_ref().unwrap()); + } } } } diff --git a/src/hyperlight_host/src/sandbox/mem_access.rs b/src/hyperlight_host/src/sandbox/mem_access.rs index 6a1e6b703..7bf4fbdc2 100644 --- a/src/hyperlight_host/src/sandbox/mem_access.rs +++ b/src/hyperlight_host/src/sandbox/mem_access.rs @@ -20,6 +20,8 @@ use tracing::{instrument, Span}; use super::mem_mgr::MemMgrWrapper; use crate::error::HyperlightError::StackOverflow; +#[cfg(gdb)] +use crate::hypervisor::handlers::{DbgMemAccessHandlerCaller, DbgMemAccessHandlerWrapper}; use crate::hypervisor::handlers::{ MemAccessHandler, MemAccessHandlerFunction, MemAccessHandlerWrapper, }; @@ -44,3 +46,42 @@ pub(crate) fn mem_access_handler_wrapper( let mem_access_hdl = MemAccessHandler::from(mem_access_func); Arc::new(Mutex::new(mem_access_hdl)) } + +#[cfg(gdb)] +struct DbgMemAccessContainer { + wrapper: MemMgrWrapper, +} + +#[cfg(gdb)] +impl DbgMemAccessHandlerCaller for DbgMemAccessContainer { + #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")] + fn read(&mut self, addr: usize, data: &mut [u8]) -> Result<()> { + self.wrapper + .unwrap_mgr_mut() + .get_shared_mem_mut() + .copy_to_slice(data, addr) + } + + #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")] + fn write(&mut self, addr: usize, data: &[u8]) -> Result<()> { + self.wrapper + .unwrap_mgr_mut() + .get_shared_mem_mut() + .copy_from_slice(data, addr) + } + + #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")] + fn get_code_offset(&mut self) -> Result { + Ok(self.wrapper.unwrap_mgr().layout.get_guest_code_address()) + } +} + +#[cfg(gdb)] +#[instrument(skip_all, parent = Span::current(), level= "Trace")] +pub(crate) fn dbg_mem_access_handler_wrapper( + wrapper: MemMgrWrapper, +) -> DbgMemAccessHandlerWrapper { + let container = DbgMemAccessContainer { wrapper }; + + Arc::new(Mutex::new(container)) +} diff --git a/src/hyperlight_host/src/sandbox/uninitialized.rs b/src/hyperlight_host/src/sandbox/uninitialized.rs index 4ca951d57..e38647d8a 100644 --- a/src/hyperlight_host/src/sandbox/uninitialized.rs +++ b/src/hyperlight_host/src/sandbox/uninitialized.rs @@ -22,6 +22,8 @@ use std::time::Duration; use tracing::{instrument, Span}; +#[cfg(gdb)] +use super::config::DebugInfo; use super::host_funcs::{default_writer_func, HostFuncsWrapper}; use super::mem_mgr::MemMgrWrapper; use super::run_options::SandboxRunOptions; @@ -52,6 +54,8 @@ pub struct UninitializedSandbox { pub(crate) max_initialization_time: Duration, pub(crate) max_execution_time: Duration, pub(crate) max_wait_for_cancellation: Duration, + #[cfg(gdb)] + pub(crate) debug_info: Option, } impl crate::sandbox_state::sandbox::UninitializedSandbox for UninitializedSandbox { @@ -161,6 +165,9 @@ impl UninitializedSandbox { } let sandbox_cfg = cfg.unwrap_or_default(); + + #[cfg(gdb)] + let debug_info = sandbox_cfg.get_guest_debug_info(); let mut mem_mgr_wrapper = { let mut mgr = UninitializedSandbox::load_guest_binary( sandbox_cfg, @@ -188,6 +195,8 @@ impl UninitializedSandbox { max_wait_for_cancellation: Duration::from_millis( sandbox_cfg.get_max_wait_for_cancellation() as u64, ), + #[cfg(gdb)] + debug_info, }; // TODO: These only here to accommodate some writer functions. diff --git a/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs b/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs index ccbed836e..f9e99d7aa 100644 --- a/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs +++ b/src/hyperlight_host/src/sandbox/uninitialized_evolve.rs @@ -20,12 +20,16 @@ use std::sync::{Arc, Mutex}; use rand::Rng; use tracing::{instrument, Span}; +#[cfg(gdb)] +use super::mem_access::dbg_mem_access_handler_wrapper; use crate::hypervisor::hypervisor_handler::{ HvHandlerConfig, HypervisorHandler, HypervisorHandlerAction, }; use crate::mem::mgr::SandboxMemoryManager; use crate::mem::ptr::RawPtr; use crate::mem::shared_mem::GuestSharedMemory; +#[cfg(gdb)] +use crate::sandbox::config::DebugInfo; use crate::sandbox::host_funcs::HostFuncsWrapper; use crate::sandbox::mem_access::mem_access_handler_wrapper; use crate::sandbox::outb::outb_handler_wrapper; @@ -66,6 +70,8 @@ where u_sbox.max_initialization_time, u_sbox.max_execution_time, u_sbox.max_wait_for_cancellation, + #[cfg(gdb)] + u_sbox.debug_info, )?; { @@ -98,9 +104,13 @@ fn hv_init( max_init_time: Duration, max_exec_time: Duration, max_wait_for_cancellation: Duration, + #[cfg(gdb)] debug_info: Option, ) -> Result { let outb_hdl = outb_handler_wrapper(hshm.clone(), host_funcs); let mem_access_hdl = mem_access_handler_wrapper(hshm.clone()); + #[cfg(gdb)] + let dbg_mem_access_hdl = dbg_mem_access_handler_wrapper(hshm.clone()); + let seed = { let mut rng = rand::rng(); rng.random::() @@ -113,6 +123,8 @@ fn hv_init( let hv_handler_config = HvHandlerConfig { outb_handler: outb_hdl, mem_access_handler: mem_access_hdl, + #[cfg(gdb)] + dbg_mem_access_handler: dbg_mem_access_hdl, seed, page_size, peb_addr, @@ -126,7 +138,11 @@ fn hv_init( let mut hv_handler = HypervisorHandler::new(hv_handler_config); - hv_handler.start_hypervisor_handler(gshm)?; + hv_handler.start_hypervisor_handler( + gshm, + #[cfg(gdb)] + debug_info, + )?; hv_handler .execute_hypervisor_handler_action(HypervisorHandlerAction::Initialise)