Skip to content

Commit 5393dd0

Browse files
committed
feat: Enable gdb debugging on x86
Enabling GDB support for debugging the guest kernel. This allows us to connect a gdb server to firecracker and debug the guest. Signed-off-by: Jack Thomson <[email protected]>
1 parent 0661dd7 commit 5393dd0

File tree

11 files changed

+1303
-11
lines changed

11 files changed

+1303
-11
lines changed

Cargo.lock

Lines changed: 32 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

resources/seccomp/x86_64-unknown-linux-musl.json

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1238,6 +1238,30 @@
12381238
}
12391239
]
12401240
},
1241+
{
1242+
"syscall": "ioctl",
1243+
"args": [
1244+
{
1245+
"index": 1,
1246+
"type": "dword",
1247+
"op": "eq",
1248+
"val": 1078505115,
1249+
"comment": "KVM_SET_GUEST_DEBUG"
1250+
}
1251+
]
1252+
},
1253+
{
1254+
"syscall": "ioctl",
1255+
"args": [
1256+
{
1257+
"index": 1,
1258+
"type": "dword",
1259+
"op": "eq",
1260+
"val": 3222843013,
1261+
"comment": "KVM_TRANSLATE"
1262+
}
1263+
]
1264+
},
12411265
{
12421266
"syscall": "ioctl",
12431267
"args": [

src/firecracker/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ serde_json = "1.0.128"
4949

5050
[features]
5151
tracing = ["log-instrument", "seccompiler/tracing", "utils/tracing", "vmm/tracing"]
52+
debug = ["vmm/debug"]
5253

5354
[lints]
5455
workspace = true

src/vmm/Cargo.toml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ crc64 = "2.0.0"
1919
derive_more = { version = "1.0.0", default-features = false, features = ["from", "display"] }
2020
displaydoc = "0.2.5"
2121
event-manager = "0.4.0"
22+
gdbstub = { version = "0.7.2", optional = true }
23+
gdbstub_arch = { version = "0.3.0", optional = true }
2224
kvm-bindings = { version = "0.9.1", features = ["fam-wrappers", "serde"] }
2325
kvm-ioctls = "0.18.0"
2426
lazy_static = "1.5.0"
@@ -56,7 +58,9 @@ itertools = "0.13.0"
5658
proptest = { version = "1.5.0", default-features = false, features = ["std"] }
5759

5860
[features]
61+
default = []
5962
tracing = ["log-instrument"]
63+
debug = ["gdbstub", "gdbstub_arch"]
6064

6165
[[bench]]
6266
name = "cpu_templates"

src/vmm/src/builder.rs

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
use std::convert::TryFrom;
88
use std::fmt::Debug;
99
use std::io::{self, Seek, SeekFrom};
10+
#[cfg(feature = "debug")]
11+
use std::sync::mpsc;
1012
use std::sync::{Arc, Mutex};
1113

1214
use event_manager::{MutEventSubscriber, SubscriberOps};
@@ -56,6 +58,8 @@ use crate::devices::virtio::net::Net;
5658
use crate::devices::virtio::rng::Entropy;
5759
use crate::devices::virtio::vsock::{Vsock, VsockUnixBackend};
5860
use crate::devices::BusDevice;
61+
#[cfg(feature = "debug")]
62+
use crate::gdb;
5963
use crate::logger::{debug, error};
6064
use crate::persist::{MicrovmState, MicrovmStateError};
6165
use crate::resources::VmResources;
@@ -304,6 +308,14 @@ pub fn build_microvm_for_boot(
304308
cpu_template.kvm_capabilities.clone(),
305309
)?;
306310

311+
#[cfg(feature = "debug")]
312+
let (gdb_tx, gdb_rx) = mpsc::channel();
313+
#[cfg(feature = "debug")]
314+
vcpus
315+
.iter_mut()
316+
.enumerate()
317+
.for_each(|(id, vcpu)| vcpu.attach_debug_info(gdb_tx.clone(), id));
318+
307319
// The boot timer device needs to be the first device attached in order
308320
// to maintain the same MMIO address referenced in the documentation
309321
// and tests.
@@ -351,16 +363,23 @@ pub fn build_microvm_for_boot(
351363
boot_cmdline,
352364
)?;
353365

366+
let vmm = Arc::new(Mutex::new(vmm));
367+
368+
#[cfg(feature = "debug")]
369+
gdb::server::gdb_thread(vmm.clone(), &vcpus, gdb_rx, entry_addr);
370+
354371
// Move vcpus to their own threads and start their state machine in the 'Paused' state.
355-
vmm.start_vcpus(
356-
vcpus,
357-
seccomp_filters
358-
.get("vcpu")
359-
.ok_or_else(|| MissingSeccompFilters("vcpu".to_string()))?
360-
.clone(),
361-
)
362-
.map_err(VmmError::VcpuStart)
363-
.map_err(Internal)?;
372+
vmm.lock()
373+
.unwrap()
374+
.start_vcpus(
375+
vcpus,
376+
seccomp_filters
377+
.get("vcpu")
378+
.ok_or_else(|| MissingSeccompFilters("vcpu".to_string()))?
379+
.clone(),
380+
)
381+
.map_err(VmmError::VcpuStart)
382+
.map_err(Internal)?;
364383

365384
// Load seccomp filters for the VMM thread.
366385
// Execution panics if filters cannot be loaded, use --no-seccomp if skipping filters
@@ -374,7 +393,6 @@ pub fn build_microvm_for_boot(
374393
.map_err(VmmError::SeccompFilters)
375394
.map_err(Internal)?;
376395

377-
let vmm = Arc::new(Mutex::new(vmm));
378396
event_manager.add_subscriber(vmm.clone());
379397

380398
Ok(vmm)

src/vmm/src/gdb/event_loop.rs

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
use std::os::unix::net::UnixStream;
5+
use std::sync::mpsc::Receiver;
6+
use std::sync::mpsc::TryRecvError::Empty;
7+
use std::sync::{Arc, Mutex};
8+
9+
use gdbstub::common::{Signal, Tid};
10+
use gdbstub::conn::{Connection, ConnectionExt};
11+
use gdbstub::stub::run_blocking::{self, WaitForStopReasonError};
12+
use gdbstub::stub::{DisconnectReason, GdbStub, MultiThreadStopReason};
13+
use gdbstub::target::Target;
14+
use vm_memory::GuestAddress;
15+
16+
use super::target::{cpuid_to_tid, Error, FirecrackerTarget};
17+
use crate::logger::trace;
18+
use crate::Vmm;
19+
20+
pub fn event_loop(
21+
connection: UnixStream,
22+
vmm: Arc<Mutex<Vmm>>,
23+
gdb_event_receiver: Receiver<usize>,
24+
entry_addr: GuestAddress,
25+
) {
26+
let target = FirecrackerTarget::new(vmm, gdb_event_receiver, entry_addr);
27+
let connection: Box<dyn ConnectionExt<Error = std::io::Error>> = { Box::new(connection) };
28+
let debugger = GdbStub::new(connection);
29+
30+
gdb_event_loop_thread(debugger, target);
31+
}
32+
33+
struct GdbBlockingEventLoop {}
34+
35+
impl run_blocking::BlockingEventLoop for GdbBlockingEventLoop {
36+
type Target = FirecrackerTarget;
37+
type Connection = Box<dyn ConnectionExt<Error = std::io::Error>>;
38+
39+
type StopReason = MultiThreadStopReason<u64>;
40+
41+
fn wait_for_stop_reason(
42+
target: &mut FirecrackerTarget,
43+
conn: &mut Self::Connection,
44+
) -> Result<
45+
run_blocking::Event<MultiThreadStopReason<u64>>,
46+
run_blocking::WaitForStopReasonError<
47+
<Self::Target as Target>::Error,
48+
<Self::Connection as Connection>::Error,
49+
>,
50+
> {
51+
loop {
52+
match target.gdb_event.try_recv() {
53+
Ok(cpu_id) => {
54+
// The VCPU reports it's id from raw_id so we straight convert here
55+
let tid = Tid::new(cpu_id).expect("Error converting cpu id to Tid");
56+
// If notify paused returns false this means we were already debugging a single
57+
// core, the target will track this for us to pick up later
58+
target.notify_paused_vcpu(tid);
59+
trace!("vcpu: {tid:?} paused from debug exit");
60+
61+
let stop_response = match target.get_stop_reason(tid) {
62+
Some(res) => res,
63+
None => {
64+
// If we returned None this is a break which should be handled by
65+
// the guest kernel (e.g. kernel int3 self testing) so we won't notify
66+
// GDB
67+
if let Err(e) = target.request_resume(tid) {
68+
return Err(WaitForStopReasonError::Target(e));
69+
}
70+
71+
trace!("Injected BP into guest early exit");
72+
continue;
73+
}
74+
};
75+
76+
trace!("Returned stop reason to gdb: {stop_response:?}");
77+
78+
return Ok(run_blocking::Event::TargetStopped(stop_response));
79+
}
80+
Err(Empty) => (),
81+
Err(_) => {
82+
return Err(WaitForStopReasonError::Target(Error::GdbQueueError));
83+
}
84+
}
85+
86+
if conn.peek().map(|b| b.is_some()).unwrap_or(false) {
87+
let byte = conn
88+
.read()
89+
.map_err(run_blocking::WaitForStopReasonError::Connection)?;
90+
return Ok(run_blocking::Event::IncomingData(byte));
91+
}
92+
}
93+
}
94+
95+
// Invoked when the GDB client sends a Ctrl-C interrupt.
96+
fn on_interrupt(
97+
target: &mut FirecrackerTarget,
98+
) -> Result<Option<MultiThreadStopReason<u64>>, <FirecrackerTarget as Target>::Error> {
99+
// notify the target that a ctrl-c interrupt has occurred.
100+
let main_core = cpuid_to_tid(0);
101+
102+
if target.request_pause(main_core).is_err() {
103+
return Err(Error::VCPURequestError);
104+
}
105+
106+
target.notify_paused_vcpu(main_core);
107+
108+
let exit_reason = MultiThreadStopReason::SignalWithThread {
109+
tid: main_core,
110+
signal: Signal::SIGINT,
111+
};
112+
Ok(Some(exit_reason))
113+
}
114+
}
115+
116+
fn gdb_event_loop_thread(
117+
debugger: GdbStub<FirecrackerTarget, Box<dyn ConnectionExt<Error = std::io::Error>>>,
118+
mut target: FirecrackerTarget,
119+
) {
120+
match debugger.run_blocking::<GdbBlockingEventLoop>(&mut target) {
121+
Ok(disconnect_reason) => match disconnect_reason {
122+
DisconnectReason::Disconnect => {
123+
println!("Client disconnected")
124+
}
125+
DisconnectReason::TargetExited(code) => {
126+
println!("Target exited with code {}", code)
127+
}
128+
DisconnectReason::TargetTerminated(sig) => {
129+
println!("Target terminated with signal {}", sig)
130+
}
131+
DisconnectReason::Kill => println!("GDB sent a kill command"),
132+
},
133+
Err(e) => {
134+
if e.is_target_error() {
135+
println!("target encountered a fatal error:")
136+
} else if e.is_connection_error() {
137+
println!("connection error: ")
138+
} else {
139+
println!("gdbstub encountered a fatal error {e:?}")
140+
}
141+
}
142+
}
143+
144+
target.shutdown();
145+
}

src/vmm/src/gdb/mod.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
mod event_loop;
5+
/// GDB Server module
6+
pub mod server;
7+
/// Target for gdb
8+
pub mod target;

0 commit comments

Comments
 (0)