Skip to content

Commit bdf5800

Browse files
dblnzdblnz
authored
Support ELF core dump creation on guest crash (#417)
* crashdump: create core dump file when a guest crashes - the core dump file is an ELF file with special segments that describe the guest's memory when it crashed, the CPU register's values and other special notes that tell the debugger how to set up a debugging session starting from the core dump Signed-off-by: Doru Blânzeanu <[email protected]> * fixup: add constants instead of hardcoded values and add more comments Signed-off-by: Doru Blânzeanu <[email protected]> * crashdump: add SandboxMetadata field that store relevant data about the sandbox - only store the binary path for now Signed-off-by: Doru Blânzeanu <[email protected]> * fixup: change default filename to "<unknown>" and aother small updates Signed-off-by: Doru Blânzeanu <[email protected]> * crashdump: add documentation on how to load a guest core dump in a gdb and lldb using vscode Signed-off-by: Doru Blânzeanu <[email protected]> * crashdump: update the use of CoreDumpBuilder to reflect elfcore change to generics Signed-off-by: Doru Blânzeanu <[email protected]> * fixup: documentation Signed-off-by: Doru Blânzeanu <[email protected]> * fixup: map_or_else to map_or Signed-off-by: Doru Blânzeanu <[email protected]> * crashdump: enable crashdump feature for release builds - Improve documentation and describe how an user can create separate files for debug information - Change the output file directory to be configurable using HYPERLIGHT_CORE_DUMP_DIR environment variable - Change output file name to include a timestamp Signed-off-by: Doru Blânzeanu <[email protected]> * crashdump: allow crashdump toggle at sandbox level - this allows a user to configure the crash dump feature at sandbox level - create a SandboxRuntimeConfig struct to contain all the configuration a sandbox would need at runtime to avoid passing the information as multiple functions arguments - add unit tests to verify crashdump behavior Signed-off-by: Doru Blânzeanu <[email protected]> * cargo: move to official elfcore crates.io 2.0 version Signed-off-by: Doru Blânzeanu <[email protected]> * Fix crashdump creation message print when the option is disabled for a sandbox - Update docs to add a note about how to disable crashdumps for specific sandboxes - Commit automatically update `Cargo.lock` for rust_guests Signed-off-by: Doru Blânzeanu <[email protected]> * docs: update crashdump docs to signal support for both debug and release builds Signed-off-by: Doru Blânzeanu <[email protected]> --------- Signed-off-by: Doru Blânzeanu <[email protected]>
1 parent d5bf65a commit bdf5800

File tree

16 files changed

+1110
-57
lines changed

16 files changed

+1110
-57
lines changed

‎.github/workflows/dep_rust.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,12 @@ jobs:
136136
RUST_LOG: debug
137137
run: just test-rust-gdb-debugging ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}}
138138

139+
- name: Run Rust Crashdump tests
140+
env:
141+
CARGO_TERM_COLOR: always
142+
RUST_LOG: debug
143+
run: just test-rust-crashdump ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}}
144+
139145
### Benchmarks ###
140146
- name: Install github-cli (Linux mariner)
141147
if: runner.os == 'Linux' && matrix.hypervisor == 'mshv'

‎Cargo.lock‎

Lines changed: 27 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Justfile‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,9 @@ test-like-ci config=default-target hypervisor="kvm":
8181
@# without any driver (should fail to compile)
8282
just test-compilation-fail {{config}}
8383

84+
@# test the crashdump feature
85+
just test-rust-crashdump {{config}}
86+
8487
# runs all tests
8588
test target=default-target features="": (test-unit target features) (test-isolated target features) (test-integration "rust" target features) (test-integration "c" target features) (test-seccomp target features)
8689

@@ -123,6 +126,10 @@ test-rust-gdb-debugging target=default-target features="":
123126
cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --example guest-debugging {{ if features =="" {'--features gdb'} else { "--features gdb," + features } }}
124127
cargo test --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {'--features gdb'} else { "--features gdb," + features } }} -- test_gdb
125128

129+
# rust test for crashdump
130+
test-rust-crashdump target=default-target features="":
131+
cargo test --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {'--features crashdump'} else { "--features crashdump," + features } }} -- test_crashdump
132+
126133

127134
################
128135
### LINTING ####

‎docs/how-to-debug-a-hyperlight-guest.md‎

Lines changed: 230 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,3 +201,233 @@ involved in the gdb debugging of a Hyperlight guest running inside a **KVM** or
201201
└─┘ │ | | | │
202202
| └───────────────────────────────────────────────────────────────────────────────────────────────┘
203203
```
204+
205+
## Dumping the guest state to an ELF core dump when an unhandled crash occurs
206+
207+
When a guest crashes because of an unknown VmExit or unhandled exception, the vCPU state is dumped to an `ELF` core dump file.
208+
This can be used to inspect the state of the guest at the time of the crash.
209+
210+
To make Hyperlight dump the state of the vCPU (general purpose registers, registers) to an `ELF` core dump file, enable the `crashdump`
211+
feature and run.
212+
The feature enables the creation of core dump files for both debug and release builds of Hyperlight hosts.
213+
By default, Hyperlight places the core dumps in the temporary directory (platform specific).
214+
To change this, use the `HYPERLIGHT_CORE_DUMP_DIR` environment variable to specify a directory.
215+
The name and location of the dump file will be printed to the console and logged as an error message.
216+
217+
**NOTE**: If the directory provided by `HYPERLIGHT_CORE_DUMP_DIR` does not exist, Hyperlight places the file in the temporary directory.
218+
**NOTE**: By enabling the `crashdump` feature, you instruct Hyperlight to create core dump files for all sandboxes when an unhandled crash occurs.
219+
To selectively disable this feature for a specific sandbox, you can set the `guest_core_dump` field to `false` in the `SandboxConfiguration`.
220+
```rust
221+
let mut cfg = SandboxConfiguration::default();
222+
cfg.set_guest_core_dump(false); // Disable core dump for this sandbox
223+
```
224+
225+
### Inspecting the core dump
226+
227+
After the core dump has been created, to inspect the state of the guest, load the core dump file using `gdb` or `lldb`.
228+
**NOTE: This feature has been tested with version `15.0` of `gdb` and version `17` of `lldb`, earlier versions may not work, it is recommended to use these versions or later.**
229+
230+
To do this in vscode, the following configuration can be used to add debug configurations:
231+
232+
```vscode
233+
{
234+
"version": "0.2.0",
235+
"inputs": [
236+
{
237+
"id": "core_dump",
238+
"type": "promptString",
239+
"description": "Path to the core dump file",
240+
},
241+
{
242+
"id": "program",
243+
"type": "promptString",
244+
"description": "Path to the program to debug",
245+
}
246+
],
247+
"configurations": [
248+
{
249+
"name": "[GDB] Load core dump file",
250+
"type": "cppdbg",
251+
"request": "launch",
252+
"program": "${input:program}",
253+
"coreDumpPath": "${input:core_dump}",
254+
"cwd": "${workspaceFolder}",
255+
"MIMode": "gdb",
256+
"externalConsole": false,
257+
"miDebuggerPath": "/usr/bin/gdb",
258+
"setupCommands": [
259+
{
260+
"description": "Enable pretty-printing for gdb",
261+
"text": "-enable-pretty-printing",
262+
"ignoreFailures": true
263+
},
264+
{
265+
"description": "Set Disassembly Flavor to Intel",
266+
"text": "-gdb-set disassembly-flavor intel",
267+
"ignoreFailures": true
268+
}
269+
]
270+
},
271+
{
272+
"name": "[LLDB] Load core dump file",
273+
"type": "lldb",
274+
"request": "launch",
275+
"stopOnEntry": true,
276+
"processCreateCommands": [],
277+
"targetCreateCommands": [
278+
"target create -c ${input:core_dump} ${input:program}",
279+
],
280+
},
281+
]
282+
}
283+
```
284+
**NOTE: The `CodeLldb` debug session does not stop after launching. To see the code, stack frames and registers you need to
285+
press the `pause` button. This is a known issue with the `CodeLldb` extension [#1245](https://github.com/vadimcn/codelldb/issues/1245).
286+
The `cppdbg` extension works as expected and stops at the entry point of the program.**
287+
288+
## Compiling guests with debug information for release builds
289+
290+
This section explains how to compile a guest with debugging information but still have optimized code, and how to separate the debug information from the binary.
291+
292+
### Creating a release build with debug information
293+
294+
To create a release build with debug information, you can add a custom profile to your `Cargo.toml` file:
295+
296+
```toml
297+
[profile.release-with-debug]
298+
inherits = "release"
299+
debug = true
300+
```
301+
302+
This creates a new profile called `release-with-debug` that inherits all settings from the release profile but adds debug information.
303+
304+
### Splitting debug information from the binary
305+
306+
To reduce the binary size while still having debug information available, you can split the debug information into a separate file.
307+
This is useful for production environments where you want smaller binaries but still want to be able to debug crashes.
308+
309+
Here's a step-by-step guide:
310+
311+
1. Build your guest with the release-with-debug profile:
312+
```bash
313+
cargo build --profile release-with-debug
314+
```
315+
316+
2. Locate your binary in the target directory:
317+
```bash
318+
TARGET_DIR="target"
319+
PROFILE="release-with-debug"
320+
ARCH="x86_64-unknown-none" # Your target architecture
321+
BUILD_DIR="${TARGET_DIR}/${ARCH}/${PROFILE}"
322+
BINARY=$(find "${BUILD_DIR}" -type f -executable -name "guest-binary" | head -1)
323+
```
324+
325+
3. Extract debug information into a full debug file:
326+
```bash
327+
DEBUG_FILE_FULL="${BINARY}.debug.full"
328+
objcopy --only-keep-debug "${BINARY}" "${DEBUG_FILE_FULL}"
329+
```
330+
331+
4. Create a symbols-only debug file (smaller, but still useful for stack traces):
332+
```bash
333+
DEBUG_FILE="${BINARY}.debug"
334+
objcopy --keep-file-symbols "${DEBUG_FILE_FULL}" "${DEBUG_FILE}"
335+
```
336+
337+
5. Strip debug information from the original binary but keep function names:
338+
```bash
339+
objcopy --strip-debug "${BINARY}"
340+
```
341+
342+
6. Add a debug link to the stripped binary:
343+
```bash
344+
objcopy --add-gnu-debuglink="${DEBUG_FILE}" "${BINARY}"
345+
```
346+
347+
After these steps, you'll have:
348+
- An optimized binary with function names for basic stack traces
349+
- A symbols-only debug file for stack traces
350+
- A full debug file for complete source-level debugging
351+
352+
### Analyzing core dumps with the debug files
353+
354+
When you have a core dump from a crashed guest, you can analyze it with different levels of detail using either GDB or LLDB.
355+
356+
#### Using GDB
357+
358+
1. For basic analysis with function names (stack traces):
359+
```bash
360+
gdb ${BINARY} -c /path/to/core.dump
361+
```
362+
363+
2. For full source-level debugging:
364+
```bash
365+
gdb -s ${DEBUG_FILE_FULL} ${BINARY} -c /path/to/core.dump
366+
```
367+
368+
#### Using LLDB
369+
370+
LLDB provides similar capabilities with slightly different commands:
371+
372+
1. For basic analysis with function names (stack traces):
373+
```bash
374+
lldb ${BINARY} -c /path/to/core.dump
375+
```
376+
377+
2. For full source-level debugging:
378+
```bash
379+
lldb -o "target create -c /path/to/core.dump ${BINARY}" -o "add-dsym ${DEBUG_FILE_FULL}"
380+
```
381+
382+
3. If your debug symbols are in a separate file:
383+
```bash
384+
lldb ${BINARY} -c /path/to/core.dump
385+
(lldb) add-dsym ${DEBUG_FILE_FULL}
386+
```
387+
388+
### VSCode Debug Configurations
389+
390+
You can configure VSCode (in `.vscode/launch.json`) to use these files by modifying the debug configurations:
391+
392+
#### For GDB
393+
394+
```json
395+
{
396+
"name": "[GDB] Load core dump with full debug symbols",
397+
"type": "cppdbg",
398+
"request": "launch",
399+
"program": "${input:program}",
400+
"coreDumpPath": "${input:core_dump}",
401+
"cwd": "${workspaceFolder}",
402+
"MIMode": "gdb",
403+
"externalConsole": false,
404+
"miDebuggerPath": "/usr/bin/gdb",
405+
"setupCommands": [
406+
{
407+
"description": "Enable pretty-printing for gdb",
408+
"text": "-enable-pretty-printing",
409+
"ignoreFailures": true
410+
}
411+
]
412+
}
413+
```
414+
415+
#### For LLDB
416+
417+
```json
418+
{
419+
"name": "[LLDB] Load core dump with full debug symbols",
420+
"type": "lldb",
421+
"request": "launch",
422+
"program": "${input:program}",
423+
"cwd": "${workspaceFolder}",
424+
"processCreateCommands": [],
425+
"targetCreateCommands": [
426+
"target create -c ${input:core_dump} ${input:program}"
427+
],
428+
"postRunCommands": [
429+
// if debug symbols are in a different file
430+
"add-dsym ${input:debug_file_path}"
431+
]
432+
}
433+
```

‎src/hyperlight_host/Cargo.toml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,10 +38,11 @@ vmm-sys-util = "0.14.0"
3838
crossbeam = "0.8.0"
3939
crossbeam-channel = "0.5.15"
4040
thiserror = "2.0.12"
41-
tempfile = { version = "3.20", optional = true }
41+
chrono = { version = "0.4", optional = true }
4242
anyhow = "1.0"
4343
metrics = "0.24.2"
4444
serde_json = "1.0"
45+
elfcore = "2.0"
4546

4647
[target.'cfg(windows)'.dependencies]
4748
windows = { version = "0.61", features = [
@@ -124,7 +125,8 @@ function_call_metrics = []
124125
executable_heap = []
125126
# This feature enables printing of debug information to stdout in debug builds
126127
print_debug = []
127-
crashdump = ["dep:tempfile"] # Dumps the VM state to a file on unexpected errors or crashes. The path of the file will be printed on stdout and logged. This feature can only be used in debug builds.
128+
# Dumps the VM state to a file on unexpected errors or crashes. The path of the file will be printed on stdout and logged.
129+
crashdump = ["dep:chrono"]
128130
kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"]
129131
mshv2 = ["dep:mshv-bindings2", "dep:mshv-ioctls2"]
130132
mshv3 = ["dep:mshv-bindings3", "dep:mshv-ioctls3"]

‎src/hyperlight_host/build.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,8 +93,7 @@ fn main() -> Result<()> {
9393
gdb: { all(feature = "gdb", debug_assertions, any(feature = "kvm", feature = "mshv2", feature = "mshv3"), target_os = "linux") },
9494
kvm: { all(feature = "kvm", target_os = "linux") },
9595
mshv: { all(any(feature = "mshv2", feature = "mshv3"), target_os = "linux") },
96-
// crashdump feature is aliased with debug_assertions to make it only available in debug-builds.
97-
crashdump: { all(feature = "crashdump", debug_assertions) },
96+
crashdump: { all(feature = "crashdump") },
9897
// print_debug feature is aliased with debug_assertions to make it only available in debug-builds.
9998
print_debug: { all(feature = "print_debug", debug_assertions) },
10099
// the following features are mutually exclusive but rather than enforcing that here we are enabling mshv3 to override mshv2 when both are enabled

0 commit comments

Comments
 (0)