From 68ac57e0b54716ad73b7e0898b4be5019c15813d Mon Sep 17 00:00:00 2001 From: Riccardo Mancini Date: Fri, 21 Nov 2025 14:21:03 +0000 Subject: [PATCH 1/3] test(gdb): add integ test to verify gdb works on x86 The integ test opens gdb on a subprocess which connects to Firecracker, sets a breakpoint and continues until boot is complete. If the VM boots correctly and the breakpoint gets hit, then the test will succeed. This test currently only works on x86 as GDB requires a vmlinux image (we use a uImage on ARM). Signed-off-by: Riccardo Mancini --- tests/framework/microvm.py | 7 ++ tests/integration_tests/build/test_gdb.py | 78 ++++++++++++++++++++++- 2 files changed, 84 insertions(+), 1 deletion(-) diff --git a/tests/framework/microvm.py b/tests/framework/microvm.py index 74ae180950c..681c397d444 100644 --- a/tests/framework/microvm.py +++ b/tests/framework/microvm.py @@ -287,6 +287,8 @@ def __init__( self.help = MicrovmHelpers(self) + self.gdb_socket = None + def __repr__(self): return f"" @@ -1198,6 +1200,11 @@ def wait_for_ssh_up(self): # run commands. The actual connection retry loop happens in SSHConnection._init_connection _ = self.ssh_iface(0) + def enable_gdb(self): + """Enables GDB debugging""" + self.gdb_socket = "gdb.socket" + self.api.machine_config.patch(gdb_socket_path=self.gdb_socket) + class MicroVMFactory: """MicroVM factory""" diff --git a/tests/integration_tests/build/test_gdb.py b/tests/integration_tests/build/test_gdb.py index 3c9ebbb6d87..872f392f620 100644 --- a/tests/integration_tests/build/test_gdb.py +++ b/tests/integration_tests/build/test_gdb.py @@ -2,15 +2,91 @@ # SPDX-License-Identifier: Apache-2.0 """A test that ensures that firecracker builds with GDB feature enabled at integration time.""" +import os import platform +import signal +import subprocess +import tempfile +from pathlib import Path import host_tools.cargo_build as host +import pytest +from framework.defs import LOCAL_BUILD_PATH +from framework.microvm import MicroVMFactory MACHINE = platform.machine() TARGET = "{}-unknown-linux-musl".format(MACHINE) +BUILD_PATH = LOCAL_BUILD_PATH / "gdb" + + +def build_gdb(): + """Builds Firecracker with GDB feature enabled""" + + host.cargo( + "build", + f"--features gdb --target {TARGET} --all", + env={"CARGO_TARGET_DIR": BUILD_PATH}, + ) def test_gdb_compiles(): """Checks that Firecracker compiles with GDB enabled""" - host.cargo("build", f"--features gdb --target {TARGET}") + build_gdb() + + +@pytest.mark.skipif( + platform.machine() != "x86_64", + reason="GDB requires a vmlinux but we ship a uImage for ARM in our CI", +) +def test_gdb_connects(guest_kernel_linux_6_1, rootfs): + """Checks that GDB works in a FC VM""" + + build_gdb() + + vmfcty = MicroVMFactory(BUILD_PATH / TARGET / "debug") + kernel_dbg = guest_kernel_linux_6_1.parent / "debug" / guest_kernel_linux_6_1.name + uvm = vmfcty.build(kernel_dbg, rootfs) + uvm.spawn(validate_api=False) + uvm.add_net_iface() + uvm.basic_config() + uvm.enable_gdb() + + chroot_gdb_socket = Path(uvm.jailer.chroot_path(), uvm.gdb_socket) + + gdb_commands = f""" + target remote {chroot_gdb_socket} + hbreak start_kernel + # continue to start_kernel + continue + # continue boot until interrupted + continue + """ + + with tempfile.NamedTemporaryFile( + mode="w", suffix=".gdb", delete=False, prefix="fc_gdb_" + ) as f: + f.write(gdb_commands) + gdb_script = f.name + + gdb_proc = subprocess.Popen( + f""" + until [ -S {chroot_gdb_socket} ]; do + echo 'waiting for {chroot_gdb_socket}'; + sleep 1; + done; + gdb {kernel_dbg} -batch -x {gdb_script} + """, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + uvm.start() + os.kill(uvm.firecracker_pid, signal.SIGKILL) + gdb_proc.terminate() + uvm.mark_killed() + stdout, stderr = gdb_proc.communicate(timeout=10) + assert ( + "hit Breakpoint 1, start_kernel" in stdout + ), f"Breakpoint wasn't hit:\nstdout:\n{stdout}\n\nstderr:\n{stderr}" From a9106ae938e51a49f60aed3c665ba5abe74c0a8c Mon Sep 17 00:00:00 2001 From: Riccardo Mancini Date: Fri, 21 Nov 2025 14:22:41 +0000 Subject: [PATCH 2/3] feat(sandbox): add --gdb option to aumatically connect to GDB This adds a sandbox option to setup GDB automatically and open it in a new tmux window. This makes it easier to use GDB with firecracker using common tooling. Signed-off-by: Riccardo Mancini --- tests/framework/microvm_helpers.py | 28 ++++++++++++++++ tools/sandbox.py | 51 ++++++++++++++++++++---------- 2 files changed, 63 insertions(+), 16 deletions(-) diff --git a/tests/framework/microvm_helpers.py b/tests/framework/microvm_helpers.py index f42b63222fb..7c7aa4fd0b1 100644 --- a/tests/framework/microvm_helpers.py +++ b/tests/framework/microvm_helpers.py @@ -7,6 +7,7 @@ import os import platform import subprocess +import tempfile from pathlib import Path @@ -245,3 +246,30 @@ def trace_cmd_guest(self, fns, cmd, port=4321): f"trace-cmd record -N {host_ip}:{port} -p function {' '.join(fns)} {cmd}" ) return list(Path(".").glob("trace.*.dat")) + + def tmux_gdb(self): + """Run GDB on a new tmux window""" + chroot_gdb_socket = Path(self.vm.jailer.chroot_path(), self.vm.gdb_socket) + + with tempfile.NamedTemporaryFile( + mode="w", suffix=".gdb", delete=False, prefix="fc_gdb_" + ) as f: + f.write( + f""" + target remote {chroot_gdb_socket} + directory resources/linux + hbreak start_kernel + continue + """ + ) + gdb_script = f.name + + self.tmux_neww( + f""" + until [ -S {chroot_gdb_socket} ]; do + echo 'waiting for {chroot_gdb_socket}'; + sleep 1; + done; + gdb {self.vm.kernel_file} -x {gdb_script} + """ + ) diff --git a/tools/sandbox.py b/tools/sandbox.py index 8cdf3277352..ad61ea7057c 100755 --- a/tools/sandbox.py +++ b/tools/sandbox.py @@ -10,11 +10,13 @@ import argparse import json +import platform import re from pathlib import Path +import host_tools.cargo_build as build_tools from framework.artifacts import disks, kernels -from framework.defs import DEFAULT_BINARY_DIR +from framework.defs import DEFAULT_BINARY_DIR, LOCAL_BUILD_PATH from framework.microvm import MicroVMFactory kernels = list(kernels("vmlinux-*")) @@ -58,12 +60,31 @@ def parse_byte_size(param): parser.add_argument("--rootfs-size", type=parse_byte_size, default=1 * 2**30) # 1GB parser.add_argument("--binary-dir", help="Path to the firecracker binaries") parser.add_argument("--cpu-template-path", help="CPU template to use", type=Path) +parser.add_argument( + "--debug", action="store_true", default=False, help="Use debug kernel" +) +parser.add_argument( + "--gdb", action="store_true", default=False, help="Connect to Firecracker guest GDB" +) args = parser.parse_args() print(args) binary_dir = None if args.binary_dir: binary_dir = Path(args.binary_dir).resolve() +elif args.gdb: + # Build Firecracker with GDB feature if needed + print("Building Firecracker with GDB feature...") + machine = platform.machine() + target = f"{machine}-unknown-linux-musl" + build_dir = LOCAL_BUILD_PATH / "gdb" + build_tools.cargo( + "build", + f"--features gdb --target {target} --all", + env={"CARGO_TARGET_DIR": build_dir}, + ) + print("Build complete!") + binary_dir = build_dir / target / "debug" else: binary_dir = DEFAULT_BINARY_DIR @@ -72,28 +93,26 @@ def parse_byte_size(param): cpu_template = json.loads(args.cpu_template_path.read_text()) vmfcty = MicroVMFactory(binary_dir) -print(f"uvm with kernel {args.kernel} ...") -uvm = vmfcty.build(args.kernel, args.rootfs) +if args.debug or args.gdb: + kernel = args.kernel.parent / "debug" / args.kernel.name +else: + kernel = args.kernel + +print(f"uvm with kernel {kernel} ...") +uvm = vmfcty.build(kernel, args.rootfs) uvm.help.enable_console() uvm.help.resize_disk(uvm.rootfs_file, args.rootfs_size) -uvm.spawn(log_show_level=True) +uvm.spawn(log_show_level=True, validate_api=False) uvm.help.print_log() uvm.add_net_iface() uvm.basic_config(vcpu_count=args.vcpus, mem_size_mib=args.guest_mem_size // 2**20) if cpu_template is not None: uvm.api.cpu_config.put(**cpu_template) print(cpu_template) + +if args.gdb: + uvm.enable_gdb() + uvm.help.tmux_gdb() + uvm.start() uvm.get_all_metrics() - -kernel_dbg_dir = args.kernel.parent / "debug" -kernel_dbg = kernel_dbg_dir / args.kernel.name -print(f"uvm2 with kernel {kernel_dbg} ...") -uvm2 = vmfcty.build(kernel_dbg, args.rootfs) -uvm2.spawn() -uvm2.add_net_iface() -uvm2.basic_config(vcpu_count=args.vcpus, mem_size_mib=args.guest_mem_size // 2**20) -uvm2.start() -# trace-cmd needs this (DNS resolution?) -uvm2.help.enable_ip_forwarding() -files = uvm2.help.trace_cmd_guest(["-l", "read_msr"], cmd="sleep 5") From 04ed0a1f2c08c676ac93da756dea3cb572510f19 Mon Sep 17 00:00:00 2001 From: Riccardo Mancini Date: Fri, 21 Nov 2025 15:41:20 +0000 Subject: [PATCH 3/3] chore(devtool): fix black command in cmd_fmt The cmd_fmt wouldn't format like the CI test would like it. This commit fixes the issue. Signed-off-by: Riccardo Mancini --- tools/devtool | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/devtool b/tools/devtool index 5bac70d0310..af0afe352af 100755 --- a/tools/devtool +++ b/tools/devtool @@ -944,7 +944,7 @@ cmd_test_debug() { cmd_fmt() { cmd_sh "cargo fmt --all -- --config $(tr '\n' ','