Skip to content

Commit 6f2ce51

Browse files
syntacticallydblnz
authored andcommitted
[hyperlight_host] Plumb a trace file descriptor around
This adds (unused) support for creating trace files for sandboxes and passing them around to relevant sandbox event handler code. This will be used for collecting debug trace and profiling information. Signed-off-by: Lucy Menon <[email protected]> Signed-off-by: Doru Blânzeanu <[email protected]>
1 parent ba24f16 commit 6f2ce51

File tree

10 files changed

+157
-21
lines changed

10 files changed

+157
-21
lines changed

Cargo.lock

Lines changed: 10 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

src/hyperlight_host/Cargo.toml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ anyhow = "1.0"
4444
metrics = "0.24.2"
4545
serde_json = "1.0"
4646
elfcore = "2.0"
47+
uuid = { version = "1.17.0", features = ["v4"] }
4748

4849
[target.'cfg(windows)'.dependencies]
4950
windows = { version = "0.61", features = [
@@ -79,6 +80,7 @@ mshv-ioctls3 = { package="mshv-ioctls", version = "=0.3.2", optional = true}
7980
[dev-dependencies]
8081
uuid = { version = "1.17.0", features = ["v4"] }
8182
signal-hook-registry = "1.4.5"
83+
envy = { version = "0.4.2" }
8284
serde = "1.0"
8385
proptest = "1.7.0"
8486
tempfile = "3.20.0"
@@ -126,6 +128,7 @@ executable_heap = []
126128
print_debug = []
127129
# Dumps the VM state to a file on unexpected errors or crashes. The path of the file will be printed on stdout and logged.
128130
crashdump = ["dep:chrono"]
131+
trace_guest = []
129132
kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"]
130133
mshv2 = ["dep:mshv-bindings2", "dep:mshv-ioctls2"]
131134
mshv3 = ["dep:mshv-bindings3", "dep:mshv-ioctls3"]

src/hyperlight_host/src/hypervisor/handlers.rs

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,21 @@ use std::sync::{Arc, Mutex};
1818

1919
use tracing::{Span, instrument};
2020

21+
#[cfg(feature = "trace_guest")]
22+
use super::Hypervisor;
2123
use crate::{Result, new_error};
2224

2325
/// The trait representing custom logic to handle the case when
2426
/// a Hypervisor's virtual CPU (vCPU) informs Hyperlight the guest
2527
/// has initiated an outb operation.
2628
pub(crate) trait OutBHandlerCaller: Sync + Send {
2729
/// Function that gets called when an outb operation has occurred.
28-
fn call(&mut self, port: u16, payload: u32) -> Result<()>;
30+
fn call(
31+
&mut self,
32+
#[cfg(feature = "trace_guest")] hv: &mut dyn Hypervisor,
33+
port: u16,
34+
payload: u32,
35+
) -> Result<()>;
2936
}
3037

3138
/// A convenient type representing a common way `OutBHandler` implementations
@@ -36,6 +43,10 @@ pub(crate) trait OutBHandlerCaller: Sync + Send {
3643
/// a &mut self).
3744
pub(crate) type OutBHandlerWrapper = Arc<Mutex<dyn OutBHandlerCaller>>;
3845

46+
#[cfg(feature = "trace_guest")]
47+
pub(crate) type OutBHandlerFunction =
48+
Box<dyn FnMut(&mut dyn Hypervisor, u16, u32) -> Result<()> + Send>;
49+
#[cfg(not(feature = "trace_guest"))]
3950
pub(crate) type OutBHandlerFunction = Box<dyn FnMut(u16, u32) -> Result<()> + Send>;
4051

4152
/// A `OutBHandler` implementation using a `OutBHandlerFunction`
@@ -52,12 +63,22 @@ impl From<OutBHandlerFunction> for OutBHandler {
5263

5364
impl OutBHandlerCaller for OutBHandler {
5465
#[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
55-
fn call(&mut self, port: u16, payload: u32) -> Result<()> {
66+
fn call(
67+
&mut self,
68+
#[cfg(feature = "trace_guest")] hv: &mut dyn Hypervisor,
69+
port: u16,
70+
payload: u32,
71+
) -> Result<()> {
5672
let mut func = self
5773
.0
5874
.try_lock()
5975
.map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))?;
60-
func(port, payload)
76+
func(
77+
#[cfg(feature = "trace_guest")]
78+
hv,
79+
port,
80+
payload,
81+
)
6182
}
6283
}
6384

src/hyperlight_host/src/hypervisor/hyperv_linux.rs

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@ use crate::HyperlightError;
7272
use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
7373
use crate::mem::ptr::{GuestPtr, RawPtr};
7474
use crate::sandbox::SandboxConfiguration;
75+
#[cfg(feature = "trace_guest")]
76+
use crate::sandbox::TraceInfo;
7577
#[cfg(crashdump)]
7678
use crate::sandbox::uninitialized::SandboxRuntimeConfig;
7779
use crate::{Result, log_then_return, new_error};
@@ -310,6 +312,9 @@ pub(crate) struct HypervLinuxDriver {
310312
gdb_conn: Option<DebugCommChannel<DebugResponse, DebugMsg>>,
311313
#[cfg(crashdump)]
312314
rt_cfg: SandboxRuntimeConfig,
315+
#[cfg(feature = "trace_guest")]
316+
#[allow(dead_code)]
317+
trace_info: TraceInfo,
313318
}
314319

315320
impl HypervLinuxDriver {
@@ -321,6 +326,8 @@ impl HypervLinuxDriver {
321326
/// the underlying virtual CPU after this function returns. Call the
322327
/// `apply_registers` method to do that, or more likely call
323328
/// `initialise` to do it for you.
329+
#[allow(clippy::too_many_arguments)]
330+
// TODO: refactor this function to take fewer arguments. Add trace_info to rt_cfg
324331
#[instrument(skip_all, parent = Span::current(), level = "Trace")]
325332
pub(crate) fn new(
326333
mem_regions: Vec<MemoryRegion>,
@@ -330,6 +337,7 @@ impl HypervLinuxDriver {
330337
config: &SandboxConfiguration,
331338
#[cfg(gdb)] gdb_conn: Option<DebugCommChannel<DebugResponse, DebugMsg>>,
332339
#[cfg(crashdump)] rt_cfg: SandboxRuntimeConfig,
340+
#[cfg(feature = "trace_guest")] trace_info: TraceInfo,
333341
) -> Result<Self> {
334342
let mshv = Mshv::new()?;
335343
let pr = Default::default();
@@ -436,6 +444,8 @@ impl HypervLinuxDriver {
436444
gdb_conn,
437445
#[cfg(crashdump)]
438446
rt_cfg,
447+
#[cfg(feature = "trace_guest")]
448+
trace_info,
439449
};
440450

441451
// Send the interrupt handle to the GDB thread if debugging is enabled
@@ -611,7 +621,12 @@ impl Hypervisor for HypervLinuxDriver {
611621
outb_handle_fn
612622
.try_lock()
613623
.map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))?
614-
.call(port, val)?;
624+
.call(
625+
#[cfg(feature = "trace_guest")]
626+
self,
627+
port,
628+
val,
629+
)?;
615630

616631
// update rip
617632
self.vcpu_fd.set_reg(&[hv_register_assoc {
@@ -1105,6 +1120,8 @@ mod tests {
11051120
#[cfg(crashdump)]
11061121
guest_core_dump: true,
11071122
},
1123+
#[cfg(feature = "trace_guest")]
1124+
TraceInfo::new().unwrap(),
11081125
)
11091126
.unwrap();
11101127
}

src/hyperlight_host/src/hypervisor/hyperv_windows.rs

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,9 +57,11 @@ use crate::hypervisor::fpu::FP_CONTROL_WORD_DEFAULT;
5757
use crate::hypervisor::wrappers::WHvGeneralRegisters;
5858
use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
5959
use crate::mem::ptr::{GuestPtr, RawPtr};
60+
#[cfg(feature = "trace_guest")]
61+
use crate::sandbox::TraceInfo;
6062
#[cfg(crashdump)]
6163
use crate::sandbox::uninitialized::SandboxRuntimeConfig;
62-
use crate::{Result, debug, new_error};
64+
use crate::{Result, debug, log_then_return, new_error};
6365

6466
#[cfg(gdb)]
6567
mod debug {
@@ -280,6 +282,9 @@ pub(crate) struct HypervWindowsDriver {
280282
gdb_conn: Option<DebugCommChannel<DebugResponse, DebugMsg>>,
281283
#[cfg(crashdump)]
282284
rt_cfg: SandboxRuntimeConfig,
285+
#[cfg(feature = "trace_guest")]
286+
#[allow(dead_code)]
287+
trace_info: TraceInfo,
283288
}
284289
/* This does not automatically impl Send/Sync because the host
285290
* address of the shared memory region is a raw pointer, which are
@@ -291,6 +296,7 @@ unsafe impl Sync for HypervWindowsDriver {}
291296

292297
impl HypervWindowsDriver {
293298
#[allow(clippy::too_many_arguments)]
299+
// TODO: refactor this function to take fewer arguments. Add trace_info to rt_cfg
294300
#[instrument(err(Debug), skip_all, parent = Span::current(), level = "Trace")]
295301
pub(crate) fn new(
296302
mem_regions: Vec<MemoryRegion>,
@@ -301,6 +307,7 @@ impl HypervWindowsDriver {
301307
mmap_file_handle: HandleWrapper,
302308
#[cfg(gdb)] gdb_conn: Option<DebugCommChannel<DebugResponse, DebugMsg>>,
303309
#[cfg(crashdump)] rt_cfg: SandboxRuntimeConfig,
310+
#[cfg(feature = "trace_guest")] trace_info: TraceInfo,
304311
) -> Result<Self> {
305312
// create and setup hypervisor partition
306313
let mut partition = VMPartition::new(1)?;
@@ -351,6 +358,8 @@ impl HypervWindowsDriver {
351358
gdb_conn,
352359
#[cfg(crashdump)]
353360
rt_cfg,
361+
#[cfg(feature = "trace_guest")]
362+
trace_info,
354363
};
355364

356365
// Send the interrupt handle to the GDB thread if debugging is enabled
@@ -659,7 +668,12 @@ impl Hypervisor for HypervWindowsDriver {
659668
outb_handle_fn
660669
.try_lock()
661670
.map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))?
662-
.call(port, val)?;
671+
.call(
672+
#[cfg(feature = "trace_guest")]
673+
self,
674+
port,
675+
val,
676+
)?;
663677

664678
let mut regs = self.processor.get_regs()?;
665679
regs.rip = rip + instruction_length;

src/hyperlight_host/src/hypervisor/kvm.rs

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ use crate::HyperlightError;
4646
use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
4747
use crate::mem::ptr::{GuestPtr, RawPtr};
4848
use crate::sandbox::SandboxConfiguration;
49+
#[cfg(feature = "trace_guest")]
50+
use crate::sandbox::TraceInfo;
4951
#[cfg(crashdump)]
5052
use crate::sandbox::uninitialized::SandboxRuntimeConfig;
5153
use crate::{Result, log_then_return, new_error};
@@ -297,12 +299,17 @@ pub(crate) struct KVMDriver {
297299
gdb_conn: Option<DebugCommChannel<DebugResponse, DebugMsg>>,
298300
#[cfg(crashdump)]
299301
rt_cfg: SandboxRuntimeConfig,
302+
#[cfg(feature = "trace_guest")]
303+
#[allow(dead_code)]
304+
trace_info: TraceInfo,
300305
}
301306

302307
impl KVMDriver {
303308
/// Create a new instance of a `KVMDriver`, with only control registers
304309
/// set. Standard registers will not be set, and `initialise` must
305310
/// be called to do so.
311+
#[allow(clippy::too_many_arguments)]
312+
// TODO: refactor this function to take fewer arguments. Add trace_info to rt_cfg
306313
#[instrument(err(Debug), skip_all, parent = Span::current(), level = "Trace")]
307314
pub(crate) fn new(
308315
mem_regions: Vec<MemoryRegion>,
@@ -312,6 +319,7 @@ impl KVMDriver {
312319
config: &SandboxConfiguration,
313320
#[cfg(gdb)] gdb_conn: Option<DebugCommChannel<DebugResponse, DebugMsg>>,
314321
#[cfg(crashdump)] rt_cfg: SandboxRuntimeConfig,
322+
#[cfg(feature = "trace_guest")] trace_info: TraceInfo,
315323
) -> Result<Self> {
316324
let kvm = Kvm::new()?;
317325

@@ -390,6 +398,8 @@ impl KVMDriver {
390398
gdb_conn,
391399
#[cfg(crashdump)]
392400
rt_cfg,
401+
#[cfg(feature = "trace_guest")]
402+
trace_info,
393403
};
394404

395405
// Send the interrupt handle to the GDB thread if debugging is enabled
@@ -555,7 +565,12 @@ impl Hypervisor for KVMDriver {
555565
outb_handle_fn
556566
.try_lock()
557567
.map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))?
558-
.call(port, value)?;
568+
.call(
569+
#[cfg(feature = "trace_guest")]
570+
self,
571+
port,
572+
value,
573+
)?;
559574
}
560575

561576
Ok(())

src/hyperlight_host/src/hypervisor/mod.rs

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -494,10 +494,12 @@ pub(crate) mod tests {
494494

495495
use hyperlight_testing::dummy_guest_as_string;
496496

497-
use super::handlers::{MemAccessHandler, OutBHandler};
497+
use super::handlers::{MemAccessHandler, OutBHandler, OutBHandlerFunction};
498498
#[cfg(gdb)]
499499
use crate::hypervisor::DbgMemAccessHandlerCaller;
500500
use crate::mem::ptr::RawPtr;
501+
#[cfg(feature = "trace_guest")]
502+
use crate::sandbox::TraceInfo;
501503
use crate::sandbox::uninitialized::GuestBinary;
502504
#[cfg(any(crashdump, gdb))]
503505
use crate::sandbox::uninitialized::SandboxRuntimeConfig;
@@ -529,11 +531,6 @@ pub(crate) mod tests {
529531
return Ok(());
530532
}
531533

532-
let outb_handler: Arc<Mutex<OutBHandler>> = {
533-
let func: Box<dyn FnMut(u16, u32) -> Result<()> + Send> =
534-
Box::new(|_, _| -> Result<()> { Ok(()) });
535-
Arc::new(Mutex::new(OutBHandler::from(func)))
536-
};
537534
let mem_access_handler = {
538535
let func: Box<dyn FnMut() -> Result<()> + Send> = Box::new(|| -> Result<()> { Ok(()) });
539536
Arc::new(Mutex::new(MemAccessHandler::from(func)))
@@ -554,7 +551,17 @@ pub(crate) mod tests {
554551
&config,
555552
#[cfg(any(crashdump, gdb))]
556553
&rt_cfg,
554+
#[cfg(feature = "trace_guest")]
555+
TraceInfo::new()?,
557556
)?;
557+
let outb_handler: Arc<Mutex<OutBHandler>> = {
558+
#[cfg(feature = "trace_guest")]
559+
#[allow(clippy::type_complexity)]
560+
let func: OutBHandlerFunction = Box::new(|_, _, _| -> Result<()> { Ok(()) });
561+
#[cfg(not(feature = "trace_guest"))]
562+
let func: OutBHandlerFunction = Box::new(|_, _| -> Result<()> { Ok(()) });
563+
Arc::new(Mutex::new(OutBHandler::from(func)))
564+
};
558565
vm.initialise(
559566
RawPtr::from(0x230000),
560567
1234567890,

src/hyperlight_host/src/sandbox/mod.rs

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,9 @@ pub(crate) mod uninitialized_evolve;
4040
/// Trait used by the macros to paper over the differences between hyperlight and hyperlight-wasm
4141
mod callable;
4242

43+
#[cfg(feature = "trace_guest")]
44+
use std::sync::{Arc, Mutex};
45+
4346
/// Trait used by the macros to paper over the differences between hyperlight and hyperlight-wasm
4447
pub use callable::Callable;
4548
/// Re-export for `SandboxConfiguration` type
@@ -86,6 +89,35 @@ pub fn is_hypervisor_present() -> bool {
8689
hypervisor::get_available_hypervisor().is_some()
8790
}
8891

92+
#[cfg(feature = "trace_guest")]
93+
#[derive(Clone)]
94+
/// The information that trace collection requires in order to write
95+
/// an accurate trace.
96+
pub(crate) struct TraceInfo {
97+
/// The epoch against which trace events are timed; at least as
98+
/// early as the creation of the sandbox being traced.
99+
#[allow(dead_code)]
100+
pub epoch: std::time::Instant,
101+
/// The file to which the trace is being written
102+
#[allow(dead_code)]
103+
pub file: Arc<Mutex<std::fs::File>>,
104+
}
105+
#[cfg(feature = "trace_guest")]
106+
impl TraceInfo {
107+
/// Create a new TraceInfo by saving the current time as the epoch
108+
/// and generating a random filename.
109+
pub fn new() -> crate::Result<Self> {
110+
let mut path = std::env::current_dir()?;
111+
path.push("trace");
112+
path.push(uuid::Uuid::new_v4().to_string());
113+
path.set_extension("trace");
114+
Ok(Self {
115+
epoch: std::time::Instant::now(),
116+
file: Arc::new(Mutex::new(std::fs::File::create_new(path)?)),
117+
})
118+
}
119+
}
120+
89121
pub(crate) trait WrapperGetter {
90122
#[allow(dead_code)]
91123
fn get_mgr_wrapper(&self) -> &MemMgrWrapper<HostSharedMemory>;

0 commit comments

Comments
 (0)