[trace-host] handle a TraceBatch from the guest

- Parse the spans and events coming from the guest and
create corresponding spans and events from the host that
mimics a single call from host

Signed-off-by: Doru Blânzeanu <[email protected]>

Author: dblnz

Cargo.lock

@@ -35,9 +35,11 @@ page_size = "0.6.0"
 termcolor = "1.2.0"
 bitflags = "2.9.3"
 log = "0.4.27"
+opentelemetry = { version = "0.30.0", optional = true }
 tracing = { version = "0.1.41", features = ["log"] }
 tracing-log = "0.2.0"
 tracing-core = "0.1.34"
+tracing-opentelemetry = { version = "0.31.0", optional = true }
 hyperlight-common = { workspace = true, default-features = true, features = [ "std" ] }
 hyperlight-guest-tracing = { workspace = true, default-features = true, optional = true }
 vmm-sys-util = "0.15.0"
@@ -98,7 +100,7 @@ tracing = "0.1.41"
 tracing-subscriber = {version = "0.3.19", features = ["std", "env-filter"]}
 tracing-opentelemetry = "0.31.0"
 opentelemetry = "0.30.0"
-opentelemetry-otlp = { version = "0.30.0", default-features = false, features = ["http-proto", "reqwest-blocking-client"] }
+opentelemetry-otlp = { version = "0.30.0", default-features = false, features = ["http-proto", "reqwest-blocking-client", "grpc-tonic"] }
 opentelemetry-semantic-conventions = "0.30"
 opentelemetry_sdk = { version = "0.30.0", features = ["rt-tokio"] }
 tokio = { version = "1.47.1", features = ["full"] }
@@ -132,7 +134,7 @@ executable_heap = []
 print_debug = []
 # Dumps the VM state to a file on unexpected errors or crashes. The path of the file will be printed on stdout and logged.
 crashdump = ["dep:chrono"]
-trace_guest = ["hyperlight-common/trace_guest", "hyperlight-guest-tracing/trace"]
+trace_guest = ["dep:opentelemetry", "dep:tracing-opentelemetry", "dep:hyperlight-guest-tracing", "hyperlight-common/trace_guest"]
 mem_profile = [ "trace_guest", "dep:framehop", "dep:fallible-iterator", "hyperlight-common/mem_profile" ]
 kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"]
 # This feature is deprecated in favor of mshv3

src/hyperlight_host/Cargo.toml

@@ -320,7 +320,6 @@ pub(crate) struct HypervLinuxDriver {
     gdb_conn: Option<DebugCommChannel<DebugResponse, DebugMsg>>,
     #[cfg(crashdump)]
     rt_cfg: SandboxRuntimeConfig,
-    #[allow(dead_code)]
     #[cfg(feature = "trace_guest")]
     trace_info: TraceInfo,
 }
@@ -775,6 +774,9 @@ impl Hypervisor for HypervLinuxDriver {
         {
             Err(mshv_ioctls::MshvError::from(libc::EINTR))
         } else {
+            #[cfg(feature = "trace_guest")]
+            self.trace_info.setup_guest_trace();
+
             // Note: if a `InterruptHandle::kill()` called while this thread is **here**
             // Then the vcpu will run, but we will keep sending signals to this thread
             // to interrupt it until `running` is set to false. The `vcpu_fd::run()` call will
@@ -923,6 +925,21 @@ impl Hypervisor for HypervLinuxDriver {
                 }
             },
         };
+
+        // If trace is enabled, process the trace batch
+        #[cfg(feature = "trace_guest")]
+        match result {
+            HyperlightExit::Halt()
+            | HyperlightExit::IoOut(_, _, _, _)
+            | HyperlightExit::Mmio(_) => {
+                // If the result is not a halt, io out, mmio or debug exit, we need to process the trace batch
+                let regs = self.read_regs()?;
+                let _ = self
+                    .trace_info
+                    .handle_trace_batch(&regs, self.mem_mgr.as_mut().unwrap());
+            }
+            _ => {}
+        }
         Ok(result)
     }
 

src/hyperlight_host/src/hypervisor/hyperv_linux.rs

@@ -293,7 +293,6 @@ pub(crate) struct HypervWindowsDriver {
     #[cfg(crashdump)]
     rt_cfg: SandboxRuntimeConfig,
     #[cfg(feature = "trace_guest")]
-    #[allow(dead_code)]
     trace_info: TraceInfo,
 }
 /* This does not automatically impl Send because the host
@@ -761,6 +760,9 @@ impl Hypervisor for HypervWindowsDriver {
                 Reserved: Default::default(),
             }
         } else {
+            #[cfg(feature = "trace_guest")]
+            self.trace_info.setup_guest_trace();
+
             self.processor.run()?
         };
         self.interrupt_handle
@@ -875,6 +877,20 @@ impl Hypervisor for HypervWindowsDriver {
             }
         };
 
+        // If trace is enabled, process the trace batch
+        #[cfg(feature = "trace_guest")]
+        match result {
+            HyperlightExit::Halt()
+            | HyperlightExit::IoOut(_, _, _, _)
+            | HyperlightExit::Mmio(_) => {
+                // If the result is not a halt, io out, mmio or debug exit, we need to process the trace batch
+                let regs = self.read_regs()?;
+                let _ = self
+                    .trace_info
+                    .handle_trace_batch(&regs, self.mem_mgr.as_mut().unwrap());
+            }
+            _ => {}
+        }
         Ok(result)
     }
 

src/hyperlight_host/src/hypervisor/hyperv_windows.rs

@@ -306,7 +306,6 @@ pub(crate) struct KVMDriver {
     #[cfg(crashdump)]
     rt_cfg: SandboxRuntimeConfig,
     #[cfg(feature = "trace_guest")]
-    #[allow(dead_code)]
     trace_info: TraceInfo,
 }
 
@@ -693,6 +692,9 @@ impl Hypervisor for KVMDriver {
         {
             Err(kvm_ioctls::Error::new(libc::EINTR))
         } else {
+            #[cfg(feature = "trace_guest")]
+            self.trace_info.setup_guest_trace();
+
             // Note: if a `InterruptHandle::kill()` called while this thread is **here**
             // Then the vcpu will run, but we will keep sending signals to this thread
             // to interrupt it until `running` is set to false. The `vcpu_fd::run()` call will
@@ -809,6 +811,21 @@ impl Hypervisor for KVMDriver {
                 HyperlightExit::Unknown(err_msg)
             }
         };
+
+        // If trace is enabled, process the trace batch
+        #[cfg(feature = "trace_guest")]
+        match result {
+            HyperlightExit::Halt()
+            | HyperlightExit::IoOut(_, _, _, _)
+            | HyperlightExit::Mmio(_) => {
+                // If the result is not a halt, io out, mmio or debug exit, we need to process the trace batch
+                let regs = self.read_regs()?;
+                let _ = self
+                    .trace_info
+                    .handle_trace_batch(&regs, self.mem_mgr.as_mut().unwrap());
+            }
+            _ => {}
+        }
         Ok(result)
     }
 

src/hyperlight_host/src/hypervisor/kvm.rs

@@ -233,7 +233,6 @@ pub(crate) trait Hypervisor: Debug + Send {
     fn check_stack_guard(&self) -> Result<bool>;
 
     /// Read a register for trace/unwind purposes
-    #[allow(dead_code)]
     #[cfg(feature = "trace_guest")]
     fn read_regs(&self) -> Result<arch::X86_64Regs>;