[guest,host/leaked_outb] Fixed inprocess execution for Windows

- Changed the ABI for the inprocess outb handler from `extern "C"` to
`extern sysv64`. This was needed because `extern "C"` caused issues on Windows.
Previously, we used `extern "win64"`, which was problematic for guests
intending to use `hyperlight_common` and not supporting the Windows ABI.
- Added a test for running inprocess on Windows w/ load library.

Signed-off-by: danbugs <[email protected]>

Author: danbugs

src/hyperlight_common/src/lib.rs

@@ -56,9 +56,9 @@ pub static mut RUNNING_MODE: peb::RunMode = peb::RunMode::None;
 /// instruction. Instead, we use a function pointer to call an `outb_handler` function.
 /// For in-process mode, we can't call the `outb` instruction directly because it is a privileged
 /// instruction. Instead, we use a function pointer to call an `outb_handler` function.
-pub static mut OUTB_HANDLER: Option<extern "C" fn(u16, u8)> = None;
+pub static mut OUTB_HANDLER: Option<extern "sysv64" fn(u16, u8)> = None;
 
-pub static mut OUTB_HANDLER_CTX: Option<extern "C" fn(*mut core::ffi::c_void, u16, u8)> = None;
+pub static mut OUTB_HANDLER_CTX: Option<extern "sysv64" fn(*mut core::ffi::c_void, u16, u8)> = None;
 
 /// Hyperlight operates with a host-guest execution model.
 ///

src/hyperlight_guest/src/entrypoint.rs

@@ -123,7 +123,7 @@ pub extern "win64" fn entrypoint(peb_address: u64, seed: u64, max_log_level: u64
             }
             RunMode::InProcessLinux | RunMode::InProcessWindows => {
                 OUTB_HANDLER = {
-                    let outb_handler: extern "C" fn(u16, u8) =
+                    let outb_handler: extern "sysv64" fn(u16, u8) =
                         core::mem::transmute((*PEB).get_outb_ptr());
                     Some(outb_handler)
                 };
@@ -133,7 +133,7 @@ pub extern "win64" fn entrypoint(peb_address: u64, seed: u64, max_log_level: u64
                 }
 
                 OUTB_HANDLER_CTX = {
-                    let outb_handler_ctx: extern "C" fn(*mut core::ffi::c_void, u16, u8) =
+                    let outb_handler_ctx: extern "sysv64" fn(*mut core::ffi::c_void, u16, u8) =
                         core::mem::transmute((*PEB).get_outb_ptr());
                     Some(outb_handler_ctx)
                 };

src/hyperlight_host/src/sandbox/leaked_outb.rs

@@ -29,8 +29,11 @@ use crate::mem::shared_mem::GuestSharedMemory;
 ///
 /// NOTE: This is not part of the C Hyperlight API , it is intended only to be
 /// called in proc through a pointer passed to the guest.
-extern "C" fn call_outb(ptr: *mut Arc<Mutex<dyn OutBHandlerCaller>>, port: u16, data: u64) {
-    let outb_handlercaller = unsafe { Box::from_raw(ptr) };
+extern "sysv64" fn call_outb(ptr: *mut core::ffi::c_void, port: u16, data: u64) {
+    // Convert from *mut core::ffi::c_void to *mut Arc<Mutex<dyn OutBHandlerCaller>>
+    let outb_handlercaller =
+        unsafe { Box::from_raw(ptr as *mut Arc<Mutex<dyn OutBHandlerCaller>>) };
+
     let res = outb_handlercaller
         .try_lock()
         .map_err(|_| crate::new_error!("Error locking"))

src/hyperlight_host/src/sandbox/sandbox_builder.rs

@@ -888,7 +888,7 @@ mod tests {
     use hyperlight_testing::simple_guest_as_string;
 
     use super::*;
-    use crate::func::{HostFunction0, HostFunction2};
+    use crate::func::HostFunction2;
     use crate::sandbox_state::sandbox::EvolvableSandbox;
     use crate::sandbox_state::transition::Noop;
     use crate::HyperlightError;
@@ -954,6 +954,39 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    #[cfg(target_os = "windows")]
+    fn test_sandbox_builder_in_process_load_library() -> Result<()> {
+        use hyperlight_testing::simple_guest_exe_as_string;
+
+        // Tests building an uninitialized sandbox w/ the sandbox builder
+        let sandbox_builder =
+            SandboxBuilder::new(GuestBinary::FilePath(simple_guest_exe_as_string()?))?
+                .set_sandbox_run_options(SandboxRunOptions::RunInProcess(true))?;
+
+        let mut uninitialized_sandbox = sandbox_builder.build()?;
+
+        // Tests registering a host function
+        fn add(a: i32, b: i32) -> Result<i32> {
+            Ok(a + b)
+        }
+        let host_function = Arc::new(Mutex::new(add));
+        host_function.register(&mut uninitialized_sandbox, "HostAdd")?;
+
+        // Tests evolving to a multi-use sandbox
+        let mut multi_use_sandbox = uninitialized_sandbox.evolve(Noop::default())?;
+
+        let result = multi_use_sandbox.call_guest_function_by_name(
+            "Add",
+            ReturnType::Int,
+            Some(vec![ParameterValue::Int(1), ParameterValue::Int(41)]),
+        )?;
+
+        assert_eq!(result, ReturnValue::Int(42));
+
+        Ok(())
+    }
+
     #[test]
     #[cfg(crashdump)]
     fn test_sandbox_builder_crashdump() -> Result<()> {
@@ -1084,6 +1117,8 @@ mod tests {
     #[ignore]
     #[cfg(target_os = "linux")]
     fn test_sandbox_builder_violate_seccomp_filters() -> Result<()> {
+        use crate::func::HostFunction0;
+
         fn make_get_pid_syscall() -> Result<u64> {
             let pid = unsafe { libc::syscall(libc::SYS_getpid) };
             Ok(pid as u64)