Merge branch 'main' into devigned-patch-1

Author: marosset

.github/workflows/CreateRelease.yml

@@ -83,7 +83,6 @@ jobs:
       [
         build-rust-ubuntu,
         build-rust-windows,
-        build-guest-binaries,
         benchmarks,
       ]
 
@@ -117,20 +116,6 @@ jobs:
           echo "HYPERLIGHT_VERSION=$version" >> $GITHUB_ENV
           echo "HYPERLIGHT_VERSION=$version"
 
-      - name: Download Guest Binaries
-        uses: actions/download-artifact@v4
-        with:
-          name: guest-binaries-${{ env.CONFIG }}
-          path: ./downloaded-guest-binaries-${{ env.CONFIG }}
-
-      - name: Copy Guest Binaries
-        run: |
-          cp ./downloaded-guest-binaries-${{ env.CONFIG }}/callbackguest ./src/tests/rust_guests/bin/${{ env.CONFIG }}/callbackguest
-          cp ./downloaded-guest-binaries-${{ env.CONFIG }}/callbackguest.exe ./src/tests/rust_guests/bin/${{ env.CONFIG }}/callbackguest.exe
-          cp ./downloaded-guest-binaries-${{ env.CONFIG }}/simpleguest ./src/tests/rust_guests/bin/${{ env.CONFIG }}/simpleguest
-          cp ./downloaded-guest-binaries-${{ env.CONFIG }}/simpleguest.exe ./src/tests/rust_guests/bin/${{ env.CONFIG }}/simpleguest.exe
-          cp ./downloaded-guest-binaries-${{ env.CONFIG }}/dummyguest ./src/tests/rust_guests/bin/${{ env.CONFIG }}/dummyguest
-
       - name: Build and archive guest library + header files
         run: |
           just tar-headers
@@ -159,20 +144,19 @@ jobs:
       - name: Extract release notes from changelog
         if: ${{ contains(github.ref, 'refs/heads/release/') }}
         run: just create-release-notes v${{ env.HYPERLIGHT_VERSION }} > RELEASE_NOTES.md
+        env:
+          GH_TOKEN: ${{ github.token }}
 
       - name: Extract prerelease notes from changelog
         if: ${{ github.ref=='refs/heads/main' }}
         run: just create-release-notes dev-latest > RELEASE_NOTES.md
+        env:
+          GH_TOKEN: ${{ github.token }}
 
       - name: Create release
         if: ${{ contains(github.ref, 'refs/heads/release/') }}
         run: |
             gh release create v${{ env.HYPERLIGHT_VERSION }} -t "Release v${{ env.HYPERLIGHT_VERSION }}" --notes-file RELEASE_NOTES.md \
-            src/tests/rust_guests/bin/${{ env.CONFIG }}/callbackguest \
-            src/tests/rust_guests/bin/${{ env.CONFIG }}/callbackguest.exe \
-            src/tests/rust_guests/bin/${{ env.CONFIG }}/simpleguest \
-            src/tests/rust_guests/bin/${{ env.CONFIG }}/simpleguest.exe \
-            src/tests/rust_guests/bin/${{ env.CONFIG }}/dummyguest \
             benchmarks_Windows_hyperv_amd.tar.gz \
             benchmarks_Windows_hyperv_intel.tar.gz \
             benchmarks_Linux_kvm_amd.tar.gz \
@@ -192,11 +176,6 @@ jobs:
         run: |
             gh release delete dev-latest -y --cleanup-tag || true
             gh release create dev-latest -t "Latest prerelease from main branch" --notes-file RELEASE_NOTES.md --latest=false -p \
-            src/tests/rust_guests/bin/${{ env.CONFIG }}/callbackguest \
-            src/tests/rust_guests/bin/${{ env.CONFIG }}/callbackguest.exe \
-            src/tests/rust_guests/bin/${{ env.CONFIG }}/simpleguest \
-            src/tests/rust_guests/bin/${{ env.CONFIG }}/simpleguest.exe \
-            src/tests/rust_guests/bin/${{ env.CONFIG }}/dummyguest \
             benchmarks_Windows_hyperv_amd.tar.gz \
             benchmarks_Windows_hyperv_intel.tar.gz \
             benchmarks_Linux_kvm_amd.tar.gz \

.github/workflows/ValidatePullRequest.yml

@@ -65,7 +65,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Spell Check Repo
-      uses: crate-ci/[email protected].1
+      uses: crate-ci/[email protected].2
 
   # Gate PR merges on this specific "join-job" which requires all other
   # jobs to run first. We need this job since we cannot gate on particular jobs

.gitignore

@@ -468,3 +468,6 @@ hyperlight_guest.h
 .mono
 
 !.gitkeep
+
+# gdb
+.gdbinit

CHANGELOG.md

@@ -5,7 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 ## [Prerelease] - Unreleased
 
 
-## [v0.4.0] - Unreleased
+## [v0.4.0] - 2025-04-30
 
 ### Changed
 - Metrics are now emitted using the [metrics](https://crates.io/crates/metrics) crate by @ludfjig in [#361](https://github.com/hyperlight-dev/hyperlight/pull/361)

Cargo.lock

@@ -21,7 +21,7 @@ exclude = [
 ]
 
 [workspace.package]
-version = "0.3.0"
+version = "0.4.0"
 edition = "2021"
 rust-version = "1.81.0"
 license = "Apache-2.0"
@@ -30,9 +30,9 @@ repository = "https://github.com/hyperlight-dev/hyperlight"
 readme = "README.md"
 
 [workspace.dependencies]
-hyperlight-common = { path = "src/hyperlight_common", version = "0.3.0", default-features = false }
-hyperlight-host = { path = "src/hyperlight_host", version = "0.3.0", default-features = false }
-hyperlight-guest = { path = "src/hyperlight_guest", version = "0.3.0", default-features = false }
+hyperlight-common = { path = "src/hyperlight_common", version = "0.4.0", default-features = false }
+hyperlight-host = { path = "src/hyperlight_host", version = "0.4.0", default-features = false }
+hyperlight-guest = { path = "src/hyperlight_guest", version = "0.4.0", default-features = false }
 hyperlight-testing = { path = "src/hyperlight_testing", default-features = false }
 
 [workspace.lints.rust]

Cargo.toml

@@ -62,6 +62,22 @@ clean-rust:
 
 # Note: most testing recipes take an optional "features" comma separated list argument. If provided, these will be passed to cargo as **THE ONLY FEATURES**, i.e. default features will be disabled.
 
+# convenience recipe to run all tests with the given target and features (similar to CI)
+test-like-ci config=default-target hypervisor="kvm":
+    @# with default features
+    just test {{config}} {{ if hypervisor == "mshv3" {"mshv3"} else {""} }}
+
+    @# with only one driver enabled + seccomp + inprocess
+    just test {{config}} inprocess,seccomp,{{ if hypervisor == "mshv" {"mshv2"} else if hypervisor == "mshv3" {"mshv3"} else {"kvm"} }}
+
+    @# make sure certain cargo features compile
+    cargo check -p hyperlight-host --features crashdump
+    cargo check -p hyperlight-host --features print_debug
+    cargo check -p hyperlight-host --features gdb
+
+    @# without any driver (should fail to compile)
+    just test-compilation-fail {{config}}
+
 # runs all tests
 test target=default-target features="": (test-unit target features) (test-isolated target features) (test-integration "rust" target features) (test-integration "c" target features) (test-seccomp target features)
 

Justfile

@@ -9,35 +9,35 @@ Hyperlight uses paging, which means the all addresses inside a Hyperlight VM are
 
 ## Host-to-Guest memory mapping
 
-Into each Hyperlight VM, memory from the host is mapped into the VM as physical memory. The physical memory inside the VM starts at address `0x200_000` and extends linearly to however much memory was mapped into the VM (depends on various parameters).
+Into each Hyperlight VM, memory from the host is mapped into the VM as physical memory. The physical memory inside the VM starts at address `0x0` and extends linearly to however much memory was mapped into the VM (depends on various parameters).
 
 ## Page table setup
 
 The following page table structs are set up in memory before running a Hyperlight VM (See [Access Flags](#access-flags) for details on access flags that are also set on each entry)
 
 ### PML4 (Page Map Level 4) Table
 
-The PML4 table is located at physical address specified in CR3. In Hyperlight we set `CR3=0x200_000`, which means the PML4 table is located at physical address `0x200_000`. The PML4 table comprises 512 64-bit entries.
+The PML4 table is located at physical address specified in CR3. In Hyperlight we set `CR3=0x0`, which means the PML4 table is located at physical address `0x0`. The PML4 table comprises 512 64-bit entries.
 
-In Hyperlight, we only initialize the first entry (at address `0x200_000`), with value `0x201_000`, implying that we only have a single PDPT.
+In Hyperlight, we only initialize the first entry (at address `0x0`), with value `0x1_000`, implying that we only have a single PDPT.
 
 ### PDPT (Page-directory-pointer Table)
 
-The first and only PDPT is located at physical address `0x201_000`. The PDPT comprises 512 64-bit entries. In Hyperlight, we only initialize the first entry of the PDPT (at address `0x201_000`), with the value `0x202_000`, implying that we only have a single PD.
+The first and only PDPT is located at physical address `0x1_000`. The PDPT comprises 512 64-bit entries. In Hyperlight, we only initialize the first entry of the PDPT (at address `0x1_000`), with the value `0x2_000`, implying that we only have a single PD.
 
 ### PD (Page Directory)
 
-The first and only PD is located at physical address `0x202_000`. The PD comprises 512 64-bit entries, each entry `i` is set to the value `(i * 0x1000) + 0x203_000`. Thus, the first entry is `0x203_000`, the second entry is `0x204_000` and so on.
+The first and only PD is located at physical address `0x2_000`. The PD comprises 512 64-bit entries, each entry `i` is set to the value `(i * 0x1000) + 0x3_000`. Thus, the first entry is `0x3_000`, the second entry is `0x4_000` and so on.
 
 ### PT (Page Table)
 
-The page tables start at physical address `0x203_000`. Each page table has 512 64-bit entries. Each entry is set to the value `p << 21|i << 12` where `p` is the page table number and `i` is the index of the entry in the page table. Thus, the first entry of the first page table is `0x000_000`, the second entry is `0x000_000 + 0x1000`, and so on. The first entry of the second page table is `0x200_000 + 0x1000`, the second entry is `0x200_000 + 0x2000`, and so on. Enough page tables are created to cover the size of memory mapped into the VM.
+The page tables start at physical address `0x3_000`. Each page table has 512 64-bit entries. Each entry is set to the value `p << 21|i << 12` where `p` is the page table number and `i` is the index of the entry in the page table. Thus, the first entry of the first page table is `0x000_000`, the second entry is `0x000_000 + 0x1000`, and so on. The first entry of the second page table is `0x200_000 + 0x1000`, the second entry is `0x200_000 + 0x2000`, and so on. Enough page tables are created to cover the size of memory mapped into the VM.
 
 ## Address Translation
 
 Given a 64-bit virtual address X, the corresponding physical address is obtained as follows:
 
-1. PML4 table's physical address is located using CR3 (CR3 is `0x200_000`).
+1. PML4 table's physical address is located using CR3 (CR3 is `0x0`).
 2. Bits 47:39 of X are used to index into PML4, giving us the address of the PDPT.
 3. Bits 38:30 of X are used to index into PDPT, giving us the address of the PD.
 4. Bits 29:21 of X are used to index into PD, giving us the address of the PT.
@@ -63,7 +63,7 @@ In addition to providing addresses, page table entries also contain access flags
 
 PML4E, PDPTE, and PD Entries have the present flag set to 1, and the rest of the flags are not set.
 
-PTE Entries all have the present flag set to 1, apart from those for the address range `0x000_000` to `0x1FF_000` which have the present flag set to 0 as we do not map memory below physical address `0x200_000`. 
+PTE Entries all have the present flag set to 1.
 
 In addition, the following flags are set according to the type of memory being mapped:
 

docs/paging-development-notes.md

@@ -116,7 +116,7 @@ proc-maps = "0.4.0"
 [build-dependencies]
 anyhow = { version = "1.0.98" }
 cfg_aliases = "0.2.1"
-built = { version = "0.7.7", features = ["chrono", "git2"] }
+built = { version = "0.8.0", features = ["chrono", "git2"] }
 
 [features]
 default = ["kvm", "mshv2", "seccomp"]

src/hyperlight_host/Cargo.toml

@@ -15,11 +15,12 @@ limitations under the License.
 */
 
 use std::sync::{Arc, Mutex};
+use std::time::Duration;
 
 use criterion::{criterion_group, criterion_main, Criterion};
 use hyperlight_common::flatbuffer_wrappers::function_types::{ParameterValue, ReturnType};
 use hyperlight_host::func::HostFunction2;
-use hyperlight_host::sandbox::{MultiUseSandbox, UninitializedSandbox};
+use hyperlight_host::sandbox::{MultiUseSandbox, SandboxConfiguration, UninitializedSandbox};
 use hyperlight_host::sandbox_state::sandbox::EvolvableSandbox;
 use hyperlight_host::sandbox_state::transition::Noop;
 use hyperlight_host::GuestBinary;
@@ -69,6 +70,40 @@ fn guest_call_benchmark(c: &mut Criterion) {
         });
     });
 
+    // This benchmark includes time to first clone a vector and string, so it is not a "pure' benchmark of the guest call, but it's still useful
+    group.bench_function("guest_call_with_large_parameters", |b| {
+        const SIZE: usize = 50 * 1024 * 1024; // 50 MB
+        let large_vec = vec![0u8; SIZE];
+        let large_string = unsafe { String::from_utf8_unchecked(large_vec.clone()) }; // Safety: indeed above vec is valid utf8
+
+        let mut config = SandboxConfiguration::default();
+        config.set_input_data_size(2 * SIZE + (1024 * 1024)); // 2 * SIZE + 1 MB, to allow 1MB for the rest of the serialized function call
+        config.set_heap_size(SIZE as u64 * 15);
+        config.set_max_execution_time(Duration::from_secs(10));
+
+        let sandbox = UninitializedSandbox::new(
+            GuestBinary::FilePath(simple_guest_as_string().unwrap()),
+            Some(config),
+            None,
+            None,
+        )
+        .unwrap();
+        let mut sandbox = sandbox.evolve(Noop::default()).unwrap();
+
+        b.iter(|| {
+            sandbox
+                .call_guest_function_by_name(
+                    "LargeParameters",
+                    ReturnType::Void,
+                    Some(vec![
+                        ParameterValue::VecBytes(large_vec.clone()),
+                        ParameterValue::String(large_string.clone()),
+                    ]),
+                )
+                .unwrap()
+        });
+    });
+
     // Benchmarks a guest function call calling into the host.
     // The benchmark does **not** include the time to reset the sandbox memory after the call.
     group.bench_function("guest_call_with_call_to_host_function", |b| {