Make benchmarks only run with release builds

.github/workflows/Benchmarks.yml

@@ -57,7 +57,7 @@ jobs:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Run Benchmarks
-        run: just bench-ci main release ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}}
+        run: just bench-ci main ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}}
 
       - uses: actions/upload-artifact@v4
         with:

.github/workflows/dep_rust.yml

@@ -151,5 +151,5 @@ jobs:
 
       - name: Run benchmarks
         run: |
-          just bench-ci main ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}}
+          just bench-ci main ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}}
         if: ${{ matrix.config == 'release' }}

Justfile

@@ -229,11 +229,13 @@ bench-download os hypervisor cpu tag="":
     tar -zxvf target/benchmarks_{{ os }}_{{ hypervisor }}_{{ cpu }}.tar.gz -C target/criterion/ --strip-components=1
 
 # Warning: compares to and then OVERWRITES the given baseline
-bench-ci baseline target=default-target features="":
-    cargo bench --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {''} else { "--features " + features } }} -- --verbose --save-baseline {{ baseline }}
+# Benchmarks only run with release builds for performance consistency
+bench-ci baseline features="":
+    cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose --save-baseline {{ baseline }}
 
-bench target=default-target features="":
-    cargo bench --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {''} else { "--features " + features } }} -- --verbose
+# Benchmarks only run with release builds for performance consistency  
+bench features="":
+    cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose
 
 ###############
 ### FUZZING ###

docs/benchmarking-hyperlight.md

@@ -72,4 +72,4 @@ Found 1 outliers among 100 measurements (1.00%)
 
 ## Running benchmarks locally
 
-Use `just bench [debug/release]` parameter to run benchmarks. Comparing local benchmarks results to github-saved benchmarks doesn't make much sense, since you'd be using different hardware, but you can use `just bench-download os hypervisor [tag] ` to download and extract the GitHub release benchmarks to the correct place folder. You can then run `just bench-ci main` to compare to (and overwrite) the previous release benchmarks. Note that `main` is the name of the baselines stored in GitHub.
+Use `just bench` to run benchmarks. Benchmarks only run with release builds for performance consistency. Comparing local benchmarks results to github-saved benchmarks doesn't make much sense, since you'd be using different hardware, but you can use `just bench-download os hypervisor [tag] ` to download and extract the GitHub release benchmarks to the correct place folder. You can then run `just bench-ci main` to compare to (and overwrite) the previous release benchmarks. Note that `main` is the name of the baselines stored in GitHub.

src/hyperlight_host/benches/benchmarks.rs

@@ -14,24 +14,37 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// Benchmarks are only meaningful and should only run with optimized builds.
+// Unoptimized builds have different performance characteristics and would not provide
+// useful benchmarking data for performance regression testing.
+
+#[cfg(optimized_build)]
 use criterion::{Criterion, criterion_group, criterion_main};
+#[cfg(optimized_build)]
 use hyperlight_host::GuestBinary;
+#[cfg(optimized_build)]
 use hyperlight_host::sandbox::{
     Callable, MultiUseSandbox, SandboxConfiguration, UninitializedSandbox,
 };
+#[cfg(optimized_build)]
 use hyperlight_host::sandbox_state::sandbox::EvolvableSandbox;
+#[cfg(optimized_build)]
 use hyperlight_host::sandbox_state::transition::Noop;
+#[cfg(optimized_build)]
 use hyperlight_testing::simple_guest_as_string;
 
+#[cfg(optimized_build)]
 fn create_uninit_sandbox() -> UninitializedSandbox {
     let path = simple_guest_as_string().unwrap();
     UninitializedSandbox::new(GuestBinary::FilePath(path), None).unwrap()
 }
 
+#[cfg(optimized_build)]
 fn create_multiuse_sandbox() -> MultiUseSandbox {
     create_uninit_sandbox().evolve(Noop::default()).unwrap()
 }
 
+#[cfg(optimized_build)]
 fn guest_call_benchmark(c: &mut Criterion) {
     let mut group = c.benchmark_group("guest_functions");
 
@@ -79,6 +92,7 @@ fn guest_call_benchmark(c: &mut Criterion) {
     group.finish();
 }
 
+#[cfg(optimized_build)]
 fn guest_call_benchmark_large_param(c: &mut Criterion) {
     let mut group = c.benchmark_group("guest_functions_with_large_parameters");
     #[cfg(target_os = "windows")]
@@ -114,6 +128,7 @@ fn guest_call_benchmark_large_param(c: &mut Criterion) {
     group.finish();
 }
 
+#[cfg(optimized_build)]
 fn sandbox_benchmark(c: &mut Criterion) {
     let mut group = c.benchmark_group("sandboxes");
 
@@ -153,9 +168,21 @@ fn sandbox_benchmark(c: &mut Criterion) {
     group.finish();
 }
 
+#[cfg(optimized_build)]
 criterion_group! {
     name = benches;
     config = Criterion::default();
     targets = guest_call_benchmark, sandbox_benchmark, guest_call_benchmark_large_param
 }
+
+#[cfg(optimized_build)]
 criterion_main!(benches);
+
+// Provide a fallback main function for unoptimized builds
+// This prevents compilation errors while providing a clear message
+#[cfg(unoptimized_build)]
+fn main() {
+    panic!(
+        "Benchmarks must be run with optimized builds only. Use `cargo bench --release` or `just bench`."
+    );
+}

src/hyperlight_host/build.rs

@@ -85,6 +85,24 @@ fn main() -> Result<()> {
         );
     }
 
+    // Set a cfg flag based on optimization level for benchmarks
+    // Benchmarks should only run with optimized builds (opt-level 1+)
+    println!("cargo:rustc-check-cfg=cfg(unoptimized_build)");
+    println!("cargo:rustc-check-cfg=cfg(optimized_build)");
+
+    if let Ok(opt_level) = std::env::var("OPT_LEVEL") {
+        if opt_level == "0" {
+            // Unoptimized build - benchmarks should not run
+            println!("cargo:rustc-cfg=unoptimized_build");
+        } else {
+            // Optimized build - benchmarks can run
+            println!("cargo:rustc-cfg=optimized_build");
+        }
+    } else {
+        // Fallback: if we can't determine opt level, assume unoptimized to be safe
+        println!("cargo:rustc-cfg=unoptimized_build");
+    }
+
     // Makes #[cfg(kvm)] == #[cfg(all(feature = "kvm", target_os = "linux"))]
     // and #[cfg(mshv)] == #[cfg(all(any(feature = "mshv2", feature = "mshv3"), target_os = "linux"))].
     // Essentially the kvm and mshv features are ignored on windows as long as you use #[cfg(kvm)] and not #[cfg(feature = "kvm")].