Enforce release builds for benchmarks in Justfile and source code

Co-authored-by: jsturtevant <[email protected]>
Signed-off-by: Simon Davies <[email protected]>

Author: 2 people

Justfile

@@ -324,11 +324,15 @@ bench-download os hypervisor cpu tag="":
     tar -zxvf target/benchmarks_{{ os }}_{{ hypervisor }}_{{ cpu }}.tar.gz -C target/criterion/ --strip-components=1
 
 # Warning: compares to and then OVERWRITES the given baseline
-bench-ci baseline target=default-target features="":
-    cargo bench --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {''} else { "--features " + features } }} -- --verbose --save-baseline {{ baseline }}
-
-bench target=default-target features="":
-    cargo bench --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {''} else { "--features " + features } }} -- --verbose
+bench-ci baseline target="release" features="":
+    @# Benchmarks should only be run with release builds for meaningful results
+    @if [ "{{ target }}" != "release" ]; then echo "Error: Benchmarks can only be run with release builds. Use 'just bench-ci <baseline> release' instead."; exit 1; fi
+    cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose --save-baseline {{ baseline }}
+
+bench target="release" features="":
+    @# Benchmarks should only be run with release builds for meaningful results
+    @if [ "{{ target }}" != "release" ]; then echo "Error: Benchmarks can only be run with release builds. Use 'just bench release' instead."; exit 1; fi
+    cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose
 
 ###############
 ### FUZZING ###

docs/benchmarking-hyperlight.md

@@ -72,4 +72,6 @@ Found 1 outliers among 100 measurements (1.00%)
 
 ## Running benchmarks locally
 
-Use `just bench [debug/release]` parameter to run benchmarks. Comparing local benchmarks results to github-saved benchmarks doesn't make much sense, since you'd be using different hardware, but you can use `just bench-download os hypervisor [tag] ` to download and extract the GitHub release benchmarks to the correct place folder. You can then run `just bench-ci main` to compare to (and overwrite) the previous release benchmarks. Note that `main` is the name of the baselines stored in GitHub.
+Use `just bench` to run benchmarks with release builds (the only supported configuration). Comparing local benchmark results to github-saved benchmarks doesn't make much sense, since you'd be using different hardware, but you can use `just bench-download os hypervisor [tag] ` to download and extract the GitHub release benchmarks to the correct place folder. You can then run `just bench-ci main` to compare to (and overwrite) the previous release benchmarks. Note that `main` is the name of the baselines stored in GitHub.
+
+**Important**: Benchmarks can only be run with release builds to ensure meaningful performance measurements. Debug builds are not supported for benchmarking.

src/hyperlight_host/benches/benchmarks.rs

@@ -14,6 +14,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// Ensure benchmarks only run with release builds for meaningful performance measurements
+#[cfg(debug_assertions)]
+compile_error!(
+    "Benchmarks can only be run with release builds. Use '--release' flag when running benchmarks."
+);
+
 use criterion::{Criterion, criterion_group, criterion_main};
 use hyperlight_host::GuestBinary;
 use hyperlight_host::sandbox::{