bench: increase iteration for criterion benchmarks

Set a fixed runtime of 10 seconds instead of fixed sample count, to try
and increase stability because the benchmarks always fail spuriously.

Signed-off-by: Patrick Roy <[email protected]>

Author: roypat

src/vmm/benches/block_request.rs

@@ -36,7 +36,7 @@ pub fn block_request_benchmark(c: &mut Criterion) {
 
 criterion_group! {
     name = block_request_benches;
-    config = Criterion::default().sample_size(1000).noise_threshold(0.05);
+    config = Criterion::default().noise_threshold(0.05).measurement_time(std::time::Duration::from_secs(10)).sample_size(10000);
     targets = block_request_benchmark
 }
 

src/vmm/benches/cpu_templates.rs

@@ -44,7 +44,7 @@ pub fn cpu_template_benchmark(c: &mut Criterion) {
 
 criterion_group! {
     name = cpu_template_benches;
-    config = Criterion::default().sample_size(200).noise_threshold(0.05);
+    config = Criterion::default().noise_threshold(0.05).measurement_time(std::time::Duration::from_secs(10)).sample_size(10000);
     targets = cpu_template_benchmark
 }
 

src/vmm/benches/memory_access.rs

@@ -59,7 +59,7 @@ pub fn bench_2m_page_fault(c: &mut Criterion) {
 
 criterion_group! {
     name = memory_access_benches;
-    config = Criterion::default().noise_threshold(0.05);
+    config = Criterion::default().noise_threshold(0.05).measurement_time(std::time::Duration::from_secs(10)).sample_size(10000);
     targets = bench_4k_page_fault, bench_2m_page_fault
 }
 

src/vmm/benches/queue.rs

@@ -108,7 +108,7 @@ pub fn queue_benchmark(c: &mut Criterion) {
 
 criterion_group! {
     name = queue_benches;
-    config = Criterion::default().sample_size(1000).noise_threshold(0.15);
+    config = Criterion::default().noise_threshold(0.15).measurement_time(std::time::Duration::from_secs(10)).sample_size(10000);
     targets = queue_benchmark
 }
 

tests/integration_tests/performance/test_benchmarks.py

@@ -18,7 +18,7 @@
 
 
 @pytest.mark.no_block_pr
-@pytest.mark.timeout(900)
+@pytest.mark.timeout(9000)
 def test_no_regression_relative_to_target_branch():
     """
     Run the microbenchmarks in this repository, comparing results from pull