bench: increase noise threshold

Empirical data has shown that the noise of our testing framework is
quite high. Criterions default threshold of 1% yields a lot of
false-positives (to the point where multiple tests fail spuriously even
if there is no code change). Therefore, we increase it to 10%. This is
still a significantly lower threshold than what the old baseline-based
approach uses, where the deltas ranges from 16% to 400% (yes,
four-hundred). Note that at 5%, significant rates of spurious failures
were still observed.

Signed-off-by: Patrick Roy <[email protected]>

Author: roypat

src/snapshot/benches/version_map.rs

@@ -124,9 +124,7 @@ pub fn criterion_benchmark(c: &mut Criterion) {
 
 criterion_group! {
     name = benches;
-    // TODO: Fix the duration below to match the required time to collect enough samples.
-    // A warning message is generated by criterion while the accuracy of the test will be lower.
-    config = Criterion::default().measurement_time(std::time::Duration::from_secs(10));
+    config = Criterion::default().sample_size(200).noise_threshold(0.05);
     targets = criterion_benchmark
 }
 

src/vmm/benches/cpu_templates.rs

@@ -6,7 +6,6 @@
 //   * `CustomCpuTemplate` JSON serialization
 
 use std::mem::size_of_val;
-use std::path::Path;
 
 use criterion::{criterion_group, criterion_main, Criterion};
 use vmm::cpu_config::templates::test_utils::{build_test_template, TEST_TEMPLATE_JSON};
@@ -45,7 +44,7 @@ pub fn cpu_template_benchmark(c: &mut Criterion) {
 
 criterion_group! {
     name = cpu_template_benches;
-    config = Criterion::default().sample_size(200);
+    config = Criterion::default().sample_size(200).noise_threshold(0.05);
     targets = cpu_template_benchmark
 }