test(perf): Aggregate snapshot latencies to average

The statistical test used by firecracker performance tests is a
simplified t-test. This is based on the central limit theorem, and is
thus only valid for testing _averages_. Nevertheless, the snapshot tests
tried to use it for detecting deviations of P90/P50 values. This is
unsound, and the results are most definitely not what was intended. This
commit changes the aggregate collected from the snapshot tests to be an
average instead, to make the test sound.

Note that since we also emit the individual data points, we can always
reconstruct percentiles.

Signed-off-by: Patrick Roy <[email protected]>

Author: roypat

tests/integration_tests/performance/configs/test_snapshot_restore_performance_config_4.14.json

@@ -4475,14 +4475,7 @@
         "latency": {
             "statistics": [
                 {
-                    "criteria": "EqualWith",
-                    "function": "Percentile50",
-                    "name": "P50"
-                },
-                {
-                    "criteria": "EqualWith",
-                    "function": "Percentile90",
-                    "name": "P90"
+                    "function": "Avg"
                 }
             ],
             "unit": "ms"

tests/integration_tests/performance/configs/test_snapshot_restore_performance_config_5.10.json

@@ -4149,14 +4149,7 @@
         "latency": {
             "statistics": [
                 {
-                    "criteria": "EqualWith",
-                    "function": "Percentile50",
-                    "name": "P50"
-                },
-                {
-                    "criteria": "EqualWith",
-                    "function": "Percentile90",
-                    "name": "P90"
+                    "function": "Avg"
                 }
             ],
             "unit": "ms"

tests/integration_tests/performance/configs/test_snapshot_restore_performance_config_6.1.json

@@ -4149,12 +4149,7 @@
         "latency": {
             "statistics": [
                 {
-                    "function": "Percentile50",
-                    "name": "P50"
-                },
-                {
-                    "function": "Percentile90",
-                    "name": "P90"
+                    "function": "Avg"
                 }
             ],
             "unit": "ms"

tools/parse_baselines/providers/snapshot_restore.py

@@ -26,22 +26,15 @@ def __init__(self, data_provider: Iterator):
         super().__init__(
             data_provider,
             [
-                "latency/P50",
-                "latency/P90",
+                "latency/Avg",
             ],
         )
 
     def calculate_baseline(self, data: List[float]) -> dict:
         """Return the target and delta values, given a list of data points."""
         avg = statistics.mean(data)
-        min_ = min(data)
-        max_ = max(data)
-
-        min_delta = 100 * abs(avg - min_) / avg
-        max_delta = 100 * abs(avg - max_) / avg
-        delta = max(max_delta, min_delta)
-
+        stddev = statistics.stdev(data)
         return {
-            "target": round(avg, 3),
-            "delta_percentage": math.ceil(delta) + DELTA_EXTRA_MARGIN,
+            "target": math.ceil(round(avg, 2)),
+            "delta_percentage": math.ceil(3 * stddev / avg * 100) + DELTA_EXTRA_MARGIN,
         }