test: add a test to measure Firecracker memory overhead

Our current binary size test measures space in disk, as a proxy for used
memory. This is not accurate, as parts of the binary are not loaded into
memory.

This commits adds a new test that measures memory overhead directly and
published metrics for different guest configurations.

Also remove thresholds from test_binary_size.py test, as we measure
memory overhead directly now.

Signed-off-by: Pablo Barbáchano <[email protected]>

Author: pb8o

tests/integration_tests/build/test_binary_size.py

@@ -1,56 +1,29 @@
 # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
-"""Tests that check if the release binary sizes fall within expected size."""
+"""Tests that check if the release binary sizes fall within expected size.
+
+This is not representative of the actual memory overhead of Firecracker.
+
+A more representative test is file:../performance/test_memory_overhead.py
+"""
 
-import os
 import platform
 
 import pytest
 
 import host_tools.cargo_build as host
 
 MACHINE = platform.machine()
-""" Platform definition used to select the correct size target"""
-
-SIZES_DICT = {
-    "x86_64": {
-        "FC_BINARY_SIZE_TARGET": 3063120,
-        "JAILER_BINARY_SIZE_TARGET": 1080088,
-    },
-    "aarch64": {
-        "FC_BINARY_SIZE_TARGET": 2441008,
-        "JAILER_BINARY_SIZE_TARGET": 1040928,
-    },
-}
-
-FC_BINARY_SIZE_TARGET = SIZES_DICT[MACHINE]["FC_BINARY_SIZE_TARGET"]
-"""Firecracker target binary size in bytes"""
-
-JAILER_BINARY_SIZE_TARGET = SIZES_DICT[MACHINE]["JAILER_BINARY_SIZE_TARGET"]
-"""Jailer target binary size in bytes"""
-
-BINARY_SIZE_TOLERANCE = 0.05
-"""Tolerance of 5% allowed for binary size"""
 
 
 @pytest.mark.timeout(500)
 def test_firecracker_binary_size(record_property, metrics):
     """
     Test if the size of the firecracker binary is within expected ranges.
     """
-    fc_binary, _ = host.get_firecracker_binaries()
-
-    result = check_binary_size(
-        "firecracker",
-        fc_binary,
-        FC_BINARY_SIZE_TARGET,
-        BINARY_SIZE_TOLERANCE,
-    )
-
-    record_property(
-        "firecracker_binary_size",
-        f"{result}B ({FC_BINARY_SIZE_TARGET}B ±{BINARY_SIZE_TOLERANCE:.0%})",
-    )
+    fc_binary = host.get_binary("firecracker")
+    result = fc_binary.stat().st_size
+    record_property("firecracker_binary_size", f"{result}B")
     metrics.set_dimensions({"cpu_arch": MACHINE})
     metrics.put_metric("firecracker_binary_size", result, unit="Bytes")
 
@@ -60,51 +33,8 @@ def test_jailer_binary_size(record_property, metrics):
     """
     Test if the size of the jailer binary is within expected ranges.
     """
-    _, jailer_binary = host.get_firecracker_binaries()
-
-    result = check_binary_size(
-        "jailer",
-        jailer_binary,
-        JAILER_BINARY_SIZE_TARGET,
-        BINARY_SIZE_TOLERANCE,
-    )
-
-    record_property(
-        "jailer_binary_size",
-        f"{result}B ({JAILER_BINARY_SIZE_TARGET}B ±{BINARY_SIZE_TOLERANCE:.0%})",
-    )
+    jailer_binary = host.get_binary("jailer")
+    result = jailer_binary.stat().st_size
+    record_property("jailer_binary_size", f"{result}B")
     metrics.set_dimensions({"cpu_arch": MACHINE})
     metrics.put_metric("jailer_binary_size", result, unit="Bytes")
-
-
-def check_binary_size(name, binary_path, size_target, tolerance):
-    """Check if the specified binary falls within the expected range."""
-    # Get the size of the release binary.
-    binary_size = os.path.getsize(binary_path)
-
-    # Get the name of the variable that needs updating.
-    namespace = globals()
-    size_target_name = [name for name in namespace if namespace[name] is size_target][0]
-
-    # Compute concrete binary size difference.
-    delta_size = size_target - binary_size
-
-    binary_low_msg = (
-        "Current {} binary size of {} bytes is below the target"
-        " of {} bytes with {} bytes.\n"
-        "Update the {} threshold".format(
-            name, binary_size, size_target, delta_size, size_target_name
-        )
-    )
-
-    assert binary_size > size_target * (1 - tolerance), binary_low_msg
-
-    binary_high_msg = (
-        "Current {} binary size of {} bytes is above the target"
-        " of {} bytes with {} bytes.\n".format(
-            name, binary_size, size_target, -delta_size
-        )
-    )
-
-    assert binary_size < size_target * (1 + tolerance), binary_high_msg
-    return binary_size

tests/integration_tests/performance/test_memory_overhead.py

@@ -0,0 +1,92 @@
+# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Track Firecracker memory overhead
+
+Because Firecracker is a static binary, and is copied before execution, no
+memory is shared across many different processes. It is thus important to track
+how much memory overhead Firecracker adds.
+
+These tests output metrics to capture the memory overhead that Firecracker adds,
+both from the binary file (file-backed) and what Firecracker allocates during
+the process lifetime.
+
+The memory overhead of the jailer is not important as it is short-lived.
+"""
+
+from collections import defaultdict
+from pathlib import Path
+
+import psutil
+import pytest
+
+from framework.properties import global_props
+
+# If guest memory is >3328MB, it is split in a 2nd region
+X86_MEMORY_GAP_START = 3328 * 2**20
+
+
+@pytest.mark.parametrize(
+    "vcpu_count,mem_size_mib",
+    [(1, 128), (1, 1024), (2, 2024), (4, 4096)],
+)
+def test_memory_overhead(
+    microvm_factory, guest_kernel, rootfs, vcpu_count, mem_size_mib, metrics
+):
+    """Track Firecracker memory overhead.
+
+    We take a single measurement as it only varies by a few KiB each run.
+    """
+
+    microvm = microvm_factory.build(guest_kernel, rootfs)
+    microvm.spawn()
+    microvm.basic_config(vcpu_count=vcpu_count, mem_size_mib=mem_size_mib)
+    microvm.add_net_iface()
+    microvm.start()
+
+    # check that the vm is running
+    microvm.ssh.run("true")
+
+    guest_mem_bytes = mem_size_mib * 2**20
+    guest_mem_splits = {
+        guest_mem_bytes,
+        X86_MEMORY_GAP_START,
+    }
+    if guest_mem_bytes > X86_MEMORY_GAP_START:
+        guest_mem_splits.add(guest_mem_bytes - X86_MEMORY_GAP_START)
+
+    mem_stats = defaultdict(int)
+    mem_stats["guest_memory"] = guest_mem_bytes
+    ps = psutil.Process(microvm.jailer_clone_pid)
+
+    for pmmap in ps.memory_maps(grouped=False):
+        # We publish 'size' and 'rss' (resident). size would be the worst case,
+        # whereas rss is the current paged-in memory.
+
+        mem_stats["total_size"] += pmmap.size
+        mem_stats["total_rss"] += pmmap.rss
+        pmmap_path = Path(pmmap.path)
+        if pmmap_path.exists() and pmmap_path.name.startswith("firecracker"):
+            mem_stats["binary_size"] += pmmap.size
+            mem_stats["binary_rss"] += pmmap.rss
+
+        if pmmap.size not in guest_mem_splits:
+            mem_stats["overhead_size"] += pmmap.size
+            mem_stats["overhead_rss"] += pmmap.rss
+
+    dimensions = {
+        # "instance": global_props.instance,
+        # "cpu_model": global_props.cpu_model,
+        "architecture": global_props.cpu_architecture,
+        "host_kernel": "linux-" + global_props.host_linux_version,
+        "guest_kernel": guest_kernel.name,
+        "rootfs": rootfs.name,
+    }
+    metrics.set_dimensions(dimensions)
+    for key, value in mem_stats.items():
+        metrics.put_metric(key, value, unit="Bytes")
+
+    mem_info = ps.memory_full_info()
+    for metric in ["uss", "text"]:
+        val = getattr(mem_info, metric)
+        metrics.put_metric(metric, val, unit="Bytes")