test: Move FcMetricsMonitor initialization into Microvm

This way, the lifecycle of the monitor perfectly matches that of the
firecracker process (as it is started up when the Firecracker process
starts, and gets terminated when the microvm is killed).

This also means that the assertion the previous commit introduces will
only trigger during teardown (or when explicitly killing microVMs), at
which will be _after_ all other test-specific metrics have been emitted
(which means that triggering the assertion will not lead to loosing
performance metrics).

Also just start the monitor thread in the constructor, since all
call-sites just immediately started it anyway.

Signed-off-by: Patrick Roy <[email protected]>

Author: roypat

tests/framework/microvm.py

@@ -39,6 +39,7 @@
 from framework.microvm_helpers import MicrovmHelpers
 from framework.properties import global_props
 from framework.utils_drive import VhostUserBlkBackend, VhostUserBlkBackendType
+from host_tools.fcmetrics import FCMetricsMonitor
 from host_tools.memory import MemoryMonitor
 
 LOG = logging.getLogger("microvm")
@@ -558,6 +559,7 @@ def spawn(
         log_show_level=False,
         log_show_origin=False,
         metrics_path="fc.ndjson",
+        emit_metrics: bool = False,
     ):
         """Start a microVM as a daemon or in a screen session."""
         # pylint: disable=subprocess-run-check
@@ -583,6 +585,8 @@ def spawn(
             self.metrics_file.touch()
             self.create_jailed_resource(self.metrics_file)
             self.jailer.extra_args.update({"metrics-path": self.metrics_file.name})
+        else:
+            assert not emit_metrics
 
         if self.metadata_file:
             if os.path.exists(self.metadata_file):
@@ -619,6 +623,9 @@ def spawn(
 
         self._spawned = True
 
+        if emit_metrics:
+            self.monitors.append(FCMetricsMonitor(self))
+
         # Wait for the jailer to create resources needed, and Firecracker to
         # create its API socket.
         # We expect the jailer to start within 80 ms. However, we wait for

tests/host_tools/fcmetrics.py

@@ -500,6 +500,7 @@ def __init__(self, vm, timer=60):
                 "guest_kernel": vm.kernel_file.stem[2:],
             }
         )
+        self.start()
 
     def _flush_metrics(self):
         """

tests/integration_tests/performance/test_block_ab.py

@@ -11,7 +11,6 @@
 
 import host_tools.drive as drive_tools
 from framework.utils import CmdBuilder, check_output, track_cpu_utilization
-from host_tools.fcmetrics import FCMetricsMonitor
 
 # size of the block device used in the test, in MB
 BLOCK_DEVICE_SIZE_MB = 2048
@@ -155,7 +154,7 @@ def test_block_performance(
     Execute block device emulation benchmarking scenarios.
     """
     vm = microvm_factory.build(guest_kernel, rootfs, monitor_memory=False)
-    vm.spawn(log_level="Info")
+    vm.spawn(log_level="Info", emit_metrics=True)
     vm.basic_config(vcpu_count=vcpus, mem_size_mib=GUEST_MEM_MIB)
     vm.add_net_iface()
     # Add a secondary block device for benchmark tests.
@@ -174,8 +173,6 @@ def test_block_performance(
             **vm.dimensions,
         }
     )
-    fcmetrics = FCMetricsMonitor(vm)
-    fcmetrics.start()
 
     vm.pin_threads(0)
 
@@ -187,8 +184,6 @@ def test_block_performance(
         for value in values:
             metrics.put_metric(f"cpu_utilization_{thread_name}", value, "Percent")
 
-    fcmetrics.stop()
-
 
 @pytest.mark.nonci
 @pytest.mark.parametrize("vcpus", [1, 2], ids=["1vcpu", "2vcpu"])
@@ -208,7 +203,7 @@ def test_block_vhost_user_performance(
     """
 
     vm = microvm_factory.build(guest_kernel, rootfs, monitor_memory=False)
-    vm.spawn(log_level="Info")
+    vm.spawn(log_level="Info", emit_metrics=True)
     vm.basic_config(vcpu_count=vcpus, mem_size_mib=GUEST_MEM_MIB)
     vm.add_net_iface()
 
@@ -226,8 +221,6 @@ def test_block_vhost_user_performance(
             **vm.dimensions,
         }
     )
-    fcmetrics = FCMetricsMonitor(vm)
-    fcmetrics.start()
 
     next_cpu = vm.pin_threads(0)
     vm.disks_vhost_user["scratch"].pin(next_cpu)
@@ -239,5 +232,3 @@ def test_block_vhost_user_performance(
     for thread_name, values in cpu_util.items():
         for value in values:
             metrics.put_metric(f"cpu_utilization_{thread_name}", value, "Percent")
-
-    fcmetrics.stop()

tests/integration_tests/performance/test_memory_overhead.py

@@ -20,8 +20,6 @@
 import psutil
 import pytest
 
-from host_tools.fcmetrics import FCMetricsMonitor
-
 # If guest memory is >3328MB, it is split in a 2nd region
 X86_MEMORY_GAP_START = 3328 * 2**20
 
@@ -40,15 +38,13 @@ def test_memory_overhead(
 
     for _ in range(5):
         microvm = microvm_factory.build(guest_kernel, rootfs)
-        microvm.spawn()
+        microvm.spawn(emit_metrics=True)
         microvm.basic_config(vcpu_count=vcpu_count, mem_size_mib=mem_size_mib)
         microvm.add_net_iface()
         microvm.start()
         metrics.set_dimensions(
             {"performance_test": "test_memory_overhead", **microvm.dimensions}
         )
-        fcmetrics = FCMetricsMonitor(microvm)
-        fcmetrics.start()
         microvm.wait_for_up()
 
         guest_mem_bytes = mem_size_mib * 2**20
@@ -84,4 +80,3 @@ def test_memory_overhead(
         for metric in ["uss", "text"]:
             val = getattr(mem_info, metric)
             metrics.put_metric(metric, val, unit="Bytes")
-        fcmetrics.stop()

tests/integration_tests/performance/test_network_ab.py

@@ -7,7 +7,6 @@
 import pytest
 
 from framework.utils_iperf import IPerf3Test, emit_iperf3_metrics
-from host_tools.fcmetrics import FCMetricsMonitor
 
 
 def consume_ping_output(ping_putput, request_per_round):
@@ -45,7 +44,7 @@ def network_microvm(request, microvm_factory, guest_kernel, rootfs):
     guest_vcpus = request.param
 
     vm = microvm_factory.build(guest_kernel, rootfs, monitor_memory=False)
-    vm.spawn(log_level="Info")
+    vm.spawn(log_level="Info", emit_metrics=True)
     vm.basic_config(vcpu_count=guest_vcpus, mem_size_mib=guest_mem_mib)
     vm.add_net_iface()
     vm.start()
@@ -71,8 +70,6 @@ def test_network_latency(network_microvm, metrics):
             **network_microvm.dimensions,
         }
     )
-    fcmetrics = FCMetricsMonitor(network_microvm)
-    fcmetrics.start()
 
     samples = []
     host_ip = network_microvm.iface["eth0"]["iface"].host_ip
@@ -83,7 +80,6 @@ def test_network_latency(network_microvm, metrics):
         )
 
         samples.extend(consume_ping_output(ping_output, request_per_round))
-    fcmetrics.stop()
 
     for sample in samples:
         metrics.put_metric("ping_latency", sample, "Milliseconds")
@@ -124,8 +120,6 @@ def test_network_tcp_throughput(
             **network_microvm.dimensions,
         }
     )
-    fcmetrics = FCMetricsMonitor(network_microvm)
-    fcmetrics.start()
 
     test = IPerf3Test(
         microvm=network_microvm,
@@ -140,4 +134,3 @@ def test_network_tcp_throughput(
     data = test.run_test(network_microvm.vcpus_count + 2)
 
     emit_iperf3_metrics(metrics, data, warmup_sec)
-    fcmetrics.stop()

tests/integration_tests/performance/test_snapshot_ab.py

@@ -10,7 +10,6 @@
 
 import host_tools.drive as drive_tools
 from framework.microvm import Microvm
-from host_tools.fcmetrics import FCMetricsMonitor
 
 USEC_IN_MSEC = 1000
 ITERATIONS = 30
@@ -53,7 +52,7 @@ def configure_vm(
             rootfs,
             monitor_memory=False,
         )
-        vm.spawn(log_level="Info")
+        vm.spawn(log_level="Info", emit_metrics=True)
         vm.time_api_requests = False
         vm.basic_config(
             vcpu_count=self.vcpus,
@@ -88,11 +87,9 @@ def sample_latency(
                 kernel=guest_kernel_linux_4_14,
                 monitor_memory=False,
             )
-            microvm.spawn()
+            microvm.spawn(emit_metrics=True)
             snapshot_copy = microvm.restore_from_snapshot(snapshot, resume=True)
 
-            fcmetrics = FCMetricsMonitor(microvm)
-            fcmetrics.start()
             microvm.wait_for_up()
 
             value = 0
@@ -106,7 +103,6 @@ def sample_latency(
                     break
             assert value > 0
             values.append(value)
-            fcmetrics.stop()
             microvm.kill()
             snapshot_copy.delete()
 
@@ -154,11 +150,8 @@ def test_restore_latency(
             **vm.dimensions,
         }
     )
-    fcmetrics = FCMetricsMonitor(vm)
-    fcmetrics.start()
 
     snapshot = vm.snapshot_full()
-    fcmetrics.stop()
     vm.kill()
 
     samples = test_setup.sample_latency(

tests/integration_tests/performance/test_vsock_ab.py

@@ -8,7 +8,6 @@
 
 from framework.utils_iperf import IPerf3Test, emit_iperf3_metrics
 from framework.utils_vsock import VSOCK_UDS_PATH, make_host_port_path
-from host_tools.fcmetrics import FCMetricsMonitor
 
 
 class VsockIPerf3Test(IPerf3Test):
@@ -87,7 +86,7 @@ def test_vsock_throughput(
 
     mem_size_mib = 1024
     vm = microvm_factory.build(guest_kernel, rootfs, monitor_memory=False)
-    vm.spawn(log_level="Info")
+    vm.spawn(log_level="Info", emit_metrics=True)
     vm.basic_config(vcpu_count=vcpus, mem_size_mib=mem_size_mib)
     vm.add_net_iface()
     # Create a vsock device
@@ -102,13 +101,10 @@ def test_vsock_throughput(
             **vm.dimensions,
         }
     )
-    fcmetrics = FCMetricsMonitor(vm)
-    fcmetrics.start()
 
     vm.pin_threads(0)
 
     test = VsockIPerf3Test(vm, mode, payload_length)
     data = test.run_test(vm.vcpus_count + 2)
 
-    fcmetrics.stop()
     emit_iperf3_metrics(metrics, data, VsockIPerf3Test.WARMUP_SEC)