test: run throughput perf tests with secret freedom enabled

Aadditionally parametrize some of our throughput performance tests
(network, block and vsock) by memory config, so that they run with
secret freedom (and hence bounce buffering) enabled. Also add it to the
boottime test, because bouncing can impact the time taken to read the
rootfs.

Skip them on m6g.metal because secret freedom does not work here for
architectural reasons (and our patches do not take this into account, so
trying to use secret freedom here would result in host kernel panics).

Signed-off-by: Patrick Roy <[email protected]>

Author: roypat

tests/conftest.py

@@ -386,6 +386,22 @@ def io_engine(request):
     return request.param
 
 
+secret_free_test_cases = [False]
+if (
+    global_props.host_linux_version_metrics == "next"
+    and global_props.instance != "m6g.metal"
+):
+    secret_free_test_cases.append(True)
+
+
+@pytest.fixture(params=secret_free_test_cases)
+def memory_config(request):
+    """Differently configured swiotlb regions. Only supported on aarch64"""
+    if not request.param:
+        return None
+    return {"secret_free": request.param}
+
+
 @pytest.fixture
 def results_dir(request):
     """

tests/framework/microvm.py

@@ -249,6 +249,7 @@ def __init__(
         self.disks_vhost_user = {}
         self.vcpus_count = None
         self.mem_size_bytes = None
+        self.memory_config = None
         self.cpu_template_name = "None"
         # The given custom CPU template will be set in basic_config() but could
         # be overwritten via set_cpu_template().
@@ -463,6 +464,11 @@ def dimensions(self):
             "rootfs": self.rootfs_file.name,
             "vcpus": str(self.vcpus_count),
             "guest_memory": f"{self.mem_size_bytes / (1024 * 1024)}MB",
+            **(
+                {k: str(v) for k, v in self.memory_config.items()}
+                if self.memory_config
+                else {}
+            ),
         }
 
     @property
@@ -730,6 +736,7 @@ def basic_config(
         rootfs_io_engine=None,
         cpu_template: Optional[str] = None,
         enable_entropy_device=False,
+        memory_config=None,
     ):
         """Shortcut for quickly configuring a microVM.
 
@@ -748,15 +755,23 @@ def basic_config(
 
         Reference: file:../../src/vmm/src/vmm_config/boot_source.rs::DEFAULT_KERNEL_CMDLINE
         """
+        # Have to do it this way as otherwise A/B-tests fail if the 'A' revision
+        # of Firecracker doesn't know about the mem_config parameter.
+        kwargs = {}
+        if memory_config is not None:
+            kwargs["mem_config"] = memory_config
+
         self.api.machine_config.put(
             vcpu_count=vcpu_count,
             smt=smt,
             mem_size_mib=mem_size_mib,
             track_dirty_pages=track_dirty_pages,
             huge_pages=huge_pages,
+            **kwargs,
         )
         self.vcpus_count = vcpu_count
         self.mem_size_bytes = mem_size_mib * 2**20
+        self.memory_config = memory_config
 
         if self.custom_cpu_template is not None:
             self.set_cpu_template(self.custom_cpu_template)

tests/integration_tests/performance/test_block_ab.py

@@ -163,14 +163,20 @@ def test_block_performance(
     fio_block_size,
     fio_engine,
     io_engine,
+    memory_config,
     metrics,
 ):
     """
     Execute block device emulation benchmarking scenarios.
     """
+    if memory_config is not None and "6.1" not in guest_kernel_acpi.name:
+        pytest.skip("swiotlb only supported on aarch64/6.1")
+
     vm = microvm_factory.build(guest_kernel_acpi, rootfs, monitor_memory=False)
     vm.spawn(log_level="Info", emit_metrics=True)
-    vm.basic_config(vcpu_count=vcpus, mem_size_mib=GUEST_MEM_MIB)
+    vm.basic_config(
+        vcpu_count=vcpus, mem_size_mib=GUEST_MEM_MIB, memory_config=memory_config
+    )
     vm.add_net_iface()
     # Add a secondary block device for benchmark tests.
     fs = drive_tools.FilesystemFile(

tests/integration_tests/performance/test_boottime.py

@@ -8,8 +8,6 @@
 
 import pytest
 
-from framework.properties import global_props
-
 # Regex for obtaining boot time from some string.
 TIMESTAMP_LOG_REGEX = r"Guest-boot-time\s+\=\s+(\d+)\s+us"
 
@@ -19,14 +17,6 @@
 )
 
 
-DIMENSIONS = {
-    "instance": global_props.instance,
-    "cpu_model": global_props.cpu_model,
-    "host_os": global_props.host_os,
-    "host_kernel": "linux-" + global_props.host_linux_version_metrics,
-}
-
-
 def _get_microvm_boottime(vm):
     """Auxiliary function for asserting the expected boot time."""
     boot_time_us = None
@@ -75,19 +65,18 @@ def find_events(log_data):
 )
 @pytest.mark.nonci
 def test_boottime(
-    microvm_factory, guest_kernel_acpi, rootfs_rw, vcpu_count, mem_size_mib, metrics
+    microvm_factory,
+    guest_kernel_acpi,
+    rootfs_rw,
+    vcpu_count,
+    mem_size_mib,
+    memory_config,
+    metrics,
 ):
     """Test boot time with different guest configurations"""
 
-    metrics.set_dimensions(
-        {
-            **DIMENSIONS,
-            "performance_test": "test_boottime",
-            "guest_kernel": guest_kernel_acpi.name,
-            "vcpus": str(vcpu_count),
-            "mem_size_mib": str(mem_size_mib),
-        }
-    )
+    if memory_config is not None and "6.1" not in guest_kernel_acpi.name:
+        pytest.skip("swiotlb only supported on aarch64/6.1")
 
     for _ in range(10):
         vm = microvm_factory.build(guest_kernel_acpi, rootfs_rw)
@@ -98,10 +87,14 @@ def test_boottime(
             mem_size_mib=mem_size_mib,
             boot_args=DEFAULT_BOOT_ARGS + " init=/usr/local/bin/init",
             enable_entropy_device=True,
+            memory_config=memory_config,
         )
         vm.add_net_iface()
         vm.start()
         vm.pin_threads(0)
+
+        metrics.set_dimensions({"performance_test": "test_boottime", **vm.dimensions})
+
         boottime_us = _get_microvm_boottime(vm)
         metrics.put_metric("boot_time", boottime_us, unit="Microseconds")
         timestamps = find_events(vm.log_data)

tests/integration_tests/performance/test_network_ab.py

@@ -36,16 +36,21 @@ def consume_ping_output(ping_putput, request_per_round):
 
 
 @pytest.fixture
-def network_microvm(request, microvm_factory, guest_kernel_acpi, rootfs):
+def network_microvm(request, microvm_factory, guest_kernel_acpi, rootfs, memory_config):
     """Creates a microvm with the networking setup used by the performance tests in this file.
     This fixture receives its vcpu count via indirect parameterization"""
 
+    if memory_config is not None and "6.1" not in guest_kernel_acpi.name:
+        pytest.skip("swiotlb only supported on aarch64/6.1")
+
     guest_mem_mib = 1024
     guest_vcpus = request.param
 
     vm = microvm_factory.build(guest_kernel_acpi, rootfs, monitor_memory=False)
     vm.spawn(log_level="Info", emit_metrics=True)
-    vm.basic_config(vcpu_count=guest_vcpus, mem_size_mib=guest_mem_mib)
+    vm.basic_config(
+        vcpu_count=guest_vcpus, mem_size_mib=guest_mem_mib, memory_config=memory_config
+    )
     vm.add_net_iface()
     vm.start()
     vm.pin_threads(0)

tests/integration_tests/performance/test_vsock_ab.py

@@ -73,7 +73,14 @@ def guest_command(self, port_offset):
 @pytest.mark.parametrize("payload_length", ["64K", "1024K"], ids=["p64K", "p1024K"])
 @pytest.mark.parametrize("mode", ["g2h", "h2g", "bd"])
 def test_vsock_throughput(
-    microvm_factory, guest_kernel_acpi, rootfs, vcpus, payload_length, mode, metrics
+    microvm_factory,
+    guest_kernel_acpi,
+    rootfs,
+    vcpus,
+    payload_length,
+    mode,
+    metrics,
+    memory_config,
 ):
     """
     Test vsock throughput for multiple vm configurations.
@@ -84,10 +91,15 @@ def test_vsock_throughput(
     if mode == "bd" and vcpus < 2:
         pytest.skip("bidrectional test only done with at least 2 vcpus")
 
+    if memory_config is not None and "6.1" not in guest_kernel_acpi.name:
+        pytest.skip("swiotlb only supported on aarch64/6.1")
+
     mem_size_mib = 1024
     vm = microvm_factory.build(guest_kernel_acpi, rootfs, monitor_memory=False)
     vm.spawn(log_level="Info", emit_metrics=True)
-    vm.basic_config(vcpu_count=vcpus, mem_size_mib=mem_size_mib)
+    vm.basic_config(
+        vcpu_count=vcpus, mem_size_mib=mem_size_mib, memory_config=memory_config
+    )
     vm.add_net_iface()
     # Create a vsock device
     vm.api.vsock.put(vsock_id="vsock0", guest_cid=3, uds_path="/" + VSOCK_UDS_PATH)