test: run throughput perf tests with swiotlb enabled

On aarch64, additionally parametrize our throughput performance tests
(network, block and vsock) by swiotlb size, to get a feeling for the
performance impact bounce buffering will have. Try out different sizes
of swiotlb region to see whether we can identify a cut off.

Signed-off-by: Patrick Roy <[email protected]>

Author: roypat

tests/conftest.py

@@ -376,6 +376,16 @@ def io_engine(request):
     return request.param
 
 
+@pytest.fixture(
+    params=[None, 16, 32, 64] if platform.machine() == "aarch64" else [None]
+)
+def memory_config(request):
+    """Differently configured swiotlb regions. Only supported on aarch64"""
+    if request.param is None:
+        return None
+    return {"initial_swiotlb_size": request.param}
+
+
 @pytest.fixture
 def results_dir(request):
     """

tests/framework/microvm.py

@@ -248,6 +248,7 @@ def __init__(
         self.disks_vhost_user = {}
         self.vcpus_count = None
         self.mem_size_bytes = None
+        self.memory_config = None
         self.cpu_template_name = None
         # The given custom CPU template will be set in basic_config() but could
         # be overwritten via set_cpu_template().
@@ -462,6 +463,7 @@ def dimensions(self):
             "rootfs": self.rootfs_file.name,
             "vcpus": str(self.vcpus_count),
             "guest_memory": f"{self.mem_size_bytes / (1024 * 1024)}MB",
+            **(self.memory_config if self.memory_config else {}),
         }
 
     @property
@@ -729,6 +731,7 @@ def basic_config(
         rootfs_io_engine=None,
         cpu_template: Optional[str] = None,
         enable_entropy_device=False,
+        memory_config=None,
     ):
         """Shortcut for quickly configuring a microVM.
 
@@ -747,15 +750,23 @@ def basic_config(
 
         Reference: file:../../src/vmm/src/vmm_config/boot_source.rs::DEFAULT_KERNEL_CMDLINE
         """
+        # Have to do it this way as otherwise A/B-tests fail if the 'A' revision
+        # of Firecracker doesn't knwo about the mem_config parameter.
+        kwargs = {}
+        if memory_config is not None:
+            kwargs["mem_config"] = memory_config
+
         self.api.machine_config.put(
             vcpu_count=vcpu_count,
             smt=smt,
             mem_size_mib=mem_size_mib,
             track_dirty_pages=track_dirty_pages,
             huge_pages=huge_pages,
+            **kwargs,
         )
         self.vcpus_count = vcpu_count
         self.mem_size_bytes = mem_size_mib * 2**20
+        self.memory_config = memory_config
 
         if self.custom_cpu_template is not None:
             self.set_cpu_template(self.custom_cpu_template)

tests/integration_tests/performance/test_block_ab.py

@@ -148,14 +148,20 @@ def test_block_performance(
     fio_mode,
     fio_block_size,
     io_engine,
+    memory_config,
     metrics,
 ):
     """
     Execute block device emulation benchmarking scenarios.
     """
+    if memory_config is not None and "6.1" not in guest_kernel_acpi:
+        pytest.skip("swiotlb only supported on aarch64/6.1")
+
     vm = microvm_factory.build(guest_kernel_acpi, rootfs, monitor_memory=False)
     vm.spawn(log_level="Info", emit_metrics=True)
-    vm.basic_config(vcpu_count=vcpus, mem_size_mib=GUEST_MEM_MIB)
+    vm.basic_config(
+        vcpu_count=vcpus, mem_size_mib=GUEST_MEM_MIB, memory_config=memory_config
+    )
     vm.add_net_iface()
     # Add a secondary block device for benchmark tests.
     fs = drive_tools.FilesystemFile(

tests/integration_tests/performance/test_network_ab.py

@@ -36,16 +36,21 @@ def consume_ping_output(ping_putput, request_per_round):
 
 
 @pytest.fixture
-def network_microvm(request, microvm_factory, guest_kernel_acpi, rootfs):
+def network_microvm(request, microvm_factory, guest_kernel_acpi, rootfs, memory_config):
     """Creates a microvm with the networking setup used by the performance tests in this file.
     This fixture receives its vcpu count via indirect parameterization"""
 
+    if memory_config is not None and "6.1" not in guest_kernel_acpi:
+        pytest.skip("swiotlb only supported on aarch64/6.1")
+
     guest_mem_mib = 1024
     guest_vcpus = request.param
 
     vm = microvm_factory.build(guest_kernel_acpi, rootfs, monitor_memory=False)
     vm.spawn(log_level="Info", emit_metrics=True)
-    vm.basic_config(vcpu_count=guest_vcpus, mem_size_mib=guest_mem_mib)
+    vm.basic_config(
+        vcpu_count=guest_vcpus, mem_size_mib=guest_mem_mib, memory_config=memory_config
+    )
     vm.add_net_iface()
     vm.start()
     vm.pin_threads(0)

tests/integration_tests/performance/test_vsock_ab.py

@@ -73,7 +73,14 @@ def guest_command(self, port_offset):
 @pytest.mark.parametrize("payload_length", ["64K", "1024K"], ids=["p64K", "p1024K"])
 @pytest.mark.parametrize("mode", ["g2h", "h2g", "bd"])
 def test_vsock_throughput(
-    microvm_factory, guest_kernel_acpi, rootfs, vcpus, payload_length, mode, metrics
+    microvm_factory,
+    guest_kernel_acpi,
+    rootfs,
+    vcpus,
+    payload_length,
+    mode,
+    metrics,
+    memory_config,
 ):
     """
     Test vsock throughput for multiple vm configurations.
@@ -84,10 +91,15 @@ def test_vsock_throughput(
     if mode == "bd" and vcpus < 2:
         pytest.skip("bidrectional test only done with at least 2 vcpus")
 
+    if memory_config is not None and "6.1" not in guest_kernel_acpi:
+        pytest.skip("swiotlb only supported on aarch64/6.1")
+
     mem_size_mib = 1024
     vm = microvm_factory.build(guest_kernel_acpi, rootfs, monitor_memory=False)
     vm.spawn(log_level="Info", emit_metrics=True)
-    vm.basic_config(vcpu_count=vcpus, mem_size_mib=mem_size_mib)
+    vm.basic_config(
+        vcpu_count=vcpus, mem_size_mib=mem_size_mib, memory_config=memory_config
+    )
     vm.add_net_iface()
     # Create a vsock device
     vm.api.vsock.put(vsock_id="vsock0", guest_cid=3, uds_path="/" + VSOCK_UDS_PATH)