feat(tests): integrate with xcp-ng fixtures

- fix(disk_perf): add configurable numjob
- fix(disk_perf): styling and readability
- misc(disk_perf): simplify fixtures
- misc(disk_perf): styling
- fix(disk_perf): usage of double quote in nested string

Signed-off-by: Mathieu Labourier <[email protected]>

Author: Millefeuille42

tests/storage/benchmarks/conftest.py

@@ -3,46 +3,90 @@
 import os
 import logging
 
-from helpers import load_results_from_csv
+from lib.commands import SSHCommandFailed
+from .helpers import load_results_from_csv
 
-@pytest.fixture(scope='package')
-def ext_sr(host, sr_disk):
-    sr = host.sr_create('ext', "EXT-local-SR-test", {'device': '/dev/' + sr_disk})
-    yield sr
-    # teardown
-    sr.destroy()
-
-@pytest.fixture(scope='module', params=['raw', 'vhd', 'qcow2'])
-def disk_on_ext_sr(request, ext_sr):
-    disk_type = request.param
-    disk = {}
-    if disk_type == 'raw':
-        ...
-    elif disk_type == 'vhd':
-        ...
-    elif disk_type == 'qcow2':
-        ...
+MAX_LENGTH = 64 * (1024**3) # 64GiB
+
+# use vhd, qcow2, raw... when image_format support will be available
+@pytest.fixture(scope="module", params=['vdi'])
+def image_format(request):
+    return request.param
+
+@pytest.fixture(scope="module")
+def running_unix_vm_with_fio(running_unix_vm):
+    vm = running_unix_vm
+    install_cmds = (
+        ("command -v apt", "apt update && apt install -y fio", "apt remove -y fio"),
+        ("command -v dnf", "dnf install -y fio", "dnf remove -y fio"),
+        ("command -v yum", "yum install -y fio", "yum remove -y fio"),
+        ("command -v apk", "apk add fio", "apk del fio")
+    )
+
+    for check_cmd, install_cmd, remove in install_cmds:
+        try:
+            vm.ssh(check_cmd, check=True)
+            logging.info(f">> Installing fio with {install_cmd}")
+            vm.ssh(install_cmd, check=True)
+            remove_cmd = remove
+            break
+        except SSHCommandFailed:
+            ...
     else:
-        raise ValueError(f"Unsupported disk type: {disk_type}")
+        raise RuntimeError("Unsupported package manager: could not install fio")
 
-    yield disk
+    yield vm
 
     # teardown
-    ...
+    logging.info(f"<< Removing fio with {remove_cmd}")
+    vm.ssh(remove_cmd, check=False)
+
+
+@pytest.fixture(scope="module")
+def vdi_on_local_sr(host, local_sr_on_hostA1, image_format):
+    sr = local_sr_on_hostA1
+    vdi = sr.create_vdi("testVDI", MAX_LENGTH)
+    vdi.image_format = image_format
+    logging.info(f">> Created VDI {vdi.uuid} of type {image_format}")
+
+    yield vdi
 
-@pytest.fixture(scope='module')
-def vm_on_ext_sr(host, ext_sr, vm_ref):
-    vm = host.import_vm(vm_ref, sr_uuid=ext_sr.uuid)
-    yield vm
     # teardown
-    logging.info("<< Destroy VM")
-    vm.destroy(verify=True)
+    logging.info(f"<< Destroying VDI {vdi.uuid}")
+    vdi.destroy()
 
-@pytest.fixture
-def temp_dir():
+@pytest.fixture(scope="module")
+def plugged_vbd(vdi_on_local_sr, running_unix_vm_with_fio):
+    vm = running_unix_vm_with_fio
+    vdi = vdi_on_local_sr
+    vbd = vm.create_vbd("autodetect", vdi.uuid)
+
+    logging.info(f">> Plugging VDI {vdi.uuid} on VM {vm.uuid}")
+    vbd.plug()
+
+    yield vbd
+
+    # teardown
+    logging.info(f"<< Unplugging VDI {vdi.uuid} from VM {vm.uuid}")
+    vbd.unplug()
+    vbd.destroy()
+
+@pytest.fixture(scope="module")
+def local_temp_dir():
     with tempfile.TemporaryDirectory() as tmpdir:
         yield tmpdir
 
+@pytest.fixture(scope="module")
+def temp_dir(running_unix_vm_with_fio):
+    vm = running_unix_vm_with_fio
+    tempdir = vm.ssh("mktemp -d")
+
+    yield tempdir
+
+    # teardown
+    vm.ssh(f"rm -r {tempdir}")
+
+
 def pytest_addoption(parser):
     parser.addoption(
         "--prev-csv",
@@ -57,4 +101,4 @@ def prev_results(request):
     results = {}
     if csv_path and os.path.exists(csv_path):
         load_results_from_csv(csv_path)
-    return results
+    return results

tests/storage/benchmarks/helpers.py

@@ -45,4 +45,4 @@ def load_results_from_csv(csv_path):
 def mean(data, key):
     return statistics.mean(
         [float(x[key]) for x in data if key in x]
-    )
+    )

tests/storage/benchmarks/test_disk_perf.py

@@ -2,55 +2,54 @@
 import os
 import json
 import statistics
-import subprocess
 import pytest
 import logging
 from datetime import datetime
 
-from helpers import load_results_from_csv, log_result_csv, mean
+from lib.commands import SSHCommandFailed
+from .helpers import load_results_from_csv, log_result_csv, mean
 
-### Tests default settings ###
+# Tests default settings #
 
-CSV_FILE = f"/tmp/results_{datetime.now().strftime("%Y-%m-%d_%H:%M:%S")}.csv"
+CSV_FILE = f"/tmp/results_{datetime.now().strftime('%Y-%m-%d_%H:%M:%S')}.csv"
 
 DEFAULT_SAMPLES_NUM = 10
 DEFAULT_SIZE = "1G"
 DEFAULT_BS = "4k"
 DEFAULT_IODEPTH = 1
+DEFAULT_NUMJOBS = 1
 DEFAULT_FILE = "fio-testfile"
 
-### Tests parameters
+# Tests parameters #
 
 system_memory = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
 
 block_sizes = ("4k", "16k", "64k", "1M")
-file_sizes = ("1G", "4G", f"{(system_memory/(1024.**3))*2}G")
+file_sizes = ("1G", "4G", f"{int((system_memory // (1024.**3)) * 2)}G")
+
 modes = (
-        "read",
-        "randread",
-        "write",
-        "randwrite"
+    "read",
+    "randread",
+    "write",
+    "randwrite"
 )
 
-test_types = {
-    "read": "seq_read",
-    "randread": "rand_read",
-    "write": "seq_write",
-    "randwrite": "rand_write"
-}
-
-### End of tests parameters ###
+# End of tests parameters #
 
 def run_fio(
+        vm,
         test_name,
         rw_mode,
         temp_dir,
+        local_temp_dir,
         bs=DEFAULT_BS,
         iodepth=DEFAULT_IODEPTH,
         size=DEFAULT_SIZE,
+        numjobs=DEFAULT_NUMJOBS,
         file_path="",
 ):
     json_output_path = os.path.join(temp_dir, f"{test_name}.json")
+    local_json_path = os.path.join(local_temp_dir, f"{test_name}.json")
     if not file_path:
         file_path = os.path.join(temp_dir, DEFAULT_FILE)
     fio_cmd = [
@@ -64,17 +63,19 @@ def run_fio(
         "--direct=1",
         "--end_fsync=1",
         "--fsync_on_close=1",
-        "--numjobs=1",
+        f"--numjobs={numjobs}",
         "--group_reporting",
         "--output-format=json",
         f"--output={json_output_path}"
     ]
-
-    result = subprocess.run(fio_cmd, capture_output=True, text=True)
-    if result.returncode != 0:
-        raise RuntimeError(f"fio failed for {test_name}:\n{result.stderr}")
-
-    with open(json_output_path) as f:
+    logging.debug(f"Running {fio_cmd}")
+    try:
+        vm.ssh(fio_cmd, check=True)
+    except SSHCommandFailed as e:
+        raise RuntimeError(f"fio failed for {test_name}:{e}")
+    vm.scp(json_output_path, local_json_path, local_dest=True)
+    logging.debug(f"Stored json at {local_json_path}")
+    with open(local_json_path) as f:
         return json.load(f)
 
 def assert_performance_not_degraded(current, previous, threshold=10):
@@ -86,7 +87,7 @@ def assert_performance_not_degraded(current, previous, threshold=10):
         except statistics.StatisticsError:
             logging.info(f"Missing metric ({metric}), skipping comparison")
             continue
-        diff = (curr-prev if metric == "latency" else prev-curr) / (prev * 100)
+        diff = (curr - prev if metric == "latency" else prev - curr) / (prev * 100)
         assert diff <= threshold, \
             f"{metric} changed by {diff:.2f}% (allowed {threshold}%)"
         diffs[metric] = diff
@@ -97,27 +98,46 @@ def assert_performance_not_degraded(current, previous, threshold=10):
         logging.info(f"- {k}: {sign}{abs(v):.2f}%")
 
 
-class TestDiskPerfDestroy: ...
-
-
 class TestDiskPerf:
     test_cases = itertools.product(block_sizes, file_sizes, modes)
 
     @pytest.mark.parametrize("block_size,file_size,rw_mode", test_cases)
     def test_disk_benchmark(
             self,
             temp_dir,
+            local_temp_dir,
             prev_results,
             block_size,
             file_size,
-            rw_mode
+            rw_mode,
+            running_unix_vm_with_fio,
+            plugged_vbd,
+            image_format
     ):
-        test_type = test_types[rw_mode]
+        vm = running_unix_vm_with_fio
+        vbd = plugged_vbd
+        device = f"/dev/{vbd.param_get(param_name='device')}"
+        test_type = "{}-{}-{}-{}".format(
+            block_size,
+            file_size,
+            rw_mode,
+            image_format
+        )
+
         for i in range(DEFAULT_SAMPLES_NUM):
-            result = run_fio(test_type, rw_mode, temp_dir)
+            result = run_fio(
+                vm,
+                test_type,
+                rw_mode,
+                temp_dir,
+                local_temp_dir,
+                file_path=device,
+                bs=block_size,
+                size=file_size
+            )
             summary = log_result_csv(test_type, rw_mode, result, CSV_FILE)
             assert summary["IOPS"] > 0
-        results = load_results_from_csv(CSV_FILE)
         key = (test_type, rw_mode)
         if prev_results and key in prev_results:
+            results = load_results_from_csv(CSV_FILE)
             assert_performance_not_degraded(results[key], prev_results[key])