feat(tests): add disk performance benchmark tests

Signed-off-by: Mathieu Labourier <[email protected]>

Author: Millefeuille42

tests/storage/benchmarks/__init__.py

@@ -0,0 +1,60 @@
+import pytest
+import tempfile
+import os
+import logging
+
+from helpers import load_results_from_csv
+
+@pytest.fixture(scope='package')
+def ext_sr(host, sr_disk):
+    sr = host.sr_create('ext', "EXT-local-SR-test", {'device': '/dev/' + sr_disk})
+    yield sr
+    # teardown
+    sr.destroy()
+
+@pytest.fixture(scope='module', params=['raw', 'vhd', 'qcow2'])
+def disk_on_ext_sr(request, ext_sr):
+    disk_type = request.param
+    disk = {}
+    if disk_type == 'raw':
+        ...
+    elif disk_type == 'vhd':
+        ...
+    elif disk_type == 'qcow2':
+        ...
+    else:
+        raise ValueError(f"Unsupported disk type: {disk_type}")
+
+    yield disk
+
+    # teardown
+    ...
+
+@pytest.fixture(scope='module')
+def vm_on_ext_sr(host, ext_sr, vm_ref):
+    vm = host.import_vm(vm_ref, sr_uuid=ext_sr.uuid)
+    yield vm
+    # teardown
+    logging.info("<< Destroy VM")
+    vm.destroy(verify=True)
+
+@pytest.fixture
+def temp_dir():
+    with tempfile.TemporaryDirectory() as tmpdir:
+        yield tmpdir
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--prev-csv",
+        action="store",
+        default=None,
+        help="Path to previous CSV results file for comparison",
+    )
+
+@pytest.fixture(scope="session")
+def prev_results(request):
+    csv_path = request.config.getoption("--prev-csv")
+    results = {}
+    if csv_path and os.path.exists(csv_path):
+        load_results_from_csv(csv_path)
+    return results

tests/storage/benchmarks/conftest.py

@@ -0,0 +1,48 @@
+import os
+import statistics
+import csv
+from datetime import datetime
+
+
+def log_result_csv(test_type, rw_mode, result_json, csv_path):
+    job = result_json["jobs"][0]
+    op_data = job[rw_mode.replace("rand", "")]
+    bw_kbps = op_data["bw"]
+    iops = op_data["iops"]
+    latency = op_data["lat_ns"]["mean"]
+
+    result = {
+        "timestamp": datetime.now().isoformat(),
+        "test": test_type,
+        "mode": rw_mode,
+        "bw_MBps": round(bw_kbps / 1024, 2),
+        "IOPS": round(iops, 2),
+        "latency": round(latency, 2),
+    }
+
+    file_exists = os.path.exists(csv_path)
+    with open(csv_path, "a", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=result.keys())
+        if not file_exists:
+            writer.writeheader()
+        writer.writerow(result)
+
+    return result
+
+
+def load_results_from_csv(csv_path):
+    results = {}
+    with open(csv_path, newline="") as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            key = (row["test"], row["mode"])
+            if key not in results:
+                results[key] = []
+            results[key].append(row)
+    return results
+
+
+def mean(data, key):
+    return statistics.mean(
+        [float(x[key]) for x in data if key in x]
+    )

tests/storage/benchmarks/helpers.py

@@ -0,0 +1,123 @@
+import itertools
+import os
+import json
+import statistics
+import subprocess
+import pytest
+import logging
+from datetime import datetime
+
+from helpers import load_results_from_csv, log_result_csv, mean
+
+### Tests default settings ###
+
+CSV_FILE = f"/tmp/results_{datetime.now().strftime("%Y-%m-%d_%H:%M:%S")}.csv"
+
+DEFAULT_SAMPLES_NUM = 10
+DEFAULT_SIZE = "1G"
+DEFAULT_BS = "4k"
+DEFAULT_IODEPTH = 1
+DEFAULT_FILE = "fio-testfile"
+
+### Tests parameters
+
+system_memory = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
+
+block_sizes = ("4k", "16k", "64k", "1M")
+file_sizes = ("1G", "4G", f"{(system_memory/(1024.**3))*2}G")
+modes = (
+        "read",
+        "randread",
+        "write",
+        "randwrite"
+)
+
+test_types = {
+    "read": "seq_read",
+    "randread": "rand_read",
+    "write": "seq_write",
+    "randwrite": "rand_write"
+}
+
+### End of tests parameters ###
+
+def run_fio(
+        test_name,
+        rw_mode,
+        temp_dir,
+        bs=DEFAULT_BS,
+        iodepth=DEFAULT_IODEPTH,
+        size=DEFAULT_SIZE,
+        file_path="",
+):
+    json_output_path = os.path.join(temp_dir, f"{test_name}.json")
+    if not file_path:
+        file_path = os.path.join(temp_dir, DEFAULT_FILE)
+    fio_cmd = [
+        "fio",
+        f"--name={test_name}",
+        f"--rw={rw_mode}",
+        f"--bs={bs}",
+        f"--iodepth={iodepth}",
+        f"--size={size}",
+        f"--filename={file_path}",
+        "--direct=1",
+        "--end_fsync=1",
+        "--fsync_on_close=1",
+        "--numjobs=1",
+        "--group_reporting",
+        "--output-format=json",
+        f"--output={json_output_path}"
+    ]
+
+    result = subprocess.run(fio_cmd, capture_output=True, text=True)
+    if result.returncode != 0:
+        raise RuntimeError(f"fio failed for {test_name}:\n{result.stderr}")
+
+    with open(json_output_path) as f:
+        return json.load(f)
+
+def assert_performance_not_degraded(current, previous, threshold=10):
+    diffs = {}
+    for metric in ("bw_MBps", "IOPS", "latency"):
+        try:
+            curr = mean(current, metric)
+            prev = mean(previous, metric)
+        except statistics.StatisticsError:
+            logging.info(f"Missing metric ({metric}), skipping comparison")
+            continue
+        diff = (curr-prev if metric == "latency" else prev-curr) / (prev * 100)
+        assert diff <= threshold, \
+            f"{metric} changed by {diff:.2f}% (allowed {threshold}%)"
+        diffs[metric] = diff
+
+    logging.info("Performance difference summary:")
+    for k, v in diffs.items():
+        sign = "+" if v < 0 else "-"
+        logging.info(f"- {k}: {sign}{abs(v):.2f}%")
+
+
+class TestDiskPerfDestroy: ...
+
+
+class TestDiskPerf:
+    test_cases = itertools.product(block_sizes, file_sizes, modes)
+
+    @pytest.mark.parametrize("block_size,file_size,rw_mode", test_cases)
+    def test_disk_benchmark(
+            self,
+            temp_dir,
+            prev_results,
+            block_size,
+            file_size,
+            rw_mode
+    ):
+        test_type = test_types[rw_mode]
+        for i in range(DEFAULT_SAMPLES_NUM):
+            result = run_fio(test_type, rw_mode, temp_dir)
+            summary = log_result_csv(test_type, rw_mode, result, CSV_FILE)
+            assert summary["IOPS"] > 0
+        results = load_results_from_csv(CSV_FILE)
+        key = (test_type, rw_mode)
+        if prev_results and key in prev_results:
+            assert_performance_not_degraded(results[key], prev_results[key])