feat(disk_perf): parameterize most test variables

- fix(disk_perf): csv paths and add logging

Signed-off-by: Mathieu Labourier <[email protected]>

Author: Millefeuille42

tests/storage/benchmarks/conftest.py

@@ -1,3 +1,4 @@
+import itertools
 import logging
 import os
 import tempfile
@@ -9,7 +10,7 @@
 
 from lib.commands import SSHCommandFailed
 
-from .helpers import load_results_from_csv
+from .helpers import load_results_from_csv, str_to_tuple
 
 MAX_LENGTH = 64 * (1024**3)  # 64GiB
 
@@ -98,12 +99,57 @@ def temp_dir(running_unix_vm_with_fio):
 
 
 def pytest_addoption(parser):
+    system_memory = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
+
     parser.addoption(
         "--prev-csv",
         action="store",
         default=None,
         help="Path/URI to previous CSV results file for comparison",
     )
+    parser.addoption(
+        "--block-sizes",
+        action="store",
+        type=lambda value: str_to_tuple(value, sep=","),
+        default=("4k", "16k", "64k", "1M"),
+        help="Comma separated values of block sizes to test in disk benchmarks",
+    )
+    parser.addoption(
+        "--file-sizes",
+        action="store",
+        type=lambda value: str_to_tuple(value, sep=","),
+        default=("1G", "4G", f"{int((system_memory // (1024.**3)) * 2)}G"),
+        help="Comma separated values of file sizes to test in disk benchmarks",
+    )
+    parser.addoption(
+        "--modes",
+        action="store",
+        type=lambda value: str_to_tuple(value, sep=","),
+        default=("read", "randread", "write", "randwrite"),
+        help="Comma separated values of rw_modes to test in disk benchmarks",
+    )
+    parser.addoption(
+        "--numjobs",
+        action="store",
+        default=1,
+        help="Mapped to fio's --numjobs",
+    )
+    parser.addoption(
+        "--iodepth",
+        action="store",
+        default=1,
+        help="Mapped to fio's --iodepth",
+    )
+
+
+def pytest_generate_tests(metafunc):
+    if {"block_size", "file_size", "rw_mode"} <= set(metafunc.fixturenames):
+        block_sizes = metafunc.config.getoption("block_sizes")
+        file_sizes = metafunc.config.getoption("file_sizes")
+        modes = metafunc.config.getoption("modes")
+
+        test_cases = list(itertools.product(block_sizes, file_sizes, modes))
+        metafunc.parametrize("block_size,file_size,rw_mode", test_cases)
 
 
 @pytest.fixture(scope="session")
@@ -113,8 +159,10 @@ def prev_results(pytestconfig):
         return {}
     csv_path = csv_uri
     if urlparse(csv_uri).scheme != "":
-        csv_path = f"{uuid4()}.csv"
+        logging.info("Detected CSV path as an url")
+        csv_path = f"/tmp/{uuid4()}.csv"
         urllib.request.urlretrieve(csv_uri, csv_path)
+        logging.info(f"Fetching CSV file from {csv_uri} to {csv_path}")
     if not os.path.exists(csv_path):
         raise FileNotFoundError(csv_path)
     return load_results_from_csv(csv_path)

tests/storage/benchmarks/helpers.py

@@ -3,6 +3,8 @@
 import statistics
 from datetime import datetime
 
+system_memory = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
+
 
 def log_result_csv(test_type, rw_mode, result_json, csv_path):
     job = result_json["jobs"][0]
@@ -44,3 +46,7 @@ def load_results_from_csv(csv_path):
 
 def mean(data, key):
     return statistics.mean([float(x[key]) for x in data if key in x])
+
+
+def str_to_tuple(value, sep=","):
+    return tuple(item.strip() for item in value.split(sep))

tests/storage/benchmarks/test_disk_perf.py

@@ -1,4 +1,3 @@
-import itertools
 import json
 import logging
 import os
@@ -13,7 +12,7 @@
 
 # Tests default settings #
 
-CSV_FILE = f"/tmp/results_{datetime.now().strftime('%Y-%m-%d_%H:%M:%S')}.csv"
+CSV_FILE = f"/tmp/results_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.csv"
 
 DEFAULT_SAMPLES_NUM = 10
 DEFAULT_SIZE = "1G"
@@ -22,17 +21,6 @@
 DEFAULT_NUMJOBS = 1
 DEFAULT_FILE = "fio-testfile"
 
-# Tests parameters #
-
-system_memory = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
-
-block_sizes = ("4k", "16k", "64k", "1M")
-file_sizes = ("1G", "4G", f"{int((system_memory // (1024.**3)) * 2)}G")
-
-modes = ("read", "randread", "write", "randwrite")
-
-# End of tests parameters #
-
 
 def run_fio(
     vm,
@@ -99,11 +87,11 @@ def assert_performance_not_degraded(current, previous, threshold=10):
 
 
 class TestDiskPerf:
-    test_cases = itertools.product(block_sizes, file_sizes, modes)
 
     @pytest.mark.parametrize("block_size,file_size,rw_mode", test_cases)
     def test_disk_benchmark(
         self,
+        pytestconfig,
         temp_dir,
         local_temp_dir,
         prev_results,
@@ -129,6 +117,8 @@ def test_disk_benchmark(
                 file_path=device,
                 bs=block_size,
                 size=file_size,
+                iodepth=pytestconfig.getoption("iodepth"),
+                numjobs=pytestconfig.getoption("numjobs"),
             )
             summary = log_result_csv(test_type, rw_mode, result, CSV_FILE)
             assert summary["IOPS"] > 0