tests: refactor data_integrity test

Signed-off-by: Kamil Gierszewski <kamil.gierszewski@huawei.com>

Author: Kamil Gierszewski

test/functional/tests/data_integrity/test_data_integrity_5d.py

@@ -16,8 +16,13 @@
 from api.cas.cache_config import CacheMode
 from api.cas.ioclass_config import IoClass
 from core.test_run import TestRun
-from test_tools.fs_tools import Filesystem, create_directory, check_if_directory_exists, \
-    read_file, crc32sum
+from test_tools.fs_tools import (
+    Filesystem,
+    create_directory,
+    check_if_directory_exists,
+    read_file,
+    crc32sum,
+)
 from test_tools.fio.fio import Fio
 from test_tools.fio.fio_param import ReadWrite, IoEngine, VerifyMethod
 from storage_devices.disk import DiskType, DiskTypeSet, DiskTypeLowerThan
@@ -53,11 +58,13 @@ def test_data_integrity_5d_with_io_classification(filesystems):
     runtime = datetime.timedelta(days=5)
 
     with TestRun.step("Prepare cache and core devices"):
-        cache_device = TestRun.disks['cache']
-        core_devices = [TestRun.disks['core1'],
-                        TestRun.disks['core2'],
-                        TestRun.disks['core3'],
-                        TestRun.disks['core4']]
+        cache_device = TestRun.disks["cache"]
+        core_devices = [
+            TestRun.disks["core1"],
+            TestRun.disks["core2"],
+            TestRun.disks["core3"],
+            TestRun.disks["core4"],
+        ]
 
         cache_device.create_partitions([Size(50, Unit.GibiByte)] * len(core_devices))
 
@@ -76,8 +83,7 @@ def test_data_integrity_5d_with_io_classification(filesystems):
 
     with TestRun.step("Add one core to each cache"):
         cores = [
-            cache.add_core(core_dev=core_part)
-            for cache, core_part in zip(caches, core_partitions)
+            cache.add_core(core_dev=core_part) for cache, core_part in zip(caches, core_partitions)
         ]
 
     with TestRun.step("Load default I/O class config for each cache"):
@@ -90,7 +96,7 @@ def test_data_integrity_5d_with_io_classification(filesystems):
 
     with TestRun.step("Mount cached volumes"):
         for core in cores:
-            mount_point = core.path.replace('/dev/', '/mnt/')
+            mount_point = core.path.replace("/dev/", "/mnt/")
             if not check_if_directory_exists(mount_point):
                 create_directory(mount_point)
             core.mount(mount_point)
@@ -99,32 +105,31 @@ def test_data_integrity_5d_with_io_classification(filesystems):
     with TestRun.step("Prepare fio workload config"):
         template_io_classes = IoClass.csv_to_list(read_file(template_config_path))
         config_max_file_sizes = [
-            int(re.search(r'\d+', io_class.rule).group())
-            for io_class in template_io_classes if io_class.rule.startswith("file_size:le")
+            int(re.search(r"\d+", io_class.rule).group())
+            for io_class in template_io_classes
+            if io_class.rule.startswith("file_size:le")
         ]
         config_max_file_sizes.append(config_max_file_sizes[-1] * 2)
         io_class_section_size = Size(
-            int(core_size.get_value(Unit.GibiByte) / len(config_max_file_sizes)),
-            Unit.GibiByte
-        )
-
-        fio = Fio()
-        fio_run = fio.create_command()
-        fio.base_cmd_parameters.set_param(
-            'alloc-size', int(Size(1, Unit.GiB).get_value(Unit.KiB))
+            int(core_size.get_value(Unit.GibiByte) / len(config_max_file_sizes)), Unit.GibiByte
         )
 
-        fio_run.io_engine(IoEngine.libaio)
-        fio_run.direct()
-        fio_run.time_based()
-        fio_run.do_verify()
-        fio_run.verify(VerifyMethod.md5)
-        fio_run.verify_dump()
-        fio_run.run_time(runtime)
-        fio_run.read_write(ReadWrite.randrw)
-        fio_run.io_depth(128)
-        fio_run.blocksize_range(
-            [(Size(512, Unit.Byte).get_value(), Size(128, Unit.KibiByte).get_value())]
+        fio_run = (
+            Fio()
+            .create_command(alloc_size=Size(1, Unit.GibiByte))
+            .io_engine(IoEngine.libaio)
+            .direct()
+            .time_based()
+            .do_verify()
+            .verify(VerifyMethod.md5)
+            .verify_dump()
+            .run_time(runtime)
+            .read_write(ReadWrite.randrw)
+            .open_files(1000)
+            .io_depth(128)
+            .blocksize_range(
+                [(Size(512, Unit.Byte).get_value(), Size(128, Unit.KibiByte).get_value())]
+            )
         )
 
         for core in cores:
@@ -157,12 +162,14 @@ def test_data_integrity_5d_with_io_classification(filesystems):
         dev_crc32s = [crc32sum(dev.path, timeout=timedelta(hours=4)) for dev in core_devices]
 
     with TestRun.step("Compare crc32 sums for cores and core devices"):
-        for core_crc32, dev_crc32, mode, fs in zip(
-                core_crc32s, dev_crc32s, cache_modes, filesystems
+        for core_crc32, dev_crc32, cache_mode, filesystem in zip(
+            core_crc32s, dev_crc32s, cache_modes, filesystems
         ):
             if core_crc32 != dev_crc32:
-                TestRun.fail("Crc32 sums of core and core device do not match! "
-                             f"Cache mode: {mode} Filesystem: {fs}")
+                TestRun.fail(
+                    "Crc32 sums of core and core device do not match!\n"
+                    f"Cache mode: {cache_mode} Filesystem: {filesystem}"
+                )
 
 
 @pytest.mark.os_dependent
@@ -186,11 +193,13 @@ def test_data_integrity_5d():
     runtime = datetime.timedelta(days=5)
 
     with TestRun.step("Prepare cache and core devices"):
-        cache_device = TestRun.disks['cache']
-        core_devices = [TestRun.disks['core1'],
-                        TestRun.disks['core2'],
-                        TestRun.disks['core3'],
-                        TestRun.disks['core4']]
+        cache_device = TestRun.disks["cache"]
+        core_devices = [
+            TestRun.disks["core1"],
+            TestRun.disks["core2"],
+            TestRun.disks["core3"],
+            TestRun.disks["core4"],
+        ]
 
         cache_device.create_partitions([Size(50, Unit.GibiByte)] * len(core_devices))
 
@@ -213,19 +222,23 @@ def test_data_integrity_5d():
         ]
 
     with TestRun.step("Prepare fio workload config"):
-        fio_run = Fio().create_command()
-        fio_run.io_engine(IoEngine.libaio)
-        fio_run.direct()
-        fio_run.time_based()
-        fio_run.do_verify()
-        fio_run.verify(VerifyMethod.md5)
-        fio_run.verify_dump()
-        fio_run.run_time(runtime)
-        fio_run.read_write(ReadWrite.randrw)
-        fio_run.io_depth(128)
-        fio_run.blocksize_range(
-            [(Size(512, Unit.Byte).get_value(), Size(128, Unit.KibiByte).get_value())]
+        fio_run = (
+            Fio()
+            .create_command()
+            .io_engine(IoEngine.pvsync)
+            .direct()
+            .time_based()
+            .do_verify()
+            .verify(VerifyMethod.md5)
+            .verify_dump()
+            .run_time(runtime)
+            .read_write(ReadWrite.randrw)
+            .io_depth(128)
+            .blocksize_range(
+                [(Size(512, Unit.Byte).get_value(), Size(128, Unit.KibiByte).get_value())]
+            )
         )
+
         for core in cores:
             fio_job = fio_run.add_job()
             fio_job.target(core)
@@ -244,7 +257,9 @@ def test_data_integrity_5d():
         dev_crc32s = [crc32sum(dev.path, timeout=timedelta(hours=4)) for dev in core_devices]
 
     with TestRun.step("Compare crc32 sums for cores and core devices"):
-        for core_crc32, dev_crc32, mode in zip(core_crc32s, dev_crc32s, cache_modes):
+        for core_crc32, dev_crc32, cache_mode in zip(core_crc32s, dev_crc32s, cache_modes):
             if core_crc32 != dev_crc32:
-                TestRun.fail("Crc32 sums of core and core device do not match! "
-                             f"Cache mode: {mode}")
+                TestRun.fail(
+                    "Crc32 sums of core and core device do not match!\n" 
+                    f"Cache mode with wrong crc32 sum: {cache_mode}"
+                )