Couple of improvements for faster tests

test/functional/api/cas/cache.py

@@ -1,5 +1,5 @@
 #
-# Copyright(c) 2019-2021 Intel Corporation
+# Copyright(c) 2019-2022 Intel Corporation
 # SPDX-License-Identifier: BSD-3-Clause
 #
 
@@ -17,12 +17,12 @@ def __init__(self, device: Device):
         self.__metadata_size = None
 
     def __get_cache_id(self):
-        cmd = f"{list_cmd(by_id_path=False)} | grep {self.cache_device.get_device_id()}"
+        cmd = f"{list_cmd(by_id_path=False)} | grep {self.cache_device.device_id}"
         output = TestRun.executor.run(cmd)
         if output.exit_code == 0 and output.stdout.strip():
             return output.stdout.split()[1]
         else:
-            raise Exception(f"There is no cache started on {self.cache_device.get_device_id()}.")
+            raise Exception(f"There is no cache started on {self.cache_device.device_id}.")
 
     def get_core_devices(self):
         return get_cores(self.cache_id)

test/functional/tests/cli/test_zero_metadata_command.py

@@ -1,5 +1,5 @@
 #
-# Copyright(c) 2021 Intel Corporation
+# Copyright(c) 2021-2022 Intel Corporation
 # SPDX-License-Identifier: BSD-3-Clause
 #
 import time
@@ -110,7 +110,7 @@ def test_zero_metadata_filesystem(filesystem):
         except CmdException as e:
             cli_messages.check_stderr_msg(e.output, cli_messages.no_cas_metadata)
 
-        file_system = get_device_filesystem_type(core.get_device_id())
+        file_system = get_device_filesystem_type(core.device_id)
 
         if file_system != filesystem:
             TestRun.LOGGER.error(f"Incorrect filesystem: {file_system}; expected: {filesystem}")

test/functional/tests/conftest.py

@@ -19,16 +19,12 @@
 from api.cas import casadm
 from api.cas import git
 from api.cas.cas_service import opencas_drop_in_directory
-from storage_devices.raid import Raid
-from storage_devices.ramdisk import RamDisk
-from test_utils.os_utils import Udev, kill_all_io
-from test_utils.disk_finder import get_disk_serial_number
-from test_tools.disk_utils import PartitionTable, create_partition_table
-from test_tools.device_mapper import DeviceMapper
-from test_tools.mdadm import Mdadm
-from test_tools.fs_utils import remove
-from log.logger import create_log, Log
+from api.cas.init_config import InitConfig
+from log.logger import create_log
 from test_utils.singleton import Singleton
+from test_utils.output import CmdException
+from test_tools.fs_utils import remove
+from storage_devices.drbd import Drbd
 
 
 class Opencas(metaclass=Singleton):
@@ -57,61 +53,81 @@ def pytest_runtest_setup(item):
     #
     # User can also have own test wrapper, which runs test prepare, cleanup, etc.
     # Then it should be placed in plugins package
+    duts = []
+    for dut_file in item.config.getoption("--dut-config"):
+        try:
+            with open(dut_file) as df:
+                dut_config = yaml.safe_load(df)
+        except Exception as ex:
+            raise Exception(
+                "You need to specify DUT config. See the example_dut_config.py file"
+            ) from ex
+
+        dut_config["extra_logs"] = {"cas": "/var/log/opencas.log"}
+        duts.append(dut_config)
 
-    test_name = item.name.split('[')[0]
-    TestRun.LOGGER = create_log(item.config.getoption('--log-path'), test_name)
+    log_path = item.config.getoption("--log-path")
+    test_name = item.name.split("[")[0]
+    logger = create_log(log_path, test_name)
 
-    duts = item.config.getoption('--dut-config')
-    required_duts = next(item.iter_markers(name="multidut"), None)
-    required_duts = required_duts.args[0] if required_duts is not None else 1
-    if required_duts > len(duts):
-        raise Exception(f"Test requires {required_duts} DUTs, only {len(duts)} DUT configs "
-                        f"provided")
-    else:
-        duts = duts[:required_duts]
+    TestRun.start(logger, duts, item)
+    for dut in TestRun.use_all_duts():
+        if not installer.check_if_installed():
+            continue
+        TestRun.LOGGER.info(f"CAS cleanup on {dut.ip}")
+        remove(opencas_drop_in_directory, recursive=True, ignore_errors=True)
 
-    TestRun.duts = []
-    for dut in duts:
         try:
-            with open(dut) as cfg:
-                dut_config = yaml.safe_load(cfg)
-        except Exception as ex:
-            raise Exception(f"{ex}\n"
-                            f"You need to specify DUT config. See the example_dut_config.py file")
+            InitConfig.create_default_init_config()
+            unmount_cas_devices()
+            try:
+                casadm.stop_all_caches()
+            except CmdException:
+                TestRun.LOGGER.warning(
+                    "Failed to stop all caches, will retry after stopping DRBD"
+                )
+            casadm.remove_all_detached_cores()
+        except Exception as e:
+            raise Exception("Exception occured during CAS cleanup:\n"
+                            f"{str(e)}\n{traceback.format_exc()}")
 
-        dut_config['plugins_dir'] = os.path.join(os.path.dirname(__file__), "../lib")
-        dut_config['opt_plugins'] = {"test_wrapper": {}, "serial_log": {}, "power_control": {}}
-        dut_config['extra_logs'] = {"cas": "/var/log/opencas.log"}
+        if Drbd.is_installed():
+            # TODO: Need proper stacking devices teardown
+            TestRun.LOGGER.workaround("Stopping DRBD")
+            Drbd.down_all()
 
         try:
-            TestRun.prepare(item, dut_config)
+            casadm.stop_all_caches()
+        except CmdException:
+            TestRun.LOGGER.blocked("Failed to stop all caches")
 
-            TestRun.presetup()
-            try:
-                TestRun.executor.wait_for_connection(timedelta(seconds=20))
-            except paramiko.AuthenticationException:
-                raise
-            except Exception:
-                try:
-                    TestRun.plugin_manager.get_plugin('power_control').power_cycle()
-                    TestRun.executor.wait_for_connection()
-                except Exception:
-                    raise Exception("Failed to connect to DUT.")
-            TestRun.setup()
-        except Exception as ex:
-            raise Exception(f"Exception occurred during test setup:\n"
-                            f"{str(ex)}\n{traceback.format_exc()}")
+    TestRun.prepare()
+
+    # If some generic device was set-up on top of CAS it failed to stop, try to stop it again
+    if installer.check_if_installed():
+        casadm.stop_all_caches()
+
+    TestRun.usr = Opencas(
+        repo_dir=os.path.join(os.path.dirname(__file__), "../../.."),
+        working_dir=TestRun.working_dir)
+
+    cas_version = TestRun.config.get("cas_version") or git.get_current_commit_hash()
+    for i, dut in enumerate(TestRun.use_all_duts()):
+        if get_force_param(item) and not TestRun.usr.already_updated:
+            installer.rsync_opencas_sources()
+            installer.reinstall_opencas(cas_version)
+        elif not installer.check_if_installed(cas_version):
+            installer.rsync_opencas_sources()
+            installer.set_up_opencas(cas_version)
+        TestRun.LOGGER.info(f"DUT-{i} info: {dut}")
 
-        TestRun.usr = Opencas(
-            repo_dir=os.path.join(os.path.dirname(__file__), "../../.."),
-            working_dir=dut_config['working_dir'])
+    TestRun.usr.already_updated = True
 
-        TestRun.LOGGER.info(f"DUT info: {TestRun.dut}")
-        TestRun.dut.plugin_manager = TestRun.plugin_manager
-        TestRun.dut.executor = TestRun.executor
-        TestRun.duts.append(TestRun.dut)
+    TestRun.LOGGER.add_build_info(f'Commit hash:')
+    TestRun.LOGGER.add_build_info(f"{git.get_current_commit_hash()}")
+    TestRun.LOGGER.add_build_info(f'Commit message:')
+    TestRun.LOGGER.add_build_info(f'{git.get_current_commit_message()}')
 
-        base_prepare(item)
     TestRun.LOGGER.write_to_command_log("Test body")
     TestRun.LOGGER.start_group("Test body")
 
@@ -127,46 +143,6 @@ def pytest_runtest_teardown():
     This method is executed always in the end of each test, even if it fails or raises exception in
     prepare stage.
     """
-    TestRun.LOGGER.end_all_groups()
-
-    with TestRun.LOGGER.step("Cleanup after test"):
-        try:
-            if TestRun.executor:
-                if not TestRun.executor.is_active():
-                    TestRun.executor.wait_for_connection()
-                Udev.enable()
-                kill_all_io()
-                unmount_cas_devices()
-
-                if installer.check_if_installed():
-                    casadm.remove_all_detached_cores()
-                    casadm.stop_all_caches()
-                    from api.cas.init_config import InitConfig
-                    InitConfig.create_default_init_config()
-
-                from storage_devices.drbd import Drbd
-                if installer.check_if_installed() and Drbd.is_installed():
-                    try:
-                        casadm.stop_all_caches()
-                    finally:
-                        __drbd_cleanup()
-                elif Drbd.is_installed():
-                    Drbd.down_all()
-
-                DeviceMapper.remove_all()
-                RamDisk.remove_all()
-        except Exception as ex:
-            TestRun.LOGGER.warning(f"Exception occurred during platform cleanup.\n"
-                                   f"{str(ex)}\n{traceback.format_exc()}")
-
-    TestRun.LOGGER.end()
-    for dut in TestRun.duts:
-        with TestRun.use_dut(dut):
-            if TestRun.executor:
-                os.makedirs(os.path.join(TestRun.LOGGER.base_dir, "dut_info", dut.ip),
-                            exist_ok=True)
-                TestRun.LOGGER.get_additional_logs()
-    Log.destroy()
     TestRun.teardown()
 
 
@@ -211,80 +187,3 @@ def unmount_cas_devices():
 def get_force_param(item):
     return item.config.getoption("--force-reinstall")
 
-
-def __drbd_cleanup():
-    from storage_devices.drbd import Drbd
-    Drbd.down_all()
-    # If drbd instance had been configured on top of the CAS, the previos attempt to stop
-    # failed. As drbd has been stopped try to stop CAS one more time.
-    if installer.check_if_installed():
-        casadm.stop_all_caches()
-
-
-def base_prepare(item):
-    with TestRun.LOGGER.step("Cleanup before test"):
-        TestRun.executor.run("pkill --signal=SIGKILL fsck")
-        Udev.enable()
-        kill_all_io()
-        DeviceMapper.remove_all()
-
-        if installer.check_if_installed():
-            try:
-                from api.cas.init_config import InitConfig
-                InitConfig.create_default_init_config()
-                unmount_cas_devices()
-                casadm.stop_all_caches()
-                casadm.remove_all_detached_cores()
-            except Exception:
-                pass  # TODO: Reboot DUT if test is executed remotely
-
-        remove(opencas_drop_in_directory, recursive=True, ignore_errors=True)
-
-        from storage_devices.drbd import Drbd
-        if Drbd.is_installed():
-            __drbd_cleanup()
-
-        raids = Raid.discover()
-        for raid in raids:
-            # stop only those RAIDs, which are comprised of test disks
-            if all(map(lambda device:
-                       any(map(lambda disk_path:
-                               disk_path in device.get_device_id(),
-                               [bd.get_device_id() for bd in TestRun.dut.disks])),
-                       raid.array_devices)):
-                raid.umount_all_partitions()
-                raid.remove_partitions()
-                raid.stop()
-                for device in raid.array_devices:
-                    Mdadm.zero_superblock(os.path.join('/dev', device.get_device_id()))
-                    Udev.settle()
-
-        RamDisk.remove_all()
-
-        for disk in TestRun.dut.disks:
-            disk_serial = get_disk_serial_number(disk.path)
-            if disk.serial_number != disk_serial:
-                raise Exception(
-                    f"Serial for {disk.path} doesn't match the one from the config."
-                    f"Serial from config {disk.serial_number}, actual serial {disk_serial}"
-                )
-
-            disk.umount_all_partitions()
-            Mdadm.zero_superblock(os.path.join('/dev', disk.get_device_id()))
-            TestRun.executor.run_expect_success("udevadm settle")
-            disk.remove_partitions()
-            create_partition_table(disk, PartitionTable.gpt)
-
-        cas_version = TestRun.config.get("cas_version") or git.get_current_commit_hash()
-        if get_force_param(item) and not TestRun.usr.already_updated:
-            installer.rsync_opencas_sources()
-            installer.reinstall_opencas(cas_version)
-        elif not installer.check_if_installed(cas_version):
-            installer.rsync_opencas_sources()
-            installer.set_up_opencas(cas_version)
-
-        TestRun.usr.already_updated = True
-        TestRun.LOGGER.add_build_info(f'Commit hash:')
-        TestRun.LOGGER.add_build_info(f"{git.get_current_commit_hash()}")
-        TestRun.LOGGER.add_build_info(f'Commit message:')
-        TestRun.LOGGER.add_build_info(f'{git.get_current_commit_message()}')

test/functional/tests/misc/test_device_capabilities.py

@@ -1,5 +1,5 @@
 #
-# Copyright(c) 2020-2021 Intel Corporation
+# Copyright(c) 2020-2022 Intel Corporation
 # SPDX-License-Identifier: BSD-3-Clause
 #
 
@@ -32,7 +32,7 @@ def test_device_capabilities():
     """
 
     core_device = TestRun.disks['core']
-    max_io_size_path = os.path.join(disk_utils.get_sysfs_path(core_device.get_device_id()),
+    max_io_size_path = os.path.join(disk_utils.get_sysfs_path(core_device.device_id),
                                     'queue/max_sectors_kb')
     default_max_io_size = fs_utils.read_file(max_io_size_path)
 
@@ -104,8 +104,8 @@ def create_scsi_debug_device(sector_size: int, physblk_exp: int, dev_size_mb=102
 def prepare_cas_device(cache_device, core_device):
     cache = casadm.start_cache(cache_device, cache_line_size=CacheLineSize.LINE_64KiB, force=True)
     try:
-        cache_dev_bs = disk_utils.get_block_size(cache_device.get_device_id())
-        core_dev_bs = disk_utils.get_block_size(core_device.get_device_id())
+        cache_dev_bs = disk_utils.get_block_size(cache_device.device_id)
+        core_dev_bs = disk_utils.get_block_size(core_device.device_id)
         core = cache.add_core(core_device)
         if cache_dev_bs > core_dev_bs:
             TestRun.LOGGER.error(
@@ -145,8 +145,8 @@ def method_lcm_not_zero(a, b):
 
 def measure_capabilities(dev):
     dev_capabilities = {}
-    dev_id = dev.parent_device.get_device_id() if isinstance(dev, Partition) \
-        else dev.get_device_id()
+    dev_id = dev.parent_device.device_id if isinstance(dev, Partition) \
+        else dev.device_id
     for c in capabilities:
         path = os.path.join(disk_utils.get_sysfs_path(dev_id), 'queue', c)
         command = f"cat {path}"
@@ -165,10 +165,10 @@ def compare_capabilities(cache_device, core_device, cache, core, msg):
                                       cli_messages.try_add_core_sector_size_mismatch)
     else:
         core_dev_sectors_num = \
-            disk_utils.get_size(core_device.get_device_id()) / disk_utils.get_block_size(
-                core_device.get_device_id())
-        core_sectors_num = disk_utils.get_size(core.get_device_id()) / disk_utils.get_block_size(
-            core.get_device_id())
+            disk_utils.get_size(core_device.device_id) / disk_utils.get_block_size(
+                core_device.device_id)
+        core_sectors_num = disk_utils.get_size(core.device_id) / disk_utils.get_block_size(
+            core.device_id)
         if core_dev_sectors_num != core_sectors_num:
             TestRun.LOGGER.error(
                 "Number of sectors in CAS device and attached core device is different.")