diff --git a/test/functional/api/cas/cache.py b/test/functional/api/cas/cache.py index abc13d883..87d6c2218 100644 --- a/test/functional/api/cas/cache.py +++ b/test/functional/api/cas/cache.py @@ -1,5 +1,5 @@ # -# Copyright(c) 2019-2021 Intel Corporation +# Copyright(c) 2019-2022 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # @@ -17,12 +17,12 @@ def __init__(self, device: Device): self.__metadata_size = None def __get_cache_id(self): - cmd = f"{list_cmd(by_id_path=False)} | grep {self.cache_device.get_device_id()}" + cmd = f"{list_cmd(by_id_path=False)} | grep {self.cache_device.device_id}" output = TestRun.executor.run(cmd) if output.exit_code == 0 and output.stdout.strip(): return output.stdout.split()[1] else: - raise Exception(f"There is no cache started on {self.cache_device.get_device_id()}.") + raise Exception(f"There is no cache started on {self.cache_device.device_id}.") def get_core_devices(self): return get_cores(self.cache_id) diff --git a/test/functional/tests/cli/test_zero_metadata_command.py b/test/functional/tests/cli/test_zero_metadata_command.py index f3fbf04d1..6e0eafcab 100644 --- a/test/functional/tests/cli/test_zero_metadata_command.py +++ b/test/functional/tests/cli/test_zero_metadata_command.py @@ -1,5 +1,5 @@ # -# Copyright(c) 2021 Intel Corporation +# Copyright(c) 2021-2022 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # import time @@ -110,7 +110,7 @@ def test_zero_metadata_filesystem(filesystem): except CmdException as e: cli_messages.check_stderr_msg(e.output, cli_messages.no_cas_metadata) - file_system = get_device_filesystem_type(core.get_device_id()) + file_system = get_device_filesystem_type(core.device_id) if file_system != filesystem: TestRun.LOGGER.error(f"Incorrect filesystem: {file_system}; expected: {filesystem}") diff --git a/test/functional/tests/conftest.py b/test/functional/tests/conftest.py index b8f6ed394..e386817ea 100644 --- a/test/functional/tests/conftest.py +++ b/test/functional/tests/conftest.py @@ -19,16 +19,12 @@ from api.cas import casadm from api.cas import git from api.cas.cas_service import opencas_drop_in_directory -from storage_devices.raid import Raid -from storage_devices.ramdisk import RamDisk -from test_utils.os_utils import Udev, kill_all_io -from test_utils.disk_finder import get_disk_serial_number -from test_tools.disk_utils import PartitionTable, create_partition_table -from test_tools.device_mapper import DeviceMapper -from test_tools.mdadm import Mdadm -from test_tools.fs_utils import remove -from log.logger import create_log, Log +from api.cas.init_config import InitConfig +from log.logger import create_log from test_utils.singleton import Singleton +from test_utils.output import CmdException +from test_tools.fs_utils import remove +from storage_devices.drbd import Drbd class Opencas(metaclass=Singleton): @@ -57,61 +53,81 @@ def pytest_runtest_setup(item): # # User can also have own test wrapper, which runs test prepare, cleanup, etc. # Then it should be placed in plugins package + duts = [] + for dut_file in item.config.getoption("--dut-config"): + try: + with open(dut_file) as df: + dut_config = yaml.safe_load(df) + except Exception as ex: + raise Exception( + "You need to specify DUT config. See the example_dut_config.py file" + ) from ex + + dut_config["extra_logs"] = {"cas": "/var/log/opencas.log"} + duts.append(dut_config) - test_name = item.name.split('[')[0] - TestRun.LOGGER = create_log(item.config.getoption('--log-path'), test_name) + log_path = item.config.getoption("--log-path") + test_name = item.name.split("[")[0] + logger = create_log(log_path, test_name) - duts = item.config.getoption('--dut-config') - required_duts = next(item.iter_markers(name="multidut"), None) - required_duts = required_duts.args[0] if required_duts is not None else 1 - if required_duts > len(duts): - raise Exception(f"Test requires {required_duts} DUTs, only {len(duts)} DUT configs " - f"provided") - else: - duts = duts[:required_duts] + TestRun.start(logger, duts, item) + for dut in TestRun.use_all_duts(): + if not installer.check_if_installed(): + continue + TestRun.LOGGER.info(f"CAS cleanup on {dut.ip}") + remove(opencas_drop_in_directory, recursive=True, ignore_errors=True) - TestRun.duts = [] - for dut in duts: try: - with open(dut) as cfg: - dut_config = yaml.safe_load(cfg) - except Exception as ex: - raise Exception(f"{ex}\n" - f"You need to specify DUT config. See the example_dut_config.py file") + InitConfig.create_default_init_config() + unmount_cas_devices() + try: + casadm.stop_all_caches() + except CmdException: + TestRun.LOGGER.warning( + "Failed to stop all caches, will retry after stopping DRBD" + ) + casadm.remove_all_detached_cores() + except Exception as e: + raise Exception("Exception occured during CAS cleanup:\n" + f"{str(e)}\n{traceback.format_exc()}") - dut_config['plugins_dir'] = os.path.join(os.path.dirname(__file__), "../lib") - dut_config['opt_plugins'] = {"test_wrapper": {}, "serial_log": {}, "power_control": {}} - dut_config['extra_logs'] = {"cas": "/var/log/opencas.log"} + if Drbd.is_installed(): + # TODO: Need proper stacking devices teardown + TestRun.LOGGER.workaround("Stopping DRBD") + Drbd.down_all() try: - TestRun.prepare(item, dut_config) + casadm.stop_all_caches() + except CmdException: + TestRun.LOGGER.blocked("Failed to stop all caches") - TestRun.presetup() - try: - TestRun.executor.wait_for_connection(timedelta(seconds=20)) - except paramiko.AuthenticationException: - raise - except Exception: - try: - TestRun.plugin_manager.get_plugin('power_control').power_cycle() - TestRun.executor.wait_for_connection() - except Exception: - raise Exception("Failed to connect to DUT.") - TestRun.setup() - except Exception as ex: - raise Exception(f"Exception occurred during test setup:\n" - f"{str(ex)}\n{traceback.format_exc()}") + TestRun.prepare() + + # If some generic device was set-up on top of CAS it failed to stop, try to stop it again + if installer.check_if_installed(): + casadm.stop_all_caches() + + TestRun.usr = Opencas( + repo_dir=os.path.join(os.path.dirname(__file__), "../../.."), + working_dir=TestRun.working_dir) + + cas_version = TestRun.config.get("cas_version") or git.get_current_commit_hash() + for i, dut in enumerate(TestRun.use_all_duts()): + if get_force_param(item) and not TestRun.usr.already_updated: + installer.rsync_opencas_sources() + installer.reinstall_opencas(cas_version) + elif not installer.check_if_installed(cas_version): + installer.rsync_opencas_sources() + installer.set_up_opencas(cas_version) + TestRun.LOGGER.info(f"DUT-{i} info: {dut}") - TestRun.usr = Opencas( - repo_dir=os.path.join(os.path.dirname(__file__), "../../.."), - working_dir=dut_config['working_dir']) + TestRun.usr.already_updated = True - TestRun.LOGGER.info(f"DUT info: {TestRun.dut}") - TestRun.dut.plugin_manager = TestRun.plugin_manager - TestRun.dut.executor = TestRun.executor - TestRun.duts.append(TestRun.dut) + TestRun.LOGGER.add_build_info(f'Commit hash:') + TestRun.LOGGER.add_build_info(f"{git.get_current_commit_hash()}") + TestRun.LOGGER.add_build_info(f'Commit message:') + TestRun.LOGGER.add_build_info(f'{git.get_current_commit_message()}') - base_prepare(item) TestRun.LOGGER.write_to_command_log("Test body") TestRun.LOGGER.start_group("Test body") @@ -127,46 +143,6 @@ def pytest_runtest_teardown(): This method is executed always in the end of each test, even if it fails or raises exception in prepare stage. """ - TestRun.LOGGER.end_all_groups() - - with TestRun.LOGGER.step("Cleanup after test"): - try: - if TestRun.executor: - if not TestRun.executor.is_active(): - TestRun.executor.wait_for_connection() - Udev.enable() - kill_all_io() - unmount_cas_devices() - - if installer.check_if_installed(): - casadm.remove_all_detached_cores() - casadm.stop_all_caches() - from api.cas.init_config import InitConfig - InitConfig.create_default_init_config() - - from storage_devices.drbd import Drbd - if installer.check_if_installed() and Drbd.is_installed(): - try: - casadm.stop_all_caches() - finally: - __drbd_cleanup() - elif Drbd.is_installed(): - Drbd.down_all() - - DeviceMapper.remove_all() - RamDisk.remove_all() - except Exception as ex: - TestRun.LOGGER.warning(f"Exception occurred during platform cleanup.\n" - f"{str(ex)}\n{traceback.format_exc()}") - - TestRun.LOGGER.end() - for dut in TestRun.duts: - with TestRun.use_dut(dut): - if TestRun.executor: - os.makedirs(os.path.join(TestRun.LOGGER.base_dir, "dut_info", dut.ip), - exist_ok=True) - TestRun.LOGGER.get_additional_logs() - Log.destroy() TestRun.teardown() @@ -211,80 +187,3 @@ def unmount_cas_devices(): def get_force_param(item): return item.config.getoption("--force-reinstall") - -def __drbd_cleanup(): - from storage_devices.drbd import Drbd - Drbd.down_all() - # If drbd instance had been configured on top of the CAS, the previos attempt to stop - # failed. As drbd has been stopped try to stop CAS one more time. - if installer.check_if_installed(): - casadm.stop_all_caches() - - -def base_prepare(item): - with TestRun.LOGGER.step("Cleanup before test"): - TestRun.executor.run("pkill --signal=SIGKILL fsck") - Udev.enable() - kill_all_io() - DeviceMapper.remove_all() - - if installer.check_if_installed(): - try: - from api.cas.init_config import InitConfig - InitConfig.create_default_init_config() - unmount_cas_devices() - casadm.stop_all_caches() - casadm.remove_all_detached_cores() - except Exception: - pass # TODO: Reboot DUT if test is executed remotely - - remove(opencas_drop_in_directory, recursive=True, ignore_errors=True) - - from storage_devices.drbd import Drbd - if Drbd.is_installed(): - __drbd_cleanup() - - raids = Raid.discover() - for raid in raids: - # stop only those RAIDs, which are comprised of test disks - if all(map(lambda device: - any(map(lambda disk_path: - disk_path in device.get_device_id(), - [bd.get_device_id() for bd in TestRun.dut.disks])), - raid.array_devices)): - raid.umount_all_partitions() - raid.remove_partitions() - raid.stop() - for device in raid.array_devices: - Mdadm.zero_superblock(os.path.join('/dev', device.get_device_id())) - Udev.settle() - - RamDisk.remove_all() - - for disk in TestRun.dut.disks: - disk_serial = get_disk_serial_number(disk.path) - if disk.serial_number != disk_serial: - raise Exception( - f"Serial for {disk.path} doesn't match the one from the config." - f"Serial from config {disk.serial_number}, actual serial {disk_serial}" - ) - - disk.umount_all_partitions() - Mdadm.zero_superblock(os.path.join('/dev', disk.get_device_id())) - TestRun.executor.run_expect_success("udevadm settle") - disk.remove_partitions() - create_partition_table(disk, PartitionTable.gpt) - - cas_version = TestRun.config.get("cas_version") or git.get_current_commit_hash() - if get_force_param(item) and not TestRun.usr.already_updated: - installer.rsync_opencas_sources() - installer.reinstall_opencas(cas_version) - elif not installer.check_if_installed(cas_version): - installer.rsync_opencas_sources() - installer.set_up_opencas(cas_version) - - TestRun.usr.already_updated = True - TestRun.LOGGER.add_build_info(f'Commit hash:') - TestRun.LOGGER.add_build_info(f"{git.get_current_commit_hash()}") - TestRun.LOGGER.add_build_info(f'Commit message:') - TestRun.LOGGER.add_build_info(f'{git.get_current_commit_message()}') diff --git a/test/functional/tests/misc/test_device_capabilities.py b/test/functional/tests/misc/test_device_capabilities.py index 95e591d8c..8434629b7 100644 --- a/test/functional/tests/misc/test_device_capabilities.py +++ b/test/functional/tests/misc/test_device_capabilities.py @@ -1,5 +1,5 @@ # -# Copyright(c) 2020-2021 Intel Corporation +# Copyright(c) 2020-2022 Intel Corporation # SPDX-License-Identifier: BSD-3-Clause # @@ -32,7 +32,7 @@ def test_device_capabilities(): """ core_device = TestRun.disks['core'] - max_io_size_path = os.path.join(disk_utils.get_sysfs_path(core_device.get_device_id()), + max_io_size_path = os.path.join(disk_utils.get_sysfs_path(core_device.device_id), 'queue/max_sectors_kb') default_max_io_size = fs_utils.read_file(max_io_size_path) @@ -104,8 +104,8 @@ def create_scsi_debug_device(sector_size: int, physblk_exp: int, dev_size_mb=102 def prepare_cas_device(cache_device, core_device): cache = casadm.start_cache(cache_device, cache_line_size=CacheLineSize.LINE_64KiB, force=True) try: - cache_dev_bs = disk_utils.get_block_size(cache_device.get_device_id()) - core_dev_bs = disk_utils.get_block_size(core_device.get_device_id()) + cache_dev_bs = disk_utils.get_block_size(cache_device.device_id) + core_dev_bs = disk_utils.get_block_size(core_device.device_id) core = cache.add_core(core_device) if cache_dev_bs > core_dev_bs: TestRun.LOGGER.error( @@ -145,8 +145,8 @@ def method_lcm_not_zero(a, b): def measure_capabilities(dev): dev_capabilities = {} - dev_id = dev.parent_device.get_device_id() if isinstance(dev, Partition) \ - else dev.get_device_id() + dev_id = dev.parent_device.device_id if isinstance(dev, Partition) \ + else dev.device_id for c in capabilities: path = os.path.join(disk_utils.get_sysfs_path(dev_id), 'queue', c) command = f"cat {path}" @@ -165,10 +165,10 @@ def compare_capabilities(cache_device, core_device, cache, core, msg): cli_messages.try_add_core_sector_size_mismatch) else: core_dev_sectors_num = \ - disk_utils.get_size(core_device.get_device_id()) / disk_utils.get_block_size( - core_device.get_device_id()) - core_sectors_num = disk_utils.get_size(core.get_device_id()) / disk_utils.get_block_size( - core.get_device_id()) + disk_utils.get_size(core_device.device_id) / disk_utils.get_block_size( + core_device.device_id) + core_sectors_num = disk_utils.get_size(core.device_id) / disk_utils.get_block_size( + core.device_id) if core_dev_sectors_num != core_sectors_num: TestRun.LOGGER.error( "Number of sectors in CAS device and attached core device is different.")