From c094842a96527109a653b902a896e4e2be5130e1 Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Wed, 31 Dec 2025 05:42:37 +0000 Subject: [PATCH 1/7] DAOS-18387 test: recovery/ddb.py test_recovery_ddb_ls MD-on-SSD Support To support MD-on-SSD for ddb, we need to support two commands. ddb prov_mem and ddb ls with --db_path. Update ddb_utils.py to support the new commands. Add check_ram_used in recovery_utils.py to detect whether the system is MD-on-SSD. Update test_recovery_ddb_ls to support MD-on-SSD with the new ddb commands. We need to update the test yaml to run on MD-on-SSD/HW Medium, but that will break other tests in ddb.py because they don't support MD-on-SSD yet. Keep the original tests as ddb_pmem.py and ddb_pmem.yaml and keep running them on VM (except test_recovery_ddb_ls because that's updated in this PR). Skip-unit-tests: true Skip-fault-injection-test: true Skip-func-hw-test-medium: false Test-tag: test_recovery_ddb_ls DdbPMEMTest Signed-off-by: Makito Kano --- src/tests/ftest/recovery/ddb.py | 471 ++++--------------------- src/tests/ftest/recovery/ddb.yaml | 11 +- src/tests/ftest/recovery/ddb_pmem.py | 468 ++++++++++++++++++++++++ src/tests/ftest/recovery/ddb_pmem.yaml | 27 ++ src/tests/ftest/util/ddb_utils.py | 30 +- src/tests/ftest/util/recovery_utils.py | 24 ++ 6 files changed, 622 insertions(+), 409 deletions(-) create mode 100644 src/tests/ftest/recovery/ddb_pmem.py create mode 100644 src/tests/ftest/recovery/ddb_pmem.yaml diff --git a/src/tests/ftest/recovery/ddb.py b/src/tests/ftest/recovery/ddb.py index 17993cfba05..4b598d13fbb 100644 --- a/src/tests/ftest/recovery/ddb.py +++ b/src/tests/ftest/recovery/ddb.py @@ -1,6 +1,6 @@ """ (C) Copyright 2022-2024 Intel Corporation. - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -10,12 +10,10 @@ from apricot import TestWithServers from ddb_utils import DdbCommand -from exception_utils import CommandFailure -from file_utils import distribute_files -from general_utils import (DaosTestError, create_string_buffer, get_random_string, report_errors, - run_command) +from general_utils import create_string_buffer, get_random_string, report_errors from pydaos.raw import DaosObjClass, IORequest -from run_utils import get_clush_command +from recovery_utils import check_ram_used +from run_utils import command_as_user, run_remote def insert_objects(context, container, object_count, dkey_count, akey_count, base_dkey, @@ -78,35 +76,6 @@ def insert_objects(context, container, object_count, dkey_count, akey_count, bas return (ioreqs, dkeys, akeys, data_list) -def copy_remote_to_local(remote_file_path, test_dir, remote): - """Copy the given file from the server node to the local test node and retrieve - the original name. - - Args: - remote_file_path (str): File path to copy to local. - test_dir (str): Test directory. Usually self.test_dir. - remote (str): Remote hostname to copy file from. - """ - # Use clush --rcopy to copy the file from the remote server node to the local test - # node. clush will append . to the file when copying. - args = f"--rcopy {remote_file_path} --dest {test_dir}" - clush_command = get_clush_command(hosts=remote, args=args, timeout=60) - try: - run_command(command=clush_command, timeout=None) - except DaosTestError as error: - raise DaosTestError( - f"ERROR: Copying {remote_file_path} from {remote}: {error}") from error - - # Remove the appended . from the copied file. - current_file_path = "".join([remote_file_path, ".", remote]) - mv_command = f"mv {current_file_path} {remote_file_path}" - try: - run_command(command=mv_command) - except DaosTestError as error: - raise DaosTestError( - f"ERROR: Moving {current_file_path} to {remote_file_path}: {error}") from error - - class DdbTest(TestWithServers): """Test ddb subcommands. @@ -125,6 +94,18 @@ def __init__(self, *args, **kwargs): self.random_akey = get_random_string(10) self.random_data = get_random_string(10) + def run_cmd_check_result(self, command): + """Run given command as root and check its result. + + Args: + command (str): Command to execute. + """ + command_root = command_as_user(command=command, user="root") + command_result = run_remote( + log=self.log, hosts=self.hostlist_servers, command=command_root).passed + if not command_result: + self.fail(f"{command} failed!") + def test_recovery_ddb_ls(self): """Test ddb ls. @@ -138,35 +119,64 @@ def test_recovery_ddb_ls(self): 6. Reset the container and the pool to prepare for the cleanup. :avocado: tags=all,full_regression - :avocado: tags=vm + :avocado: tags=hw,medium :avocado: tags=recovery :avocado: tags=DdbTest,ddb_cmd,test_recovery_ddb_ls """ - # Create a pool and a container. + # This is where we load pool for MD-on-SSD. It's called tmpfs_mount in ddb + # prov_mem documentation, but use daos_load_path here for clarity. + daos_load_path = "/mnt/daos_load" + md_on_ssd = check_ram_used(server_manager=self.server_managers[0], log=self.log) + if md_on_ssd: + self.log_step("MD-on-SSD: Create a directory to load pool data under /mnt.") + mkdir_command = f"mkdir {daos_load_path}" + self.run_cmd_check_result(command=mkdir_command) + + self.log_step("Create a pool and a container.") pool = self.get_pool() container = self.get_container(pool) - # Find the vos file name. e.g., /mnt/daos0//vos-0. - vos_paths = self.server_managers[0].get_vos_files(pool) - if not vos_paths: - self.fail(f"vos file wasn't found in {self.server_managers[0].get_vos_path(pool)}") - ddb_command = DdbCommand(self.server_managers[0].hosts[0:1], self.bin, vos_paths[0]) + if md_on_ssd: + ddb_command = DdbCommand( + server_host=self.server_managers[0].hosts[0:1], path=self.bin, + vos_path="\"\"") + else: + # Find the vos file name. e.g., /mnt/daos0//vos-0. + vos_paths = self.server_managers[0].get_vos_files(pool) + if not vos_paths: + self.fail(f"vos file wasn't found in {self.server_managers[0].get_vos_path(pool)}") + ddb_command = DdbCommand(self.server_managers[0].hosts[0:1], self.bin, vos_paths[0]) errors = [] object_count = self.object_count dkey_count = self.dkey_count akey_count = self.akey_count - # Insert objects with API. + self.log_step("Insert objects with API.") insert_objects( context=self.context, container=container, object_count=object_count, dkey_count=dkey_count, akey_count=akey_count, base_dkey=self.random_dkey, base_akey=self.random_akey, base_data=self.random_data) - # Need to stop the server to use ddb. + self.log_step("Stop server to use ddb.") self.get_dmg_command().system_stop() - # 1. Verify container UUID. + db_path = None + if md_on_ssd: + self.log_step("MD-on-SSD: Load pool dir to %s", daos_load_path) + db_path = os.path.join( + self.log_dir, "control_metadata", "daos_control", "engine0") + ddb_command.prov_mem(db_path=db_path, tmpfs_mount=daos_load_path) + + self.log_step("Verify container UUID.") + if md_on_ssd: + # "ddb ls" command for MD-on-SSD is quite different. + # PMEM: ddb /mnt/daos//vos-0 ls + # MD-on-SSD: ddb --db_path=/var/tmp/daos_testing/control_metadata/daos_control/engine0 /mnt/daos_load//vos-0 ls + db_path_arg = " ".join(["--db_path", db_path]) + ddb_command.db_path.update(value=db_path_arg) + vos_path = os.path.join(daos_load_path, pool.uuid.lower(), "vos-0") + ddb_command.vos_path.update(value=vos_path) cmd_result = ddb_command.list_component() # Sample output. # Listing contents of '/' @@ -186,7 +196,7 @@ def test_recovery_ddb_ls(self): msg = f"Unexpected container UUID! Expected = {expected_uuid}; Actual = {actual_uuid}" errors.append(msg) - # 2. Verify object count in the container. + self.log_step("Verify object count in the container.") cmd_result = ddb_command.list_component(component_path="[0]") # Sample output. # Listing contents of 'CONT: (/[0]) /3082b7d3-32f9-41ea-bcbf-5d6450c1b34f' @@ -204,8 +214,9 @@ def test_recovery_ddb_ls(self): f"Unexpected object count! Expected = {object_count}; " f"Actual = {actual_object_count}") - # 3. Verify there are two dkeys for every object. Also verify the dkey string and - # the size. + msg = ("Verify there are two dkeys for every object. Also verify the dkey string " + "and the size.") + self.log_step(msg) dkey_regex = f"/{uuid_regex}/{object_id_regex}/(.*)" actual_dkey_count = 0 for obj_index in range(object_count): @@ -227,15 +238,16 @@ def test_recovery_ddb_ls(self): f"Actual = {actual_dkey}") errors.append(msg) - # Verify there are two dkeys for every object. + self.log_step("Verify there are two dkeys for every object.") expected_dkey_count = object_count * dkey_count if actual_dkey_count != expected_dkey_count: msg = (f"Unexpected number of dkeys! Expected = {expected_dkey_count}; " f"Actual = {actual_dkey_count}") errors.append(msg) - # 4. Verify there is one akey for every dkey. Also verify the key string and the - # size. + msg = ("Verify there is one akey for every dkey. Also verify the key string and " + "the size.") + self.log_step(msg) akey_count = 0 for obj_index in range(object_count): for dkey_index in range(dkey_count): @@ -260,365 +272,26 @@ def test_recovery_ddb_ls(self): f"Expected = {self.random_akey}; Actual = {actual_akey}") errors.append(msg) - # Verify there is one akey for every dkey. + self.log_step("Verify there is one akey for every dkey.") if expected_dkey_count != akey_count: msg = (f"Unexpected number of akeys! Expected = {expected_dkey_count}; " f"Actual = {akey_count}") errors.append(msg) - # 5. Restart the server for the cleanup. - self.get_dmg_command().system_start() + if md_on_ssd: + self.log_step("MD-on-SSD: Clean %s", daos_load_path) + umount_cmd = f"umount {daos_load_path}" + self.run_cmd_check_result(command=umount_cmd) + rm_command = f"rm -rf {daos_load_path}" + self.run_cmd_check_result(command=rm_command) - # 6. Reset the container and the pool to prepare for the cleanup. - container.close() - pool.disconnect() - pool.connect() - container.open() + self.log_step("Restart the server for the cleanup.") self.get_dmg_command().system_start() - self.log.info("##### Errors #####") - report_errors(test=self, errors=errors) - self.log.info("##################") - - def test_recovery_ddb_rm(self): - """Test rm. - - 1. Create a pool and a container. Insert objects, dkeys, and akeys. - 2. Stop the server to use ddb. - 3. Find the vos file name. e.g., /mnt/daos0//vos-0. - 4. Call ddb rm to remove the akey. - 5. Restart the server to use the API. - 6. Reset the object, container, and pool to use the API after server restart. - 7. Call list_akey() in pydaos API to verify that the akey was removed. - 8. Stop the server to use ddb. - 9. Call ddb rm to remove the dkey. - 10. Restart the server to use the API. - 11. Reset the object, container, and pool to use the API after server restart. - 12. Call list_dkey() in pydaos API to verify that the dkey was removed. - 13. Stop the server to use ddb. - 14. Call ddb rm to remove the object. - 15. Restart the server to use daos command. - 16. Reset the container and pool so that cleanup works. - 17. Call "daos container list-objects " to verify that the - object was removed. - - :avocado: tags=all,full_regression - :avocado: tags=vm - :avocado: tags=recovery - :avocado: tags=DdbTest,ddb_cmd,test_recovery_ddb_rm - """ - # 1. Create a pool and a container. Insert objects, dkeys, and akeys. - pool = self.get_pool(connect=True) - container = self.get_container(pool) - - # Insert one object with one dkey and one akey with API. - obj_dataset = insert_objects( - context=self.context, container=container, object_count=1, - dkey_count=1, akey_count=2, base_dkey=self.random_dkey, - base_akey=self.random_akey, base_data=self.random_data) - ioreqs = obj_dataset[0] - dkeys_inserted = obj_dataset[1] - akeys_inserted = obj_dataset[2] - - # For debugging/reference, check that the dkey and the akey we just inserted are - # returned from the API. - akeys_api = ioreqs[0].list_akey(dkey=dkeys_inserted[0]) - self.log.info("akeys from API (before) = %s", akeys_api) - dkeys_api = ioreqs[0].list_dkey() - self.log.info("dkeys from API (before) = %s", dkeys_api) - - # For debugging/reference, check that the object was inserted using daos command. - list_obj_out = self.get_daos_command().container_list_objects( - pool=pool.identifier, cont=container.uuid) - self.log.info("Object list (before) = %s", list_obj_out["response"]) - - # 2. Need to stop the server to use ddb. - dmg_command = self.get_dmg_command() - dmg_command.system_stop() - - # 3. Find the vos file name. - vos_paths = self.server_managers[0].get_vos_files(pool) - if not vos_paths: - self.fail(f"vos file wasn't found in {self.server_managers[0].get_vos_path(pool)}") - ddb_command = DdbCommand(self.server_managers[0].hosts[0:1], self.bin, vos_paths[0]) - - # 4. Call ddb rm to remove the akey. - cmd_result = ddb_command.remove_component(component_path="[0]/[0]/[0]/[0]") - self.log.info("rm akey stdout = %s", cmd_result.joined_stdout) - - # 5. Restart the server to use the API. - dmg_command.system_start() - - # 6. Reset the object, container, and pool to use the API after server restart. - ioreqs[0].obj.close() - container.close() - pool.disconnect() - pool.connect() - container.open() - ioreqs[0].obj.open() - - # 7. Call list_akey() in pydaos API to verify that the akey was removed. - akeys_api = ioreqs[0].list_akey(dkey=dkeys_inserted[0]) - self.log.info("akeys from API (after) = %s", akeys_api) - - errors = [] - expected_len = len(akeys_inserted) - 1 - actual_len = len(akeys_api) - if actual_len != expected_len: - msg = (f"Unexpected number of akeys after ddb rm! Expected = {expected_len}; " - f"Actual = {actual_len}") - errors.append(msg) - - # 8. Stop the server to use ddb. - dmg_command.system_stop() - - # 9. Call ddb rm to remove the dkey. - cmd_result = ddb_command.remove_component(component_path="[0]/[0]/[0]") - self.log.info("rm dkey stdout = %s", cmd_result.joined_stdout) - - # 10. Restart the server to use the API. - dmg_command.system_start() - - # 11. Reset the object, container, and pool to use the API after server restart. - ioreqs[0].obj.close() - container.close() - pool.disconnect() - pool.connect() - container.open() - ioreqs[0].obj.open() - - # 12. Call list_dkey() in pydaos API to verify that the dkey was removed. - dkeys_api = ioreqs[0].list_dkey() - self.log.info("dkeys from API (after) = %s", dkeys_api) - - expected_len = len(dkeys_inserted) - 1 - actual_len = len(dkeys_api) - if actual_len != expected_len: - msg = (f"Unexpected number of dkeys after ddb rm! Expected = {expected_len}; " - f"Actual = {actual_len}") - errors.append(msg) - - # 13. Stop the server to use ddb. - dmg_command.system_stop() - - # 14. Call ddb rm to remove the object. - cmd_result = ddb_command.remove_component(component_path="[0]/[0]") - self.log.info("rm object stdout = %s", cmd_result.joined_stdout) - - # 15. Restart the server to use daos command. - dmg_command.system_start() - - # 16. Reset the container and pool so that cleanup works. - container.close() - pool.disconnect() - pool.connect() - container.open() - - # 17. Call "daos container list-objects " to verify that - # the object was removed. - list_obj_out = self.get_daos_command().container_list_objects( - pool=pool.identifier, cont=container.uuid) - obj_list = list_obj_out["response"] - self.log.info("Object list (after) = %s", obj_list) - - expected_len = len(ioreqs) - 1 - if obj_list: - actual_len = len(obj_list) - else: - actual_len = 0 - if actual_len != expected_len: - msg = (f"Unexpected number of objects after ddb rm! Expected = {expected_len}; " - f"Actual = {actual_len}") - errors.append(msg) - - self.log.info("##### Errors #####") - report_errors(test=self, errors=errors) - self.log.info("##################") - - def test_recovery_ddb_load(self): - """Test ddb value_load. - - 1. Create a pool and a container. - 2. Insert one object with one dkey with the API. - 3. Stop the server to use ddb. - 4. Find the vos file name. e.g., /mnt/daos0//vos-0. - 5. Load new data into [0]/[0]/[0]/[0] - 6. Restart the server. - 7. Reset the object, container, and pool to use the API. - 8. Verify the data in the akey with single_fetch(). - - :avocado: tags=all,full_regression - :avocado: tags=vm - :avocado: tags=recovery - :avocado: tags=DdbTest,ddb_cmd,test_recovery_ddb_load - """ - # 1. Create a pool and a container. - pool = self.get_pool(connect=True) - container = self.get_container(pool) - - # 2. Insert one object with one dkey with API. - obj_dataset = insert_objects( - context=self.context, container=container, object_count=1, - dkey_count=1, akey_count=1, base_dkey=self.random_dkey, - base_akey=self.random_akey, base_data=self.random_data) - ioreqs = obj_dataset[0] - dkeys_inserted = obj_dataset[1] - akeys_inserted = obj_dataset[2] - data_list = obj_dataset[3] - - # For debugging/reference, call single_fetch and get the data just inserted. - # Pass in size + 1 to single_fetch to avoid the no-space error. - data_size = len(data_list[0]) + 1 - data = ioreqs[0].single_fetch( - dkey=dkeys_inserted[0], akey=akeys_inserted[0], size=data_size) - self.log.info("data (before) = %s", data.value.decode('utf-8')) - - # 3. Stop the server to use ddb. - dmg_command = self.get_dmg_command() - dmg_command.system_stop() - - # 4. Find the vos file name. - host = self.server_managers[0].hosts[0:1] - vos_paths = self.server_managers[0].get_vos_files(pool) - if not vos_paths: - self.fail(f"vos file wasn't found in {self.server_managers[0].get_vos_path(pool)}") - ddb_command = DdbCommand(host, self.bin, vos_paths[0]) - - # 5. Load new data into [0]/[0]/[0]/[0] - # Create a file in test node. - load_file_path = os.path.join(self.test_dir, "new_data.txt") - new_data = "New akey data 0123456789" - with open(load_file_path, "w", encoding="utf-8") as file: - file.write(new_data) - - # Copy the created file to server node. - result = distribute_files(self.log, host, load_file_path, load_file_path, False) - if not result.passed: - raise CommandFailure(f"ERROR: Copying new_data.txt to {result.failed_hosts}") - - # The file with the new data is ready. Run ddb load. - ddb_command.value_load(component_path="[0]/[0]/[0]/[0]", load_file_path=load_file_path) - - # 6. Restart the server. - dmg_command.system_start() - - # 7. Reset the object, container, and pool to use the API after server restart. - ioreqs[0].obj.close() - container.close() - pool.disconnect() - pool.connect() - container.open() - ioreqs[0].obj.open() - - # 8. Verify the data in the akey with single_fetch(). - data_size = len(new_data) + 1 - data = ioreqs[0].single_fetch( - dkey=dkeys_inserted[0], akey=akeys_inserted[0], size=data_size) - actual_data = data.value.decode('utf-8') - self.log.info("data (after) = %s", actual_data) - - errors = [] - if new_data != actual_data: - msg = f"ddb load failed! Expected = {new_data}; Actual = {actual_data}" - errors.append(msg) - - self.log.info("##### Errors #####") - report_errors(test=self, errors=errors) - self.log.info("##################") - - def test_recovery_ddb_dump_value(self): - """Test ddb dump_value. - - 1. Create a pool and a container. - 2. Insert one object with one dkey with API. - 3. Stop the server to use ddb. - 4. Find the vos file name. e.g., /mnt/daos0//vos-0. - 5. Dump the two akeys to files. - 6. Verify the content of the files. - 7. Restart the server for the cleanup. - 8. Reset the object, container, and pool to prepare for the cleanup. - - :avocado: tags=all,full_regression - :avocado: tags=vm - :avocado: tags=recovery - :avocado: tags=DdbTest,ddb_cmd,test_recovery_ddb_dump_value - """ - # 1. Create a pool and a container. - pool = self.get_pool(connect=True) - container = self.get_container(pool) - - # 2. Insert one object with one dkey with API. - obj_dataset = insert_objects( - context=self.context, container=container, object_count=1, - dkey_count=1, akey_count=2, base_dkey=self.random_dkey, - base_akey=self.random_akey, base_data=self.random_data) - ioreqs = obj_dataset[0] - data_list = obj_dataset[3] - - # 3. Stop the server to use ddb. - dmg_command = self.get_dmg_command() - dmg_command.system_stop() - - # 4. Find the vos file name. - vos_paths = self.server_managers[0].get_vos_files(pool) - if not vos_paths: - self.fail(f"vos file wasn't found in {self.server_managers[0].get_vos_path(pool)}") - ddb_command = DdbCommand(self.server_managers[0].hosts[0:1], self.bin, vos_paths[0]) - - # 5. Dump the two akeys to files. - akey1_file_path = os.path.join(self.test_dir, "akey1.txt") - ddb_command.value_dump( - component_path="[0]/[0]/[0]/[0]", out_file_path=akey1_file_path) - akey2_file_path = os.path.join(self.test_dir, "akey2.txt") - ddb_command.value_dump( - component_path="[0]/[0]/[0]/[1]", out_file_path=akey2_file_path) - - # Copy them from remote server node to local test node. - copy_remote_to_local( - remote_file_path=akey1_file_path, test_dir=self.test_dir, - remote=self.hostlist_servers[0]) - copy_remote_to_local( - remote_file_path=akey2_file_path, test_dir=self.test_dir, - remote=self.hostlist_servers[0]) - - # 6. Verify the content of the files. - actual_akey1_data = None - with open(akey1_file_path, "r", encoding="utf-8") as file: - actual_akey1_data = file.readlines()[0] - actual_akey2_data = None - with open(akey2_file_path, "r", encoding="utf-8") as file: - actual_akey2_data = file.readlines()[0] - - errors = [] - str_data_list = [] - # Convert the data to string. - for data in data_list: - str_data_list.append(data.value.decode("utf-8")) - # Verify that we were able to obtain the data and akey1 and akey2 aren't the same. - if actual_akey1_data is None or actual_akey2_data is None or \ - actual_akey1_data == actual_akey2_data: - msg = (f"Invalid dumped value! Dumped akey1 data = {actual_akey1_data}; " - f"Dumped akey2 data = {actual_akey2_data}") - errors.append(msg) - # Verify that the data we obtained with ddb are the ones we wrote. The order isn't - # deterministic, so check with "in". - if actual_akey1_data not in str_data_list or \ - actual_akey2_data not in str_data_list: - msg = (f"Unexpected dumped value! Dumped akey data 1 = {actual_akey1_data}; Dumped " - f"akey data 2 = {actual_akey2_data}; Expected data list = {str_data_list}") - errors.append(msg) - - # 7. Restart the server for the cleanup. - dmg_command.system_start() - - # 8. Reset the object, container, and pool to prepare for the cleanup. - ioreqs[0].obj.close() + self.log_step("Reset the container and the pool to prepare for the cleanup.") container.close() pool.disconnect() pool.connect() container.open() - ioreqs[0].obj.open() - self.log.info("##### Errors #####") report_errors(test=self, errors=errors) - self.log.info("##################") diff --git a/src/tests/ftest/recovery/ddb.yaml b/src/tests/ftest/recovery/ddb.yaml index a89fa7beb29..6d31aa6daf3 100644 --- a/src/tests/ftest/recovery/ddb.yaml +++ b/src/tests/ftest/recovery/ddb.yaml @@ -9,12 +9,9 @@ server_config: engines_per_host: 1 engines: 0: - targets: 1 - storage: - 0: - class: ram - scm_mount: /mnt/daos - system_ram_reserved: 1 + log_file: daos_server0.log + nr_xs_helpers: 1 + storage: auto # In CI, all tests in ddb.py are ran in a single launch.py execution. In that case, the # test_dir (/var/tmp/daos_testing/) in the server node will not be created @@ -24,4 +21,4 @@ setup: start_servers_once: False pool: - scm_size: 1G + scm_size: 50G diff --git a/src/tests/ftest/recovery/ddb_pmem.py b/src/tests/ftest/recovery/ddb_pmem.py new file mode 100644 index 00000000000..4e5faaeb0b4 --- /dev/null +++ b/src/tests/ftest/recovery/ddb_pmem.py @@ -0,0 +1,468 @@ +""" + (C) Copyright 2022-2024 Intel Corporation. + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP + + SPDX-License-Identifier: BSD-2-Clause-Patent +""" +import ctypes +import os + +from apricot import TestWithServers +from ddb_utils import DdbCommand +from exception_utils import CommandFailure +from file_utils import distribute_files +from general_utils import (DaosTestError, create_string_buffer, get_random_string, report_errors, + run_command) +from pydaos.raw import DaosObjClass, IORequest +from run_utils import get_clush_command + + +def insert_objects(context, container, object_count, dkey_count, akey_count, base_dkey, + base_akey, base_data): + """Insert objects, dkeys, akeys, and data into the container. + + Args: + context (DaosContext): + container (TestContainer): Container to insert objects. + object_count (int): Number of objects to insert. + dkey_count (int): Number of dkeys to insert. + akey_count (int): Number of akeys to insert. + base_dkey (str): Base dkey. Index numbers will be appended to it. + base_akey (str):Base akey. Index numbers will be appended to it. + base_data (str):Base data that goes inside akey. Index numbers will be appended + to it. + + Returns: + tuple: Inserted objects, dkeys, akeys, and data as (ioreqs, dkeys, akeys, + data_list) + + """ + ioreqs = [] + dkeys = [] + akeys = [] + data_list = [] + + container.open() + + for obj_index in range(object_count): + # Insert object. + ioreqs.append(IORequest( + context=context, container=container.container, obj=None, + objtype=DaosObjClass.OC_S1)) + + for dkey_index in range(dkey_count): + # Prepare the dkey to insert into the object. + dkey_str = " ".join( + [base_dkey, str(obj_index), str(dkey_index)]).encode("utf-8") + dkeys.append(create_string_buffer(value=dkey_str, size=len(dkey_str))) + + for akey_index in range(akey_count): + # Prepare the akey to insert into the dkey. + akey_str = " ".join( + [base_akey, str(obj_index), str(dkey_index), + str(akey_index)]).encode("utf-8") + akeys.append(create_string_buffer(value=akey_str, size=len(akey_str))) + + # Prepare the data to insert into the akey. + data_str = " ".join( + [base_data, str(obj_index), str(dkey_index), + str(akey_index)]).encode("utf-8") + data_list.append(create_string_buffer(value=data_str, size=len(data_str))) + c_size = ctypes.c_size_t(ctypes.sizeof(data_list[-1])) + + # Insert dkeys, akeys, and the data. + ioreqs[-1].single_insert( + dkey=dkeys[-1], akey=akeys[-1], value=data_list[-1], size=c_size) + + return (ioreqs, dkeys, akeys, data_list) + + +def copy_remote_to_local(remote_file_path, test_dir, remote): + """Copy the given file from the server node to the local test node and retrieve + the original name. + + Args: + remote_file_path (str): File path to copy to local. + test_dir (str): Test directory. Usually self.test_dir. + remote (str): Remote hostname to copy file from. + """ + # Use clush --rcopy to copy the file from the remote server node to the local test + # node. clush will append . to the file when copying. + args = f"--rcopy {remote_file_path} --dest {test_dir}" + clush_command = get_clush_command(hosts=remote, args=args, timeout=60) + try: + run_command(command=clush_command, timeout=None) + except DaosTestError as error: + raise DaosTestError( + f"ERROR: Copying {remote_file_path} from {remote}: {error}") from error + + # Remove the appended . from the copied file. + current_file_path = "".join([remote_file_path, ".", remote]) + mv_command = f"mv {current_file_path} {remote_file_path}" + try: + run_command(command=mv_command) + except DaosTestError as error: + raise DaosTestError( + f"ERROR: Moving {current_file_path} to {remote_file_path}: {error}") from error + + +class DdbPMEMTest(TestWithServers): + """Test ddb subcommands. + + :avocado: recursive + """ + + def __init__(self, *args, **kwargs): + """Initialize a DdbTest object.""" + super().__init__(*args, **kwargs) + # how many objects and keys to insert/expect + self.object_count = 5 + self.dkey_count = 2 + self.akey_count = 1 + # Generate random keys and data to insert into the object. + self.random_dkey = get_random_string(10) + self.random_akey = get_random_string(10) + self.random_data = get_random_string(10) + + def test_recovery_ddb_rm(self): + """Test rm. + + 1. Create a pool and a container. Insert objects, dkeys, and akeys. + 2. Stop the server to use ddb. + 3. Find the vos file name. e.g., /mnt/daos0//vos-0. + 4. Call ddb rm to remove the akey. + 5. Restart the server to use the API. + 6. Reset the object, container, and pool to use the API after server restart. + 7. Call list_akey() in pydaos API to verify that the akey was removed. + 8. Stop the server to use ddb. + 9. Call ddb rm to remove the dkey. + 10. Restart the server to use the API. + 11. Reset the object, container, and pool to use the API after server restart. + 12. Call list_dkey() in pydaos API to verify that the dkey was removed. + 13. Stop the server to use ddb. + 14. Call ddb rm to remove the object. + 15. Restart the server to use daos command. + 16. Reset the container and pool so that cleanup works. + 17. Call "daos container list-objects " to verify that the + object was removed. + + :avocado: tags=all,full_regression + :avocado: tags=vm + :avocado: tags=recovery + :avocado: tags=DdbPMEMTest,ddb_cmd,test_recovery_ddb_rm + """ + # 1. Create a pool and a container. Insert objects, dkeys, and akeys. + pool = self.get_pool(connect=True) + container = self.get_container(pool) + + # Insert one object with one dkey and one akey with API. + obj_dataset = insert_objects( + context=self.context, container=container, object_count=1, + dkey_count=1, akey_count=2, base_dkey=self.random_dkey, + base_akey=self.random_akey, base_data=self.random_data) + ioreqs = obj_dataset[0] + dkeys_inserted = obj_dataset[1] + akeys_inserted = obj_dataset[2] + + # For debugging/reference, check that the dkey and the akey we just inserted are + # returned from the API. + akeys_api = ioreqs[0].list_akey(dkey=dkeys_inserted[0]) + self.log.info("akeys from API (before) = %s", akeys_api) + dkeys_api = ioreqs[0].list_dkey() + self.log.info("dkeys from API (before) = %s", dkeys_api) + + # For debugging/reference, check that the object was inserted using daos command. + list_obj_out = self.get_daos_command().container_list_objects( + pool=pool.identifier, cont=container.uuid) + self.log.info("Object list (before) = %s", list_obj_out["response"]) + + # 2. Need to stop the server to use ddb. + dmg_command = self.get_dmg_command() + dmg_command.system_stop() + + # 3. Find the vos file name. + vos_paths = self.server_managers[0].get_vos_files(pool) + if not vos_paths: + self.fail(f"vos file wasn't found in {self.server_managers[0].get_vos_path(pool)}") + ddb_command = DdbCommand(self.server_managers[0].hosts[0:1], self.bin, vos_paths[0]) + + # 4. Call ddb rm to remove the akey. + cmd_result = ddb_command.remove_component(component_path="[0]/[0]/[0]/[0]") + self.log.info("rm akey stdout = %s", cmd_result.joined_stdout) + + # 5. Restart the server to use the API. + dmg_command.system_start() + + # 6. Reset the object, container, and pool to use the API after server restart. + ioreqs[0].obj.close() + container.close() + pool.disconnect() + pool.connect() + container.open() + ioreqs[0].obj.open() + + # 7. Call list_akey() in pydaos API to verify that the akey was removed. + akeys_api = ioreqs[0].list_akey(dkey=dkeys_inserted[0]) + self.log.info("akeys from API (after) = %s", akeys_api) + + errors = [] + expected_len = len(akeys_inserted) - 1 + actual_len = len(akeys_api) + if actual_len != expected_len: + msg = (f"Unexpected number of akeys after ddb rm! Expected = {expected_len}; " + f"Actual = {actual_len}") + errors.append(msg) + + # 8. Stop the server to use ddb. + dmg_command.system_stop() + + # 9. Call ddb rm to remove the dkey. + cmd_result = ddb_command.remove_component(component_path="[0]/[0]/[0]") + self.log.info("rm dkey stdout = %s", cmd_result.joined_stdout) + + # 10. Restart the server to use the API. + dmg_command.system_start() + + # 11. Reset the object, container, and pool to use the API after server restart. + ioreqs[0].obj.close() + container.close() + pool.disconnect() + pool.connect() + container.open() + ioreqs[0].obj.open() + + # 12. Call list_dkey() in pydaos API to verify that the dkey was removed. + dkeys_api = ioreqs[0].list_dkey() + self.log.info("dkeys from API (after) = %s", dkeys_api) + + expected_len = len(dkeys_inserted) - 1 + actual_len = len(dkeys_api) + if actual_len != expected_len: + msg = (f"Unexpected number of dkeys after ddb rm! Expected = {expected_len}; " + f"Actual = {actual_len}") + errors.append(msg) + + # 13. Stop the server to use ddb. + dmg_command.system_stop() + + # 14. Call ddb rm to remove the object. + cmd_result = ddb_command.remove_component(component_path="[0]/[0]") + self.log.info("rm object stdout = %s", cmd_result.joined_stdout) + + # 15. Restart the server to use daos command. + dmg_command.system_start() + + # 16. Reset the container and pool so that cleanup works. + container.close() + pool.disconnect() + pool.connect() + container.open() + + # 17. Call "daos container list-objects " to verify that + # the object was removed. + list_obj_out = self.get_daos_command().container_list_objects( + pool=pool.identifier, cont=container.uuid) + obj_list = list_obj_out["response"] + self.log.info("Object list (after) = %s", obj_list) + + expected_len = len(ioreqs) - 1 + if obj_list: + actual_len = len(obj_list) + else: + actual_len = 0 + if actual_len != expected_len: + msg = (f"Unexpected number of objects after ddb rm! Expected = {expected_len}; " + f"Actual = {actual_len}") + errors.append(msg) + + self.log.info("##### Errors #####") + report_errors(test=self, errors=errors) + self.log.info("##################") + + def test_recovery_ddb_load(self): + """Test ddb value_load. + + 1. Create a pool and a container. + 2. Insert one object with one dkey with the API. + 3. Stop the server to use ddb. + 4. Find the vos file name. e.g., /mnt/daos0//vos-0. + 5. Load new data into [0]/[0]/[0]/[0] + 6. Restart the server. + 7. Reset the object, container, and pool to use the API. + 8. Verify the data in the akey with single_fetch(). + + :avocado: tags=all,full_regression + :avocado: tags=vm + :avocado: tags=recovery + :avocado: tags=DdbPMEMTest,ddb_cmd,test_recovery_ddb_load + """ + # 1. Create a pool and a container. + pool = self.get_pool(connect=True) + container = self.get_container(pool) + + # 2. Insert one object with one dkey with API. + obj_dataset = insert_objects( + context=self.context, container=container, object_count=1, + dkey_count=1, akey_count=1, base_dkey=self.random_dkey, + base_akey=self.random_akey, base_data=self.random_data) + ioreqs = obj_dataset[0] + dkeys_inserted = obj_dataset[1] + akeys_inserted = obj_dataset[2] + data_list = obj_dataset[3] + + # For debugging/reference, call single_fetch and get the data just inserted. + # Pass in size + 1 to single_fetch to avoid the no-space error. + data_size = len(data_list[0]) + 1 + data = ioreqs[0].single_fetch( + dkey=dkeys_inserted[0], akey=akeys_inserted[0], size=data_size) + self.log.info("data (before) = %s", data.value.decode('utf-8')) + + # 3. Stop the server to use ddb. + dmg_command = self.get_dmg_command() + dmg_command.system_stop() + + # 4. Find the vos file name. + host = self.server_managers[0].hosts[0:1] + vos_paths = self.server_managers[0].get_vos_files(pool) + if not vos_paths: + self.fail(f"vos file wasn't found in {self.server_managers[0].get_vos_path(pool)}") + ddb_command = DdbCommand(host, self.bin, vos_paths[0]) + + # 5. Load new data into [0]/[0]/[0]/[0] + # Create a file in test node. + load_file_path = os.path.join(self.test_dir, "new_data.txt") + new_data = "New akey data 0123456789" + with open(load_file_path, "w", encoding="utf-8") as file: + file.write(new_data) + + # Copy the created file to server node. + result = distribute_files(self.log, host, load_file_path, load_file_path, False) + if not result.passed: + raise CommandFailure(f"ERROR: Copying new_data.txt to {result.failed_hosts}") + + # The file with the new data is ready. Run ddb load. + ddb_command.value_load(component_path="[0]/[0]/[0]/[0]", load_file_path=load_file_path) + + # 6. Restart the server. + dmg_command.system_start() + + # 7. Reset the object, container, and pool to use the API after server restart. + ioreqs[0].obj.close() + container.close() + pool.disconnect() + pool.connect() + container.open() + ioreqs[0].obj.open() + + # 8. Verify the data in the akey with single_fetch(). + data_size = len(new_data) + 1 + data = ioreqs[0].single_fetch( + dkey=dkeys_inserted[0], akey=akeys_inserted[0], size=data_size) + actual_data = data.value.decode('utf-8') + self.log.info("data (after) = %s", actual_data) + + errors = [] + if new_data != actual_data: + msg = f"ddb load failed! Expected = {new_data}; Actual = {actual_data}" + errors.append(msg) + + self.log.info("##### Errors #####") + report_errors(test=self, errors=errors) + self.log.info("##################") + + def test_recovery_ddb_dump_value(self): + """Test ddb dump_value. + + 1. Create a pool and a container. + 2. Insert one object with one dkey with API. + 3. Stop the server to use ddb. + 4. Find the vos file name. e.g., /mnt/daos0//vos-0. + 5. Dump the two akeys to files. + 6. Verify the content of the files. + 7. Restart the server for the cleanup. + 8. Reset the object, container, and pool to prepare for the cleanup. + + :avocado: tags=all,full_regression + :avocado: tags=vm + :avocado: tags=recovery + :avocado: tags=DdbPMEMTest,ddb_cmd,test_recovery_ddb_dump_value + """ + # 1. Create a pool and a container. + pool = self.get_pool(connect=True) + container = self.get_container(pool) + + # 2. Insert one object with one dkey with API. + obj_dataset = insert_objects( + context=self.context, container=container, object_count=1, + dkey_count=1, akey_count=2, base_dkey=self.random_dkey, + base_akey=self.random_akey, base_data=self.random_data) + ioreqs = obj_dataset[0] + data_list = obj_dataset[3] + + # 3. Stop the server to use ddb. + dmg_command = self.get_dmg_command() + dmg_command.system_stop() + + # 4. Find the vos file name. + vos_paths = self.server_managers[0].get_vos_files(pool) + if not vos_paths: + self.fail(f"vos file wasn't found in {self.server_managers[0].get_vos_path(pool)}") + ddb_command = DdbCommand(self.server_managers[0].hosts[0:1], self.bin, vos_paths[0]) + + # 5. Dump the two akeys to files. + akey1_file_path = os.path.join(self.test_dir, "akey1.txt") + ddb_command.value_dump( + component_path="[0]/[0]/[0]/[0]", out_file_path=akey1_file_path) + akey2_file_path = os.path.join(self.test_dir, "akey2.txt") + ddb_command.value_dump( + component_path="[0]/[0]/[0]/[1]", out_file_path=akey2_file_path) + + # Copy them from remote server node to local test node. + copy_remote_to_local( + remote_file_path=akey1_file_path, test_dir=self.test_dir, + remote=self.hostlist_servers[0]) + copy_remote_to_local( + remote_file_path=akey2_file_path, test_dir=self.test_dir, + remote=self.hostlist_servers[0]) + + # 6. Verify the content of the files. + actual_akey1_data = None + with open(akey1_file_path, "r", encoding="utf-8") as file: + actual_akey1_data = file.readlines()[0] + actual_akey2_data = None + with open(akey2_file_path, "r", encoding="utf-8") as file: + actual_akey2_data = file.readlines()[0] + + errors = [] + str_data_list = [] + # Convert the data to string. + for data in data_list: + str_data_list.append(data.value.decode("utf-8")) + # Verify that we were able to obtain the data and akey1 and akey2 aren't the same. + if actual_akey1_data is None or actual_akey2_data is None or \ + actual_akey1_data == actual_akey2_data: + msg = (f"Invalid dumped value! Dumped akey1 data = {actual_akey1_data}; " + f"Dumped akey2 data = {actual_akey2_data}") + errors.append(msg) + # Verify that the data we obtained with ddb are the ones we wrote. The order isn't + # deterministic, so check with "in". + if actual_akey1_data not in str_data_list or \ + actual_akey2_data not in str_data_list: + msg = (f"Unexpected dumped value! Dumped akey data 1 = {actual_akey1_data}; Dumped " + f"akey data 2 = {actual_akey2_data}; Expected data list = {str_data_list}") + errors.append(msg) + + # 7. Restart the server for the cleanup. + dmg_command.system_start() + + # 8. Reset the object, container, and pool to prepare for the cleanup. + ioreqs[0].obj.close() + container.close() + pool.disconnect() + pool.connect() + container.open() + ioreqs[0].obj.open() + + self.log.info("##### Errors #####") + report_errors(test=self, errors=errors) + self.log.info("##################") diff --git a/src/tests/ftest/recovery/ddb_pmem.yaml b/src/tests/ftest/recovery/ddb_pmem.yaml new file mode 100644 index 00000000000..a89fa7beb29 --- /dev/null +++ b/src/tests/ftest/recovery/ddb_pmem.yaml @@ -0,0 +1,27 @@ +hosts: + test_servers: 1 + test_clients: 1 + +timeout: 1800 + +server_config: + name: daos_server + engines_per_host: 1 + engines: + 0: + targets: 1 + storage: + 0: + class: ram + scm_mount: /mnt/daos + system_ram_reserved: 1 + +# In CI, all tests in ddb.py are ran in a single launch.py execution. In that case, the +# test_dir (/var/tmp/daos_testing/) in the server node will not be created +# for each test if "start_servers_once: False" isn't set. test_load() needs this +# directory, so we need to set it. +setup: + start_servers_once: False + +pool: + scm_size: 1G diff --git a/src/tests/ftest/util/ddb_utils.py b/src/tests/ftest/util/ddb_utils.py index 53f9601653e..3768ff58047 100644 --- a/src/tests/ftest/util/ddb_utils.py +++ b/src/tests/ftest/util/ddb_utils.py @@ -34,11 +34,14 @@ def __init__(self, server_host, path, verbose=True, timeout=None, sudo=True): # Write mode that's necessary for the commands that alters the data such as load. self.write_mode = FormattedParameter("-w", default=False) - # Command to run on the VOS file that contains container, object info, etc. - self.single_command = BasicParameter(None, position=2) + # Path to the system database. Used for MD-on-SSD. + self.db_path = BasicParameter(None, position=1) # VOS file path. - self.vos_path = BasicParameter(None, position=1) + self.vos_path = BasicParameter(None, position=2) + + # Command to run on the VOS file that contains container, object info, etc. + self.single_command = BasicParameter(None, position=3) # Members needed for run(). self.verbose = verbose @@ -282,3 +285,24 @@ def dtx_cmt_clear(self, component_path="[0]"): self.single_command.value = " ".join(["dtx_cmt_clear", component_path]) return self.run() + + def prov_mem(self, db_path, tmpfs_mount): + """Call ddb "" prov_mem . + + Before calling this method, "" (two double quotes) needs to be set to + self.vos_path. + + Args: + db_path (str): Path to the system database. e.g., + /var/tmp/daos_testing/control_metadata/daos_control/engine0 + tmpfs_mount (str): Path to the tmpfs mount point. Directory that needs to be + created beforehand. e.g., /mnt/daos_load + + Returns: + CommandResult: groups of command results from the same hosts with the same + return status + """ + cmd = ["prov_mem", db_path, tmpfs_mount] + self.single_command.value = " ".join(cmd) + + return self.run() diff --git a/src/tests/ftest/util/recovery_utils.py b/src/tests/ftest/util/recovery_utils.py index 3eb732cd25c..1b5fc5a4202 100644 --- a/src/tests/ftest/util/recovery_utils.py +++ b/src/tests/ftest/util/recovery_utils.py @@ -122,3 +122,27 @@ def check_policies(dmg_command, interact_count): msg = f"Unexpected policy for {class_name}! Expected = DEFAULT, Actual = {policy}" raise CommandFailure(msg) return policies + + +def check_ram_used(server_manager, log): + """Check whether 'ram' field is used in the storage section of the server config. + + Args: + server_manager (ServerManager): + log (logging.Logger): Used to print helpful logs. + + Returns: + bool: If 'ram' field is found in 'storage', return True. Otherwise return False. + """ + server_config_file = server_manager.manager.job.yaml_data + log.info("server_config_file = %s", server_config_file) + engine = server_config_file["engines"][0] + storage_list = engine["storage"] + ram_used = False + for storage in storage_list: + log.info("storage = %s", storage) + if storage["class"] == "ram": + log.info("ram found in %s", storage) + ram_used = True + break + return ram_used From 975427f7bd2877cbea21f51cadc6cc1eca56d423 Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Wed, 31 Dec 2025 06:06:50 +0000 Subject: [PATCH 2/7] DAOS-18387 test: Fix pylint Skip-unit-tests: true Skip-fault-injection-test: true Skip-func-hw-test-medium: false Test-tag: test_recovery_ddb_ls DdbPMEMTest Signed-off-by: Makito Kano --- src/tests/ftest/recovery/ddb.py | 20 ++++++++++---------- src/tests/ftest/util/ddb_utils.py | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/tests/ftest/recovery/ddb.py b/src/tests/ftest/recovery/ddb.py index 4b598d13fbb..7bc1f270a81 100644 --- a/src/tests/ftest/recovery/ddb.py +++ b/src/tests/ftest/recovery/ddb.py @@ -129,8 +129,7 @@ def test_recovery_ddb_ls(self): md_on_ssd = check_ram_used(server_manager=self.server_managers[0], log=self.log) if md_on_ssd: self.log_step("MD-on-SSD: Create a directory to load pool data under /mnt.") - mkdir_command = f"mkdir {daos_load_path}" - self.run_cmd_check_result(command=mkdir_command) + self.run_cmd_check_result(command=f"mkdir {daos_load_path}") self.log_step("Create a pool and a container.") pool = self.get_pool() @@ -166,15 +165,18 @@ def test_recovery_ddb_ls(self): self.log_step("MD-on-SSD: Load pool dir to %s", daos_load_path) db_path = os.path.join( self.log_dir, "control_metadata", "daos_control", "engine0") - ddb_command.prov_mem(db_path=db_path, tmpfs_mount=daos_load_path) + ddb_command.prov_mem( + db_path=os.path.join( + self.log_dir, "control_metadata", "daos_control", "engine0"), + mpfs_mount=daos_load_path) self.log_step("Verify container UUID.") if md_on_ssd: # "ddb ls" command for MD-on-SSD is quite different. # PMEM: ddb /mnt/daos//vos-0 ls - # MD-on-SSD: ddb --db_path=/var/tmp/daos_testing/control_metadata/daos_control/engine0 /mnt/daos_load//vos-0 ls - db_path_arg = " ".join(["--db_path", db_path]) - ddb_command.db_path.update(value=db_path_arg) + # MD-on-SSD: ddb --db_path=/var/tmp/daos_testing/control_metadata/daos_control + # /engine0 /mnt/daos_load//vos-0 ls + ddb_command.db_path.update(value=" ".join(["--db_path", db_path])) vos_path = os.path.join(daos_load_path, pool.uuid.lower(), "vos-0") ddb_command.vos_path.update(value=vos_path) cmd_result = ddb_command.list_component() @@ -280,10 +282,8 @@ def test_recovery_ddb_ls(self): if md_on_ssd: self.log_step("MD-on-SSD: Clean %s", daos_load_path) - umount_cmd = f"umount {daos_load_path}" - self.run_cmd_check_result(command=umount_cmd) - rm_command = f"rm -rf {daos_load_path}" - self.run_cmd_check_result(command=rm_command) + self.run_cmd_check_result(command=f"umount {daos_load_path}") + self.run_cmd_check_result(command=f"rm -rf {daos_load_path}") self.log_step("Restart the server for the cleanup.") self.get_dmg_command().system_start() diff --git a/src/tests/ftest/util/ddb_utils.py b/src/tests/ftest/util/ddb_utils.py index 3768ff58047..7bbe72082f4 100644 --- a/src/tests/ftest/util/ddb_utils.py +++ b/src/tests/ftest/util/ddb_utils.py @@ -287,7 +287,7 @@ def dtx_cmt_clear(self, component_path="[0]"): return self.run() def prov_mem(self, db_path, tmpfs_mount): - """Call ddb "" prov_mem . + """Call ddb "" prov_mem . Before calling this method, "" (two double quotes) needs to be set to self.vos_path. From 70db49311281d5453883a900cf57b9c0cebaa541 Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Wed, 31 Dec 2025 06:15:10 +0000 Subject: [PATCH 3/7] DAOS-18387 test: Fix pylint Skip-unit-tests: true Skip-fault-injection-test: true Skip-func-hw-test-medium: false Test-tag: test_recovery_ddb_ls DdbPMEMTest Signed-off-by: Makito Kano --- src/tests/ftest/recovery/ddb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ftest/recovery/ddb.py b/src/tests/ftest/recovery/ddb.py index 7bc1f270a81..03d0d7a6a28 100644 --- a/src/tests/ftest/recovery/ddb.py +++ b/src/tests/ftest/recovery/ddb.py @@ -168,7 +168,7 @@ def test_recovery_ddb_ls(self): ddb_command.prov_mem( db_path=os.path.join( self.log_dir, "control_metadata", "daos_control", "engine0"), - mpfs_mount=daos_load_path) + mpfs_mount=daos_load_path) self.log_step("Verify container UUID.") if md_on_ssd: From 517afe8e7f91635415258a1316ad571a884ee3e1 Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Wed, 31 Dec 2025 06:26:58 +0000 Subject: [PATCH 4/7] DAOS-18387 test: Fix pylint Skip-unit-tests: true Skip-fault-injection-test: true Skip-func-hw-test-medium: false Test-tag: test_recovery_ddb_ls DdbPMEMTest Signed-off-by: Makito Kano --- src/tests/ftest/recovery/ddb.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/tests/ftest/recovery/ddb.py b/src/tests/ftest/recovery/ddb.py index 03d0d7a6a28..f9f368bf0fb 100644 --- a/src/tests/ftest/recovery/ddb.py +++ b/src/tests/ftest/recovery/ddb.py @@ -165,10 +165,7 @@ def test_recovery_ddb_ls(self): self.log_step("MD-on-SSD: Load pool dir to %s", daos_load_path) db_path = os.path.join( self.log_dir, "control_metadata", "daos_control", "engine0") - ddb_command.prov_mem( - db_path=os.path.join( - self.log_dir, "control_metadata", "daos_control", "engine0"), - mpfs_mount=daos_load_path) + ddb_command.prov_mem(db_path=db_path, tmpfs_mount=daos_load_path) self.log_step("Verify container UUID.") if md_on_ssd: @@ -195,8 +192,9 @@ def test_recovery_ddb_ls(self): actual_uuid = match.group(1).lower() expected_uuid = container.uuid.lower() if actual_uuid != expected_uuid: - msg = f"Unexpected container UUID! Expected = {expected_uuid}; Actual = {actual_uuid}" - errors.append(msg) + errors.append( + f"Unexpected container UUID! Expected = {expected_uuid}; Actual = " + f"{actual_uuid}") self.log_step("Verify object count in the container.") cmd_result = ddb_command.list_component(component_path="[0]") @@ -243,9 +241,9 @@ def test_recovery_ddb_ls(self): self.log_step("Verify there are two dkeys for every object.") expected_dkey_count = object_count * dkey_count if actual_dkey_count != expected_dkey_count: - msg = (f"Unexpected number of dkeys! Expected = {expected_dkey_count}; " - f"Actual = {actual_dkey_count}") - errors.append(msg) + errors.append( + f"Unexpected number of dkeys! Expected = {expected_dkey_count}; " + f"Actual = {actual_dkey_count}") msg = ("Verify there is one akey for every dkey. Also verify the key string and " "the size.") From 5327dc49e8c59d5df8f439fb8f45a66f783b7395 Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Wed, 31 Dec 2025 06:36:00 +0000 Subject: [PATCH 5/7] DAOS-18387 test: Fix pylint Skip-unit-tests: true Skip-fault-injection-test: true Skip-func-hw-test-medium: false Test-tag: test_recovery_ddb_ls DdbPMEMTest Signed-off-by: Makito Kano --- src/tests/ftest/recovery/ddb.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/tests/ftest/recovery/ddb.py b/src/tests/ftest/recovery/ddb.py index f9f368bf0fb..a199b1bf5ea 100644 --- a/src/tests/ftest/recovery/ddb.py +++ b/src/tests/ftest/recovery/ddb.py @@ -174,8 +174,8 @@ def test_recovery_ddb_ls(self): # MD-on-SSD: ddb --db_path=/var/tmp/daos_testing/control_metadata/daos_control # /engine0 /mnt/daos_load//vos-0 ls ddb_command.db_path.update(value=" ".join(["--db_path", db_path])) - vos_path = os.path.join(daos_load_path, pool.uuid.lower(), "vos-0") - ddb_command.vos_path.update(value=vos_path) + ddb_command.vos_path.update( + value=os.path.join(daos_load_path, pool.uuid.lower(), "vos-0")) cmd_result = ddb_command.list_component() # Sample output. # Listing contents of '/' @@ -245,9 +245,9 @@ def test_recovery_ddb_ls(self): f"Unexpected number of dkeys! Expected = {expected_dkey_count}; " f"Actual = {actual_dkey_count}") - msg = ("Verify there is one akey for every dkey. Also verify the key string and " - "the size.") - self.log_step(msg) + self.log_step( + "Verify there is one akey for every dkey. Also verify the key string and " + "the size.") akey_count = 0 for obj_index in range(object_count): for dkey_index in range(dkey_count): From 069c12679f54f80cc95dac03cda6c0e4a9d2dedd Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Fri, 2 Jan 2026 05:16:05 +0000 Subject: [PATCH 6/7] DAOS-18387 test: Add targets: 1 to ddb.yaml Skip-unit-tests: true Skip-fault-injection-test: true Skip-func-hw-test-medium: false Test-tag: test_recovery_ddb_ls DdbPMEMTest Signed-off-by: Makito Kano --- src/tests/ftest/recovery/ddb.py | 4 ++-- src/tests/ftest/recovery/ddb.yaml | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/tests/ftest/recovery/ddb.py b/src/tests/ftest/recovery/ddb.py index a199b1bf5ea..9cd0920a404 100644 --- a/src/tests/ftest/recovery/ddb.py +++ b/src/tests/ftest/recovery/ddb.py @@ -162,7 +162,7 @@ def test_recovery_ddb_ls(self): db_path = None if md_on_ssd: - self.log_step("MD-on-SSD: Load pool dir to %s", daos_load_path) + self.log_step(f"MD-on-SSD: Load pool dir to {daos_load_path}") db_path = os.path.join( self.log_dir, "control_metadata", "daos_control", "engine0") ddb_command.prov_mem(db_path=db_path, tmpfs_mount=daos_load_path) @@ -279,7 +279,7 @@ def test_recovery_ddb_ls(self): errors.append(msg) if md_on_ssd: - self.log_step("MD-on-SSD: Clean %s", daos_load_path) + self.log_step(f"MD-on-SSD: Clean {daos_load_path}") self.run_cmd_check_result(command=f"umount {daos_load_path}") self.run_cmd_check_result(command=f"rm -rf {daos_load_path}") diff --git a/src/tests/ftest/recovery/ddb.yaml b/src/tests/ftest/recovery/ddb.yaml index 6d31aa6daf3..007ad3cbd0f 100644 --- a/src/tests/ftest/recovery/ddb.yaml +++ b/src/tests/ftest/recovery/ddb.yaml @@ -11,6 +11,9 @@ server_config: 0: log_file: daos_server0.log nr_xs_helpers: 1 + # Objects are placed in different targets, or in different vos-x, so we need to use + # 1 target to make the test steps simpler. + targets: 1 storage: auto # In CI, all tests in ddb.py are ran in a single launch.py execution. In that case, the From a4106c022cded7d29f330dd43cf6aed9051f7739 Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Fri, 2 Jan 2026 05:21:00 +0000 Subject: [PATCH 7/7] DAOS-18387 test: Fix pylint Skip-unit-tests: true Skip-fault-injection-test: true Skip-func-hw-test-medium: false Test-tag: test_recovery_ddb_ls DdbPMEMTest Signed-off-by: Makito Kano --- src/tests/ftest/util/ddb_utils.py | 2 +- src/tests/ftest/util/recovery_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/ftest/util/ddb_utils.py b/src/tests/ftest/util/ddb_utils.py index 7bbe72082f4..242b1b61790 100644 --- a/src/tests/ftest/util/ddb_utils.py +++ b/src/tests/ftest/util/ddb_utils.py @@ -1,6 +1,6 @@ """ (C) Copyright 2022 Intel Corporation. - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ diff --git a/src/tests/ftest/util/recovery_utils.py b/src/tests/ftest/util/recovery_utils.py index 1b5fc5a4202..1ca6e71c393 100644 --- a/src/tests/ftest/util/recovery_utils.py +++ b/src/tests/ftest/util/recovery_utils.py @@ -1,6 +1,6 @@ """ (C) Copyright 2024 Intel Corporation. - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """