diff --git a/src/tests/ftest/pool/eviction_metrics.py b/src/tests/ftest/pool/eviction_metrics.py new file mode 100644 index 00000000000..33b5524327d --- /dev/null +++ b/src/tests/ftest/pool/eviction_metrics.py @@ -0,0 +1,124 @@ +""" + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP + + SPDX-License-Identifier: BSD-2-Clause-Patent +""" +import json +import math + +from job_manager_utils import get_job_manager +from mdtest_utils import MDTEST_NAMESPACE, run_mdtest +from telemetry_test_base import TestWithTelemetry + + +class EvictionMetrics(TestWithTelemetry): + """ + Tests DAOS client eviction from a pool that the client is using. + + :avocado: recursive + """ + + def test_eviction_metrics(self): + """Verify page eviction on the pool + + 1. Create a pool with a mem ratio of 100% (for pmem or phase 1) or 25% (for phase 2) + 2. Collect a baseline for the pool eviction metrics + 3. Run mdtest -a DFS to generate many small files larger than mem size + 4. Collect new page eviction metrics + 5. Verify page eviction + + :avocado: tags=all,daily_regression + :avocado: tags=hw,medium + :avocado: tags=pool + :avocado: tags=EvictionMetrics,test_eviction_metrics + """ + write_bytes = self.params.get('write_bytes', MDTEST_NAMESPACE, None) + processes = self.params.get('processes', MDTEST_NAMESPACE, None) + ppn = self.params.get('ppn', MDTEST_NAMESPACE, None) + + evict_metrics = list(self.telemetry.ENGINE_POOL_VOS_CACHE_METRICS) + + self.log_step('Creating a pool (dmg pool create)') + pool = self.get_pool(connect=False) + try: + _result = json.loads(pool.dmg.result.stdout) + tier_bytes_scm = int(_result["response"]["tier_bytes"][0]) + mem_file_bytes = int(_result["response"]["mem_file_bytes"]) + except Exception as error: # pylint: disable=broad-except + self.fail(f"Error extracting data for dmg pool create output: {error}") + + # Calculate the mdtest files_per_process based upon the scm size and other mdtest params + _write_processes = processes + if ppn is not None: + _write_processes = ppn * len(self.host_info.clients.hosts) + files_per_process = math.floor(mem_file_bytes / (write_bytes * _write_processes)) + if tier_bytes_scm > mem_file_bytes: + # Write more (125%) files to exceed mem_file_bytes and cause eviction + mdtest_params = {"num_of_files_dirs": math.ceil(files_per_process * 1.25)} + else: + # Write less (75%) files to avoid out of space errors + mdtest_params = {"num_of_files_dirs": math.floor(files_per_process * 0.75)} + + self.log.debug("-" * 60) + self.log.debug("Pool %s create data:", pool) + self.log.debug(" tier_bytes_scm: %s", tier_bytes_scm) + self.log.debug(" mem_file_bytes: %s", mem_file_bytes) + self.log.debug(" mem_ratio.value: %s", pool.mem_ratio.value) + self.log.debug("Mdtest write parameters:") + self.log.debug(" write_bytes: %s", write_bytes) + if ppn is not None: + self.log.debug(" ppn / nodes: %s / %s", ppn, len(self.host_info.clients.hosts)) + else: + self.log.debug(" processes: %s", processes) + self.log.debug(" files_per_process: %s", files_per_process) + self.log.debug(" num_of_files_dirs: %s", mdtest_params["num_of_files_dirs"]) + self.log.debug(" expected to write: %s", + _write_processes * write_bytes * mdtest_params["num_of_files_dirs"]) + self.log.debug("-" * 60) + + self.log_step('Creating a container (dmg container create)') + container = self.get_container(pool) + + self.log_step( + 'Collect pool eviction metrics after creating a pool (dmg telemetry metrics query)') + expected_ranges = self.telemetry.collect_data(evict_metrics) + for metric in sorted(expected_ranges): + for label in expected_ranges[metric]: + if pool.mem_ratio.value is not None and metric.endswith('_hit'): + expected_ranges[metric][label] = [0, 100] # 0-100 (phase 2) + elif pool.mem_ratio.value is not None and metric.endswith('_miss'): + expected_ranges[metric][label] = [0, 5] # 0-5 (phase 2) + elif pool.mem_ratio.value is not None and metric.endswith('_ne'): + expected_ranges[metric][label] = [0, 5] # 0-5 (phase 2) + else: + expected_ranges[metric][label] = [0, 0] # 0 only + self.log.debug("%s expected_ranges: %s", pool, expected_ranges) + + self.log_step('Verify pool eviction metrics after pool creation') + if not self.telemetry.verify_data(expected_ranges): + self.fail('Pool eviction metrics verification failed after pool creation') + + self.log_step('Writing data to the pool (mdtest -a DFS)') + manager = get_job_manager(self, subprocess=False, timeout=None) + run_mdtest( + self, self.hostlist_clients, self.workdir, None, container, processes, ppn, manager, + mdtest_params=mdtest_params) + + self.log_step( + 'Collect pool eviction metrics after writing data (dmg telemetry metrics query)') + expected_ranges = self.telemetry.collect_data(evict_metrics) + for metric in sorted(expected_ranges): + for label in expected_ranges[metric]: + if pool.mem_ratio.value is None: + expected_ranges[metric][label] = [0, 0] # 0 only (phase 1) + elif metric.endswith('_page_flush'): + expected_ranges[metric][label] = [0] # 0 or greater (phase 2) + else: + expected_ranges[metric][label] = [1, 10000000] # 1-10,000,000 (phase 2) + self.log.debug("%s expected_ranges: %s", pool, expected_ranges) + + self.log_step('Verify pool eviction metrics after writing data') + if not self.telemetry.verify_data(expected_ranges): + self.fail('Pool eviction metrics verification failed after writing data') + + self.log_step('Test passed') diff --git a/src/tests/ftest/pool/eviction_metrics.yaml b/src/tests/ftest/pool/eviction_metrics.yaml new file mode 100644 index 00000000000..b90e3515daa --- /dev/null +++ b/src/tests/ftest/pool/eviction_metrics.yaml @@ -0,0 +1,42 @@ +launch: + !filter-only : /run/pool/default # yamllint disable-line rule:colons + +hosts: + test_servers: 1 + test_clients: 3 + +timeout: 16000 + +server_config: + name: daos_server + engines_per_host: 1 + engines: + 0: + targets: 4 + nr_xs_helpers: 0 + storage: auto + +pool: !mux + default: + size: 10% + md_on_ssd_p2: + size: 10% + mem_ratio: 25 + +container: + type: POSIX + oclass: S1 + dir_oclass: SX + +mdtest: + dfs_oclass: S1 + dfs_dir_oclass: SX + dfs_destroy: False + manager: "MPICH" + ppn: 32 + test_dir: "/" + api: DFS + flags: "-C -F -G 27 -N 1 -Y -u -L" + branching_factor: 1 + write_bytes: 1024 + read_bytes: 1024 diff --git a/src/tests/ftest/util/mdtest_utils.py b/src/tests/ftest/util/mdtest_utils.py index 97e5d75d088..1a736649d39 100644 --- a/src/tests/ftest/util/mdtest_utils.py +++ b/src/tests/ftest/util/mdtest_utils.py @@ -1,5 +1,6 @@ """ (C) Copyright 2019-2024 Intel Corporation. + (C) Copyright 2025 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -8,20 +9,115 @@ import re from command_utils import ExecutableCommand -from command_utils_base import FormattedParameter, LogParameter +from command_utils_base import BasicParameter, FormattedParameter, LogParameter +from exception_utils import CommandFailure from general_utils import get_log_file +from job_manager_utils import get_job_manager + +MDTEST_NAMESPACE = "/run/mdtest/*" + + +def get_mdtest(test, hosts, manager=None, path=None, slots=None, namespace=MDTEST_NAMESPACE, + mdtest_params=None): + """Get a Mdtest object. + + Args: + test (Test): avocado Test object + hosts (NodeSet): hosts on which to run the mdtest command + manager (JobManager, optional): command to manage the multi-host execution of mdtest. + Defaults to None, which will get a default job manager. + path (str, optional): hostfile path. Defaults to None. + slots (int, optional): hostfile number of slots per host. Defaults to None. + namespace (str, optional): path to yaml parameters. Defaults to MDTEST_NAMESPACE. + mdtest_params (dict, optional): parameters to update the mdtest command. Defaults to None. + + Returns: + Mdtest: the Mdtest object requested + """ + mdtest = Mdtest(test, hosts, manager, path, slots, namespace) + if mdtest_params: + for name, value in mdtest_params.items(): + mdtest.update(name, value) + return mdtest + + +def run_mdtest(test, hosts, path, slots, container, processes, ppn=None, manager=None, + log_file=None, intercept=None, display_space=True, namespace=MDTEST_NAMESPACE, + mdtest_params=None): + # pylint: disable=too-many-arguments + """Run Mdtest on multiple hosts. + + Args: + test (Test): avocado Test object + hosts (NodeSet): hosts on which to run the mdtest command + path (str): hostfile path. + slots (int): hostfile number of slots per host. + container (TestContainer): DAOS test container object. + processes (int): number of processes to run + ppn (int, optional): number of processes per node to run. If specified it will override + the processes input. Defaults to None. + manager (JobManager, optional): command to manage the multi-host execution of mdtest. + Defaults to None, which will get a default job manager. + log_file (str, optional): log file name. Defaults to None, which will result in a log file + name containing the test, pool, and container IDs. + intercept (str, optional): path to interception library. Defaults to None. + display_space (bool, optional): Whether to display the pool space. Defaults to True. + namespace (str, optional): path to yaml parameters. Defaults to MDTEST_NAMESPACE. + mdtest_params (dict, optional): dictionary of MdtestCommand attributes to override from + get_params(). Defaults to None. + + Raises: + CommandFailure: if there is an error running the mdtest command + + Returns: + CmdResult: result of the ior command + + """ + mdtest = get_mdtest(test, hosts, manager, path, slots, namespace, mdtest_params) + if log_file is None: + log_file = mdtest.get_unique_log(container) + mdtest.update_log_file(log_file) + return mdtest.run(container, processes, ppn, intercept, display_space) + + +def write_mdtest_data(test, container, namespace=MDTEST_NAMESPACE, **mdtest_run_params): + """Write data to the container/dfuse using mdtest. + + Simple method for test classes to use to write data with mdtest. While not required, this is + setup by default to pull in mdtest parameters from the test yaml. + + Args: + test (Test): avocado Test object + container (TestContainer): the container to populate + namespace (str, optional): path to mdtest yaml parameters. Defaults to MDTEST_NAMESPACE. + mdtest_run_params (dict): optional params for the Mdtest.run() command. + + Returns: + Mdtest: the Mdtest object used to populate the container + """ + mdtest = get_mdtest(test, test.hostlist_clients, None, test.workdir, None, namespace) + mdtest.update_log_file(mdtest.get_unique_log(container)) + + if 'processes' not in mdtest_run_params: + mdtest_run_params['processes'] = test.params.get('processes', namespace, None) + elif 'ppn' not in mdtest_run_params: + mdtest_run_params['ppn'] = test.params.get('ppn', namespace, None) + + mdtest.run(container, **mdtest_run_params) + return mdtest class MdtestCommand(ExecutableCommand): """Defines a object representing a mdtest command.""" - def __init__(self, log_dir): + def __init__(self, log_dir, namespace="/run/mdtest/*"): """Create an MdtestCommand object. Args: log_dir (str): directory in which to put log files + namespace (str, optional): path to yaml parameters. Defaults to "/run/mdtest/*". """ - super().__init__("/run/mdtest/*", "mdtest") + super().__init__(namespace, "mdtest") self._log_dir = log_dir @@ -137,6 +233,147 @@ def get_default_env(self, manager_cmd, log_file=None): return env +class Mdtest: + """Defines a class that runs the mdtest command through a job manager, e.g. mpirun.""" + + def __init__(self, test, hosts, manager=None, path=None, slots=None, + namespace=MDTEST_NAMESPACE): + """Initialize an Mdtest object. + + Args: + test (Test): avocado Test object + hosts (NodeSet): hosts on which to run the mdtest command + manager (JobManager, optional): command to manage the multi-host execution of mdtest. + Defaults to None, which will get a default job manager. + path (str, optional): hostfile path. Defaults to None. + slots (int, optional): hostfile number of slots per host. Defaults to None. + namespace (str, optional): path to yaml parameters. Defaults to MDTEST_NAMESPACE. + """ + if manager is None: + manager = get_job_manager(test, subprocess=False, timeout=60) + self.manager = manager + self.manager.assign_hosts(hosts, path, slots) + self.manager.job = MdtestCommand(test.test_env.log_dir, namespace) + self.manager.job.get_params(test) + self.manager.output_check = "both" + self.timeout = test.params.get("timeout", namespace, None) + self.label_generator = test.label_generator + self.test_id = test.test_id + self.env = self.command.get_default_env(str(self.manager)) + + @property + def command(self): + """Get the MdtestCommand object. + + Returns: + MdtestCommand: the MdtestCommand object managed by the JobManager + + """ + return self.manager.job + + def update(self, name, value): + """Update a MdtestCommand BasicParameter with a new value. + + Args: + name (str): name of the MdtestCommand BasicParameter to update + value (str): value to assign to the MdtestCommand BasicParameter + """ + param = getattr(self.command, name, None) + if param: + if isinstance(param, BasicParameter): + param.update(value, ".".join([self.command.command, name])) + + def update_log_file(self, log_file): + """Update the log file for the mdtest command. + + Args: + log_file (str): new mdtest log file + """ + self.command.env["D_LOG_FILE"] = get_log_file( + log_file or f"{self.command.command}_daos.log") + + def get_unique_log(self, container): + """Get a unique mdtest log file name. + + Args: + container (TestContainer): container involved with the command + + Returns: + str: a log file name + """ + label = self.label_generator.get_label("mdtest") + parts = [self.test_id, container.pool.identifier, container.identifier, label] + return '.'.join(['_'.join(parts), 'log']) + + def update_daos_params(self, pool, container): + """Set the mdtest parameters for the pool and container. + + Optionally also set the DAOS pool and container environment variables for mdtest. + + Args: + pool (TestPool): the pool to use with the mdtest command + container (TestContainer): the container to use with the mdtest command + """ + self.command.update_params(dfs_pool=pool.identifier, dfs_cont=container.identifier) + + if "mpirun" in str(self.manager) or "srun" in str(self.manager): + self.env["DAOS_POOL"] = self.command.dfs_pool.value + self.env["DAOS_CONT"] = self.command.dfs_cont.value + self.env["IOR_HINT__MPI__romio_daos_obj_class"] = self.command.dfs_oclass.value + + def run(self, container, processes, ppn=None, intercept=None, display_space=True): + """Run mdtest. + + Args: + container (TestContainer): DAOS test container object. + processes (int): number of processes to run + ppn (int, optional): number of processes per node to run. If specified it will override + the processes input. Defaults to None. + intercept (str, optional): path to interception library. Defaults to None. + display_space (bool, optional): Whether to display the pool space. Defaults to True. + + Raises: + CommandFailure: if there is an error running the mdtest command + + Returns: + CmdResult: result of the mdtest command + """ + result = None + error_message = None + + self.update_daos_params(container.pool, container) + + if intercept: + self.env["LD_PRELOAD"] = intercept + if "D_LOG_MASK" not in self.env: + self.env["D_LOG_MASK"] = "INFO" + + # Pass only processes or ppn to be compatible with previous behavior + if ppn is not None: + self.manager.assign_processes(ppn=ppn) + else: + self.manager.assign_processes(processes=processes) + + self.manager.assign_environment(self.env) + + try: + if display_space: + container.pool.display_space() + result = self.manager.run() + + except CommandFailure as error: + error_message = "Mdtest Failed:\n {}".format("\n ".join(str(error).split("\n"))) + + finally: + if not self.manager.run_as_subprocess and display_space: + container.pool.display_space() + + if error_message: + raise CommandFailure(error_message) + + return result + + class MdtestMetrics(): # pylint: disable=too-few-public-methods """Represents metrics from mdtest output. diff --git a/src/tests/ftest/util/telemetry_utils.py b/src/tests/ftest/util/telemetry_utils.py index 8937db87788..5230fba5a46 100644 --- a/src/tests/ftest/util/telemetry_utils.py +++ b/src/tests/ftest/util/telemetry_utils.py @@ -159,6 +159,12 @@ class TelemetryUtils(): "engine_pool_vos_wal_replay_size", "engine_pool_vos_wal_replay_time", "engine_pool_vos_wal_replay_transactions"] + ENGINE_POOL_VOS_CACHE_METRICS = [ + "engine_pool_vos_cache_page_evict", + "engine_pool_vos_cache_page_flush", + "engine_pool_vos_cache_page_hit", + "engine_pool_vos_cache_page_miss", + "engine_pool_vos_cache_page_ne"] ENGINE_POOL_SVC_METRICS = [ "engine_pool_svc_degraded_ranks", "engine_pool_svc_disabled_targets", @@ -179,6 +185,7 @@ class TelemetryUtils(): ENGINE_POOL_VOS_SPACE_METRICS + \ ENGINE_POOL_VOS_WAL_METRICS + \ ENGINE_POOL_VOS_WAL_REPLAY_METRICS +\ + ENGINE_POOL_VOS_CACHE_METRICS +\ ENGINE_POOL_SVC_METRICS ENGINE_EVENT_METRICS = [ "engine_events_dead_ranks", diff --git a/src/vos/vos_internal.h b/src/vos/vos_internal.h index 428051203ed..c3ef2867ebb 100644 --- a/src/vos/vos_internal.h +++ b/src/vos/vos_internal.h @@ -263,7 +263,7 @@ struct vos_cache_metrics { struct d_tm_node_t *vcm_obj_hit; }; -void vos_cache_metrics_init(struct vos_cache_metrics *vc_metrcis, const char *path, int tgt_id); +void vos_cache_metrics_init(struct vos_cache_metrics *vc_metrics, const char *path, int tgt_id); struct vos_pool_metrics { void *vp_vea_metrics;