diff --git a/.github/workflows/bullseye-coverage.yml b/.github/workflows/bullseye-coverage.yml index 52c9b5dea81..cadf7e889df 100644 --- a/.github/workflows/bullseye-coverage.yml +++ b/.github/workflows/bullseye-coverage.yml @@ -563,11 +563,13 @@ jobs: STAGE_TAGS+=",provider" if [[ '${{ matrix.stage }}' = *\ Verbs\ * ]]; then FTEST_ARG+=' --provider ofi+verbs' + INST_RPMS+=' mercury-libfabric' elif [[ '${{ matrix.stage }}' = *\ UCX\ * ]]; then FTEST_ARG+=' --provider ucx+dc_x' INST_RPMS+=' mercury-ucx' elif [[ '${{ matrix.stage }}' = *\ TCP\ * ]]; then FTEST_ARG+=' --provider ofi+tcp' + INST_RPMS+=' mercury-libfabric' else echo 'Unknown provider in ${{ matrix.stage }}' exit 1 diff --git a/.github/workflows/rpm-build-and-test.yml b/.github/workflows/rpm-build-and-test.yml index 00eb9b021a8..b6938a8194f 100644 --- a/.github/workflows/rpm-build-and-test.yml +++ b/.github/workflows/rpm-build-and-test.yml @@ -572,11 +572,13 @@ jobs: STAGE_TAGS+=",provider" if [[ '${{ matrix.stage }}' = *\ Verbs\ * ]]; then FTEST_ARG+=' --provider ofi+verbs' + INST_RPMS+=' mercury-libfabric' elif [[ '${{ matrix.stage }}' = *\ UCX\ * ]]; then FTEST_ARG+=' --provider ucx+dc_x' INST_RPMS+=' mercury-ucx' elif [[ '${{ matrix.stage }}' = *\ TCP\ * ]]; then FTEST_ARG+=' --provider ofi+tcp' + INST_RPMS+=' mercury-libfabric' else echo 'Unknown provider in ${{ matrix.stage }}' exit 1 diff --git a/Jenkinsfile b/Jenkinsfile index 756892b901d..0fb13828cd9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,7 +5,7 @@ /* groovylint-disable ParameterName, VariableName */ /* Copyright 2019-2024 Intel Corporation /* Copyright 2025 Google LLC - * Copyright 2025 Hewlett Packard Enterprise Development LP + * Copyright 2025-2026 Hewlett Packard Enterprise Development LP * All rights reserved. * * This file is part of the DAOS Project. It is subject to the license terms @@ -18,7 +18,7 @@ // To use a test branch (i.e. PR) until it lands to master // I.e. for testing library changes -//@Library(value='pipeline-lib@your_branch') _ +@Library(value='pipeline-lib@soumagne/mercury_libfabric') _ /* groovylint-disable-next-line CompileStatic */ job_status_internal = [:] @@ -1048,7 +1048,8 @@ pipeline { steps { job_step_update( testRpm(inst_repos: daosRepos(), - daos_pkg_version: daosPackagesVersion(next_version())) + daos_pkg_version: daosPackagesVersion(next_version()), + inst_rpms: 'mercury-libfabric') ) } post { @@ -1100,7 +1101,8 @@ pipeline { } */ job_step_update( testRpm(inst_repos: daosRepos(), - daos_pkg_version: daosPackagesVersion(next_version())) + daos_pkg_version: daosPackagesVersion(next_version()), + inst_rpms: 'mercury-libfabric') ) } post { diff --git a/ci/provisioning/post_provision_config_common.sh b/ci/provisioning/post_provision_config_common.sh index 257e6dfe2d9..3cac657efaf 100755 --- a/ci/provisioning/post_provision_config_common.sh +++ b/ci/provisioning/post_provision_config_common.sh @@ -1,7 +1,7 @@ #!/bin/bash # # Copyright 2021-2023 Intel Corporation. -# Copyright 2025 Hewlett Packard Enterprise Development LP +# Copyright 2025-2026 Hewlett Packard Enterprise Development LP # # SPDX-License-Identifier: BSD-2-Clause-Patent # @@ -32,7 +32,7 @@ fi # shellcheck disable=SC1091 . /etc/os-release # shellcheck disable=SC2034 -EXCLUDE_UPGRADE=mercury,daos,daos-\* +EXCLUDE_UPGRADE=mercury,mercury-\*,daos,daos-\* if rpm -qa | grep mlnx; then # packages not to allow upgrading if MLNX OFED is installed EXCLUDE_UPGRADE+=,openmpi,\*mlnx\*,\*ucx\* diff --git a/ci/unit/required_packages.sh b/ci/unit/required_packages.sh index bbd3313155a..5a57b0cb054 100755 --- a/ci/unit/required_packages.sh +++ b/ci/unit/required_packages.sh @@ -1,5 +1,10 @@ #!/bin/bash - +# +# (C) Copyright 2025 Google LLC +# Copyright 2025-2026 Hewlett Packard Enterprise Development LP +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# set -eu # No longer used but provided by pipeline-lib @@ -24,6 +29,7 @@ pkgs="$(utils/rpms/package_version.sh argobots lib) \ $(utils/rpms/package_version.sh libfabric debug) \ $(utils/rpms/package_version.sh mercury dev) \ $(utils/rpms/package_version.sh mercury debug) \ + $(utils/rpms/package_version.sh mercury lib mercury_libfabric) \ $(utils/rpms/package_version.sh pmdk lib pmemobj) \ $(utils/rpms/package_version.sh pmdk debug pmemobj) \ $(utils/rpms/package_version.sh pmdk debug pmem) \ diff --git a/deps/patches/mercury/0001_dep_versions.patch b/deps/patches/mercury/0001_dep_versions.patch new file mode 100644 index 00000000000..195775f0f95 --- /dev/null +++ b/deps/patches/mercury/0001_dep_versions.patch @@ -0,0 +1,71 @@ +From c0d62973299b4ba98a9aaf9f82ed000a11c0f7b0 Mon Sep 17 00:00:00 2001 +From: Jerome Soumagne +Date: Mon, 24 Nov 2025 16:44:14 -0600 +Subject: [PATCH] NA OFI/UCX: print version infos + +NA OFI: fix log warning +--- + src/na/na_ofi.c | 12 +++++++----- + src/na/na_ucx.c | 10 ++++++++++ + 2 files changed, 17 insertions(+), 5 deletions(-) + +diff --git a/src/na/na_ofi.c b/src/na/na_ofi.c +index c7c3e0b3..682efe65 100644 +--- a/src/na/na_ofi.c ++++ b/src/na/na_ofi.c +@@ -8048,8 +8048,10 @@ na_ofi_check_protocol(const char *protocol_name) + uint32_t runtime_version = fi_version(); + na_return_t na_ret; + +- NA_LOG_SUBSYS_DEBUG(cls, "Querying info on libfabric v%d.%d", +- FI_MAJOR(runtime_version), FI_MINOR(runtime_version)); ++ NA_LOG_SUBSYS_INFO(cls, ++ "Querying info on libfabric (runtime v%d.%d, API v%d.%d)", ++ FI_MAJOR(runtime_version), FI_MINOR(runtime_version), ++ FI_MAJOR(FI_COMPILE_VERSION), FI_MINOR(FI_COMPILE_VERSION)); + NA_CHECK_SUBSYS_ERROR(cls, FI_VERSION_LT(runtime_version, NA_OFI_VERSION), + out, accept, false, + "runtime libfabric version (v%d.%d) is lower than required version " +@@ -9105,7 +9107,7 @@ na_ofi_mem_handle_create(na_class_t NA_UNUSED *na_class, void *buf, + + NA_LOG_SUBSYS_DEBUG(mem, + "Created mem handle %p (iov_base=%p, iov_len=%zu, iovcnt=1, " +- "flags=0x%lx, len=%zu)", ++ "flags=0x%" PRIx8 ", len=%" PRIu64 ")", + (void *) na_ofi_mem_handle, na_ofi_mem_handle->desc.iov.s[0].iov_base, + na_ofi_mem_handle->desc.iov.s[0].iov_len, + na_ofi_mem_handle->desc.info.flags, na_ofi_mem_handle->desc.info.len); +@@ -9444,8 +9446,8 @@ na_ofi_mem_handle_deserialize(na_class_t NA_UNUSED *na_class, + na_ofi_mem_handle->desc.info.iovcnt); + + NA_LOG_SUBSYS_DEBUG(mem, +- "Deserialized mem handle %p (iov_base=%p, iov_len=%zu, iovcnt=%zu, " +- "flags=0x%lx, len=%zu)", ++ "Deserialized mem handle %p (iov_base=%p, iov_len=%zu, iovcnt=%" PRIu64 ++ ", flags=0x%" PRIx8 ", len=%" PRIu64 ")", + (void *) na_ofi_mem_handle, na_ofi_mem_handle->desc.iov.s[0].iov_base, + na_ofi_mem_handle->desc.iov.s[0].iov_len, + na_ofi_mem_handle->desc.info.iovcnt, na_ofi_mem_handle->desc.info.flags, +diff --git a/src/na/na_ucx.c b/src/na/na_ucx.c +index 96501b27..7c0ac8d4 100644 +--- a/src/na/na_ucx.c ++++ b/src/na/na_ucx.c +@@ -3433,8 +3433,18 @@ na_ucx_check_protocol(const char *protocol_name) + .field_mask = UCP_PARAM_FIELD_FEATURES, .features = NA_UCX_FEATURES}; + ucp_context_h context = NULL; + ucs_status_t status; ++ unsigned int runtime_major_version, runtime_minor_version, ++ runtime_patch_version; + bool accept = false; + ++ ucp_get_version( ++ &runtime_major_version, &runtime_minor_version, &runtime_patch_version); ++ ++ NA_LOG_SUBSYS_INFO(cls, ++ "Querying info on UCX (runtime v%u.%u.%u, API v%u.%u)", ++ runtime_major_version, runtime_minor_version, runtime_patch_version, ++ UCP_API_MAJOR, UCP_API_MINOR); ++ + status = ucp_config_read(NULL, NULL, &config); + NA_CHECK_SUBSYS_ERROR_NORET(cls, status != UCS_OK, done, + "ucp_config_read() failed (%s)", ucs_status_string(status)); diff --git a/deps/patches/mercury/0001_na_ucx.patch b/deps/patches/mercury/0001_na_ucx.patch deleted file mode 100644 index 57b39feef9d..00000000000 --- a/deps/patches/mercury/0001_na_ucx.patch +++ /dev/null @@ -1,110 +0,0 @@ -diff --git a/src/na/na_ucx.c b/src/na/na_ucx.c -index 84eb8b0..e4b6676 100644 ---- a/src/na/na_ucx.c -+++ b/src/na/na_ucx.c -@@ -614,7 +614,7 @@ na_ucx_addr_map_update(struct na_ucx_class *na_ucx_class, - */ - static na_return_t - na_ucx_addr_map_remove( -- struct na_ucx_map *na_ucx_map, ucs_sock_addr_t *addr_key); -+ struct na_ucx_map *na_ucx_map, struct na_ucx_addr *remove_addr); - - /** - * Hash connection ID. -@@ -1688,8 +1688,12 @@ na_ucp_listener_conn_cb(ucp_conn_request_h conn_request, void *arg) - .addr = (const struct sockaddr *) &conn_request_attrs.client_address, - .addrlen = sizeof(conn_request_attrs.client_address)}; - na_ucx_addr = na_ucx_addr_map_lookup(&na_ucx_class->addr_map, &addr_key); -- NA_CHECK_SUBSYS_ERROR_NORET(addr, na_ucx_addr != NULL, error, -- "An entry is already present for this address"); -+ -+ if (na_ucx_addr != NULL) { -+ NA_LOG_SUBSYS_WARNING(addr, -+ "An entry is already present for this address"); -+ na_ucx_addr_map_remove(&na_ucx_class->addr_map, na_ucx_addr); -+ } - - /* Insert new entry and create new address */ - na_ret = na_ucx_addr_map_insert(na_ucx_class, &na_ucx_class->addr_map, -@@ -1937,10 +1941,14 @@ na_ucp_ep_error_cb( - static void - na_ucp_ep_close(ucp_ep_h ep) - { -- ucs_status_ptr_t status_ptr = ucp_ep_close_nb(ep, UCP_EP_CLOSE_MODE_FORCE); -+ const ucp_request_param_t close_params = { -+ .op_attr_mask = UCP_OP_ATTR_FIELD_FLAGS, -+ .flags = UCP_EP_CLOSE_FLAG_FORCE}; -+ ucs_status_ptr_t status_ptr = ucp_ep_close_nbx(ep, &close_params); -+ - NA_CHECK_SUBSYS_ERROR_DONE(addr, - status_ptr != NULL && UCS_PTR_IS_ERR(status_ptr), -- "ucp_ep_close_nb() failed (%s)", -+ "ucp_ep_close_nbx() failed (%s)", - ucs_status_string(UCS_PTR_STATUS(status_ptr))); - } - -@@ -2722,7 +2730,7 @@ unlock: - - /*---------------------------------------------------------------------------*/ - static na_return_t --na_ucx_addr_map_remove(struct na_ucx_map *na_ucx_map, ucs_sock_addr_t *addr_key) -+na_ucx_addr_map_remove(struct na_ucx_map *na_ucx_map, struct na_ucx_addr *remove_addr) - { - struct na_ucx_addr *na_ucx_addr = NULL; - na_return_t ret = NA_SUCCESS; -@@ -2731,13 +2739,14 @@ na_ucx_addr_map_remove(struct na_ucx_map *na_ucx_map, ucs_sock_addr_t *addr_key) - hg_thread_rwlock_wrlock(&na_ucx_map->lock); - - na_ucx_addr = hg_hash_table_lookup( -- na_ucx_map->key_map, (hg_hash_table_key_t) addr_key); -- if (na_ucx_addr == HG_HASH_TABLE_NULL) -+ na_ucx_map->key_map, (hg_hash_table_key_t) &remove_addr->addr_key); -+ -+ if (na_ucx_addr == HG_HASH_TABLE_NULL || na_ucx_addr->ucp_ep != remove_addr->ucp_ep) - goto unlock; - - /* Remove addr key from primary map */ - rc = hg_hash_table_remove( -- na_ucx_map->key_map, (hg_hash_table_key_t) addr_key); -+ na_ucx_map->key_map, (hg_hash_table_key_t) &na_ucx_addr->addr_key); - NA_CHECK_SUBSYS_ERROR(addr, rc != 1, unlock, ret, NA_NOENTRY, - "hg_hash_table_remove() failed"); - -@@ -2841,7 +2850,7 @@ na_ucx_addr_release(struct na_ucx_addr *na_ucx_addr) - NA_UCX_PRINT_ADDR_KEY_INFO("Removing address", &na_ucx_addr->addr_key); - - na_ucx_addr_map_remove( -- &na_ucx_addr->na_ucx_class->addr_map, &na_ucx_addr->addr_key); -+ &na_ucx_addr->na_ucx_class->addr_map, na_ucx_addr); - } - - if (na_ucx_addr->ucp_ep != NULL) { -@@ -3023,6 +3032,18 @@ na_ucx_rma(struct na_ucx_class NA_UNUSED *na_ucx_class, na_context_t *context, - - /* There is no need to have a fully resolved address to start an RMA. - * This is only necessary for two-sided communication. */ -+ /* The above assumption is now in question, so the following will resolve -+ * the address if required. */ -+ -+ /* Check addr to ensure the EP for that addr is still valid */ -+ if (!(hg_atomic_get32(&na_ucx_addr->status) & NA_UCX_ADDR_RESOLVED)) { -+ ret = na_ucx_addr_map_update( -+ na_ucx_class, &na_ucx_class->addr_map, na_ucx_addr); -+ NA_CHECK_SUBSYS_NA_ERROR( -+ addr, error, ret, "Could not update NA UCX address"); -+ } -+ NA_CHECK_SUBSYS_ERROR(msg, na_ucx_addr->ucp_ep == NULL, error, ret, -+ NA_ADDRNOTAVAIL, "UCP endpoint is NULL for that address"); - - /* TODO UCX requires the remote key to be bound to the origin, do we need a - * new API? */ -@@ -3061,6 +3082,9 @@ na_ucx_rma_key_resolve(ucp_ep_h ep, struct na_ucx_mem_handle *na_ucx_mem_handle, - - hg_thread_mutex_lock(&na_ucx_mem_handle->rkey_unpack_lock); - -+ NA_CHECK_SUBSYS_ERROR( -+ mem, ep == NULL, error, ret, NA_INVALID_ARG, "Invalid endpoint (%p)", ep); -+ - switch (hg_atomic_get32(&na_ucx_mem_handle->type)) { - case NA_UCX_MEM_HANDLE_REMOTE_PACKED: { - ucs_status_t status = ucp_ep_rkey_unpack(ep, diff --git a/deps/patches/mercury/0002_na_ucx_ep_flush.patch b/deps/patches/mercury/0002_na_ucx_ep_flush.patch deleted file mode 100644 index f7b38d304aa..00000000000 --- a/deps/patches/mercury/0002_na_ucx_ep_flush.patch +++ /dev/null @@ -1,64 +0,0 @@ -diff --git a/src/na/na_ucx.c b/src/na/na_ucx.c -index 6e9c3b0..2f157da 100644 ---- a/src/na/na_ucx.c -+++ b/src/na/na_ucx.c -@@ -441,6 +441,12 @@ na_ucp_ep_create(ucp_worker_h worker, ucp_ep_params_t *ep_params, - static void - na_ucp_ep_error_cb(void *arg, ucp_ep_h ep, ucs_status_t status); - -+/** -+ * Flush endpoint. -+ */ -+static ucs_status_ptr_t -+na_ucp_ep_flush(ucp_ep_h ep); -+ - /** - * Close endpoint. - */ -@@ -1940,6 +1946,21 @@ na_ucp_ep_error_cb( - na_ucx_addr_ref_decr(na_ucx_addr); - } - -+/*---------------------------------------------------------------------------*/ -+static ucs_status_ptr_t -+na_ucp_ep_flush(ucp_ep_h ep) -+{ -+ const ucp_request_param_t flush_params = { -+ .op_attr_mask = 0}; -+ ucs_status_ptr_t status_ptr = ucp_ep_flush_nbx(ep, &flush_params); -+ -+ NA_CHECK_SUBSYS_ERROR_DONE(addr, -+ status_ptr != NULL && UCS_PTR_IS_ERR(status_ptr), -+ "ucp_ep_flush_nb() failed (%s)", -+ ucs_status_string(UCS_PTR_STATUS(status_ptr))); -+ return status_ptr; -+} -+ - /*---------------------------------------------------------------------------*/ - static void - na_ucp_ep_close(ucp_ep_h ep) -@@ -2859,8 +2880,23 @@ na_ucx_addr_release(struct na_ucx_addr *na_ucx_addr) - if (na_ucx_addr->ucp_ep != NULL) { - /* NB. for deserialized addresses that are not "connected" addresses, do - * not close the EP */ -- if (na_ucx_addr->worker_addr == NULL) -+ if (na_ucx_addr->worker_addr == NULL) { -+ if (!na_ucx_addr->na_ucx_class->ucp_listener) { -+ ucs_status_ptr_t status_ptr = na_ucp_ep_flush(na_ucx_addr->ucp_ep); -+ -+ if (UCS_PTR_IS_PTR(status_ptr)) { -+ ucs_status_t status; -+ -+ do { -+ ucp_worker_progress(na_ucx_addr->na_ucx_class->ucp_worker); -+ status = ucp_request_check_status(status_ptr); -+ } while (status == UCS_INPROGRESS); -+ ucp_request_free(status_ptr); -+ } -+ } -+ - na_ucp_ep_close(na_ucx_addr->ucp_ep); -+ } - na_ucx_addr->ucp_ep = NULL; - } - diff --git a/deps/patches/mercury/0002_ofi_counters.patch b/deps/patches/mercury/0002_ofi_counters.patch new file mode 100644 index 00000000000..5afdab2ff6a --- /dev/null +++ b/deps/patches/mercury/0002_ofi_counters.patch @@ -0,0 +1,1180 @@ +From e9eff75beee31bce6e8d6ea6841652f453a26d71 Mon Sep 17 00:00:00 2001 +From: Jerome Soumagne +Date: Thu, 18 Dec 2025 17:47:03 -0600 +Subject: [PATCH] NA OFI: add counters for monitoring tx/rx/rma/cq counts + +Monitor mr and addr counts + +NA: add NA_Diag_dump_counters() routine to dump counters +if HG_LOG_LEVEL>=min_debug is set + +HG Core: clean up counters + +HG util: add ability to remove counters +--- + CMakeLists.txt | 2 +- + src/mercury.c | 1 + + src/mercury_core.c | 40 ++--- + src/na/CMakeLists.txt | 5 + + src/na/na.c | 9 + + src/na/na.h | 6 + + src/na/na_config.h.in | 1 + + src/na/na_ofi.c | 359 +++++++++++++++++++++++++++++++++------- + src/util/mercury_dlog.c | 37 ++++- + src/util/mercury_dlog.h | 18 ++ + src/util/mercury_log.c | 14 +- + src/util/mercury_log.h | 8 + + 12 files changed, 411 insertions(+), 89 deletions(-) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index e4e79711..e71944f1 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -192,7 +192,7 @@ endif() + # Enable diagnostics counters separately from debug. + #------------------------------------------------------------------------------ + option(MERCURY_ENABLE_COUNTERS "Enable diagnostics counters." OFF) +-if(MERCURY_ENABLE_COUNTERS) ++if(MERCURY_ENABLE_COUNTERS AND NOT WIN32) + set(HG_HAS_DIAG 1) + else() + set(HG_HAS_DIAG 0) +diff --git a/src/mercury.c b/src/mercury.c +index 2c062384..6f0376ee 100644 +--- a/src/mercury.c ++++ b/src/mercury.c +@@ -1178,6 +1178,7 @@ HG_Diag_dump_counters(void) + #ifndef _WIN32 + hg_log_dump_counters(&HG_LOG_OUTLET(hg_diag)); + #endif ++ NA_Diag_dump_counters(); + } + + /*---------------------------------------------------------------------------*/ +diff --git a/src/mercury_core.c b/src/mercury_core.c +index 8ac34bd0..06404586 100644 +--- a/src/mercury_core.c ++++ b/src/mercury_core.c +@@ -185,7 +185,7 @@ struct hg_core_private_class { + struct hg_core_map rpc_map; /* RPC Map */ + struct hg_core_more_data_cb more_data_cb; /* More data callbacks */ + na_tag_t request_max_tag; /* Max value for tag */ +-#if defined(HG_HAS_DIAG) && !defined(_WIN32) ++#ifdef HG_HAS_DIAG + struct hg_core_counters counters; /* Diag counters */ + #endif + hg_atomic_int32_t n_contexts; /* Total number of contexts */ +@@ -369,7 +369,7 @@ struct hg_core_private_handle { + uint8_t cookie; /* Cookie */ + bool multi_recv_copy; /* Copy on multi-recv */ + bool reuse; /* Re-use handle once ref_count is 0 */ +-#if defined(HG_HAS_DIAG) && !defined(_WIN32) ++#ifdef HG_HAS_DIAG + bool active; + #endif + }; +@@ -405,7 +405,7 @@ hg_core_op_type_to_string(enum hg_core_op_type op_type); + /** + * Init counters. + */ +-#if defined(HG_HAS_DIAG) && !defined(_WIN32) ++#ifdef HG_HAS_DIAG + static void + hg_core_counters_init(struct hg_core_counters *hg_core_counters); + #endif +@@ -447,7 +447,7 @@ hg_core_finalize(struct hg_core_private_class *hg_core_class); + /** + * Get counters. + */ +-#if defined(HG_HAS_DIAG) && !defined(_WIN32) ++#ifdef HG_HAS_DIAG + static void + hg_core_class_get_counters(const struct hg_core_counters *counters, + struct hg_diag_counters *diag_counters); +@@ -1091,7 +1091,7 @@ hg_core_op_type_to_string(enum hg_core_op_type op_type) + #endif + + /*---------------------------------------------------------------------------*/ +-#if defined(HG_HAS_DIAG) && !defined(_WIN32) ++#ifdef HG_HAS_DIAG + static void + hg_core_counters_init(struct hg_core_counters *hg_core_counters) + { +@@ -1326,7 +1326,7 @@ hg_core_init(const char *na_info_string, bool na_listen, unsigned int version, + hg_core_class->init_info.listen = na_listen; + + /* Stats / counters */ +-#if defined(HG_HAS_DIAG) && !defined(_WIN32) ++#ifdef HG_HAS_DIAG + hg_core_counters_init(&hg_core_class->counters); + #endif + +@@ -1522,7 +1522,7 @@ hg_core_finalize(struct hg_core_private_class *hg_core_class) + } + + /*---------------------------------------------------------------------------*/ +-#if defined(HG_HAS_DIAG) && !defined(_WIN32) ++#ifdef HG_HAS_DIAG + static void + hg_core_class_get_counters(const struct hg_core_counters *counters, + struct hg_diag_counters *diag_counters) +@@ -3446,7 +3446,7 @@ hg_core_destroy(struct hg_core_private_handle *hg_core_handle) + return HG_SUCCESS; /* Cannot free yet */ + } + +-#if defined(HG_HAS_DIAG) && !defined(_WIN32) ++#ifdef HG_HAS_DIAG + if (hg_core_handle->active) { + hg_atomic_decr64(HG_CORE_HANDLE_CLASS(hg_core_handle) + ->counters.rpc_req_recv_active_count); +@@ -4049,7 +4049,7 @@ hg_core_forward(struct hg_core_private_handle *hg_core_handle, + hg_core_handle->request_callback = callback; + hg_core_handle->request_arg = arg; + +-#if defined(HG_HAS_DIAG) && !defined(_WIN32) ++#ifdef HG_HAS_DIAG + /* Increment counter */ + hg_atomic_incr64( + HG_CORE_HANDLE_CLASS(hg_core_handle)->counters.rpc_req_sent_count); +@@ -4264,7 +4264,7 @@ hg_core_respond(struct hg_core_private_handle *hg_core_handle, + hg_core_handle->response_callback = callback; + hg_core_handle->response_arg = arg; + +-#if defined(HG_HAS_DIAG) && !defined(_WIN32) ++#ifdef HG_HAS_DIAG + /* Increment counter */ + hg_atomic_incr64( + HG_CORE_HANDLE_CLASS(hg_core_handle)->counters.rpc_resp_sent_count); +@@ -4500,7 +4500,7 @@ hg_core_recv_input_cb(const struct na_cb_info *callback_info) + hg_thread_spin_lock(&hg_core_handle_pool->pending_list.lock); + LIST_REMOVE(hg_core_handle, pending); + hg_thread_spin_unlock(&hg_core_handle_pool->pending_list.lock); +-#if defined(HG_HAS_DIAG) && !defined(_WIN32) ++#ifdef HG_HAS_DIAG + /* Increment counter */ + hg_atomic_incr64(HG_CORE_HANDLE_CLASS(hg_core_handle) + ->counters.rpc_req_recv_active_count); +@@ -4609,7 +4609,7 @@ hg_core_multi_recv_input_cb(const struct na_cb_info *callback_info) + ret = hg_core_handle_pool_get(context->handle_pool, &hg_core_handle); + HG_CHECK_SUBSYS_HG_ERROR( + rpc, error, ret, "Could not get handle from pool"); +-#if defined(HG_HAS_DIAG) && !defined(_WIN32) ++#ifdef HG_HAS_DIAG + /* Increment counter */ + hg_atomic_incr64(HG_CORE_HANDLE_CLASS(hg_core_handle) + ->counters.rpc_req_recv_active_count); +@@ -4666,7 +4666,7 @@ hg_core_multi_recv_input_cb(const struct na_cb_info *callback_info) + "Copying multi-recv payload of size %zu for handle (%p)", + hg_core_handle->core_handle.in_buf_used, + (void *) hg_core_handle); +-#if defined(HG_HAS_DIAG) && !defined(_WIN32) ++#ifdef HG_HAS_DIAG + /* Increment counter */ + hg_atomic_incr64(HG_CORE_CONTEXT_CLASS(context) + ->counters.rpc_multi_recv_copy_count); +@@ -4764,7 +4764,7 @@ hg_core_process_input(struct hg_core_private_handle *hg_core_handle) + uint32_t flags = (uint32_t) hg_atomic_get32(&hg_core_handle->flags); + hg_return_t ret; + +-#if defined(HG_HAS_DIAG) && !defined(_WIN32) ++#ifdef HG_HAS_DIAG + /* Increment counter */ + hg_atomic_incr64(hg_core_class->counters.rpc_req_recv_count); + #endif +@@ -4813,7 +4813,7 @@ hg_core_process_input(struct hg_core_private_handle *hg_core_handle) + "Handle (%p) expected_count incr to %" PRId32, + (void *) hg_core_handle, expected_count); + +-#if defined(HG_HAS_DIAG) && !defined(_WIN32) ++#ifdef HG_HAS_DIAG + /* Increment counter */ + hg_atomic_incr64(hg_core_class->counters.rpc_req_extra_count); + #endif +@@ -4937,7 +4937,7 @@ hg_core_process_output(struct hg_core_private_handle *hg_core_handle) + uint32_t flags = (uint32_t) hg_atomic_get32(&hg_core_handle->flags); + hg_return_t ret; + +-#if defined(HG_HAS_DIAG) && !defined(_WIN32) ++#ifdef HG_HAS_DIAG + /* Increment counter */ + hg_atomic_incr64(hg_core_class->counters.rpc_resp_recv_count); + #endif +@@ -4981,7 +4981,7 @@ hg_core_process_output(struct hg_core_private_handle *hg_core_handle) + "Handle (%p) expected_count incr to %" PRId32, + (void *) hg_core_handle, expected_count); + +-#if defined(HG_HAS_DIAG) && !defined(_WIN32) ++#ifdef HG_HAS_DIAG + /* Increment counter */ + hg_atomic_incr64(hg_core_class->counters.rpc_resp_extra_count); + #endif +@@ -5320,7 +5320,7 @@ hg_core_completion_add(struct hg_core_context *core_context, + struct hg_core_completion_queue *backfill_queue = &context->backfill_queue; + int rc; + +-#if defined(HG_HAS_DIAG) && !defined(_WIN32) ++#ifdef HG_HAS_DIAG + /* Increment counter */ + if (hg_completion_entry->op_type == HG_BULK) + hg_atomic_incr64(HG_CORE_CONTEXT_CLASS(context)->counters.bulk_count); +@@ -6213,7 +6213,7 @@ hg_return_t + HG_Core_class_get_counters(const hg_core_class_t *hg_core_class, + struct hg_diag_counters *diag_counters) + { +-#if defined(HG_HAS_DIAG) && !defined(_WIN32) ++#ifdef HG_HAS_DIAG + const struct hg_core_private_class *private_class = + (const struct hg_core_private_class *) hg_core_class; + #endif +@@ -6223,7 +6223,7 @@ HG_Core_class_get_counters(const hg_core_class_t *hg_core_class, + HG_INVALID_ARG, "NULL HG core class"); + HG_CHECK_SUBSYS_ERROR(cls, diag_counters == NULL, error, ret, + HG_INVALID_ARG, "NULL pointer to diag_counters"); +-#if defined(HG_HAS_DIAG) && !defined(_WIN32) ++#ifdef HG_HAS_DIAG + hg_core_class_get_counters(&private_class->counters, diag_counters); + #else + HG_LOG_SUBSYS_ERROR(cls, "Counters not supported in current build, please " +diff --git a/src/na/CMakeLists.txt b/src/na/CMakeLists.txt +index 115e70bb..d99ed05d 100644 +--- a/src/na/CMakeLists.txt ++++ b/src/na/CMakeLists.txt +@@ -59,6 +59,11 @@ set(NA_BUILD_INCLUDE_DEPENDENCIES + #------------------------------------------------------------------------------ + # Internal dependencies + #------------------------------------------------------------------------------ ++# Diagnostics counters ++if(HG_HAS_DIAG) ++ set(NA_HAS_DIAG 1) ++endif() ++ + # Multi progress + if(NOT HG_ALLOW_MULTI_PROGRESS) + option(NA_ALLOW_MULTI_PROGRESS "Allow concurrent progress on single context." ON) +diff --git a/src/na/na.c b/src/na/na.c +index d54db626..61abf0a3 100644 +--- a/src/na/na.c ++++ b/src/na/na.c +@@ -1010,6 +1010,15 @@ NA_Set_log_level(const char *level) + hg_log_set_subsys_level(NA_SUBSYS_NAME_STRING, hg_log_name_to_level(level)); + } + ++/*---------------------------------------------------------------------------*/ ++void ++NA_Diag_dump_counters(void) ++{ ++#ifndef _WIN32 ++ hg_log_dump_counters(&HG_LOG_OUTLET(NA_SUBSYS_NAME)); ++#endif ++} ++ + /*---------------------------------------------------------------------------*/ + na_context_t * + NA_Context_create(na_class_t *na_class) +diff --git a/src/na/na.h b/src/na/na.h +index ff65ceb2..3d335fe7 100644 +--- a/src/na/na.h ++++ b/src/na/na.h +@@ -153,6 +153,12 @@ NA_Has_opt_feature( + NA_PUBLIC void + NA_Set_log_level(const char *level); + ++/** ++ * Dump diagnostic counters into the existing log stream. ++ */ ++NA_PUBLIC void ++NA_Diag_dump_counters(void); ++ + /** + * Return the name of the NA class. + * +diff --git a/src/na/na_config.h.in b/src/na/na_config.h.in +index 8468de8c..1419bee6 100644 +--- a/src/na/na_config.h.in ++++ b/src/na/na_config.h.in +@@ -85,6 +85,7 @@ + + /* Build Options */ + #cmakedefine NA_HAS_DEBUG ++#cmakedefine NA_HAS_DIAG + #cmakedefine NA_HAS_MULTI_PROGRESS + + /* HWLOC */ +diff --git a/src/na/na_ofi.c b/src/na/na_ofi.c +index 682efe65..19f83cac 100644 +--- a/src/na/na_ofi.c ++++ b/src/na/na_ofi.c +@@ -262,6 +262,9 @@ static unsigned long const na_ofi_prov_flags[] = {NA_OFI_PROV_TYPES}; + /* Prov info array init count */ + #define NA_OFI_PROV_INFO_COUNT (32) + ++/* Max counter name length */ ++#define NA_OFI_MAX_COUNTER_NAME (64) ++ + /* Address / URI max len */ + #define NA_OFI_MAX_URI_LEN (128) + +@@ -787,7 +790,6 @@ struct na_ofi_domain { + hg_atomic_int64_t requested_key; /* Requested key if not FI_MR_PROV_KEY */ + int64_t max_key; /* Max key if not FI_MR_PROV_KEY */ + uint64_t max_tag; /* Max tag from CQ data size */ +- hg_atomic_int32_t mr_reg_count; /* Number of MR registered */ + bool no_wait; /* Wait disabled on domain */ + bool av_auth_key; /* Use FI_AV_AUTH_KEY */ + bool av_user_id; /* Use FI_AV_USER_ID */ +@@ -830,15 +832,26 @@ struct na_ofi_verify_info { + enum na_ofi_prov_type prov_type; /* Provider type */ + }; + +-/* OFI class */ +-struct na_ofi_class { +- struct na_ofi_addr_pool addr_pool; /* Addr pool */ +- struct fi_info *fi_info; /* OFI info */ +- struct na_ofi_fabric *fabric; /* Fabric pointer */ +- struct na_ofi_domain *domain; /* Domain pointer */ +- struct na_ofi_endpoint *endpoint; /* Endpoint pointer */ +- struct hg_mem_pool *send_pool; /* Msg send buf pool */ +- struct hg_mem_pool *recv_pool; /* Msg recv buf pool */ ++#ifdef NA_HAS_DIAG ++/* OFI counters */ ++struct na_ofi_counters { ++ char tx_count_string[NA_OFI_MAX_COUNTER_NAME]; /* TX count string */ ++ char rx_count_string[NA_OFI_MAX_COUNTER_NAME]; /* RX count string */ ++ char rma_count_string[NA_OFI_MAX_COUNTER_NAME]; /* RMA count string */ ++ char mr_count_string[NA_OFI_MAX_COUNTER_NAME]; /* MR count string */ ++ char addr_count_string[NA_OFI_MAX_COUNTER_NAME]; /* Addr count string */ ++ char cq_count_string[NA_OFI_MAX_COUNTER_NAME]; /* CQ count string */ ++ hg_atomic_int32_t *tx_count; /* Number of active sends */ ++ hg_atomic_int32_t *rx_count; /* Number of active receives */ ++ hg_atomic_int32_t *rma_count; /* Number of active RMAs */ ++ hg_atomic_int32_t *mr_count; /* Number of active MRs */ ++ hg_atomic_int32_t *addr_count; /* Number of addresses inserted */ ++ hg_atomic_int32_t *cq_count; /* Number of CQ events */ ++}; ++#endif ++ ++/* OFI ops */ ++struct na_ofi_ops { + na_return_t (*msg_send_unexpected)( + struct fid_ep *, const struct na_ofi_msg_info *, void *); + na_return_t (*msg_recv_unexpected)( +@@ -853,14 +866,29 @@ struct na_ofi_class { + const char *msg_recv_unexpected_string; /* Error log string */ + const char *msg_send_expected_string; /* Error log string */ + const char *msg_recv_expected_string; /* Error log string */ +- unsigned long opt_features; /* Optional feature flags */ +- hg_atomic_int32_t n_contexts; /* Number of context */ +- unsigned int op_retry_timeout; /* Retry timeout */ +- unsigned int op_retry_period; /* Time elapsed until next retry */ +- uint8_t context_max; /* Max number of contexts */ +- bool no_wait; /* Ignore wait object */ +- bool use_sep; /* Use scalable endpoints */ +- bool finalizing; /* Class being destroyed */ ++}; ++ ++/* OFI class */ ++struct na_ofi_class { ++ struct na_ofi_addr_pool addr_pool; /* Addr pool */ ++ struct fi_info *fi_info; /* OFI info */ ++ struct na_ofi_fabric *fabric; /* Fabric pointer */ ++ struct na_ofi_domain *domain; /* Domain pointer */ ++ struct na_ofi_endpoint *endpoint; /* Endpoint pointer */ ++ struct hg_mem_pool *send_pool; /* Msg send buf pool */ ++ struct hg_mem_pool *recv_pool; /* Msg recv buf pool */ ++ struct na_ofi_ops ops; /* OFI operations */ ++#ifdef NA_HAS_DIAG ++ struct na_ofi_counters counters; /* OFI counters */ ++#endif ++ unsigned long opt_features; /* Optional feature flags */ ++ hg_atomic_int32_t n_contexts; /* Number of context */ ++ unsigned int op_retry_timeout; /* Retry timeout */ ++ unsigned int op_retry_period; /* Time elapsed until next retry */ ++ uint8_t context_max; /* Max number of contexts */ ++ bool no_wait; /* Ignore wait object */ ++ bool use_sep; /* Use scalable endpoints */ ++ bool finalizing; /* Class being destroyed */ + }; + + /********************/ +@@ -1145,6 +1173,20 @@ na_ofi_class_alloc(void); + static na_return_t + na_ofi_class_free(struct na_ofi_class *na_ofi_class); + ++#ifdef NA_HAS_DIAG ++/** ++ * Init counters. ++ */ ++static void ++na_ofi_counters_init(struct na_ofi_counters *counters, int class_id); ++ ++/** ++ * Finalize counters. ++ */ ++static void ++na_ofi_counters_finalize(struct na_ofi_counters *counters); ++#endif ++ + /** + * Configure class parameters from environment variables. + */ +@@ -1762,6 +1804,14 @@ static void + na_ofi_op_retry_abort_addr( + struct na_ofi_context *na_ofi_context, fi_addr_t fi_addr, na_return_t ret); + ++/** ++ * Process counters. ++ */ ++#ifdef NA_HAS_DIAG ++static void ++na_ofi_cq_process_counters(struct na_ofi_op_id *na_ofi_op_id); ++#endif ++ + /** + * Complete operation ID. + */ +@@ -3307,6 +3357,12 @@ na_ofi_addr_map_insert(struct na_ofi_class *na_ofi_class, + na_ofi_errno_to_na(-rc), + "fi_av_remove(%" PRIu64 ") failed, rc: %d (%s)", + na_ofi_addr->fi_addr, rc, fi_strerror(-rc)); ++ ++#ifdef NA_HAS_DIAG ++ /* Counters */ ++ hg_atomic_decr32(na_ofi_class->counters.addr_count); ++#endif ++ + addr_map_exist = true; + } + } else { +@@ -3353,6 +3409,11 @@ na_ofi_addr_map_insert(struct na_ofi_class *na_ofi_class, + addr_str, &addr_str_len), + rc); + ++#ifdef NA_HAS_DIAG ++ /* Counters */ ++ hg_atomic_incr32(na_ofi_class->counters.addr_count); ++#endif ++ + #if FI_VERSION_GE(FI_COMPILE_VERSION, FI_VERSION(1, 20)) + if (na_ofi_class->domain->av_auth_key) { + size_t addrlen = sizeof(na_ofi_addr->addr_key.addr); +@@ -3458,6 +3519,11 @@ na_ofi_addr_map_remove( + "fi_av_remove(%" PRIu64 ") failed, rc: %d (%s)", na_ofi_addr->fi_addr, + rc, fi_strerror(-rc)); + ++#ifdef NA_HAS_DIAG ++ /* Counters */ ++ hg_atomic_decr32(na_ofi_addr->class->counters.addr_count); ++#endif ++ + NA_LOG_SUBSYS_DEBUG( + addr, "Removed addr for FI addr %" PRIu64, na_ofi_addr->fi_addr); + +@@ -4313,6 +4379,50 @@ na_ofi_class_free(struct na_ofi_class *na_ofi_class) + return ret; + } + ++/*---------------------------------------------------------------------------*/ ++#ifdef NA_HAS_DIAG ++static void ++na_ofi_counters_init(struct na_ofi_counters *counters, int class_id) ++{ ++ snprintf(counters->tx_count_string, sizeof(counters->tx_count_string), ++ "[%d] na_ofi_tx_count ", class_id); ++ snprintf(counters->rx_count_string, sizeof(counters->rx_count_string), ++ "[%d] na_ofi_rx_count ", class_id); ++ snprintf(counters->rma_count_string, sizeof(counters->rma_count_string), ++ "[%d] na_ofi_rma_count ", class_id); ++ snprintf(counters->mr_count_string, sizeof(counters->mr_count_string), ++ "[%d] na_ofi_mr_count ", class_id); ++ snprintf(counters->addr_count_string, sizeof(counters->addr_count_string), ++ "[%d] na_ofi_addr_count", class_id); ++ snprintf(counters->cq_count_string, sizeof(counters->cq_count_string), ++ "[%d] na_ofi_cq_count ", class_id); ++ HG_LOG_ADD_COUNTER32(na, &counters->tx_count, counters->tx_count_string, ++ "Number of active sends"); ++ HG_LOG_ADD_COUNTER32(na, &counters->rx_count, counters->rx_count_string, ++ "Number of active recvs"); ++ HG_LOG_ADD_COUNTER32(na, &counters->rma_count, counters->rma_count_string, ++ "Number of active RMAs"); ++ HG_LOG_ADD_COUNTER32(na, &counters->mr_count, counters->mr_count_string, ++ "Number of active MRs"); ++ HG_LOG_ADD_COUNTER32(na, &counters->addr_count, counters->addr_count_string, ++ "Number of addresses inserted"); ++ HG_LOG_ADD_COUNTER32(na, &counters->cq_count, counters->cq_count_string, ++ "Number of events still in CQ"); ++} ++ ++/*---------------------------------------------------------------------------*/ ++static void ++na_ofi_counters_finalize(struct na_ofi_counters *counters) ++{ ++ HG_LOG_DEL_COUNTER32(na, counters->tx_count); ++ HG_LOG_DEL_COUNTER32(na, counters->rx_count); ++ HG_LOG_DEL_COUNTER32(na, counters->rma_count); ++ HG_LOG_DEL_COUNTER32(na, counters->mr_count); ++ HG_LOG_DEL_COUNTER32(na, counters->addr_count); ++ HG_LOG_DEL_COUNTER32(na, counters->cq_count); ++} ++#endif ++ + /*---------------------------------------------------------------------------*/ + static na_return_t + na_ofi_class_env_config(struct na_ofi_class *na_ofi_class) +@@ -4323,26 +4433,26 @@ na_ofi_class_env_config(struct na_ofi_class *na_ofi_class) + /* Set unexpected msg callbacks */ + env = getenv("NA_OFI_UNEXPECTED_TAG_MSG"); + if (env == NULL || env[0] == '0' || tolower(env[0]) == 'n') { +- na_ofi_class->msg_send_unexpected = na_ofi_msg_send; +- na_ofi_class->msg_send_unexpected_string = "fi_senddata"; +- na_ofi_class->msg_recv_unexpected = na_ofi_msg_recv; +- na_ofi_class->msg_recv_unexpected_string = "fi_recv"; ++ na_ofi_class->ops.msg_send_unexpected = na_ofi_msg_send; ++ na_ofi_class->ops.msg_send_unexpected_string = "fi_senddata"; ++ na_ofi_class->ops.msg_recv_unexpected = na_ofi_msg_recv; ++ na_ofi_class->ops.msg_recv_unexpected_string = "fi_recv"; + } else { + NA_LOG_SUBSYS_DEBUG(cls, + "NA_OFI_UNEXPECTED_TAG_MSG set to %s, forcing unexpected messages " + "to use tagged recvs", + env); +- na_ofi_class->msg_send_unexpected = na_ofi_tag_send; +- na_ofi_class->msg_send_unexpected_string = "fi_tsend"; +- na_ofi_class->msg_recv_unexpected = na_ofi_tag_recv; +- na_ofi_class->msg_recv_unexpected_string = "fi_trecv"; ++ na_ofi_class->ops.msg_send_unexpected = na_ofi_tag_send; ++ na_ofi_class->ops.msg_send_unexpected_string = "fi_tsend"; ++ na_ofi_class->ops.msg_recv_unexpected = na_ofi_tag_recv; ++ na_ofi_class->ops.msg_recv_unexpected_string = "fi_trecv"; + } + + /* Set expected msg callbacks */ +- na_ofi_class->msg_send_expected = na_ofi_tag_send; +- na_ofi_class->msg_send_expected_string = "fi_tsend"; +- na_ofi_class->msg_recv_expected = na_ofi_tag_recv; +- na_ofi_class->msg_recv_expected_string = "fi_trecv"; ++ na_ofi_class->ops.msg_send_expected = na_ofi_tag_send; ++ na_ofi_class->ops.msg_send_expected_string = "fi_tsend"; ++ na_ofi_class->ops.msg_recv_expected = na_ofi_tag_recv; ++ na_ofi_class->ops.msg_recv_expected_string = "fi_trecv"; + + /* Default retry timeouts in ms */ + if ((env = getenv("NA_OFI_OP_RETRY_TIMEOUT")) != NULL) { +@@ -5073,7 +5183,6 @@ na_ofi_domain_open(const struct na_ofi_fabric *na_ofi_fabric, + hg_atomic_init64(&na_ofi_domain->requested_key, 0); + /* No need to take a refcount on fabric */ + na_ofi_domain->fabric = na_ofi_fabric; +- hg_atomic_init32(&na_ofi_domain->mr_reg_count, 0); + + /* Dup name */ + na_ofi_domain->name = strdup(domain_attr->name); +@@ -6056,6 +6165,11 @@ na_ofi_mem_buf_register(const void *buf, size_t len, unsigned long flags, + + /* Register memory if FI_MR_LOCAL is set and provider uses it */ + if (na_ofi_class->fi_info->domain_attr->mr_mode & FI_MR_LOCAL) { ++#ifdef NA_HAS_DIAG ++ int32_t mr_cnt = hg_atomic_get32(na_ofi_class->counters.mr_count); ++#else ++ int32_t mr_cnt = -1; ++#endif + struct fid_mr *mr_hdl = NULL; + uint64_t access = 0; + int rc; +@@ -6072,10 +6186,11 @@ na_ofi_mem_buf_register(const void *buf, size_t len, unsigned long flags, + NA_CHECK_SUBSYS_ERROR(mem, rc != 0, out, ret, HG_UTIL_FAIL, + "fi_mr_reg(buf=%p, len=%zu, flags=%lu) failed, rc: %d (%s), " + "mr_reg_count: %d", +- buf, len, flags, rc, fi_strerror(-rc), +- hg_atomic_get32(&na_ofi_class->domain->mr_reg_count)); ++ buf, len, flags, rc, fi_strerror(-rc), mr_cnt); + +- hg_atomic_incr32(&na_ofi_class->domain->mr_reg_count); ++#ifdef NA_HAS_DIAG ++ hg_atomic_incr32(na_ofi_class->counters.mr_count); ++#endif + *handle_p = (void *) mr_hdl; + } else + *handle_p = NULL; +@@ -6093,11 +6208,17 @@ na_ofi_mem_buf_deregister(void *handle, void *arg) + /* Release MR handle is there was any */ + if (handle) { + struct fid_mr *mr_hdl = (struct fid_mr *) handle; ++#ifdef NA_HAS_DIAG + struct na_ofi_class *na_ofi_class = (struct na_ofi_class *) arg; ++#else ++ (void) arg; ++#endif + int rc = fi_close(&mr_hdl->fid); + NA_CHECK_SUBSYS_ERROR(mem, rc != 0, out, ret, HG_UTIL_FAIL, + "fi_close() mr_hdl failed, rc: %d (%s)", rc, fi_strerror(-rc)); +- hg_atomic_decr32(&na_ofi_class->domain->mr_reg_count); ++#ifdef NA_HAS_DIAG ++ hg_atomic_decr32(na_ofi_class->counters.mr_count); ++#endif + } + + out: +@@ -6159,6 +6280,11 @@ na_ofi_msg_send_common(struct na_ofi_class *na_ofi_class, + if ((int) na_ofi_class->fi_info->addr_format == FI_ADDR_OPX) + na_ofi_op_id->fi_ctx[0].internal[0] = &na_ofi_addr->addr_key.addr.opx; + ++#ifdef NA_HAS_DIAG ++ /* Counters */ ++ hg_atomic_incr32(na_ofi_class->counters.tx_count); ++#endif ++ + ret = msg_op( + na_ofi_context->fi_tx, &na_ofi_op_id->info.msg, &na_ofi_op_id->fi_ctx); + if (ret != NA_SUCCESS) { +@@ -6166,8 +6292,12 @@ na_ofi_msg_send_common(struct na_ofi_class *na_ofi_class, + na_ofi_op_id->retry_op.msg = msg_op; + na_ofi_op_retry( + na_ofi_context, na_ofi_class->op_retry_timeout, na_ofi_op_id); +- } else ++ } else { ++#ifdef NA_HAS_DIAG ++ hg_atomic_decr32(na_ofi_class->counters.tx_count); ++#endif + NA_GOTO_SUBSYS_ERROR_NORET(msg, release, "Could not post msg send"); ++ } + } + + return NA_SUCCESS; +@@ -6224,6 +6354,11 @@ na_ofi_msg_recv_common(struct na_ofi_class *na_ofi_class, + .tag = tag, + .tag_mask = tag_mask}; + ++#ifdef NA_HAS_DIAG ++ /* Counters */ ++ hg_atomic_incr32(na_ofi_class->counters.rx_count); ++#endif ++ + ret = msg_op( + na_ofi_context->fi_rx, &na_ofi_op_id->info.msg, &na_ofi_op_id->fi_ctx); + if (ret != NA_SUCCESS) { +@@ -6231,8 +6366,12 @@ na_ofi_msg_recv_common(struct na_ofi_class *na_ofi_class, + na_ofi_op_id->retry_op.msg = msg_op; + na_ofi_op_retry( + na_ofi_context, na_ofi_class->op_retry_timeout, na_ofi_op_id); +- } else ++ } else { ++#ifdef NA_HAS_DIAG ++ hg_atomic_decr32(na_ofi_class->counters.rx_count); ++#endif + NA_GOTO_SUBSYS_ERROR_NORET(msg, release, "Could not post msg recv"); ++ } + } + + return NA_SUCCESS; +@@ -6620,6 +6759,11 @@ na_ofi_rma_common(struct na_ofi_class *na_ofi_class, na_context_t *context, + NA_OFI_SEP_RX_CTX_BITS) + : na_ofi_addr->fi_addr; + ++#ifdef NA_HAS_DIAG ++ /* Counters */ ++ hg_atomic_incr32(na_ofi_class->counters.rma_count); ++#endif ++ + /* Post the OFI RMA operation */ + ret = + na_ofi_rma_post(na_ofi_context->fi_tx, rma_info, &na_ofi_op_id->fi_ctx); +@@ -6628,8 +6772,12 @@ na_ofi_rma_common(struct na_ofi_class *na_ofi_class, na_context_t *context, + na_ofi_op_id->retry_op.rma = na_ofi_rma_post; + na_ofi_op_retry( + na_ofi_context, na_ofi_class->op_retry_timeout, na_ofi_op_id); +- } else ++ } else { ++#ifdef NA_HAS_DIAG ++ hg_atomic_decr32(na_ofi_class->counters.rma_count); ++#endif + NA_GOTO_SUBSYS_ERROR_NORET(rma, release, "Could not post RMA op"); ++ } + } + + return NA_SUCCESS; +@@ -6991,6 +7139,10 @@ na_ofi_cq_process_canceled(const struct na_ofi_class *na_ofi_class, + cq_err->err, fi_strerror(cq_err->err), (void *) na_ofi_op_id, + na_cb_type_to_string(na_ofi_op_id->type)); + ++#ifdef NA_HAS_DIAG ++ na_ofi_cq_process_counters(na_ofi_op_id); ++#endif ++ + /* When tearing down connections, it is possible that operations will be + canceled by libfabric itself. + NA_CHECK_SUBSYS_WARNING(op, +@@ -7014,6 +7166,10 @@ na_ofi_cq_process_canceled(const struct na_ofi_class *na_ofi_class, + &na_ofi_op_id->completion_data->callback_info.info + .multi_recv_unexpected, + complete); ++#ifdef NA_HAS_DIAG ++ if (complete) ++ hg_atomic_decr32(na_ofi_class->counters.rx_count); ++#endif + } else + complete = true; + +@@ -7151,6 +7307,10 @@ na_ofi_cq_process_error( + NA_OFI_OP_CANCELED) + return NA_SUCCESS; /* already handled */ + ++#ifdef NA_HAS_DIAG ++ na_ofi_cq_process_counters(na_ofi_op_id); ++#endif ++ + /* Abort other retries if peer is unreachable */ + if (na_ret == NA_HOSTUNREACH && na_ofi_op_id->addr) + na_ofi_op_retry_abort_addr(na_ofi_op_id->na_ofi_context, +@@ -7172,6 +7332,10 @@ na_ofi_cq_process_error( + &na_ofi_op_id->completion_data->callback_info.info + .multi_recv_unexpected, + complete); ++#ifdef NA_HAS_DIAG ++ if (complete) ++ hg_atomic_decr32(na_ofi_class->counters.rx_count); ++#endif + } else + complete = true; + +@@ -7346,6 +7510,10 @@ na_ofi_cq_process_event(struct na_ofi_class *na_ofi_class, + cq_event->op_context, cq_event->flags, cq_event->len, cq_event->buf, + cq_event->data, cq_event->tag); + ++#ifdef NA_HAS_DIAG ++ na_ofi_cq_process_counters(na_ofi_op_id); ++#endif ++ + switch (na_ofi_op_id->type) { + case NA_CB_RECV_UNEXPECTED: + /* Default to cq_event->tag for backward compatibility */ +@@ -7361,6 +7529,10 @@ na_ofi_cq_process_event(struct na_ofi_class *na_ofi_class, + break; + case NA_CB_MULTI_RECV_UNEXPECTED: + complete = cq_event->flags & FI_MULTI_RECV; ++#ifdef HG_HAS_DIAG ++ if (complete) ++ hg_atomic_decr32(na_ofi_class->counters.rx_count); ++#endif + + ret = na_ofi_cq_process_multi_recv_unexpected(na_ofi_class, + &na_ofi_op_id->info.msg, +@@ -7628,6 +7800,10 @@ na_ofi_cq_process_retries( + NA_LOG_SUBSYS_ERROR(op, "retry operation of %p (%s) failed", + (void *) na_ofi_op_id, na_cb_type_to_string(cb_type)); + ++#ifdef NA_HAS_DIAG ++ na_ofi_cq_process_counters(na_ofi_op_id); ++#endif ++ + /* Force internal completion in error mode */ + hg_atomic_or32(&na_ofi_op_id->status, NA_OFI_OP_ERRORED); + na_ofi_op_id->complete(na_ofi_op_id, true, ret); +@@ -7690,6 +7866,32 @@ na_ofi_op_retry_abort_addr( + hg_thread_spin_unlock(&op_queue->lock); + } + ++/*---------------------------------------------------------------------------*/ ++#ifdef NA_HAS_DIAG ++static void ++na_ofi_cq_process_counters(struct na_ofi_op_id *na_ofi_op_id) ++{ ++ switch (na_ofi_op_id->type) { ++ case NA_CB_RECV_UNEXPECTED: ++ case NA_CB_RECV_EXPECTED: ++ hg_atomic_decr32(na_ofi_op_id->na_ofi_class->counters.rx_count); ++ break; ++ case NA_CB_SEND_UNEXPECTED: ++ case NA_CB_SEND_EXPECTED: ++ hg_atomic_decr32(na_ofi_op_id->na_ofi_class->counters.tx_count); ++ break; ++ case NA_CB_PUT: ++ case NA_CB_GET: ++ hg_atomic_decr32(na_ofi_op_id->na_ofi_class->counters.rma_count); ++ break; ++ case NA_CB_MULTI_RECV_UNEXPECTED: ++ /* TODO currently treated outside of switch */ ++ default: ++ break; ++ } ++} ++#endif ++ + /*---------------------------------------------------------------------------*/ + static NA_INLINE void + na_ofi_op_complete_single(struct na_ofi_op_id *na_ofi_op_id, +@@ -7711,6 +7913,9 @@ na_ofi_op_complete_single(struct na_ofi_op_id *na_ofi_op_id, + completion_data->plugin_callback = na_ofi_op_release_single; + + NA_LOG_SUBSYS_DEBUG(op, "Adding completion data to queue"); ++#ifdef NA_HAS_DIAG ++ hg_atomic_incr32(na_ofi_op_id->na_ofi_class->counters.cq_count); ++#endif + + /* Add OP to NA completion queue */ + na_cb_completion_add( +@@ -7728,6 +7933,9 @@ na_ofi_op_release_single(void *arg) + (!(hg_atomic_get32(&na_ofi_op_id->status) & NA_OFI_OP_COMPLETED)), + "Releasing resources from an uncompleted operation"); + ++#ifdef NA_HAS_DIAG ++ hg_atomic_decr32(na_ofi_op_id->na_ofi_class->counters.cq_count); ++#endif + if (na_ofi_op_id->addr) { + na_ofi_addr_ref_decr(na_ofi_op_id->addr); + na_ofi_op_id->addr = NULL; +@@ -7775,6 +7983,10 @@ na_ofi_op_complete_multi( + op, na_ofi_op_id->completion_data == NULL, error, "Queue is full"); + + NA_LOG_SUBSYS_DEBUG(op, "Adding completion data to queue"); ++#ifdef NA_HAS_DIAG ++ hg_atomic_incr32(na_ofi_op_id->na_ofi_class->counters.cq_count); ++#endif ++ + /* Add OP to NA completion queue */ + na_cb_completion_add( + na_ofi_op_id->na_ofi_context->context, completion_data); +@@ -7789,6 +8001,9 @@ na_ofi_op_release_multi(void *arg) + { + struct na_ofi_op_id *na_ofi_op_id = (struct na_ofi_op_id *) arg; + ++#ifdef NA_HAS_DIAG ++ hg_atomic_decr32(na_ofi_op_id->na_ofi_class->counters.cq_count); ++#endif + na_ofi_completion_multi_pop(&na_ofi_op_id->completion_data_storage.multi); + } + +@@ -8124,6 +8339,9 @@ na_ofi_initialize( + struct na_ofi_info info = NA_OFI_INFO_INITIALIZER; + union na_ofi_auth_key base_auth_key; + struct na_loc_info *loc_info = NULL; ++#ifdef NA_HAS_DIAG ++ static int class_id = 0; ++#endif + na_return_t ret; + #ifdef NA_OFI_HAS_MEM_POOL + size_t pool_chunk_size; +@@ -8203,6 +8421,9 @@ na_ofi_initialize( + na_ofi_class = na_ofi_class_alloc(); + NA_CHECK_SUBSYS_ERROR(cls, na_ofi_class == NULL, error, ret, NA_NOMEM, + "Could not allocate NA OFI class"); ++#ifdef NA_HAS_DIAG ++ na_ofi_counters_init(&na_ofi_class->counters, class_id++); ++#endif + + /* Check env config */ + ret = na_ofi_class_env_config(na_ofi_class); +@@ -8230,15 +8451,15 @@ na_ofi_initialize( + + /* Set/check optional features */ + if ((na_ofi_prov_extra_caps[prov_type] & FI_MULTI_RECV) && +- (na_ofi_class->msg_recv_unexpected == na_ofi_msg_recv)) { ++ (na_ofi_class->ops.msg_recv_unexpected == na_ofi_msg_recv)) { + NA_CHECK_SUBSYS_ERROR(cls, + !(na_ofi_class->fi_info->caps & FI_MULTI_RECV), error, ret, + NA_PROTONOSUPPORT, "FI_MULTI_RECV is not supported by provider"); + na_ofi_class->opt_features |= NA_OPT_MULTI_RECV; + } +- na_ofi_class->cq_poll = (na_ofi_class->fi_info->caps & FI_SOURCE_ERR) +- ? na_ofi_cq_poll_fi_source +- : na_ofi_cq_poll_no_source; ++ na_ofi_class->ops.cq_poll = (na_ofi_class->fi_info->caps & FI_SOURCE_ERR) ++ ? na_ofi_cq_poll_fi_source ++ : na_ofi_cq_poll_no_source; + + /* Open fabric */ + ret = na_ofi_fabric_open( +@@ -8373,6 +8594,11 @@ na_ofi_finalize(na_class_t *na_class) + na_ofi_addr_ref_decr(na_ofi_addr); + } + ++#ifdef NA_HAS_DIAG ++ /* Remove counters */ ++ na_ofi_counters_finalize(&na_ofi_class->counters); ++#endif ++ + /* Free class */ + ret = na_ofi_class_free(na_ofi_class); + NA_CHECK_SUBSYS_NA_ERROR(cls, out, ret, "Coult not free NA OFI class"); +@@ -8959,8 +9185,8 @@ na_ofi_msg_send_unexpected(na_class_t *na_class, na_context_t *context, + { + return na_ofi_msg_send_common(NA_OFI_CLASS(na_class), + NA_OFI_CONTEXT(context), NA_CB_SEND_UNEXPECTED, callback, arg, +- NA_OFI_CLASS(na_class)->msg_send_unexpected, +- NA_OFI_CLASS(na_class)->msg_send_unexpected_string, buf, buf_size, ++ NA_OFI_CLASS(na_class)->ops.msg_send_unexpected, ++ NA_OFI_CLASS(na_class)->ops.msg_send_unexpected_string, buf, buf_size, + NA_OFI_CLASS(na_class)->endpoint->unexpected_msg_size_max, + (struct na_ofi_msg_buf_handle *) plugin_data, + (struct na_ofi_addr *) dest_addr, dest_id, +@@ -8975,8 +9201,8 @@ na_ofi_msg_recv_unexpected(na_class_t *na_class, na_context_t *context, + { + return na_ofi_msg_recv_common(NA_OFI_CLASS(na_class), + NA_OFI_CONTEXT(context), NA_CB_RECV_UNEXPECTED, callback, arg, +- NA_OFI_CLASS(na_class)->msg_recv_unexpected, +- NA_OFI_CLASS(na_class)->msg_recv_unexpected_string, buf, buf_size, ++ NA_OFI_CLASS(na_class)->ops.msg_recv_unexpected, ++ NA_OFI_CLASS(na_class)->ops.msg_recv_unexpected_string, buf, buf_size, + NA_OFI_CLASS(na_class)->endpoint->unexpected_msg_size_max, + (struct na_ofi_msg_buf_handle *) plugin_data, NULL, 0, + NA_OFI_UNEXPECTED_TAG, NA_OFI_TAG_MASK, (struct na_ofi_op_id *) op_id); +@@ -9027,9 +9253,17 @@ na_ofi_msg_multi_recv_unexpected(na_class_t *na_class, na_context_t *context, + .tag = 0 /* unused */, + .tag_mask = 0 /* unused */}; + ++#ifdef NA_HAS_DIAG ++ /* Counters */ ++ hg_atomic_incr32(na_ofi_class->counters.rx_count); ++#endif ++ + ret = na_ofi_msg_multi_recv( + na_ofi_context->fi_rx, &na_ofi_op_id->info.msg, &na_ofi_op_id->fi_ctx); + if (ret != NA_SUCCESS) { ++#ifdef NA_HAS_DIAG ++ hg_atomic_decr32(na_ofi_class->counters.rx_count); ++#endif + if (ret == NA_AGAIN) { + na_ofi_op_id->retry_op.msg = na_ofi_msg_multi_recv; + na_ofi_op_retry( +@@ -9061,8 +9295,8 @@ na_ofi_msg_send_expected(na_class_t *na_class, na_context_t *context, + { + return na_ofi_msg_send_common(NA_OFI_CLASS(na_class), + NA_OFI_CONTEXT(context), NA_CB_SEND_EXPECTED, callback, arg, +- NA_OFI_CLASS(na_class)->msg_send_expected, +- NA_OFI_CLASS(na_class)->msg_send_expected_string, buf, buf_size, ++ NA_OFI_CLASS(na_class)->ops.msg_send_expected, ++ NA_OFI_CLASS(na_class)->ops.msg_send_expected_string, buf, buf_size, + NA_OFI_CLASS(na_class)->endpoint->expected_msg_size_max, + (struct na_ofi_msg_buf_handle *) plugin_data, + (struct na_ofi_addr *) dest_addr, dest_id, (uint64_t) tag, +@@ -9077,8 +9311,8 @@ na_ofi_msg_recv_expected(na_class_t *na_class, na_context_t *context, + { + return na_ofi_msg_recv_common(NA_OFI_CLASS(na_class), + NA_OFI_CONTEXT(context), NA_CB_RECV_EXPECTED, callback, arg, +- NA_OFI_CLASS(na_class)->msg_recv_expected, +- NA_OFI_CLASS(na_class)->msg_recv_expected_string, buf, buf_size, ++ NA_OFI_CLASS(na_class)->ops.msg_recv_expected, ++ NA_OFI_CLASS(na_class)->ops.msg_recv_expected_string, buf, buf_size, + NA_OFI_CLASS(na_class)->endpoint->expected_msg_size_max, + (struct na_ofi_msg_buf_handle *) plugin_data, + (struct na_ofi_addr *) source_addr, source_id, (uint64_t) tag, 0, +@@ -9214,7 +9448,11 @@ na_ofi_mem_register(na_class_t *na_class, na_mem_handle_t *mem_handle, + (struct na_ofi_mem_handle *) mem_handle; + struct na_ofi_domain *domain = NA_OFI_CLASS(na_class)->domain; + const struct fi_info *fi_info = NA_OFI_CLASS(na_class)->fi_info; +- int32_t mr_cnt = hg_atomic_get32(&domain->mr_reg_count); ++#ifdef NA_HAS_DIAG ++ int32_t mr_cnt = hg_atomic_get32(NA_OFI_CLASS(na_class)->counters.mr_count); ++#else ++ int32_t mr_cnt = -1; ++#endif + struct fi_mr_attr fi_mr_attr = { + .mr_iov = NA_OFI_IOV( + na_ofi_mem_handle->desc.iov, na_ofi_mem_handle->desc.info.iovcnt), +@@ -9283,7 +9521,9 @@ na_ofi_mem_register(na_class_t *na_class, na_mem_handle_t *mem_handle, + fi_mr_attr.mr_iov[0].iov_base, fi_mr_attr.mr_iov[0].iov_len, + fi_mr_attr.iov_count, fi_mr_attr.access, fi_mr_attr.iface, + fi_mr_attr.requested_key, rc, fi_strerror(-rc), mr_cnt); +- mr_cnt = hg_atomic_incr32(&domain->mr_reg_count); ++#ifdef NA_HAS_DIAG ++ mr_cnt = hg_atomic_incr32(NA_OFI_CLASS(na_class)->counters.mr_count); ++#endif + + /* Attach MR to endpoint when provider requests it */ + if (fi_info->domain_attr->mr_mode & FI_MR_ENDPOINT) { +@@ -9327,7 +9567,9 @@ na_ofi_mem_register(na_class_t *na_class, na_mem_handle_t *mem_handle, + error: + if (na_ofi_mem_handle->fi_mr) { + (void) fi_close(&na_ofi_mem_handle->fi_mr->fid); +- hg_atomic_decr32(&domain->mr_reg_count); ++#ifdef NA_HAS_DIAG ++ hg_atomic_decr32(NA_OFI_CLASS(na_class)->counters.mr_count); ++#endif + } + return ret; + } +@@ -9336,7 +9578,6 @@ na_ofi_mem_register(na_class_t *na_class, na_mem_handle_t *mem_handle, + static na_return_t + na_ofi_mem_deregister(na_class_t *na_class, na_mem_handle_t *mem_handle) + { +- struct na_ofi_domain *domain = NA_OFI_CLASS(na_class)->domain; + struct na_ofi_mem_handle *na_ofi_mem_handle = + (struct na_ofi_mem_handle *) mem_handle; + na_return_t ret; +@@ -9344,14 +9585,18 @@ na_ofi_mem_deregister(na_class_t *na_class, na_mem_handle_t *mem_handle) + + /* close MR handle */ + if (na_ofi_mem_handle->fi_mr != NULL) { +- int32_t NA_DEBUG_LOG_USED mr_cnt; ++ int32_t NA_DEBUG_LOG_USED mr_cnt = -1; + const struct iovec NA_DEBUG_LOG_USED *mr_iov = NA_OFI_IOV( + na_ofi_mem_handle->desc.iov, na_ofi_mem_handle->desc.info.iovcnt); + + rc = fi_close(&na_ofi_mem_handle->fi_mr->fid); + NA_CHECK_SUBSYS_ERROR(mem, rc != 0, error, ret, na_ofi_errno_to_na(-rc), + "fi_close() mr_hdl failed, rc: %d (%s)", rc, fi_strerror(-rc)); +- mr_cnt = hg_atomic_decr32(&domain->mr_reg_count); ++#ifdef NA_HAS_DIAG ++ mr_cnt = hg_atomic_decr32(NA_OFI_CLASS(na_class)->counters.mr_count); ++#else ++ (void) na_class; ++#endif + + NA_LOG_SUBSYS_DEBUG(mem, + "Deregistered memory region: mr_iov[0].iov_base=%p, " +@@ -9575,7 +9820,7 @@ na_ofi_poll(na_class_t *na_class, na_context_t *context, unsigned int *count_p) + return NA_SUCCESS; + + /* Read from CQ and process events */ +- ret = na_ofi_class->cq_poll(na_ofi_class, na_ofi_context, &count); ++ ret = na_ofi_class->ops.cq_poll(na_ofi_class, na_ofi_context, &count); + NA_CHECK_SUBSYS_NA_ERROR(poll, error, ret, "Could not poll context CQ"); + + /* Attempt to process retries */ +diff --git a/src/util/mercury_dlog.c b/src/util/mercury_dlog.c +index 042a0157..fbdc114f 100644 +--- a/src/util/mercury_dlog.c ++++ b/src/util/mercury_dlog.c +@@ -138,6 +138,40 @@ hg_dlog_mkcount64(struct hg_dlog *d, hg_atomic_int64_t **cptr, const char *name, + hg_thread_mutex_unlock(&d->dlock); + } + ++/*---------------------------------------------------------------------------*/ ++void ++hg_dlog_rmcount32(struct hg_dlog *d, hg_atomic_int32_t *cptr) ++{ ++ struct hg_dlog_dcount32 *dcnt; ++ ++ hg_thread_mutex_lock(&d->dlock); ++ TAILQ_FOREACH (dcnt, &d->cnts32, l) { ++ if (&dcnt->c == cptr) { ++ TAILQ_REMOVE(&d->cnts32, dcnt, l); ++ free(dcnt); ++ break; ++ } ++ } ++ hg_thread_mutex_unlock(&d->dlock); ++} ++ ++/*---------------------------------------------------------------------------*/ ++void ++hg_dlog_rmcount64(struct hg_dlog *d, hg_atomic_int64_t *cptr) ++{ ++ struct hg_dlog_dcount64 *dcnt; ++ ++ hg_thread_mutex_lock(&d->dlock); ++ TAILQ_FOREACH (dcnt, &d->cnts64, l) { ++ if (&dcnt->c == cptr) { ++ TAILQ_REMOVE(&d->cnts64, dcnt, l); ++ free(dcnt); ++ break; ++ } ++ } ++ hg_thread_mutex_unlock(&d->dlock); ++} ++ + /*---------------------------------------------------------------------------*/ + unsigned int + hg_dlog_addlog(struct hg_dlog *d, const char *file, unsigned int line, +@@ -210,8 +244,9 @@ hg_dlog_dump(struct hg_dlog *d, int (*log_func)(FILE *, const char *, ...), + "### (%s) debug log summary\n" + "### ----------------------\n", + (d->dlog_magic + strlen(HG_DLOG_STDMAGIC))); +- if (!TAILQ_EMPTY(&d->cnts32) && !TAILQ_EMPTY(&d->cnts64)) { ++ if (!TAILQ_EMPTY(&d->cnts32) || !TAILQ_EMPTY(&d->cnts64)) { + log_func(stream, "# Counters\n"); ++ + TAILQ_FOREACH (dc32, &d->cnts32, l) { + log_func(stream, "# %s: %" PRId32 " [%s]\n", dc32->name, + hg_atomic_get32(&dc32->c), dc32->descr); +diff --git a/src/util/mercury_dlog.h b/src/util/mercury_dlog.h +index 88944b05..43c30a4f 100644 +--- a/src/util/mercury_dlog.h ++++ b/src/util/mercury_dlog.h +@@ -155,6 +155,15 @@ HG_UTIL_PUBLIC void + hg_dlog_mkcount32(struct hg_dlog *d, hg_atomic_int32_t **cptr, const char *name, + const char *descr); + ++/** ++ * remove a 32-bit counter from a dlog. ++ * ++ * \param d [IN] dlog to remove the counter from ++ * \param cptr [IN] pointer to counter to remove ++ */ ++HG_UTIL_PUBLIC void ++hg_dlog_rmcount32(struct hg_dlog *d, hg_atomic_int32_t *cptr); ++ + /** + * make a named atomic64 counter in a dlog and return a pointer to + * it. we use the dlock to ensure a counter under a given name only +@@ -178,6 +187,15 @@ HG_UTIL_PUBLIC void + hg_dlog_mkcount64(struct hg_dlog *d, hg_atomic_int64_t **cptr, const char *name, + const char *descr); + ++/** ++ * remove a 64-bit counter from a dlog. ++ * ++ * \param d [IN] dlog to remove the counter from ++ * \param cptr [IN] pointer to counter to remove ++ */ ++HG_UTIL_PUBLIC void ++hg_dlog_rmcount64(struct hg_dlog *d, hg_atomic_int64_t *cptr); ++ + /** + * attempt to add a log record to a dlog. the id and msg should point + * to static strings that are valid throughout the life of the program +diff --git a/src/util/mercury_log.c b/src/util/mercury_log.c +index 483922bf..2fc13548 100644 +--- a/src/util/mercury_log.c ++++ b/src/util/mercury_log.c +@@ -500,16 +500,9 @@ hg_log_outlet_deregister(struct hg_log_outlet *hg_log_outlet) + + if (hg_log_outlet->debug_log && + !(hg_log_outlet->parent && +- hg_log_outlet->parent->debug_log == hg_log_outlet->debug_log)) { +- if (hg_log_outlet->level >= HG_LOG_LEVEL_MIN_DEBUG) { +- FILE *stream = hg_log_streams_g[hg_log_outlet->level] +- ? hg_log_streams_g[hg_log_outlet->level] +- : *hg_log_std_streams_g[hg_log_outlet->level]; +- hg_dlog_dump_counters( +- hg_log_outlet->debug_log, hg_log_func_g, stream, 0); +- } ++ hg_log_outlet->parent->debug_log == hg_log_outlet->debug_log)) + hg_dlog_free(hg_log_outlet->debug_log); +- } ++ + STAILQ_REMOVE(&hg_log_outlets_g, hg_log_outlet, hg_log_outlet, entry); + hg_log_outlet->registered = false; + } +@@ -593,7 +586,8 @@ hg_log_vwrite(struct hg_log_outlet *hg_log_outlet, enum hg_log_level log_level, + no_return ? "" : "\n"); + #endif + +- if (log_level == HG_LOG_LEVEL_ERROR && hg_log_outlet->debug_log && ++ if ((log_level == HG_LOG_LEVEL_ERROR || log_level == HG_LOG_LEVEL_FATAL) && ++ hg_log_outlet->debug_log && + hg_log_outlet->level >= HG_LOG_LEVEL_MIN_DEBUG) { + hg_dlog_dump(hg_log_outlet->debug_log, hg_log_func_g, stream, 0); + hg_dlog_resetlog(hg_log_outlet->debug_log); +diff --git a/src/util/mercury_log.h b/src/util/mercury_log.h +index ceba0c7a..1ed01429 100644 +--- a/src/util/mercury_log.h ++++ b/src/util/mercury_log.h +@@ -308,11 +308,19 @@ + hg_dlog_mkcount32(HG_LOG_OUTLET(name).debug_log, counter_ptr, \ + counter_name, counter_desc) + ++/* HG_LOG_DEL_COUNTER32: delete 32-bit debug log counter */ ++#define HG_LOG_DEL_COUNTER32(name, counter_ptr) \ ++ hg_dlog_rmcount32(HG_LOG_OUTLET(name).debug_log, counter_ptr) ++ + /* HG_LOG_ADD_COUNTER64: add 64-bit debug log counter */ + #define HG_LOG_ADD_COUNTER64(name, counter_ptr, counter_name, counter_desc) \ + hg_dlog_mkcount64(HG_LOG_OUTLET(name).debug_log, counter_ptr, \ + counter_name, counter_desc) + ++/* HG_LOG_DEL_COUNTER64: delete 64-bit debug log counter */ ++#define HG_LOG_DEL_COUNTER64(name, counter_ptr) \ ++ hg_dlog_rmcount64(HG_LOG_OUTLET(name).debug_log, counter_ptr) ++ + /*************************************/ + /* Public Type and Struct Definition */ + /*************************************/ diff --git a/deps/patches/mercury/0003_combined_plugin_path.patch b/deps/patches/mercury/0003_combined_plugin_path.patch deleted file mode 100644 index 476598dbc84..00000000000 --- a/deps/patches/mercury/0003_combined_plugin_path.patch +++ /dev/null @@ -1,260 +0,0 @@ -diff --git a/src/na/CMakeLists.txt b/src/na/CMakeLists.txt -index 5c4ec2f..625c07c 100644 ---- a/src/na/CMakeLists.txt -+++ b/src/na/CMakeLists.txt -@@ -82,9 +82,10 @@ if(NA_USE_DYNAMIC_PLUGINS) - if(NOT BUILD_SHARED_LIBS) - message(FATAL_ERROR "Using dynamic plugins requires BUILD_SHARED_LIBS to be ON.") - endif() -+ cmake_path(SET NA_PLUGIN_RELATIVE_PATH ${NA_INSTALL_PLUGIN_DIR}) -+ cmake_path(RELATIVE_PATH NA_PLUGIN_RELATIVE_PATH BASE_DIRECTORY ${NA_INSTALL_LIB_DIR}) -+ message(STATUS "NA plugin install directory: ${NA_INSTALL_PLUGIN_DIR} (relative path to libraries: ${NA_PLUGIN_RELATIVE_PATH})") - set(NA_HAS_DYNAMIC_PLUGINS 1) -- set(NA_DEFAULT_PLUGIN_PATH ${NA_INSTALL_PLUGIN_DIR} CACHE PATH "Default path used to load plugins.") -- mark_as_advanced(NA_DEFAULT_PLUGIN_PATH) - endif() - - # BMI -diff --git a/src/na/na.c b/src/na/na.c -index 781f4c8..b60d305 100644 ---- a/src/na/na.c -+++ b/src/na/na.c -@@ -20,6 +20,9 @@ - # include - # else - # include -+# include -+# include -+# include - # endif - #endif - -@@ -138,6 +141,10 @@ na_plugin_check_protocol(const struct na_class_ops *const class_ops[], - const struct na_class_ops **ops_p); - - #ifdef NA_HAS_DYNAMIC_PLUGINS -+/* Resolve plugin search path */ -+static na_return_t -+na_plugin_resolve_path(const char *offset, char *path, size_t path_size); -+ - /* Scan a given path and return a list of plugins */ - static na_return_t - na_plugin_scan_path(const char *path, struct na_plugin_entry **entries_p); -@@ -265,15 +272,26 @@ static void - na_initialize(void) - { - const char *plugin_path = getenv("NA_PLUGIN_PATH"); -+ char resolved_path[NA_PLUGIN_PATH_MAX]; - na_return_t ret; - -- if (plugin_path == NULL) -- plugin_path = NA_DEFAULT_PLUGIN_PATH; -+ if (plugin_path == NULL) { -+ ret = na_plugin_resolve_path( -+ NA_PLUGIN_RELATIVE_PATH, resolved_path, sizeof(resolved_path)); -+ NA_CHECK_SUBSYS_NA_ERROR(cls, done, ret, -+ "Could not resolve plugin path using offset (%s)", -+ NA_PLUGIN_RELATIVE_PATH); -+ plugin_path = resolved_path; -+ } - - ret = na_plugin_scan_path(plugin_path, &na_plugin_dynamic_g); - NA_CHECK_SUBSYS_WARNING(fatal, ret != NA_SUCCESS, -- "No plugin found in path (%s), consider setting NA_PLUGIN_PATH.", -+ "No usable plugin found in path (%s), consider setting NA_PLUGIN_PATH " -+ "if path indicated is not valid.", - plugin_path); -+ -+done: -+ return; - } - - /*---------------------------------------------------------------------------*/ -@@ -472,6 +490,44 @@ error: - /*---------------------------------------------------------------------------*/ - #ifdef NA_HAS_DYNAMIC_PLUGINS - # ifdef _WIN32 -+# define PATH_MAX MAX_PATH -+# define realpath(N, R) _fullpath((R), (N), PATH_MAX) -+# endif -+static na_return_t -+na_plugin_resolve_path(const char *offset, char *path, size_t path_size) -+{ -+ static int placeholder; -+ char libpath[PATH_MAX]; -+ char *slash; -+ na_return_t ret; -+ int rc; -+ -+ rc = hg_dl_get_path(&placeholder, path, path_size); -+ NA_CHECK_SUBSYS_ERROR( -+ cls, rc != 0, error, ret, NA_NOENTRY, "hg_dl_get_path() failed"); -+ -+ NA_CHECK_SUBSYS_ERROR(cls, realpath(path, libpath) == NULL, error, ret, -+ NA_NOENTRY, "realpath() failed, %s", strerror(errno)); -+ -+ slash = strrchr(libpath, '/'); -+ NA_CHECK_SUBSYS_ERROR(cls, slash == NULL, error, ret, NA_INVALID_ARG, -+ "Could not find last '/' in %s", libpath); -+ *slash = '\0'; -+ -+ rc = snprintf(path, path_size, "%s/%s", libpath, offset); -+ NA_CHECK_SUBSYS_ERROR(cls, rc < 0 || rc > (int) path_size, error, ret, -+ NA_OVERFLOW, -+ "snprintf() failed or name truncated, rc: %d (expected %zu)", rc, -+ path_size); -+ -+ return NA_SUCCESS; -+ -+error: -+ return ret; -+} -+ -+/*---------------------------------------------------------------------------*/ -+# ifdef _WIN32 - static na_return_t - na_plugin_scan_path(const char *path, struct na_plugin_entry **entries_p) - { -@@ -494,7 +550,7 @@ na_plugin_scan_path(const char *path, struct na_plugin_entry **entries_p) - struct dirent **plugin_list; - struct na_plugin_entry *entries = NULL; - na_return_t ret; -- int n, n_entries = 0; -+ int n, opened_plugins = 0; - - n = scandir(path, &plugin_list, na_plugin_filter, alphasort); - NA_CHECK_SUBSYS_ERROR( -@@ -504,16 +560,20 @@ na_plugin_scan_path(const char *path, struct na_plugin_entry **entries_p) - (struct na_plugin_entry *) calloc((size_t) n + 1, sizeof(*entries)); - NA_CHECK_SUBSYS_ERROR(cls, entries == NULL, error, ret, NA_NOMEM, - "Could not allocate %d plugin entries", n); -- n_entries = n; - - while (n--) { - ret = na_plugin_open(path, plugin_list[n]->d_name, &entries[n]); - free(plugin_list[n]); -- NA_CHECK_SUBSYS_NA_ERROR(cls, error, ret, "Could not open plugin (%s)", -- plugin_list[n]->d_name); -+ if (ret == NA_SUCCESS) -+ opened_plugins++; -+ else -+ NA_CHECK_SUBSYS_NA_ERROR( -+ cls, error, ret, "Could not open plugin (%s)", plugin_list[n]->d_name); - } - - free(plugin_list); -+ NA_CHECK_SUBSYS_ERROR(cls, opened_plugins == 0, error, ret, NA_NOENTRY, -+ "No usable plugin found in path (%s)", path); - - *entries_p = entries; - -@@ -521,19 +581,11 @@ na_plugin_scan_path(const char *path, struct na_plugin_entry **entries_p) - - error: - if (n > 0) { -- if (entries != NULL) { -- int i; -- -- /* close entry */ -- for (i = n + 1; i < n_entries; i++) -- na_plugin_close(&entries[i]); -- free(entries); -- } -- - while (n--) - free(plugin_list[n]); - free(plugin_list); - } -+ free(entries); - - return ret; - } -diff --git a/src/na/na_config.h.in b/src/na/na_config.h.in -index 30d0e08..50dd443 100644 ---- a/src/na/na_config.h.in -+++ b/src/na/na_config.h.in -@@ -80,7 +80,7 @@ - # define NA_PLUGIN - # define NA_PLUGIN_VISIBILITY NA_PRIVATE - #endif --#cmakedefine NA_DEFAULT_PLUGIN_PATH "@NA_DEFAULT_PLUGIN_PATH@" -+#cmakedefine NA_PLUGIN_RELATIVE_PATH "@NA_PLUGIN_RELATIVE_PATH@" - - /* Build Options */ - #cmakedefine NA_HAS_DEBUG -diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt -index 44fe6b9..32f51d1 100644 ---- a/src/util/CMakeLists.txt -+++ b/src/util/CMakeLists.txt -@@ -193,6 +193,7 @@ configure_file( - #------------------------------------------------------------------------------ - set(MERCURY_UTIL_SRCS - ${CMAKE_CURRENT_SOURCE_DIR}/mercury_atomic_queue.c -+ ${CMAKE_CURRENT_SOURCE_DIR}/mercury_dl.c - ${CMAKE_CURRENT_SOURCE_DIR}/mercury_dlog.c - ${CMAKE_CURRENT_SOURCE_DIR}/mercury_event.c - ${CMAKE_CURRENT_SOURCE_DIR}/mercury_hash_table.c -diff --git a/src/util/mercury_dl.h b/src/util/mercury_dl.h -index b86932a..0cda094 100644 ---- a/src/util/mercury_dl.h -+++ b/src/util/mercury_dl.h -@@ -62,6 +62,18 @@ hg_dl_close(HG_DL_HANDLE handle); - static HG_UTIL_INLINE void * - hg_dl_sym(HG_DL_HANDLE handle, const char *name); - -+/** -+ * Retrieve library path. -+ * -+ * \param addr [IN] address of the symbol -+ * \param path [OUT] buffer to store the path -+ * \param path_size [IN] size of the buffer -+ * -+ * \return Non-negative on success or negative on failure -+ */ -+HG_UTIL_PUBLIC int -+hg_dl_get_path(const void *addr, char *path, size_t path_size); -+ - /*---------------------------------------------------------------------------*/ - static HG_UTIL_INLINE const char * - hg_dl_error(void) -diff --git a/src/util/mercury_dl.c b/src/util/mercury_dl.c -new file mode 100644 -index 0000000..6ed4666 ---- /dev/null -+++ b/src/util/mercury_dl.c -@@ -0,0 +1,35 @@ -+/** -+ * Copyright (c) 2013-2022 UChicago Argonne, LLC and The HDF Group. -+ * Copyright (c) 2022-2023 Intel Corporation. -+ * -+ * SPDX-License-Identifier: BSD-3-Clause -+ */ -+ -+#if !defined(_WIN32) && !defined(_GNU_SOURCE) -+# define _GNU_SOURCE -+#endif -+#include "mercury_dl.h" -+ -+#include -+ -+/*---------------------------------------------------------------------------*/ -+int -+hg_dl_get_path(const void *addr, char *path, size_t path_size) -+{ -+#ifdef _WIN32 -+ HMODULE module; -+ if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | -+ GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, -+ (LPCSTR) addr, &module)) -+ return GetModuleFileNameA(module, path, (DWORD) path_size); -+#else -+ Dl_info info; -+ if (dladdr(addr, &info) && info.dli_fname) { -+ strncpy(path, info.dli_fname, path_size); -+ path[path_size - 1] = '\0'; -+ return HG_UTIL_SUCCESS; -+ } -+#endif -+ -+ return HG_UTIL_FAIL; -+} diff --git a/docs/admin/hardware.md b/docs/admin/hardware.md index 3f4658cf40b..c35b00af025 100644 --- a/docs/admin/hardware.md +++ b/docs/admin/hardware.md @@ -35,9 +35,10 @@ validated on a regular basis. An RDMA-capable fabric is preferred for best performance. The DAOS data plane relies on [OFI libfabric](https://ofiwg.github.io/libfabric/) and supports OFI providers for Ethernet/tcp and InfiniBand/verbs. -Starting with a Technology Preview in DAOS 2.2, [UCX](https://www.openucx.org/) -is also supported as an alternative network stack for DAOS. -Refer to [UCX Fabric Support (DAOS 2.2 Technology Preview)](./ucx.md) +[UCX](https://www.openucx.org/) +is also supported as an alternative network stack for DAOS on InfiniBand/verbs +platforms. +Refer to [UCX Fabric Support](./ucx.md) for details on setting up DAOS with UCX support. DAOS supports multiple network interfaces on the servers diff --git a/docs/admin/ucx.md b/docs/admin/ucx.md index b6529c7f1ee..c366fb1cf06 100644 --- a/docs/admin/ucx.md +++ b/docs/admin/ucx.md @@ -16,33 +16,26 @@ the following steps are needed: for information about supported MLNX\_OFED releases. * The `mercury-ucx` RPM package needs to be **manually** selected for - installation: - - - The base `mercury` RPM package ships with the libfabric plugin. - This RPM will be installed by default and is a dependency of the - `mercury-ucx` RPM. - - - The additional `mercury-ucx` RPM is also provided. This RPM contains - the UCX plugin that is required for enabling UCX support. - This RPM **must** be used in - InfiniBand environments when the intention is to use - UCX. - Attempts to install this RPM in non-Infiniband environments - will fail, because it has a dependency on UCX packages. + installation. The base `mercury` RPM package ships with no plugins. + The `mercury-ucx` RPM contains the UCX plugin that is required for + enabling UCX support. + This RPM **must** be used in InfiniBand environments when the intention + is to use UCX. Attempts to install this RPM in non-Infiniband environments + will fail, because it has a dependency on UCX packages. * At DAOS **installation** time, to enable UCX support the `mercury-ucx` RPM package must be explicitly listed. - For example, using the `yum`/`dnf` package manager on EL8: + For example, using the `dnf`/`yum` package manager on EL8: ```bash # on DAOS_ADMIN nodes: - yum install mercury-ucx daos-admin + dnf install mercury-ucx daos-admin # on DAOS_SERVER nodes: - yum install mercury-ucx daos-server + dnf install mercury-ucx daos-server # on DAOS_CLIENT nodes: - yum install mercury-ucx daos-client + dnf install mercury-ucx daos-client ``` After UCX support has been enabled by installing the `mercury-ucx` diff --git a/site_scons/components/__init__.py b/site_scons/components/__init__.py index da2518a9928..f6ab99d6245 100644 --- a/site_scons/components/__init__.py +++ b/site_scons/components/__init__.py @@ -1,6 +1,6 @@ # Copyright 2016-2024 Intel Corporation # Copyright 2025 Google LLC -# Copyright 2025 Hewlett Packard Enterprise Development LP +# Copyright 2025-2026 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -202,17 +202,13 @@ def define_mercury(reqs): '-DMERCURY_USE_SYSTEM_BOOST:BOOL=ON', '-DMERCURY_USE_CHECKSUMS:BOOL=OFF', '-DMERCURY_ENABLE_COUNTERS:BOOL=ON', + '-DMERCURY_ENABLE_DEBUG:BOOL=ON', '-DNA_USE_DYNAMIC_PLUGINS:BOOL=ON', '-DNA_USE_SM:BOOL=ON', '-DNA_USE_OFI:BOOL=ON', '-DNA_USE_UCX:BOOL=ON', '../mercury'] - if reqs.target_type == 'debug': - mercury_build.append('-DMERCURY_ENABLE_DEBUG:BOOL=ON') - else: - mercury_build.append('-DMERCURY_ENABLE_DEBUG:BOOL=OFF') - reqs.define('mercury', retriever=GitRepoRetriever(True), commands=[mercury_build, diff --git a/src/cart/utils/memcheck-cart.supp b/src/cart/utils/memcheck-cart.supp index b29fbe80ace..d235ae4457e 100644 --- a/src/cart/utils/memcheck-cart.supp +++ b/src/cart/utils/memcheck-cart.supp @@ -512,6 +512,14 @@ fun:hg_dlog_mkcount32 ... } +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:malloc + fun:hg_dlog_mkcount64 + ... +} { FI leak 9 Memcheck:Leak @@ -527,18 +535,6 @@ fun:HG_Init_opt fun:crt_hg_class_init } -{ - Tcp provider - Memcheck:Param - sendmsg(msg.msg_iov[1]) - ... - fun:sendmsg - fun:ofi_sockapi_sendv_socket - fun:ofi_bsock_sendv - ... - fun:fi_senddata - ... -} { Tcp provider with ofi rxm Memcheck:Param @@ -546,20 +542,14 @@ ... fun:ofi_bsock_sendv ... - fun:fi_tsend - ... } { Tcp provider with ofi rxm 2 Memcheck:Param sendmsg(msg.msg_iov[2]) ... - fun:sendmsg - fun:ofi_sockapi_sendv_socket fun:ofi_bsock_sendv ... - fun:fi_tsend - ... } { Go syscall. @@ -777,21 +767,3 @@ fun:__tsan_go_atomic64_fetch_add fun:racecall } -{ - DAOS-17006 - mercury leak - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:hg_dlog_mkcount64 - fun:hg_core_counters_init - fun:hg_core_init - fun:HG_Core_init_opt2 - fun:HG_Init_opt2 - fun:crt_hg_class_init - fun:crt_hg_ctx_init - fun:crt_context_provider_create - fun:daos_eq_lib_init - fun:daos_init - fun:_cgo_b590e4e2531a_Cfunc_daos_init - fun:runtime.asmcgocall.abi0 -} diff --git a/utils/build.config b/utils/build.config index c36fbd2a043..42e12cc7c03 100644 --- a/utils/build.config +++ b/utils/build.config @@ -9,7 +9,7 @@ isal=v2.31.1 isal_crypto=v2.25.0 spdk=v24.09 ofi=v1.22.0 -mercury=v2.4.0 +mercury=v2.4.1 protobufc=v1.3.3 ucx=v1.14.1 @@ -27,5 +27,5 @@ ucx=https://github.com/openucx/ucx.git [patch_versions] spdk=0001_3428322b812fe31cc3e1d0308a7f5bd4b06b9886.diff,0002_spdk_rwf_nowait.patch,0003_external_isal.patch -mercury=0001_na_ucx.patch,0002_na_ucx_ep_flush.patch,0003_combined_plugin_path.patch +mercury=0001_dep_versions.patch,0002_ofi_counters.patch argobots=0001_411e5b344642ebc82190fd8b125db512e5b449d1.diff,0002_bb0c908abfac4bfe37852eee621930634183c6aa.diff diff --git a/utils/rpms/daos.changelog b/utils/rpms/daos.changelog index 40e3d18fde6..3a0a53f5251 100644 --- a/utils/rpms/daos.changelog +++ b/utils/rpms/daos.changelog @@ -1,4 +1,8 @@ %changelog +* Fri Jan 16 2026 Jerome Soumagne 2.7.103-2 +- Drop libfabric-devel build requirement +- Drop libfabric requirement that is already provided by mercury-libfabric + * Fri Dec 19 2025 Dalton Bohning 2.7.103-1 - Bump version to 2.7.103 diff --git a/utils/rpms/daos.sh b/utils/rpms/daos.sh index 8a28b2a187d..1171a079746 100755 --- a/utils/rpms/daos.sh +++ b/utils/rpms/daos.sh @@ -1,6 +1,10 @@ #!/bin/bash -# (C) Copyright 2025 Google LLC -# WORK IN PROGRESS +# +# (C) Copyright 2025 Google LLC +# Copyright 2025-2026 Hewlett Packard Enterprise Development LP +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# set -eEuo pipefail root="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" . "${root}/fpm_common.sh" @@ -64,7 +68,7 @@ install_list+=("${tmp}${sysconfdir}/daos/certs=${sysconfdir}/daos") EXTRA_OPTS+=("--rpm-attr" "0755,root,root:${sysconfdir}/daos/certs") -DEPENDS=( "mercury >= ${mercury_full}" "${libfabric_lib} >= ${libfabric_full}" ) +DEPENDS=( "mercury >= ${mercury_version}" ) DEPENDS+=( "${isal_crypto_lib} >= ${isal_crypto_version}" ) build_package "daos" diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index a8f5da38d1f..2a1827b0176 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -12,7 +12,6 @@ %global daos_build_args client test %endif %global mercury_version 2.4 -%global libfabric_version 1.15.1-1 %global argobots_version 1.2 %global __python %{__python3} %global daos_log_dir "/var/log/daos" @@ -25,7 +24,7 @@ Name: daos Version: 2.7.103 -Release: 1%{?relval}%{?dist} +Release: 2%{?relval}%{?dist} Summary: DAOS Storage Engine License: BSD-2-Clause-Patent @@ -40,7 +39,6 @@ BuildRequires: python3-scons >= 2.4 %else BuildRequires: scons >= 2.4 %endif -BuildRequires: libfabric-devel >= %{libfabric_version} BuildRequires: mercury-devel >= %{mercury_version} BuildRequires: gcc-c++ %if (0%{?rhel} >= 8) @@ -167,12 +165,10 @@ Requires: ndctl %if (0%{?suse_version} >= 1500) Requires: ipmctl >= 03.00.00.0423 Requires: libpmemobj1 >= 2.1.0-1.suse1500 -Requires: libfabric1 >= %{libfabric_version} %else Requires: ipmctl >= 03.00.00.0468 Requires: libpmemobj >= 2.1.0-1%{?dist} %endif -Requires: libfabric >= %{libfabric_version} Requires: mercury >= %{mercury_version} Requires(post): /sbin/ldconfig Requires(postun): /sbin/ldconfig @@ -195,10 +191,6 @@ This package contains DAOS administrative tools (e.g. dmg). Summary: The DAOS client Requires: %{name}%{?_isa} = %{version}-%{release} Requires: mercury >= %{mercury_version} -Requires: libfabric >= %{libfabric_version} -%if (0%{?suse_version} >= 1500) -Requires: libfabric1 >= %{libfabric_version} -%endif Requires: /usr/bin/fusermount3 %{?systemd_requires} diff --git a/utils/rpms/mercury.changelog b/utils/rpms/mercury.changelog new file mode 100644 index 00000000000..9afbeda4e23 --- /dev/null +++ b/utils/rpms/mercury.changelog @@ -0,0 +1,273 @@ +%changelog +* Fri Jan 16 2026 Jerome Soumagne - 2.4.1-1 +- Update to 2.4.1 +- Separate libfabric plugin from main build to align with ucx plugin +- Add patches for runtime version checks and libfabric plugin counters + +* Wed Jun 25 2025 Joseph Moore - 2.4.0-5 +- Update release number to differentiate from test RPMs for prior issue. + +* Tue Mar 11 2025 Joseph Moore - 2.4.0-4 +- Change to addr_release for handling of "already present" warning. + +* Wed Jan 15 2025 Joseph Moore - 2.4.0-3 +- Add patch to na_ucx.c to flush end point prior to close. + +* Tue Jan 07 2025 Joseph Moore - 2.4.0-2 +- Enable debug RPMs for Leap sub-packages. + +* Mon Nov 04 2024 Jerome Soumagne - 2.4.0-1 +- Update to 2.4.0 +- Update required libfabric version (>= 1.20) + +* Mon Oct 07 2024 Joseph Moore - 2.4.0~rc5-5 +- Update patch to na_ucx.c to set thread-safe on clients. + +* Thu Sep 26 2024 Joseph Moore - 2.4.0~rc5-4 +- Update patch to na_ucx.c to add fix for connection accept. + +* Wed Sep 04 2024 Brian J. Murrell - 2.4.0~rc5-3 +- Add --without ucx build switch + +* Thu Aug 29 2024 Joseph Moore - 2.4.0~rc5-2 +- Add patch to na_ucx.c to check ep in key_resolve. + +* Mon Aug 26 2024 Jerome Soumagne - 2.4.0~rc5-1 +- Update to 2.4.0rc5 + +* Fri Aug 02 2024 Jerome Soumagne - 2.4.0~rc4-1 +- Update to 2.4.0rc4 +- Remove previous patches now included in 2.4 +- Require libfabric >= 1.15 + +* Tue Mar 19 2024 Jerome Soumagne - 2.3.1-3 +- Add patch to fix ucx hg_info +- Add patch to remove ofi cxi MR warnings +- Add patch to fix potential segfault on log free + +* Wed Nov 22 2023 Jerome Soumagne - 2.3.1-2 +- Rebuild for EL 8.8 and Leap 15.5 + +* Fri Oct 27 2023 Jerome Soumagne - 2.3.1-1 +- Update to 2.3.1 +- Add json-c dependency for hg_info JSON output support +- Drop support for CentOS7 + +* Tue Sep 26 2023 Joseph Moore - 2.3.1~rc1-2 +- Add patch to na_ucx.c to force retry of out-of-memory error. + +* Tue Aug 29 2023 Jerome Soumagne - 2.3.1~rc1-1 +- Update to 2.3.1rc1 + +* Thu Jun 22 2023 Brian J. Murrell - 2.3.0-2 +- Rebuild for EL9 + +* Wed Jun 7 2023 Jerome Soumagne - 2.3.0-1 +- Update to 2.3.0 +- Add hg_info tool +- Fix pie flags on CentOS7 +- Remove na_ucx_src_port.patch and old patches + +* Tue Apr 25 2023 Jerome Soumagne - 2.3.0~rc5-1 +- Update to 2.3.0rc5 +- Remove na_ucx.c patch and add temporary na_ucx_src_port.patch +- Update build to make use of NA dynamic plugins +- Fix source URL and package perf tests + +* Thu Dec 22 2022 Joseph Moore - 2.2.0-6 +- Regenerate packages for LEAP15.4 + +* Thu Nov 17 2022 Joseph Moore - 2.2.0-5 +- Update na_ucx.c patch to support reconnection following a disconnect. + +* Wed Oct 05 2022 Joseph Moore - 2.2.0-4 +- Update na_ucx.c patch to include UCX status to NA error mapping. + +* Tue Sep 20 2022 Joseph Moore - 2.2.0-3 +- Fix defect in connect function. + +* Fri Sep 09 2022 Joseph Moore - 2.2.0-2 +- Add na_ucx.c patch to change ep creation for single IB device. + +* Fri Aug 5 2022 Jerome Soumagne - 2.2.0-1 +- Update to 2.2.0 + +* Mon Aug 1 2022 Jerome Soumagne - 2.2.0~rc6-2 +- Rebuild after libfabric rpm dropped CXI compat patch +- Drop CXI compat patch + +* Mon Jun 27 2022 Jerome Soumagne - 2.2.0~rc6-1 +- Update to 2.2.0rc6 +- Skip install rpath, enable debug log. +- Remove openpa dependency. + +* Fri Apr 22 2022 Joseph Moore - 2.1.0~rc4-9 +- Change ucx unified mode to off (updated UCX patch file). + +* Fri Apr 1 2022 Brian J. Murrell - 2.1.0~rc4-8 +- Build with ucx subpackage on supported platforms +- Removed invalid build options: + * MERCURY_ENABLE_VERBOSE_ERROR + * MERCURY_USE_SELF_FORWARD + +* Thu Mar 31 2022 Joseph Moore - 2.1.0~rc4-7 +- Apply daos-9679 address parsing change and active message revision to na_ucx.c. + +* Fri Mar 11 2022 Alexander Oganezov - 2.1.0~rc4-6 +- Apply cxi provider patch + +* Tue Feb 22 2022 Alexander Oganezov - 2.1.0~rc4-5 +- Apply doas-9561 workaround + +* Thu Feb 17 2022 Brian J. Murrell - 2.1.0~rc4-4 +- Fix issues with %%post* ldconfig + - No lines are allowed after %%post -p + - These are not needed on EL8 as it's glibc does the work + +* Thu Dec 23 2021 Alexander Oganezov - 2.1.0~rc4-3 +- Remove daos-9173 workaround +- Apply cpu usage fix to mercury + +* Tue Dec 7 2021 Alexander Oganezov - 2.1.0~rc4-2 +- Apply DAOS-9173 workaround patch to na_ofi.c + +* Tue Nov 30 2021 Alexander Oganezov - 2.1.0~rc4-1 +- Update to version v2.1.0rc4 + +* Tue Oct 12 2021 Alexander Oganezov - 2.1.0~rc2-1 +- Update to version v2.1.0rc2 + +* Fri May 14 2021 Alexander Oganezov - 2.0.1-1 +- Update to version v2.0.1 + +* Mon May 10 2021 Brian J. Murryyell - 2.0.1~rc1-2 +- Enable debuginfo package building for SUSE + +* Wed Jan 20 2021 Alexander Oganezov - 2.0.1~rc1-1 +- Update to version v2.0.1rc1 + +* Wed Nov 18 2020 Alexander Oganezov - 2.0.0-1 +- Update to release v2.0.0 + +* Wed Oct 28 2020 Alexander Oganezov - 2.0.0~rc3-1 +- Update to release v2.0.0rc3 + +* Mon Oct 12 2020 Alexander Oganezov - 2.0.0~rc2-1 +- Update to release v2.0.0rc2 + +* Tue Aug 18 2020 Brian J. Murryyell - 2.0.0~rc1-2 +- Use release tarball and not individual submodule tarballs + +* Mon Jul 6 2020 Alexander A Oganezov - 2.0.0~rc1-1 +- Update to release v2.0.0rc1 + +* Mon Jun 22 2020 Brian J. Murryyell - 2.0.0~a1-2 +- Fix License: +- Add %%license + +* Thu May 07 2020 Brian J. Murrell - 2.0.0~a1-1 +- Fix pre-release tag in Version: +- Add Requires: libfabric-devel to devel package + +* Thu Apr 9 2020 Alexander A Oganezov - 2.0.0a1-0.8 +- Update to 4871023058887444d47ead4d089c99db979f3d93 + +* Tue Mar 17 2020 Alexander A Oganezov - 2.0.0a1-0.7 +- Update to 41caa143a07ed179a3149cac4af0dc7aa3f946fd + +* Thu Mar 12 2020 Alexander A Oganezov - 2.0.0a1-0.6 +- Update to 299b06d47e6c1d59a45985dcbbebe3caca0189d0 + +* Tue Mar 10 2020 Alexander A Oganezov - 2.0.0a1-0.5 +- Updated to ad5a3b3dbf171a97e1ca5f1683299db1c69b03ea + +* Thu Mar 05 2020 Vikram Chhabra - 2.0.0a1-0.4 +- Updated to latest master with HG_Forward fix. + +* Tue Feb 11 2020 Yulu Jia - 2.0.0a1-0.3 +- Remove nameserver patch + +* Sun Feb 09 2020 Yulu Jia - 2.0.0a1-0.2 +- Update patch to enable ip:port URI format for psm2 + +* Tue Feb 04 2020 Brian J. Murrell - 2.0.0a1-0.1 +- Update to 2.0.0a1 + +* Tue Jan 28 2020 Yulu Jia - 1.0.1-22 +- Update to c2c2628 +- Apply patch to enable ip:port URI format for psm2 + +* Mon Dec 2 2019 Alexander Oganezov - 1.0.1-21 +- Removed sl_patch on top of 7b529b +- Updated to 9889a0 + +* Thu Oct 31 2019 Alexander Oganezov - 1.0.1-20 +- sl_patch on top of 7b529b + +* Wed Oct 23 2019 Alexander Oganezov - 1.0.1-19 +- Update to 7b529b + +* Tue Oct 22 2019 Alexander Oganezov - 1.0.1-18 +- Reverting from 6a8b693 due to mercury segfaults + +* Mon Oct 21 2019 Alexander Oganezov - 1.0.1-17 +- Update to 6a8b693 + +* Wed Oct 16 2019 Alexander Oganezov - 1.0.1-16 +- Fixed spec to apply patch for 616fee properly + +* Tue Oct 15 2019 Alexander Oganezov - 1.0.1-15 +- Update to 616fee to get latest changes + +* Wed Oct 02 2019 Brian J. Murrell - 1.0.1-14 +- Update to cc0807 to include the HG_Cancel() fix. +- Update to f0b9f9 to get latest changes + +* Wed Oct 02 2019 Brian J. Murrell - 1.0.1-13 +- Once again revert previous update + +* Wed Oct 02 2019 Brian J. Murrell - 1.0.1-12 +- Update to cc0807 to include the HG_Cancel() fix. +- Update to f0b9f9 to get latest changes + +* Wed Sep 25 2019 Brian J. Murrell - 1.0.1-11 +- Back out previous update + - not all consumers are ready for it yet so they need to + pin their BR + +* Fri Sep 20 2019 Brian J. Murrell - 1.0.1-10 +- Update to cc0807 to include the HG_Cancel() fix. +- Update to f0b9f9 to get latest changes + +* Thu Aug 08 2019 Brian J. Murrell - 1.0.1-9 +- Revert previous update + +* Fri Aug 02 2019 Yulu Jia - 1.0.1-8 +- Update to cc0807 to include the HG_Cancel() fix. +- Roll the version number back to 1.0.1 + +* Fri Aug 02 2019 Brian J. Murrell - 1.0.1-7 +- Revert back to the 1.0.1-4 release as the upgrade included + in -5 (and the subsequent fix in -6) was premature + +* Thu Aug 01 2019 Brian J. Murrell - 1.0.1-6 +- Roll the version number back to 1.0.1 + +* Fri Jul 26 2019 Yulu Jia - 1.0.1-5 +- Update to cc0807 to include the HG_Cancel() fix. + +* Thu May 02 2019 Brian J. Murrell - 1.0.1-4 +- Devel package needs to require the lib package + +* Fri Mar 15 2019 Brian J. Murrell - 1.0.1-2 +- Add patch to revert back to Dec 06, 2018 c68870f + +* Mon Mar 11 2019 Brian J. Murrell - 1.0.1-1 +- Update to 1.0.1 +- Add patch for "HG Core: fix missing static inline in mercury_core.h" + +* Wed Oct 24 2018 Brian J. Murrell - 0.9.0-1.git.0f8f25b +- Update mercury to git sha1 0f8f25bb3d57f117979de65cc3c05cf192cf4b31 + +* Mon Aug 20 2018 Brian J. Murrell - 0.9.0-1.git.f7f6955 +- Initial package diff --git a/utils/rpms/mercury.sh b/utils/rpms/mercury.sh index f14fe057043..5f30dd89372 100755 --- a/utils/rpms/mercury.sh +++ b/utils/rpms/mercury.sh @@ -1,5 +1,10 @@ #!/bin/bash -# (C) Copyright 2025 Google LLC +# +# (C) Copyright 2025 Google LLC +# Copyright 2025-2026 Hewlett Packard Enterprise Development LP +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# set -eEuo pipefail root="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" . "${root}/fpm_common.sh" @@ -24,6 +29,7 @@ Access (RMA). Its interface is generic and allows any function call to be serialized. Since code generation is done using the C preprocessor, no external tool is required." URL="http://mercury-hpc.github.io" +RPM_CHANGELOG="mercury.changelog" files=() TARGET_PATH="${bindir}" @@ -36,14 +42,17 @@ list_files files "${SL_MERCURY_PREFIX}/lib64/lib*.so.*" clean_bin "${files[@]}" append_install_list "${files[@]}" +ARCH="${isa}" +build_package "mercury" + TARGET_PATH="${libdir}/mercury" list_files files "${SL_MERCURY_PREFIX}/lib64/mercury/libna_plugin_ofi.so" clean_bin "${files[@]}" append_install_list "${files[@]}" ARCH="${isa}" -DEPENDS=("${libfabric_lib} >= ${libfabric_version}") -build_package "mercury" +DEPENDS=("${libfabric_lib} >= ${libfabric_min_version}") +build_package "mercury-libfabric" DEPENDS=() TARGET_PATH="${libdir}/mercury" diff --git a/utils/rpms/package_info.sh b/utils/rpms/package_info.sh index cc3be377607..ac6356c3c72 100644 --- a/utils/rpms/package_info.sh +++ b/utils/rpms/package_info.sh @@ -1,5 +1,10 @@ #!/bin/bash -# (C) Copyright 2025 Google LLC +# +# (C) Copyright 2025 Google LLC +# Copyright 2025-2026 Hewlett Packard Enterprise Development LP +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# root="$(realpath "$(dirname "$(dirname "$(dirname "${BASH_SOURCE[0]}")")")")" set_lib_name() { comp="$1"; shift @@ -35,11 +40,12 @@ daos_release="$(grep "^Release: " "${root}/utils/rpms/daos.spec" | \ sed 's/^Release: *//' | sed 's/%.*//')${DAOS_RELVAL:-}${distro_name}" export daos_release +export libfabric_min_version="1.20" export libfabric_version="1.22.0" export libfabric_release="5${distro_name}" export libfabric_full="${libfabric_version}-${libfabric_release}" -export mercury_version="2.4.0" -export mercury_release="8${distro_name}" +export mercury_version="2.4.1" +export mercury_release="1${distro_name}" export mercury_full="${mercury_version}-${mercury_release}" export argobots_version="1.2" export argobots_release="4${distro_name}" @@ -94,6 +100,8 @@ set_lib_name mercury dev mercury mercury mercury export mercury_dev set_lib_name mercury lib mercury mercury mercury export mercury_lib +set_lib_name mercury_libfabric lib mercury-libfabric mercury-libfabric mercury-libfabric +export mercury_libfabric_lib set_lib_name pmemobj lib libpmemobj libpmemobj1 libpmemobj1 set_lib_name pmemobj dev libpmemobj libpmemobj1 libpmemobj1 diff --git a/utils/test_memcheck.supp b/utils/test_memcheck.supp index 0d41b6e9a3d..c2a39de98d7 100644 --- a/utils/test_memcheck.supp +++ b/utils/test_memcheck.supp @@ -271,15 +271,11 @@ ... } { - Tcp provider - Memcheck:Param - sendmsg(msg.msg_iov[1]) - ... - fun:sendmsg - fun:ofi_sockapi_sendv_socket - fun:ofi_bsock_sendv - ... - fun:fi_senddata + + Memcheck:Leak + match-leak-kinds: reachable + fun:malloc + fun:hg_dlog_mkcount64 ... } { @@ -289,20 +285,14 @@ ... fun:ofi_bsock_sendv ... - fun:fi_tsend - ... } { Tcp provider with ofi rxm 2 Memcheck:Param sendmsg(msg.msg_iov[2]) ... - fun:sendmsg - fun:ofi_sockapi_sendv_socket fun:ofi_bsock_sendv ... - fun:fi_tsend - ... } { par_init mpi or dlopen leak