Merge pull request open-mpi#11076 from hppritcha/hmem_fix_for_cxi_prov

OFI/MTL: bypass HMEM mem reg for CXI provider

Author: hppritcha

ompi/mca/mtl/ofi/mtl_ofi.h

@@ -2,7 +2,7 @@
  * Copyright (c) 2013-2018 Intel, Inc. All rights reserved
  * Copyright (c) 2017      Los Alamos National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2019-2021 Triad National Security, LLC. All rights
+ * Copyright (c) 2019-2022 Triad National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2018-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
  *                         reserved.
@@ -306,7 +306,7 @@ int ompi_mtl_ofi_register_buffer(struct opal_convertor_t *convertor,
         return OMPI_SUCCESS;
     }
 
-    if (convertor->flags & CONVERTOR_ACCELERATOR) {
+    if ((convertor->flags & CONVERTOR_ACCELERATOR) && ompi_mtl_ofi.hmem_needs_reg) {
         /* Register buffer */
         int ret;
         struct fi_mr_attr attr = {0};

ompi/mca/mtl/ofi/mtl_ofi_component.c

@@ -777,6 +777,19 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
         *accelerator_support = false;
     } else {
         *accelerator_support = true;
+        ompi_mtl_ofi.hmem_needs_reg = true;
+        /*
+         * Workaround for the fact that the CXI provider actually doesn't need for accelerator memory to be registered
+         * for local buffers, but if one does do so using fi_mr_regattr, one actually needs to manage the
+         * requested_key field in the fi_mr_attr attr argument, and the OFI MTL doesn't track which requested_keys
+         * have already been registered. So just set a flag to disable local registration.  Note the OFI BTL doesn't
+         * have a problem here since it uses fi_mr_regattr only within the context of an rcache, and manages the
+         * requested_key field in this way.
+         */
+         if (!strncasecmp(prov->fabric_attr->prov_name, "cxi", 3)) {
+             ompi_mtl_ofi.hmem_needs_reg = false;
+         }
+
     }
 
     /**

ompi/mca/mtl/ofi/mtl_ofi_types.h

@@ -2,6 +2,8 @@
  * Copyright (c) 2013-2018 Intel, Inc. All rights reserved
  *
  * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2022 Triad National Security, LLC. All rights
+ *                    reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -97,6 +99,7 @@ typedef struct mca_mtl_ofi_module_t {
 
     bool is_initialized;
     bool has_posted_initial_buffer;
+    bool hmem_needs_reg;
 
 } mca_mtl_ofi_module_t;