ofi/mtl,btl: Add flags to track fabric/domain sharing with BTL

MTL is initialized and finalized before BTL.
When MTL and BTL share the same fabric and domain, BTL must be responsible
for closing them to ensure the correct order of resource finalization.

This commit adds flags to indicate whether the fabric and domain are shared
with BTL, allowing MTL to skip closing these objects during its finalization.

Signed-off-by: Jessie Yang <[email protected]>

Author: jiaxiyan

ompi/mca/mtl/ofi/mtl_ofi_component.c

@@ -1124,6 +1124,9 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
 
     ompi_mtl_ofi.base.mtl_flags |= MCA_MTL_BASE_FLAG_SUPPORTS_EXT_CID;
 
+    ompi_mtl_ofi.sharing_fabric = false;
+    ompi_mtl_ofi.sharing_domain = false;
+
     return &ompi_mtl_ofi.base;
 
 error:
@@ -1206,11 +1209,12 @@ ompi_mtl_ofi_finalize(struct mca_mtl_base_module_t *mtl)
         }
     }
 
-    if ((ret = fi_close((fid_t)ompi_mtl_ofi.domain))) {
+    /* BTL will close the shared domain and fabric */
+    if (!ompi_mtl_ofi.sharing_domain && (ret = fi_close((fid_t)ompi_mtl_ofi.domain))) {
         goto finalize_err;
     }
 
-    if ((ret = fi_close((fid_t)ompi_mtl_ofi.fabric))) {
+    if (!ompi_mtl_ofi.sharing_fabric && (ret = fi_close((fid_t)ompi_mtl_ofi.fabric))) {
         goto finalize_err;
     }
 

ompi/mca/mtl/ofi/mtl_ofi_types.h

@@ -108,6 +108,11 @@ typedef struct mca_mtl_ofi_module_t {
 
     /** registration cache */
     mca_rcache_base_module_t *rcache;
+
+    /** Whether fabric is shared with BTL */
+    bool sharing_fabric;
+    /** Whether domain is shared with BTL */
+    bool sharing_domain;
 } mca_mtl_ofi_module_t;
 
 extern mca_mtl_ofi_module_t ompi_mtl_ofi;

opal/mca/btl/ofi/btl_ofi_component.c

@@ -536,6 +536,7 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
     struct fid_av *av = NULL;
 
     mca_btl_ofi_module_t *module;
+    mca_mtl_ofi_module_t *mtl_ofi;
 
     module = mca_btl_ofi_module_alloc(mca_btl_ofi_component.mode);
     if (NULL == module) {
@@ -575,10 +576,17 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
     BTL_VERBOSE(
         ("initializing dev:%s provider:%s", linux_device_name, info->fabric_attr->prov_name));
 
+    if (ompi_mtl && ompi_mtl_base_selected_component && 
+        strcmp(ompi_mtl_base_selected_component->mtl_version.mca_component_name, "ofi") == 0) {
+        mtl_ofi = (mca_mtl_ofi_module_t *) ompi_mtl;
+    }
+
     /* fabric */
     if (info->fabric_attr->fabric) {
         BTL_VERBOSE(("Reusing existing fabric: %s", info->fabric_attr->name));
         fabric = info->fabric_attr->fabric;
+        if (mtl_ofi)
+            mtl_ofi->sharing_fabric = true;
     } else {
         rc = fi_fabric(ofi_info->fabric_attr, &fabric, NULL);
         if (0 != rc) {
@@ -591,6 +599,8 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
     if (info->domain_attr->domain) {
         BTL_VERBOSE(("Reusing existing domain: %s", info->domain_attr->name));
         domain = info->domain_attr->domain;
+        if (mtl_ofi)
+            mtl_ofi->sharing_domain = true;
     } else {
         rc = fi_domain(fabric, ofi_info, &domain, NULL);
         if (0 != rc) {
@@ -775,11 +785,11 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
         fi_close(&av->fid);
     }
 
-    if (NULL != domain) {
+    if (NULL != domain && (NULL == mtl_ofi || !mtl_ofi->sharing_domain)) {
         fi_close(&domain->fid);
     }
 
-    if (NULL != fabric) {
+    if (NULL != fabric && (NULL == mtl_ofi || !mtl_ofi->sharing_fabric)) {
         fi_close(&fabric->fid);
     }
     free(module);