mtl/ofi: retain inline progress function

Retain inline progress function for ofi
mtl, but have a non-inlined progress function
which is registered with the opal progress
mechanism.

 @jithinjosepkl

I've bad news about the psm provider.  I still notice
segfaults - not always - but frequently at finalize
when using the psm provider.  I don't notice this
when using the sockets provider.

Signed-off-by: Howard Pritchard <[email protected]>

Author: hppritcha

ompi/mca/mtl/ofi/mtl_ofi.h

@@ -52,6 +52,8 @@ extern int ompi_mtl_ofi_del_procs(struct mca_mtl_base_module_t *mtl,
                                   size_t nprocs,
                                   struct ompi_proc_t **procs);
 
+int ompi_mtl_ofi_progress_no_inline(void);
+
 __opal_attribute_always_inline__ static inline int
 ompi_mtl_ofi_progress(void)
 {
@@ -115,42 +117,7 @@ ompi_mtl_ofi_progress(void)
 
 
 /* MTL interface functions */
-__opal_attribute_always_inline__ static inline int
-ompi_mtl_ofi_finalize(struct mca_mtl_base_module_t *mtl)
-{
-    opal_progress_unregister(ompi_mtl_ofi_progress);
-
-    /**
-     * Close all the OFI objects
-     */
-    if (fi_close((fid_t)ompi_mtl_ofi.ep)) {
-        opal_output(ompi_mtl_base_framework.framework_output,
-                "fi_close failed: %s", strerror(errno));
-        abort();
-    }
-    if (fi_close((fid_t)ompi_mtl_ofi.cq)) {
-        opal_output(ompi_mtl_base_framework.framework_output,
-                "fi_close failed: %s", strerror(errno));
-        abort();
-    }
-    if (fi_close((fid_t)ompi_mtl_ofi.av)) {
-        opal_output(ompi_mtl_base_framework.framework_output,
-                "fi_close failed: %s", strerror(errno));
-        abort();
-    }
-    if (fi_close((fid_t)ompi_mtl_ofi.domain)) {
-        opal_output(ompi_mtl_base_framework.framework_output,
-                "fi_close failed: %s", strerror(errno));
-        abort();
-    }
-    if (fi_close((fid_t)ompi_mtl_ofi.fabric)) {
-        opal_output(ompi_mtl_base_framework.framework_output,
-                "fi_close failed: %s", strerror(errno));
-        abort();
-    }
-
-    return OMPI_SUCCESS;
-}
+int ompi_mtl_ofi_finalize(struct mca_mtl_base_module_t *mtl);
 
 __opal_attribute_always_inline__ static inline int
 ompi_mtl_ofi_get_error(int error_num)

ompi/mca/mtl/ofi/mtl_ofi_component.c

@@ -101,6 +101,11 @@ ompi_mtl_ofi_component_close(void)
     return OMPI_SUCCESS;
 }
 
+int
+ompi_mtl_ofi_progress_no_inline(void)
+{
+	ompi_mtl_ofi_progress();
+}
 
 static mca_mtl_base_module_t*
 ompi_mtl_ofi_component_init(bool enable_progress_threads,
@@ -338,7 +343,7 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
     /**
      * Activate progress callback.
      */
-    ret = opal_progress_register(ompi_mtl_ofi_progress);
+    ret = opal_progress_register(ompi_mtl_ofi_progress_no_inline);
     if (OMPI_SUCCESS != ret) {
         opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                             "%s:%d: opal_progress_register failed: %d\n",
@@ -373,5 +378,42 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
     return NULL;
 }
 
+int
+ompi_mtl_ofi_finalize(struct mca_mtl_base_module_t *mtl)
+{
+    opal_progress_unregister(ompi_mtl_ofi_progress_no_inline);
+
+    /**
+ *      * Close all the OFI objects
+ *           */
+    if (fi_close((fid_t)ompi_mtl_ofi.ep)) {
+        opal_output(ompi_mtl_base_framework.framework_output,
+                "fi_close failed: %s", strerror(errno));
+        abort();
+    }
+    if (fi_close((fid_t)ompi_mtl_ofi.cq)) {
+        opal_output(ompi_mtl_base_framework.framework_output,
+                "fi_close failed: %s", strerror(errno));
+        abort();
+    }
+    if (fi_close((fid_t)ompi_mtl_ofi.av)) {
+        opal_output(ompi_mtl_base_framework.framework_output,
+                "fi_close failed: %s", strerror(errno));
+        abort();
+    }
+    if (fi_close((fid_t)ompi_mtl_ofi.domain)) {
+        opal_output(ompi_mtl_base_framework.framework_output,
+                "fi_close failed: %s", strerror(errno));
+        abort();
+    }
+    if (fi_close((fid_t)ompi_mtl_ofi.fabric)) {
+        opal_output(ompi_mtl_base_framework.framework_output,
+                "fi_close failed: %s", strerror(errno));
+        abort();
+    }
+
+    return OMPI_SUCCESS;
+}
+