Merge pull request open-mpi#636 from miked-mellanox/topic/v2.x_finalize_flag_and_fca_use_it

v2.x - Add in-finalize indicator, fca fall back to prev barrier if in-finalize

Author: hppritcha

ompi/mca/coll/fca/coll_fca_ops.c

@@ -153,6 +153,10 @@ int mca_coll_fca_barrier(struct ompi_communicator_t *comm,
     int ret;
 
     FCA_VERBOSE(5,"Using FCA Barrier");
+    if (OPAL_UNLIKELY(ompi_mpi_finalize_started)) {
+        FCA_VERBOSE(5, "In finalize, reverting to previous barrier");
+        goto orig_barrier;
+    }
     ret = fca_do_barrier(fca_module->fca_comm);
     if (ret < 0) {
         if (ret == -EUSEMPI) {

ompi/runtime/mpiruntime.h

@@ -55,6 +55,8 @@ OMPI_DECLSPEC extern bool ompi_mpi_initialized;
 OMPI_DECLSPEC extern bool ompi_mpi_finalized;
 /** Has the RTE been initialized? */
 OMPI_DECLSPEC extern bool ompi_rte_initialized;
+/** Did mpi start to finalize? */
+OMPI_DECLSPEC extern int32_t ompi_mpi_finalize_started;
 
 /** Do we have multiple threads? */
 OMPI_DECLSPEC extern bool ompi_mpi_thread_multiple;

ompi/runtime/ompi_mpi_finalize.c

@@ -93,7 +93,6 @@ extern bool ompi_enable_timing_ext;
 int ompi_mpi_finalize(void)
 {
     int ret;
-    static int32_t finalize_has_already_started = 0;
     opal_list_item_t *item;
     ompi_proc_t** procs;
     size_t nprocs;
@@ -106,7 +105,7 @@ int ompi_mpi_finalize(void)
        ompi_comm_free() (or run into other nasty lions, tigers, or
        bears) */
 
-    if (! opal_atomic_cmpset_32(&finalize_has_already_started, 0, 1)) {
+    if (! opal_atomic_cmpset_32(&ompi_mpi_finalize_started, 0, 1)) {
         /* Note that if we're already finalized, we cannot raise an
            MPI exception.  The best that we can do is write something
            to stderr. */

ompi/runtime/ompi_mpi_init.c

@@ -128,6 +128,7 @@ bool ompi_mpi_init_started = false;
 bool ompi_mpi_initialized = false;
 bool ompi_mpi_finalized = false;
 bool ompi_rte_initialized = false;
+int32_t ompi_mpi_finalize_started = false;
 
 bool ompi_mpi_thread_multiple = false;
 int ompi_mpi_thread_requested = MPI_THREAD_SINGLE;