ch4: remove the usage of MPIDI_global.is_initialized

Now we separated the multivci initialization from the base init, we no
longer need to use MPIDI_global.is_initialized to guard pre-mature
access to extra vcis during init.

We still need a flag to check whether the root endpoints has been setup
before we try to tear them down in OFI/UCX finalize. Use a static
initialized flag for this purpose.

Author: hzhou

src/mpid/ch4/netmod/ofi/ofi_init.c

@@ -549,6 +549,8 @@ categories :
 === END_MPI_T_CVAR_INFO_BLOCK ===
 */
 
+static bool ofi_initialized = false;
+
 static int update_global_limits(struct fi_info *prov);
 static void dump_global_settings(void);
 static int destroy_vci_context(int vci, int nic);
@@ -799,6 +801,8 @@ int MPIDI_OFI_init_world(void)
         MPIR_ERR_CHECK(mpi_errno);
     }
 
+    ofi_initialized = true;
+
   fn_exit:
     return mpi_errno;
   fn_fail:
@@ -879,10 +883,14 @@ static int flush_send_queue(void)
 {
     int mpi_errno = MPI_SUCCESS;
 
+    if (!ofi_initialized) {
+        goto fn_exit;
+    }
+
     MPIDI_OFI_dynamic_process_request_t *reqs;
     /* TODO - Iterate over each NIC in addition to each VNI when multi-NIC within the same
      * process is implemented. */
-    int num_vcis = (MPIDI_global.is_initialized ? MPIDI_OFI_global.num_vcis : 1);
+    int num_vcis = MPIDI_OFI_global.num_vcis;
     int num_reqs = num_vcis * 2;
     reqs = MPL_malloc(sizeof(MPIDI_OFI_dynamic_process_request_t) * num_reqs, MPL_MEM_OTHER);
 
@@ -937,12 +945,9 @@ int MPIDI_OFI_mpi_finalize_hook(void)
     MPIDI_OFI_mr_key_allocator_destroy();
 
     if (strcmp("sockets", MPIDI_OFI_global.prov_use[0]->fabric_attr->prov_name) == 0) {
-        /* sockets provider need flush any last lightweight send. Only do it if we initialized
-         * world. Sockets provider can't even send self messages otherwise. */
-        if (MPIDI_global.is_initialized) {
-            mpi_errno = flush_send_queue();
-            MPIR_ERR_CHECK(mpi_errno);
-        }
+        /* sockets provider need flush any last lightweight send. */
+        mpi_errno = flush_send_queue();
+        MPIR_ERR_CHECK(mpi_errno);
     } else if (MPIR_CVAR_NO_COLLECTIVE_FINALIZE) {
         /* skip collective work arounds */
     } else if (strcmp("verbs;ofi_rxm", MPIDI_OFI_global.prov_use[0]->fabric_attr->prov_name) == 0
@@ -980,12 +985,10 @@ int MPIDI_OFI_mpi_finalize_hook(void)
     /* Tearing down endpoints in reverse order they were created */
     for (int nic = MPIDI_OFI_global.num_nics - 1; nic >= 0; nic--) {
         for (int vci = MPIDI_OFI_global.num_vcis - 1; vci >= 0; vci--) {
-            if (MPIDI_global.is_initialized || (vci == 0 && nic == 0)) {
-                /* If the user has not freed all MPI objects, ofi might not shut down cleanly.
-                 * We intentionally ignore errors to avoid crashing in finalize. Debug builds
-                 * will warn about unfreed objects/memory. */
-                (void) destroy_vci_context(vci, nic);
-            }
+            /* If the user has not freed all MPI objects, ofi might not shut down cleanly.
+             * We intentionally ignore errors to avoid crashing in finalize. Debug builds
+             * will warn about unfreed objects/memory. */
+            (void) destroy_vci_context(vci, nic);
         }
     }
 

src/mpid/ch4/netmod/ucx/ucx_init.c

@@ -18,6 +18,8 @@ categories :
 === END_MPI_T_CVAR_INFO_BLOCK ===
 */
 
+static bool ucx_initialized = false;
+
 static void request_init_callback(void *request)
 {
 
@@ -211,6 +213,8 @@ int MPIDI_UCX_init_world(void)
     mpi_errno = initial_address_exchange();
     MPIR_ERR_CHECK(mpi_errno);
 
+    ucx_initialized = true;
+
   fn_exit:
     return mpi_errno;
   fn_fail:
@@ -231,13 +235,12 @@ int MPIDI_UCX_mpi_finalize_hook(void)
 {
     int mpi_errno = MPI_SUCCESS;
 
-    if (!MPIDI_global.is_initialized) {
-        /* Nothing to do */
-        return mpi_errno;
-    }
-
     ucs_status_ptr_t ucp_request;
-    ucs_status_ptr_t *pending;
+    ucs_status_ptr_t *pending = NULL;
+
+    if (!ucx_initialized) {
+        goto fn_exit;
+    }
 
     int n = MPIDI_UCX_global.num_vcis;
     pending = MPL_malloc(sizeof(ucs_status_ptr_t) * MPIR_Process.size * n * n, MPL_MEM_OTHER);

src/mpid/ch4/src/ch4_progress.h

@@ -35,7 +35,7 @@ extern MPL_TLS int global_vci_poll_count;
 
 MPL_STATIC_INLINE_PREFIX int MPIDI_do_global_progress(void)
 {
-    if (MPIDI_global.n_vcis == 1 || !MPIDI_global.is_initialized || !MPIR_CVAR_CH4_GLOBAL_PROGRESS) {
+    if (MPIDI_global.n_vcis == 1 || !MPIR_CVAR_CH4_GLOBAL_PROGRESS) {
         return 0;
     } else {
         global_vci_poll_count++;
@@ -161,16 +161,11 @@ MPL_STATIC_INLINE_PREFIX void MPIDI_progress_state_init(MPID_Progress_state * st
         state->flag |= MPIDI_PROGRESS_NM_LOCKLESS;
     }
 
-    if (!MPIDI_global.is_initialized) {
-        state->vci[0] = 0;
-        state->vci_count = 1;
-    } else {
-        /* global progress by default */
-        for (int i = 0; i < MPIDI_global.n_vcis; i++) {
-            state->vci[i] = i;
-        }
-        state->vci_count = MPIDI_global.n_vcis;
+    /* global progress by default */
+    for (int i = 0; i < MPIDI_global.n_vcis; i++) {
+        state->vci[i] = i;
     }
+    state->vci_count = MPIDI_global.n_vcis;
 }
 
 MPL_STATIC_INLINE_PREFIX int MPIDI_Progress_test(int flags)