ch4/ofi: delay setting MPIDI_OFI_global.num_nics

MPIDI_OFI_global.num_nics affects runtime paths such as ofi progress and
large message striping. Only set it in MPIDI_OFI_init_vcis so we won't
have complications when multi-nics is not ready.

Author: hzhou

src/mpid/ch4/netmod/ofi/ofi_nic.c

@@ -195,7 +195,7 @@ int MPIDI_OFI_init_multi_nic(struct fi_info *prov)
         mpi_errno = setup_single_nic();
         MPIR_ERR_CHECK(mpi_errno);
     }
-    MPIR_Assert(MPIDI_OFI_global.num_nics > 0);
+    MPIR_Assert(MPIDI_OFI_global.num_nics_available > 0);
 
   fn_exit:
     return mpi_errno;
@@ -205,7 +205,7 @@ int MPIDI_OFI_init_multi_nic(struct fi_info *prov)
 
 static int setup_single_nic(void)
 {
-    MPIDI_OFI_global.num_nics = 1;
+    MPIDI_OFI_global.num_nics_available = 1;
     MPIDI_OFI_global.nic_info[0].nic = MPIDI_OFI_global.prov_use[0];
     MPIDI_OFI_global.nic_info[0].id = 0;
     MPIDI_OFI_global.nic_info[0].close = 1;
@@ -217,7 +217,7 @@ static int setup_single_nic(void)
     MPIR_Info *info_ptr = NULL;
     MPIR_Info_get_ptr(MPI_INFO_ENV, info_ptr);
     snprintf(nics_str, 32, "%d", 1);
-    MPIR_Info_set_impl(info_ptr, "num_nics", nics_str);
+    MPIR_Info_set_impl(info_ptr, "num_nics_available", nics_str);
     snprintf(nics_str, 32, "%d", 1);
     MPIR_Info_set_impl(info_ptr, "num_close_nics", nics_str);
 
@@ -405,14 +405,14 @@ static int setup_multi_nic(int nic_count)
     for (int i = num_nics; i < nic_count; i++) {
         fi_freeinfo(MPIDI_OFI_global.prov_use[i]);
     }
-    MPIDI_OFI_global.num_nics = MPL_MIN(nic_count, num_nics);
+    MPIDI_OFI_global.num_nics_available = MPL_MIN(nic_count, num_nics);
 
     /* Set some info keys on MPI_INFO_ENV to reflect the number of available (close) NICs */
     char nics_str[32];
     MPIR_Info *info_ptr = NULL;
     MPIR_Info_get_ptr(MPI_INFO_ENV, info_ptr);
-    snprintf(nics_str, 32, "%d", MPIDI_OFI_global.num_nics);
-    MPIR_Info_set_impl(info_ptr, "num_nics", nics_str);
+    snprintf(nics_str, 32, "%d", MPIDI_OFI_global.num_nics_available);
+    MPIR_Info_set_impl(info_ptr, "num_nics_available", nics_str);
     snprintf(nics_str, 32, "%d", num_close_nics);
     MPIR_Info_set_impl(info_ptr, "num_close_nics", nics_str);
 

src/mpid/ch4/netmod/ofi/ofi_types.h

@@ -460,6 +460,7 @@ typedef struct MPIDI_GPU_RDMA_queue_t {
 typedef struct {
     /* OFI objects */
     int avtid;
+    int num_nics_available;
     struct fi_info *prov_use[MPIDI_OFI_MAX_NICS];
     MPIDI_OFI_nic_info_t nic_info[MPIDI_OFI_MAX_NICS];
     struct fid_fabric *fabric;

src/mpid/ch4/netmod/ofi/ofi_vci.c

@@ -67,6 +67,7 @@ int MPIDI_OFI_init_vcis(int num_vcis, int *num_vcis_actual)
     MPIR_Assert(num_vcis == 1 || MPIDI_OFI_ENABLE_SCALABLE_ENDPOINTS);
 #endif
 
+    MPIDI_OFI_global.num_nics = MPIDI_OFI_global.num_nics_available;
     MPIDI_OFI_global.num_vcis = num_vcis;
 
     /* All processes must have the same number of NICs */