ch4/ofi: refactor vci-related initialization code

Move code that are related to multiple-vci setup to ofi_vci.c.

Author: hzhou

src/mpid/ch4/netmod/ofi/init_addrxchg.c

@@ -38,58 +38,6 @@
  * isolates multi-nic/vci complications from bootstrapping phase.
  */
 
-/* with MPIDI_OFI_ENABLE_AV_TABLE, we potentially can omit storing av tables.
- * The following routines ensures we can do that. It is static now, but we can
- * easily export to global when we need to.
- */
-
-#if !defined(MPIDI_OFI_VNI_USE_DOMAIN) || MPIDI_CH4_MAX_VCIS == 1
-/* NOTE: with scalable endpoint as context, all vcis share the same address. */
-#define NUM_VCIS_FOR_RANK(r) 1
-#else
-#define NUM_VCIS_FOR_RANK(r) all_num_vcis[r]
-#endif
-
-ATTRIBUTE((unused))
-static int get_root_av_table_index(int rank)
-{
-    if (MPIR_CVAR_CH4_ROOTS_ONLY_PMI) {
-        /* node roots with greater ranks are inserted before this rank if it a non-node-root */
-        int num_extra = 0;
-
-        /* check node roots */
-        for (int i = 0; i < MPIR_Process.num_nodes; i++) {
-            if (MPIR_Process.node_root_map[i] == rank) {
-                return i;
-            } else if (MPIR_Process.node_root_map[i] > rank) {
-                num_extra++;
-            }
-        }
-
-        /* must be non-node-root */
-        return rank + num_extra;
-    } else {
-        return rank;
-    }
-}
-
-ATTRIBUTE((unused))
-static int get_av_table_index(int rank, int nic, int vci, int *all_num_vcis)
-{
-    if (nic == 0 && vci == 0) {
-        return get_root_av_table_index(rank);
-    } else {
-        int num_nics = MPIDI_OFI_global.num_nics;
-        int idx = 0;
-        idx += MPIR_Process.size;       /* root entries */
-        for (int i = 0; i < rank; i++) {
-            idx += num_nics * NUM_VCIS_FOR_RANK(i) - 1;
-        }
-        idx += nic * NUM_VCIS_FOR_RANK(rank) + vci - 1;
-        return idx;
-    }
-}
-
 /* Step 1: exchange root contexts */
 int MPIDI_OFI_addr_exchange_root_ctx(void)
 {
@@ -170,14 +118,6 @@ int MPIDI_OFI_addr_exchange_root_ctx(void)
         MPIR_ERR_CHECK(mpi_errno);
     }
 
-    /* check */
-    if (MPIDI_OFI_ENABLE_AV_TABLE) {
-        for (int r = 0; r < size; r++) {
-            MPIDI_av_entry_t *av ATTRIBUTE((unused)) = &MPIDIU_get_av(0, r);
-            MPIR_Assert(MPIDI_OFI_AV_ROOT_ADDR(av) == get_root_av_table_index(r));
-        }
-    }
-
   fn_exit:
     if (init_comm && !mpi_errno) {
         MPIDI_destroy_init_comm(&init_comm);
@@ -186,185 +126,3 @@ int MPIDI_OFI_addr_exchange_root_ctx(void)
   fn_fail:
     goto fn_exit;
 }
-
-/* Step 2 & 3: exchange non-root contexts */
-
-/* Macros to reduce clutter, so we can focus on the ordering logics.
- * Note: they are not perfectly wrapped, but tolerable since only used here. */
-#define GET_AV_AND_ADDRNAMES(rank) \
-    MPIDI_av_entry_t *av ATTRIBUTE((unused)) = &MPIDIU_get_av(0, rank); \
-    char *r_names = all_names + rank * max_vcis * num_nics * name_len;
-
-#define DO_AV_INSERT(ctx_idx, nic, vci) \
-    fi_addr_t addr; \
-    MPIDI_OFI_CALL(fi_av_insert(MPIDI_OFI_global.ctx[ctx_idx].av, \
-                                r_names + (vci * num_nics + nic) * name_len, 1, \
-                                &addr, 0ULL, NULL), avmap);
-
-#define SKIP_ROOT(nic, vci) \
-    if (nic == 0 && vci == 0) { \
-        continue; \
-    }
-
-int MPIDI_OFI_addr_exchange_all_ctx(void)
-{
-    int mpi_errno = MPI_SUCCESS;
-
-    MPIR_Comm *comm = MPIR_Process.comm_world;
-    int size = MPIR_Process.size;
-    int rank = MPIR_Process.rank;
-    MPIR_CHKLMEM_DECL();
-
-    int max_vcis;
-    int *all_num_vcis;
-
-#if !defined(MPIDI_OFI_VNI_USE_DOMAIN) || MPIDI_CH4_MAX_VCIS == 1
-    max_vcis = 1;
-    all_num_vcis = NULL;
-#else
-    /* Allgather num_vcis */
-    MPIR_CHKLMEM_MALLOC(all_num_vcis, sizeof(int) * size);
-    mpi_errno = MPIR_Allgather_fallback(&MPIDI_OFI_global.num_vcis, 1, MPIR_INT_INTERNAL,
-                                        all_num_vcis, 1, MPIR_INT_INTERNAL, comm, MPIR_ERR_NONE);
-    MPIR_ERR_CHECK(mpi_errno);
-
-    max_vcis = 0;
-    for (int i = 0; i < size; i++) {
-        if (max_vcis < NUM_VCIS_FOR_RANK(i)) {
-            max_vcis = NUM_VCIS_FOR_RANK(i);
-        }
-    }
-#endif
-
-    int num_vcis = NUM_VCIS_FOR_RANK(rank);
-    int num_nics = MPIDI_OFI_global.num_nics;
-
-    /* Assume num_nics are all equal */
-    if (max_vcis * num_nics == 1) {
-        goto fn_exit;
-    }
-
-    /* allocate additional av addrs */
-    for (int i = 0; i < size; i++) {
-        MPIDI_av_entry_t *av = &MPIDIU_get_av(0, i);
-        MPIDI_OFI_AV(av).all_dest = MPL_malloc(max_vcis * num_nics * sizeof(fi_addr_t),
-                                               MPL_MEM_ADDRESS);
-        MPIR_ERR_CHKANDJUMP(!MPIDI_OFI_AV(av).all_dest, mpi_errno, MPI_ERR_OTHER, "**nomem");
-    }
-
-    /* libfabric uses uniform name_len within a single provider */
-    int name_len = MPIDI_OFI_global.addrnamelen;
-    int my_len = max_vcis * num_nics * name_len;
-    char *all_names;
-    MPIR_CHKLMEM_MALLOC(all_names, size * my_len);
-    char *my_names = all_names + rank * my_len;
-
-    /* put in my addrnames */
-    for (int nic = 0; nic < num_nics; nic++) {
-        for (int vci = 0; vci < num_vcis; vci++) {
-            size_t actual_name_len = name_len;
-            char *vci_addrname = my_names + (vci * num_nics + nic) * name_len;
-            int ctx_idx = MPIDI_OFI_get_ctx_index(vci, nic);
-            MPIDI_OFI_CALL(fi_getname((fid_t) MPIDI_OFI_global.ctx[ctx_idx].ep, vci_addrname,
-                                      &actual_name_len), getname);
-            MPIR_Assert(actual_name_len == name_len);
-        }
-    }
-    /* Allgather */
-    mpi_errno = MPIR_Allgather_fallback(MPI_IN_PLACE, 0, MPIR_BYTE_INTERNAL,
-                                        all_names, my_len, MPIR_BYTE_INTERNAL, comm, MPIR_ERR_NONE);
-
-    /* Step 2: insert and store non-root nic/vci on the root context */
-    int root_ctx_idx = MPIDI_OFI_get_ctx_index(0, 0);
-    for (int r = 0; r < size; r++) {
-        GET_AV_AND_ADDRNAMES(r);
-        for (int nic = 0; nic < num_nics; nic++) {
-            for (int vci = 0; vci < NUM_VCIS_FOR_RANK(r); vci++) {
-                SKIP_ROOT(nic, vci);
-                DO_AV_INSERT(root_ctx_idx, nic, vci);
-                MPIDI_OFI_AV_ADDR(av, 0, 0, vci, nic) = addr;
-            }
-        }
-    }
-
-    /* Step 3: insert all nic/vci on non-root context, following exact order as step 1 and 2 */
-
-    int *is_node_roots = NULL;
-    if (MPIR_CVAR_CH4_ROOTS_ONLY_PMI) {
-        MPIR_CHKLMEM_MALLOC(is_node_roots, size * sizeof(int));
-        for (int r = 0; r < size; r++) {
-            is_node_roots[r] = 0;
-        }
-        for (int i = 0; i < MPIR_Process.num_nodes; i++) {
-            is_node_roots[MPIR_Process.node_root_map[i]] = 1;
-        }
-    }
-
-    for (int nic_local = 0; nic_local < num_nics; nic_local++) {
-        for (int vci_local = 0; vci_local < num_vcis; vci_local++) {
-            SKIP_ROOT(nic_local, vci_local);
-            int ctx_idx = MPIDI_OFI_get_ctx_index(vci_local, nic_local);
-
-            /* -- same order as step 1 -- */
-            if (MPIR_CVAR_CH4_ROOTS_ONLY_PMI) {
-                /* node roots */
-                for (int r = 0; r < size; r++) {
-                    if (is_node_roots[r]) {
-                        GET_AV_AND_ADDRNAMES(r);
-                        DO_AV_INSERT(ctx_idx, 0, 0);
-                        MPIR_Assert(MPIDI_OFI_AV_ROOT_ADDR(av) == addr);
-                    }
-                }
-                /* non-node-root */
-                for (int r = 0; r < size; r++) {
-                    if (!is_node_roots[r]) {
-                        GET_AV_AND_ADDRNAMES(r);
-                        DO_AV_INSERT(ctx_idx, 0, 0);
-                        MPIR_Assert(MPIDI_OFI_AV_ROOT_ADDR(av) == addr);
-                    }
-                }
-            } else {
-                /* !MPIR_CVAR_CH4_ROOTS_ONLY_PMI */
-                for (int r = 0; r < size; r++) {
-                    GET_AV_AND_ADDRNAMES(r);
-                    DO_AV_INSERT(ctx_idx, 0, 0);
-                    MPIR_Assert(MPIDI_OFI_AV_ROOT_ADDR(av) == addr);
-                }
-            }
-
-            /* -- same order as step 2 -- */
-            for (int r = 0; r < size; r++) {
-                GET_AV_AND_ADDRNAMES(r);
-                for (int nic = 0; nic < num_nics; nic++) {
-                    for (int vci = 0; vci < NUM_VCIS_FOR_RANK(r); vci++) {
-                        SKIP_ROOT(nic, vci);
-                        DO_AV_INSERT(ctx_idx, nic, vci);
-                        MPIR_Assert(MPIDI_OFI_AV_ADDR(av, 0, 0, vci, nic) == addr);
-                    }
-                }
-            }
-        }
-    }
-    mpi_errno = MPIR_Barrier_fallback(comm, MPIR_ERR_NONE);
-    MPIR_ERR_CHECK(mpi_errno);
-
-    /* check */
-#if MPIDI_CH4_MAX_VCIS > 1
-    if (MPIDI_OFI_ENABLE_AV_TABLE) {
-        for (int r = 0; r < size; r++) {
-            MPIDI_av_entry_t *av ATTRIBUTE((unused)) = &MPIDIU_get_av(0, r);
-            for (int nic = 0; nic < num_nics; nic++) {
-                for (int vci = 0; vci < NUM_VCIS_FOR_RANK(r); vci++) {
-                    MPIR_Assert(MPIDI_OFI_AV_ADDR(av, 0, 0, vci, nic) ==
-                                get_av_table_index(r, nic, vci, all_num_vcis));
-                }
-            }
-        }
-    }
-#endif
-  fn_exit:
-    MPIR_CHKLMEM_FREEALL();
-    return mpi_errno;
-  fn_fail:
-    goto fn_exit;
-}