Merge pull request pmodels#7353 from hzhou/2503_dyn_prep

misc: preparation for ch4 dynamic process revamp

Approved-by: Ken Raffenetti

Author: hzhou

CHANGES

@@ -15,12 +15,19 @@
 # Yaksa is now maintained inside MPICH rather than as an external submodule.  
 
 # Added internal builtin datatypes and external builtin datatypes are mapped to
-internal types. For example, both MPI_INT and MPI_INT32_T are mapped to internal
-type MPIR_INT32. NOTE: direct usage of external MPI types are disallowed
-in MPICH internally. For example, use MPIR_INT_INTERNAL to replace direct usage
-of MPI_INT. Commonly used types include MPI_BYTE, MPI_CHAR, MPI_AINT, use
-MPIR_BYTE_INTERNAL, MPIR_CHAR_INTERNAL, MPIR_AINT_INTERNAL instead. There is no
-impact to users.
+  internal types. For example, both MPI_INT and MPI_INT32_T are mapped to internal
+  type MPIR_INT32. NOTE: direct usage of external MPI types are disallowed
+  in MPICH internally. For example, use MPIR_INT_INTERNAL to replace direct usage
+  of MPI_INT. Commonly used types include MPI_BYTE, MPI_CHAR, MPI_AINT, use
+  MPIR_BYTE_INTERNAL, MPIR_CHAR_INTERNAL, MPIR_AINT_INTERNAL instead. There is no
+  impact to users.
+
+# ADI: MPID_Comm_get_lpid removed. Lpids are looked up from the local_group and
+  remote_group in the MPIR_Comm struct.
+
+# ADI: MPID_Intercomm_exchange_map renamed to MPID_Intercomm_exchange and
+  parameters now include tag, context_id, and will perform context_id exchange
+  and lpid exchange.
 
 # Added MPI_LOGICAL1, MPI_LOGICAL2, MPI_LOGICAL4, MPI_LOGICAL8, and MPI_LOGICAL16.
 

src/include/mpiimpl.h

@@ -169,6 +169,7 @@ typedef struct MPIR_Stream MPIR_Stream;
 #include "mpir_errhandler.h"
 #include "mpir_attr_generic.h"
 #include "mpir_contextid.h"
+#include "mpir_lpid.h"
 #include "mpir_status.h"
 #include "mpir_debugger.h"
 #include "mpir_op.h"

src/include/mpir_group.h

@@ -50,54 +50,13 @@
 #define MPIR_GROUP_WORLD_PTR (MPIR_Group_builtin + 1)
 #define MPIR_GROUP_SELF_PTR  (MPIR_Group_builtin + 2)
 
-/* Worlds -
- * We need a device-independent way of identifying processes. Assuming the concept of
- * "worlds", we can describe a process with (world_idx, world_rank).
- *
- * The world_idx is a local id because each process may not see all worlds. Thus,
- * each process only can maintain a list of worlds as it encounters them. Thus,
- * a process id derived from (world_idx, world_rank) is referred as LPID, or
- * "local process id".
- *
- * Each process should maintain a table of worlds with sufficient information so
- * processes can match worlds upon connection or making address exchange.
- */
-
-#define MPIR_NAMESPACE_MAX 128
-struct MPIR_World {
-    char namespace[MPIR_NAMESPACE_MAX];
-    /* other useful fields */
-    int num_procs;
-};
-
-extern struct MPIR_World MPIR_Worlds[];
-
-int MPIR_add_world(const char *namespace, int num_procs);
-int MPIR_find_world(const char *namespace);
-
-/* Abstract the integer type for lpid (process id). It is possible to use 32-bit
- * in principle, but 64-bit is simpler since we can trivially combine
- * (world_idx, world_rank).
- */
-typedef int64_t MPIR_Lpid;
-
-#define MPIR_LPID_WORLD_INDEX(lpid) ((lpid) >> 32)
-#define MPIR_LPID_WORLD_RANK(lpid)  ((lpid) & 0xffffffff)
-#define MPIR_LPID_FROM(world_idx, world_rank) (((uint64_t)(world_idx) << 32) | (world_rank))
-/* A dynamic mask is used for temporary lpid during establishing dynamic connections.
- *     dynamic_lpid = MPIR_LPID_DYNAMIC_MASK | index_to_dynamic_av_table
- */
-#define MPIR_LPID_DYNAMIC_MASK ((MPIR_Lpid)0x1 << 63)
-
 struct MPIR_Pmap {
-    int size;                   /* same as group->size, duplicate here so Pmap is logically complete */
     bool use_map;
     union {
         MPIR_Lpid *map;
         struct {
             MPIR_Lpid offset;
             MPIR_Lpid stride;
-            MPIR_Lpid blocksize;
         } stride;
     } u;
 };
@@ -148,8 +107,7 @@ int MPIR_Group_dup(MPIR_Group * old_group, MPIR_Session * session_ptr, MPIR_Grou
 int MPIR_Group_create_map(int size, int rank, MPIR_Session * session_ptr, MPIR_Lpid * map,
                           MPIR_Group ** new_group_ptr);
 int MPIR_Group_create_stride(int size, int rank, MPIR_Session * session_ptr,
-                             MPIR_Lpid offset, MPIR_Lpid stride, MPIR_Lpid blocksize,
-                             MPIR_Group ** new_group_ptr);
+                             MPIR_Lpid offset, MPIR_Lpid stride, MPIR_Group ** new_group_ptr);
 int MPIR_Group_lpid_to_rank(MPIR_Group * group, MPIR_Lpid lpid);
 
 int MPIR_Group_check_subset(MPIR_Group * group_ptr, MPIR_Comm * comm_ptr);
@@ -159,16 +117,14 @@ void MPIR_Group_finalize(void);
 
 MPL_STATIC_INLINE_PREFIX MPIR_Lpid MPIR_Group_rank_to_lpid(MPIR_Group * group, int rank)
 {
-    if (rank < 0 || rank >= group->pmap.size) {
+    if (rank < 0 || rank >= group->size) {
         return MPI_UNDEFINED;
     }
 
     if (group->pmap.use_map) {
         return group->pmap.u.map[rank];
     } else {
-        MPIR_Lpid i_blk = rank / group->pmap.u.stride.blocksize;
-        MPIR_Lpid r_blk = rank % group->pmap.u.stride.blocksize;
-        return group->pmap.u.stride.offset + i_blk * group->pmap.u.stride.stride + r_blk;
+        return group->pmap.u.stride.offset + rank * group->pmap.u.stride.stride;
     }
 }
 

src/include/mpir_lpid.h

@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) by Argonne National Laboratory
+ *     See COPYRIGHT in top-level directory
+ */
+
+#ifndef MPIR_LPID_H_INCLUDED
+#define MPIR_LPID_H_INCLUDED
+
+/* Worlds -
+ * We need a device-independent way of identifying processes. Assuming the concept of
+ * "worlds", we can describe a process with (world_idx, world_rank).
+ *
+ * The world_idx is a local id because each process may not see all worlds. Thus,
+ * each process only can maintain a list of worlds as it encounters them. Thus,
+ * a process id derived from (world_idx, world_rank) is referred as LPID, or
+ * "local process id".
+ *
+ * Each process should maintain a table of worlds with sufficient information so
+ * processes can match worlds upon connection or making address exchange.
+ */
+
+#define MPIR_NAMESPACE_MAX 128
+struct MPIR_World {
+    char namespace[MPIR_NAMESPACE_MAX];
+    /* other useful fields */
+    int num_procs;
+};
+
+extern struct MPIR_World MPIR_Worlds[];
+
+int MPIR_add_world(const char *namespace, int num_procs);
+int MPIR_find_world(const char *namespace);
+
+/* Abstract the integer type for lpid (process id). It is possible to use 32-bit
+ * in principle, but 64-bit is simpler since we can trivially combine
+ * (world_idx, world_rank).
+ */
+typedef int64_t MPIR_Lpid;
+
+#define MPIR_LPID_WORLD_INDEX(lpid) ((lpid) >> 32)
+#define MPIR_LPID_WORLD_RANK(lpid)  ((lpid) & 0xffffffff)
+#define MPIR_LPID_FROM(world_idx, world_rank) (((uint64_t)(world_idx) << 32) | (world_rank))
+/* A dynamic mask is used for temporary lpid during establishing dynamic connections.
+ *     dynamic_lpid = MPIR_LPID_DYNAMIC_MASK | index_to_dynamic_av_table
+ */
+#define MPIR_LPID_DYNAMIC_MASK ((MPIR_Lpid)0x1 << 62)   /* MPIR_Lpid is signed, avoid using the signed bit */
+#define MPIR_LPID_INVALID      0xffffffff
+
+#endif /* MPIR_LPID_H_INCLUDED */

src/mpi/coll/algorithms/treealgo/treeutil.c

@@ -504,10 +504,10 @@ static int MPII_Treeutil_hierarchy_populate(MPIR_Comm * comm, int rank, int nran
         MPIR_Assert(upper_level != NULL);
 
         /* Get wrank from the communicator as the coords are stored with wrank */
-        MPIR_Lpid temp = 0;
-        MPID_Comm_get_lpid(comm, r, &temp, FALSE);
-        int wrank = (int) temp;
-        if (wrank < 0)
+        MPIR_Lpid temp = MPIR_comm_rank_to_lpid(comm, r);
+        int world_idx = MPIR_LPID_WORLD_INDEX(temp);
+        int wrank = MPIR_LPID_WORLD_RANK(temp);
+        if (world_idx != 0)
             goto fn_fail;
         MPIR_Assert(0 <= wrank && wrank < MPIR_Process.size);
 

src/mpi/comm/comm_impl.c

@@ -73,8 +73,7 @@ static int comm_create_local_group(MPIR_Comm * comm_ptr)
     MPIR_Lpid *map = MPL_malloc(n * sizeof(MPIR_Lpid), MPL_MEM_GROUP);
 
     for (int i = 0; i < n; i++) {
-        mpi_errno = MPID_Comm_get_lpid(comm_ptr, i, &map[i], FALSE);
-        MPIR_ERR_CHECK(mpi_errno);
+        map[i] = MPIR_Group_rank_to_lpid(comm_ptr->local_group, i);
     }
 
     mpi_errno = MPIR_Group_create_map(n, comm_ptr->rank, comm_ptr->session_ptr, map,
@@ -238,8 +237,7 @@ int MPII_Comm_create_calculate_mapping(MPIR_Group * group_ptr,
             /* FIXME : BUBBLE SORT */
             mapping[i] = -1;
             for (j = 0; j < comm_ptr->local_size; j++) {
-                MPIR_Lpid comm_lpid;
-                MPID_Comm_get_lpid(comm_ptr, j, &comm_lpid, FALSE);
+                MPIR_Lpid comm_lpid = MPIR_Group_rank_to_lpid(comm_ptr->local_group, j);
                 if (comm_lpid == MPIR_Group_rank_to_lpid(group_ptr, i)) {
                     mapping[i] = j;
                     break;
@@ -918,8 +916,7 @@ int MPIR_Comm_remote_group_impl(MPIR_Comm * comm_ptr, MPIR_Group ** group_ptr)
         MPIR_Lpid *map = MPL_malloc(n * sizeof(MPIR_Lpid), MPL_MEM_GROUP);
 
         for (int i = 0; i < n; i++) {
-            mpi_errno = MPID_Comm_get_lpid(comm_ptr, i, &map[i], TRUE);
-            MPIR_ERR_CHECK(mpi_errno);
+            map[i] = MPIR_Group_rank_to_lpid(comm_ptr->remote_group, i);
         }
         mpi_errno = MPIR_Group_create_map(n, MPI_UNDEFINED, comm_ptr->session_ptr, map,
                                           &comm_ptr->remote_group);
@@ -952,88 +949,112 @@ int MPIR_Comm_set_info_impl(MPIR_Comm * comm_ptr, MPIR_Info * info_ptr)
     goto fn_exit;
 }
 
+#if 0
+/* arbitrarily determine which group is the low_group by comparing
+ * world namespaces and world ranks */
+static int determine_low_group(MPIR_Lpid remote_lpid, bool * is_low_group_out)
+{
+    int mpi_errno = MPI_SUCCESS;
+
+    int my_world_idx = 0;
+    int my_world_rank = MPIR_Process.rank;
+    int remote_world_idx = MPIR_LPID_WORLD_INDEX(remote_lpid);
+    int remote_world_rank = MPIR_LPID_WORLD_RANK(remote_lpid);
+
+    if (my_world_idx == remote_world_idx) {
+        /* same world, just compare world ranks */
+        MPIR_Assert(my_world_idx == 0);
+        *is_low_group_out = (my_world_rank < remote_world_rank);
+    } else {
+        /* different world, compare namespace */
+        int cmp_result = strncmp(MPIR_Worlds[my_world_idx].namespace,
+                                 MPIR_Worlds[remote_world_idx].namespace,
+                                 MPIR_NAMESPACE_MAX);
+        MPIR_Assert(cmp_result != 0);
+        if (cmp_result < 0)
+            *is_low_group_out = false;
+        else
+            *is_low_group_out = true;
+    }
+
+    return mpi_errno;
+}
+#endif
+
 int MPIR_Intercomm_create_impl(MPIR_Comm * local_comm_ptr, int local_leader,
                                MPIR_Comm * peer_comm_ptr, int remote_leader, int tag,
                                MPIR_Comm ** new_intercomm_ptr)
 {
     int mpi_errno = MPI_SUCCESS;
-    int final_context_id, recvcontext_id;
     int remote_size = 0;
     MPIR_Lpid *remote_lpids = NULL;
-    int comm_info[3];
-    int is_low_group = 0;
     MPIR_Session *session_ptr = local_comm_ptr->session_ptr;
 
     MPIR_FUNC_ENTER;
 
-    /* Shift tag into the tagged coll space */
-    tag |= MPIR_TAG_COLL_BIT;
-
-    mpi_errno = MPID_Intercomm_exchange_map(local_comm_ptr, local_leader,
-                                            peer_comm_ptr, remote_leader,
-                                            &remote_size, &remote_lpids, &is_low_group);
-    MPIR_ERR_CHECK(mpi_errno);
-
     /*
      * Create the contexts.  Each group will have a context for sending
      * to the other group. All processes must be involved.  Because
      * we know that the local and remote groups are disjoint, this
      * step will complete
      */
-    MPL_DBG_MSG_FMT(MPIR_DBG_COMM, VERBOSE,
-                    (MPL_DBG_FDEST, "About to get contextid (local_size=%d) on rank %d",
-                     local_comm_ptr->local_size, local_comm_ptr->rank));
     /* In the multi-threaded case, MPIR_Get_contextid_sparse assumes that the
      * calling routine already holds the single critical section */
     /* TODO: Make sure this is tag-safe */
+    int recvcontext_id;
     mpi_errno = MPIR_Get_contextid_sparse(local_comm_ptr, &recvcontext_id, FALSE);
     MPIR_ERR_CHECK(mpi_errno);
     MPIR_Assert(recvcontext_id != 0);
-    MPL_DBG_MSG_FMT(MPIR_DBG_COMM, VERBOSE, (MPL_DBG_FDEST, "Got contextid=%d", recvcontext_id));
-
-    /* Leaders can now swap context ids and then broadcast the value
-     * to the local group of processes */
-    if (local_comm_ptr->rank == local_leader) {
-        int remote_context_id;
 
-        mpi_errno =
-            MPIC_Sendrecv(&recvcontext_id, 1, MPIR_CONTEXT_ID_T_DATATYPE, remote_leader, tag,
-                          &remote_context_id, 1, MPIR_CONTEXT_ID_T_DATATYPE, remote_leader, tag,
-                          peer_comm_ptr, MPI_STATUS_IGNORE, MPIR_ERR_NONE);
-        MPIR_ERR_CHECK(mpi_errno);
-
-        final_context_id = remote_context_id;
+    /* Shift tag into the tagged coll space */
+    tag |= MPIR_TAG_COLL_BIT;
 
-        /* Now, send all of our local processes the remote_lpids,
-         * along with the final context id */
-        comm_info[0] = final_context_id;
-        MPL_DBG_MSG(MPIR_DBG_COMM, VERBOSE, "About to bcast on local_comm");
-        mpi_errno = MPIR_Bcast(comm_info, 1, MPIR_INT_INTERNAL, local_leader,
-                               local_comm_ptr, MPIR_ERR_NONE);
-        MPIR_ERR_CHECK(mpi_errno);
-        MPL_DBG_MSG_D(MPIR_DBG_COMM, VERBOSE, "end of bcast on local_comm of size %d",
-                      local_comm_ptr->local_size);
-    } else {
-        /* we're the other processes */
-        MPL_DBG_MSG(MPIR_DBG_COMM, VERBOSE, "About to receive bcast on local_comm");
-        mpi_errno = MPIR_Bcast(comm_info, 1, MPIR_INT_INTERNAL, local_leader,
-                               local_comm_ptr, MPIR_ERR_NONE);
-        MPIR_ERR_CHECK(mpi_errno);
+    int remote_context_id;
+    mpi_errno = MPID_Intercomm_exchange(local_comm_ptr, local_leader,
+                                        peer_comm_ptr, remote_leader, tag,
+                                        recvcontext_id, &remote_context_id,
+                                        &remote_size, &remote_lpids);
+    MPIR_ERR_CHECK(mpi_errno);
 
-        /* Extract the context and group sign information */
-        final_context_id = comm_info[0];
+    bool is_low_group;
+#if 0
+    mpi_errno = determine_low_group(remote_lpids[0], &is_low_group);
+    MPIR_ERR_CHECK(mpi_errno);
+#else
+    if (local_comm_ptr->rank == local_leader) {
+        if (MPIR_LPID_WORLD_INDEX(remote_lpids[0]) == 0) {
+            is_low_group = (MPIR_Process.rank < MPIR_LPID_WORLD_RANK(remote_lpids[0]));
+        } else {
+            char remote_namespace[MPIR_NAMESPACE_MAX];
+            mpi_errno = MPIC_Sendrecv(MPIR_Worlds[0].namespace, MPIR_NAMESPACE_MAX,
+                                      MPIR_CHAR_INTERNAL, remote_leader, tag,
+                                      remote_namespace, MPIR_NAMESPACE_MAX, MPIR_CHAR_INTERNAL,
+                                      remote_leader, tag, peer_comm_ptr, MPI_STATUS_IGNORE,
+                                      MPIR_ERR_NONE);
+            MPIR_ERR_CHECK(mpi_errno);
+            int cmp_result;
+            cmp_result = strncmp(MPIR_Worlds[0].namespace, remote_namespace, MPIR_NAMESPACE_MAX);
+            MPIR_Assert(cmp_result != 0);
+            if (cmp_result < 0)
+                is_low_group = false;
+            else
+                is_low_group = true;
+        }
     }
+    mpi_errno = MPIR_Bcast_impl(&is_low_group, 1, MPIR_C_BOOL_INTERNAL,
+                                local_leader, local_comm_ptr, MPIR_ERR_NONE);
+    MPIR_ERR_CHECK(mpi_errno);
+#endif
 
     /* At last, we now have the information that we need to build the
      * intercommunicator */
 
     /* All processes in the local_comm now build the communicator */
 
     mpi_errno = MPIR_Comm_create(new_intercomm_ptr);
-    if (mpi_errno)
-        goto fn_fail;
+    MPIR_ERR_CHECK(mpi_errno);
 
-    (*new_intercomm_ptr)->context_id = final_context_id;
+    (*new_intercomm_ptr)->context_id = remote_context_id;
     (*new_intercomm_ptr)->recvcontext_id = recvcontext_id;
     (*new_intercomm_ptr)->remote_size = remote_size;
     (*new_intercomm_ptr)->local_size = local_comm_ptr->local_size;
@@ -1055,6 +1076,7 @@ int MPIR_Intercomm_create_impl(MPIR_Comm * local_comm_ptr, int local_leader,
     /* construct remote_group */
     mpi_errno = MPIR_Group_create_map(remote_size, MPI_UNDEFINED, session_ptr, remote_lpids,
                                       &(*new_intercomm_ptr)->remote_group);
+    MPIR_ERR_CHECK(mpi_errno);
 
     MPIR_Comm_set_session_ptr(*new_intercomm_ptr, session_ptr);
 
@@ -1113,10 +1135,9 @@ int MPIR_peer_intercomm_create(int context_id, int recvcontext_id,
 
     MPIR_Session *session_ptr = NULL;   /* Can we just use NULL session since peer_intercomm is always temporary? */
     MPIR_Lpid my_lpid = MPIR_Group_rank_to_lpid(comm_self->local_group, 0);
-    mpi_errno = MPIR_Group_create_stride(1, 0, session_ptr, my_lpid, 1, 1,
-                                         &(*newcomm)->local_group);
+    mpi_errno = MPIR_Group_create_stride(1, 0, session_ptr, my_lpid, 1, &(*newcomm)->local_group);
     MPIR_ERR_CHECK(mpi_errno);
-    mpi_errno = MPIR_Group_create_stride(1, 0, session_ptr, remote_lpid, 1, 1,
+    mpi_errno = MPIR_Group_create_stride(1, 0, session_ptr, remote_lpid, 1,
                                          &(*newcomm)->remote_group);
     MPIR_ERR_CHECK(mpi_errno);