Skip to content

Commit 74357f0

Browse files
hzhouhzhou
authored
Merge pull request pmodels#7235 from hzhou/2412_group
group: refactor MPIR_Group Approved-by: Yanfei Guo Approved-by: Ken Raffenetti
2 parents 704c536 + d5d81c7 commit 74357f0

File tree

11 files changed

+495
-813
lines changed

11 files changed

+495
-813
lines changed

‎src/include/mpir_group.h‎

Lines changed: 32 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -11,18 +11,7 @@
1111
* only because they are required for the group operations (e.g.,
1212
* MPI_Group_intersection) and for the scalable RMA synchronization
1313
*---------------------------------------------------------------------------*/
14-
/* This structure is used to implement the group operations such as
15-
MPI_Group_translate_ranks */
16-
/* note: next_lpid (with idx_of_first_lpid in MPIR_Group) gives a linked list
17-
* in a sorted lpid ascending order */
18-
typedef struct MPII_Group_pmap_t {
19-
uint64_t lpid; /* local process id, from VCONN */
20-
int next_lpid; /* Index of next lpid (in lpid order) */
21-
} MPII_Group_pmap_t;
22-
23-
/* Any changes in the MPIR_Group structure must be made to the
24-
predefined value in MPIR_Group_builtin for MPI_GROUP_EMPTY in
25-
src/mpi/group/grouputil.c */
14+
2615
/*S
2716
MPIR_Group - Description of the Group data structure
2817
@@ -53,22 +42,35 @@ typedef struct MPII_Group_pmap_t {
5342
Group-DS
5443
5544
S*/
45+
46+
/* Abstract the integer type for lpid (process id). It is possible to use 32-bit
47+
* in principle, but 64-bit is simpler since we can trivially combine
48+
* (world_idx, world_rank).
49+
*/
50+
typedef int64_t MPIR_Lpid;
51+
52+
struct MPIR_Pmap {
53+
int size; /* same as group->size, duplicate here so Pmap is logically complete */
54+
bool use_map;
55+
union {
56+
MPIR_Lpid *map;
57+
struct {
58+
MPIR_Lpid offset;
59+
MPIR_Lpid stride;
60+
MPIR_Lpid blocksize;
61+
} stride;
62+
} u;
63+
};
64+
5665
struct MPIR_Group {
5766
MPIR_OBJECT_HEADER; /* adds handle and ref_count fields */
5867
int size; /* Size of a group */
59-
int rank; /* rank of this process relative to this
60-
* group */
61-
int idx_of_first_lpid;
62-
MPII_Group_pmap_t *lrank_to_lpid; /* Array mapping a local rank to local
63-
* process number */
64-
int is_local_dense_monotonic; /* see NOTE-G1 */
65-
66-
/* We may want some additional data for the RMA syncrhonization calls */
67-
/* Other, device-specific information */
68+
int rank; /* rank of this process relative to this group */
69+
struct MPIR_Pmap pmap;
70+
MPIR_Session *session_ptr; /* Pointer to session to which this group belongs */
6871
#ifdef MPID_DEV_GROUP_DECL
6972
MPID_DEV_GROUP_DECL
7073
#endif
71-
MPIR_Session * session_ptr; /* Pointer to session to which this group belongs */
7274
};
7375

7476
/* NOTE-G1: is_local_dense_monotonic will be true iff the group meets the
@@ -97,18 +99,21 @@ extern MPIR_Group *const MPIR_Group_empty;
9799
#define MPIR_Group_release_ref(_group, _inuse) \
98100
do { MPIR_Object_release_ref(_group, _inuse); } while (0)
99101

100-
void MPII_Group_setup_lpid_list(MPIR_Group *);
101102
int MPIR_Group_check_valid_ranks(MPIR_Group *, const int[], int);
102103
int MPIR_Group_check_valid_ranges(MPIR_Group *, int[][3], int);
103-
void MPIR_Group_setup_lpid_pairs(MPIR_Group *, MPIR_Group *);
104104
int MPIR_Group_create(int, MPIR_Group **);
105105
int MPIR_Group_release(MPIR_Group * group_ptr);
106106

107+
int MPIR_Group_create_map(int size, int rank, MPIR_Session * session_ptr, MPIR_Lpid * map,
108+
MPIR_Group ** new_group_ptr);
109+
int MPIR_Group_create_stride(int size, int rank, MPIR_Session * session_ptr,
110+
MPIR_Lpid offset, MPIR_Lpid stride, MPIR_Lpid blocksize,
111+
MPIR_Group ** new_group_ptr);
112+
MPIR_Lpid MPIR_Group_rank_to_lpid(MPIR_Group * group, int rank);
113+
int MPIR_Group_lpid_to_rank(MPIR_Group * group, MPIR_Lpid lpid);
114+
107115
int MPIR_Group_check_subset(MPIR_Group * group_ptr, MPIR_Comm * comm_ptr);
108116
void MPIR_Group_set_session_ptr(MPIR_Group * group_ptr, MPIR_Session * session_out);
109117
int MPIR_Group_init(void);
110118

111-
/* internal functions */
112-
void MPII_Group_setup_lpid_list(MPIR_Group *);
113-
114119
#endif /* MPIR_GROUP_H_INCLUDED */

‎src/mpi/comm/comm_impl.c‎

Lines changed: 18 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -68,36 +68,19 @@ int MPIR_Comm_test_threadcomm_impl(MPIR_Comm * comm_ptr, int *flag)
6868
static int comm_create_local_group(MPIR_Comm * comm_ptr)
6969
{
7070
int mpi_errno = MPI_SUCCESS;
71-
MPIR_Group *group_ptr;
72-
int n = comm_ptr->local_size;
73-
74-
mpi_errno = MPIR_Group_create(n, &group_ptr);
75-
MPIR_ERR_CHECK(mpi_errno);
7671

77-
/* Group belongs to the same session as communicator */
78-
MPIR_Group_set_session_ptr(group_ptr, comm_ptr->session_ptr);
79-
80-
group_ptr->is_local_dense_monotonic = TRUE;
72+
int n = comm_ptr->local_size;
73+
MPIR_Lpid *map = MPL_malloc(n * sizeof(MPIR_Lpid), MPL_MEM_GROUP);
8174

82-
int comm_world_size = MPIR_Process.size;
8375
for (int i = 0; i < n; i++) {
8476
uint64_t lpid;
8577
(void) MPID_Comm_get_lpid(comm_ptr, i, &lpid, FALSE);
86-
group_ptr->lrank_to_lpid[i].lpid = lpid;
87-
if (lpid > comm_world_size || (i > 0 && group_ptr->lrank_to_lpid[i - 1].lpid != (lpid - 1))) {
88-
group_ptr->is_local_dense_monotonic = FALSE;
89-
}
78+
map[i] = lpid;
9079
}
9180

92-
group_ptr->size = n;
93-
group_ptr->rank = comm_ptr->rank;
94-
group_ptr->idx_of_first_lpid = -1;
95-
96-
comm_ptr->local_group = group_ptr;
97-
98-
/* FIXME : Add a sanity check that the size of the group is the same as
99-
* the size of the communicator. This helps catch corrupted
100-
* communicators */
81+
mpi_errno = MPIR_Group_create_map(n, comm_ptr->rank, comm_ptr->session_ptr, map,
82+
&comm_ptr->local_group);
83+
MPIR_ERR_CHECK(mpi_errno);
10184

10285
fn_exit:
10386
return mpi_errno;
@@ -215,16 +198,13 @@ int MPII_Comm_create_calculate_mapping(MPIR_Group * group_ptr,
215198
* exactly the same as the ranks in comm world.
216199
*/
217200

218-
/* we examine the group's lpids in both the intracomm and non-comm_world cases */
219-
MPII_Group_setup_lpid_list(group_ptr);
220-
221201
/* Optimize for groups contained within MPI_COMM_WORLD. */
222202
if (comm_ptr->comm_kind == MPIR_COMM_KIND__INTRACOMM) {
223203
int wsize;
224204
subsetOfWorld = 1;
225205
wsize = MPIR_Process.size;
226206
for (i = 0; i < n; i++) {
227-
uint64_t g_lpid = group_ptr->lrank_to_lpid[i].lpid;
207+
MPIR_Lpid g_lpid = MPIR_Group_rank_to_lpid(group_ptr, i);
228208

229209
/* This mapping is relative to comm world */
230210
MPL_DBG_MSG_FMT(MPIR_DBG_COMM, VERBOSE,
@@ -261,7 +241,7 @@ int MPII_Comm_create_calculate_mapping(MPIR_Group * group_ptr,
261241
for (j = 0; j < comm_ptr->local_size; j++) {
262242
uint64_t comm_lpid;
263243
MPID_Comm_get_lpid(comm_ptr, j, &comm_lpid, FALSE);
264-
if (comm_lpid == group_ptr->lrank_to_lpid[i].lpid) {
244+
if (comm_lpid == MPIR_Group_rank_to_lpid(group_ptr, i)) {
265245
mapping[i] = j;
266246
break;
267247
}
@@ -795,7 +775,7 @@ int MPIR_Intercomm_create_from_groups_impl(MPIR_Group * local_group_ptr, int loc
795775

796776
int tag = get_tag_from_stringtag(stringtag);
797777
/* FIXME: ensure lpid is from comm_world */
798-
uint64_t remote_lpid = remote_group_ptr->lrank_to_lpid[remote_leader].lpid;
778+
MPIR_Lpid remote_lpid = MPIR_Group_rank_to_lpid(remote_group_ptr, remote_leader);
799779
MPIR_Assert(remote_lpid < MPIR_Process.size);
800780
mpi_errno = MPIR_Intercomm_create_impl(local_comm, local_leader,
801781
MPIR_Process.comm_world, (int) remote_lpid,
@@ -926,31 +906,23 @@ int MPIR_Comm_idup_with_info_impl(MPIR_Comm * comm_ptr, MPIR_Info * info,
926906
int MPIR_Comm_remote_group_impl(MPIR_Comm * comm_ptr, MPIR_Group ** group_ptr)
927907
{
928908
int mpi_errno = MPI_SUCCESS;
929-
int i, n;
930-
931909
MPIR_FUNC_ENTER;
910+
932911
/* Create a group and populate it with the local process ids */
933912
if (!comm_ptr->remote_group) {
934-
n = comm_ptr->remote_size;
935-
mpi_errno = MPIR_Group_create(n, group_ptr);
936-
MPIR_ERR_CHECK(mpi_errno);
913+
int n = comm_ptr->remote_size;
914+
MPIR_Lpid *map = MPL_malloc(n * sizeof(MPIR_Lpid), MPL_MEM_GROUP);
937915

938-
for (i = 0; i < n; i++) {
916+
for (int i = 0; i < n; i++) {
939917
uint64_t lpid;
940918
(void) MPID_Comm_get_lpid(comm_ptr, i, &lpid, TRUE);
941-
(*group_ptr)->lrank_to_lpid[i].lpid = lpid;
942-
/* TODO calculate is_local_dense_monotonic */
919+
map[i] = lpid;
943920
}
944-
(*group_ptr)->size = n;
945-
(*group_ptr)->rank = MPI_UNDEFINED;
946-
(*group_ptr)->idx_of_first_lpid = -1;
947-
948-
MPIR_Group_set_session_ptr(*group_ptr, comm_ptr->session_ptr);
949-
950-
comm_ptr->remote_group = *group_ptr;
951-
} else {
952-
*group_ptr = comm_ptr->remote_group;
921+
mpi_errno = MPIR_Group_create_map(n, MPI_UNDEFINED, comm_ptr->session_ptr, map,
922+
&comm_ptr->remote_group);
923+
MPIR_ERR_CHECK(mpi_errno);
953924
}
925+
*group_ptr = comm_ptr->remote_group;
954926
MPIR_Group_add_ref(comm_ptr->remote_group);
955927

956928
fn_exit:

‎src/mpi/comm/ulfm_impl.c‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -87,21 +87,22 @@ int MPIR_Comm_get_failed_impl(MPIR_Comm * comm_ptr, MPIR_Group ** failed_group_p
8787
/* create failed_group */
8888
int n = utarray_len(failed_procs);
8989

90+
MPIR_Lpid *map = MPL_malloc(n * sizeof(MPIR_Lpid), MPL_MEM_GROUP);
91+
9092
MPIR_Group *new_group;
91-
mpi_errno = MPIR_Group_create(n, &new_group);
92-
MPIR_ERR_CHECK(mpi_errno);
9393

94-
new_group->rank = MPI_UNDEFINED;
94+
int myrank = MPI_UNDEFINED;
9595
for (int i = 0; i < utarray_len(failed_procs); i++) {
9696
int *p = (int *) utarray_eltptr(failed_procs, i);
97-
new_group->lrank_to_lpid[i].lpid = *p;
97+
map[i] = *p;
9898
/* if calling process is part of the group, set the rank */
9999
if (*p == MPIR_Process.rank) {
100-
new_group->rank = i;
100+
myrank = i;
101101
}
102102
}
103-
new_group->size = n;
104-
new_group->idx_of_first_lpid = -1;
103+
104+
mpi_errno = MPIR_Group_create_map(n, myrank, comm_ptr->session_ptr, map, &new_group);
105+
MPIR_ERR_CHECK(mpi_errno);
105106

106107
MPIR_Group *comm_group;
107108
MPIR_Comm_group_impl(comm_ptr, &comm_group);

0 commit comments

Comments
 (0)