Skip to content

Commit e378b1a

Browse files
hppritchahppritcha
authored
Merge pull request #10065 from hppritcha/sessions_tag_ub_fix
sessions: support MPI_TAG_UB
2 parents f0f94fb + 01c2233 commit e378b1a

File tree

5 files changed

+110
-45
lines changed

5 files changed

+110
-45
lines changed

‎ompi/attribute/attribute.h‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -515,11 +515,22 @@ int ompi_attr_delete_all(ompi_attribute_type_t type, void *object,
515515
/**
516516
* \internal
517517
*
518-
* Create all the predefined attributes
518+
* Create all the predefined attribute keys
519+
* @note This routine is invoked when creating a session
520+
* so must be thread safe.
519521
*
520522
* @returns OMPI_SUCCESS
521523
*/
522-
int ompi_attr_create_predefined(void);
524+
int ompi_attr_create_predefined_keyvals(void);
525+
526+
/**
527+
* \internal
528+
*
529+
* Cache predefined attribute keys used in the World Model
530+
*
531+
* @returns OMPI_SUCCESS
532+
*/
533+
int ompi_attr_set_predefined_keyvals_for_wm(void);
523534

524535
/**
525536
* \internal

‎ompi/attribute/attribute_predefined.c‎

Lines changed: 78 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
* Copyright (c) 2020 Intel, Inc. All rights reserved.
1717
* Copyright (c) 2022 Amazon.com, Inc. or its affiliates.
1818
* All Rights reserved.
19+
* Copyright (c) 2022 Triad National Security, LLC. All rights
20+
* reserved.
1921
* $COPYRIGHT$
2022
*
2123
* Additional copyrights may follow
@@ -95,6 +97,8 @@
9597
#include "ompi/mca/pml/pml.h"
9698
#include "ompi/runtime/ompi_rte.h"
9799

100+
static bool attrs_predefined_initialized = false;
101+
98102
/*
99103
* Private functions
100104
*/
@@ -106,33 +110,53 @@ static int free_win(int keyval);
106110

107111
static int set_f(int keyval, MPI_Fint value);
108112

109-
110-
int ompi_attr_create_predefined(void)
113+
/*
114+
* We do not need a lock here as this function is invoked when the
115+
* instance_lock mutex is held.
116+
*/
117+
int ompi_attr_create_predefined_keyvals(void)
111118
{
112-
int ret;
113-
114-
/* Create all the keyvals */
115-
116-
/* DO NOT CHANGE THE ORDER OF CREATING THESE KEYVALS! This order
117-
strictly adheres to the order in mpi.h. If you change the
118-
order here, you must change the order in mpi.h as well! */
119-
120-
if (OMPI_SUCCESS != (ret = create_comm(MPI_TAG_UB, true)) ||
121-
OMPI_SUCCESS != (ret = create_comm(MPI_HOST, true)) ||
122-
OMPI_SUCCESS != (ret = create_comm(MPI_IO, true)) ||
123-
OMPI_SUCCESS != (ret = create_comm(MPI_WTIME_IS_GLOBAL, true)) ||
124-
OMPI_SUCCESS != (ret = create_comm(MPI_APPNUM, true)) ||
125-
OMPI_SUCCESS != (ret = create_comm(MPI_LASTUSEDCODE, false)) ||
126-
OMPI_SUCCESS != (ret = create_comm(MPI_UNIVERSE_SIZE, true)) ||
127-
OMPI_SUCCESS != (ret = create_win(MPI_WIN_BASE)) ||
128-
OMPI_SUCCESS != (ret = create_win(MPI_WIN_SIZE)) ||
129-
OMPI_SUCCESS != (ret = create_win(MPI_WIN_DISP_UNIT)) ||
130-
OMPI_SUCCESS != (ret = create_win(MPI_WIN_CREATE_FLAVOR)) ||
131-
OMPI_SUCCESS != (ret = create_win(MPI_WIN_MODEL)) ||
132-
OMPI_SUCCESS != (ret = create_comm(MPI_FT, false)) || /* not #if conditional on OPAL_ENABLE_FT_MPI for ABI */
133-
0) {
134-
return ret;
119+
int ret = OMPI_SUCCESS, rc;
120+
121+
if (false == attrs_predefined_initialized) {
122+
123+
attrs_predefined_initialized = true;
124+
125+
/* Create all the keyvals */
126+
127+
/* DO NOT CHANGE THE ORDER OF CREATING THESE KEYVALS! This order
128+
strictly adheres to the order in mpi.h. If you change the
129+
order here, you must change the order in mpi.h as well! */
130+
131+
if (OMPI_SUCCESS != (rc = create_comm(MPI_TAG_UB, true)) ||
132+
OMPI_SUCCESS != (rc = create_comm(MPI_HOST, true)) ||
133+
OMPI_SUCCESS != (rc = create_comm(MPI_IO, true)) ||
134+
OMPI_SUCCESS != (rc = create_comm(MPI_WTIME_IS_GLOBAL, true)) ||
135+
OMPI_SUCCESS != (rc = create_comm(MPI_APPNUM, true)) ||
136+
OMPI_SUCCESS != (rc = create_comm(MPI_LASTUSEDCODE, false)) ||
137+
OMPI_SUCCESS != (rc = create_comm(MPI_UNIVERSE_SIZE, true)) ||
138+
OMPI_SUCCESS != (rc = create_win(MPI_WIN_BASE)) ||
139+
OMPI_SUCCESS != (rc = create_win(MPI_WIN_SIZE)) ||
140+
OMPI_SUCCESS != (rc = create_win(MPI_WIN_DISP_UNIT)) ||
141+
OMPI_SUCCESS != (rc = create_win(MPI_WIN_CREATE_FLAVOR)) ||
142+
OMPI_SUCCESS != (rc = create_win(MPI_WIN_MODEL)) ||
143+
OMPI_SUCCESS != (rc = create_comm(MPI_FT, false)) || /* not #if conditional on OPAL_ENABLE_FT_MPI for ABI */
144+
0) {
145+
ret = rc;
146+
}
147+
135148
}
149+
150+
return ret;
151+
}
152+
153+
/*
154+
* This method is only invoked during MPI initialization using the world model
155+
* (MPI_Init/MPI_Init_thread) so does not need to be thread safe.
156+
*/
157+
int ompi_attr_set_predefined_keyvals_for_wm(void)
158+
{
159+
int ret = OMPI_SUCCESS;
136160

137161
/* Set default values for everything except MPI_UNIVERSE_SIZE */
138162

@@ -165,26 +189,38 @@ int ompi_attr_create_predefined(void)
165189
}
166190

167191

192+
/*
193+
* We do not need a lock here as this function is invoked when the
194+
* destructor for attr_subsys is invoked.
195+
*/
196+
168197
int ompi_attr_free_predefined(void)
169198
{
170-
int ret;
171-
172-
if (OMPI_SUCCESS != (ret = free_comm(MPI_TAG_UB)) ||
173-
OMPI_SUCCESS != (ret = free_comm(MPI_HOST)) ||
174-
OMPI_SUCCESS != (ret = free_comm(MPI_IO)) ||
175-
OMPI_SUCCESS != (ret = free_comm(MPI_WTIME_IS_GLOBAL)) ||
176-
OMPI_SUCCESS != (ret = free_comm(MPI_APPNUM)) ||
177-
OMPI_SUCCESS != (ret = free_comm(MPI_LASTUSEDCODE)) ||
178-
OMPI_SUCCESS != (ret = free_comm(MPI_UNIVERSE_SIZE)) ||
179-
OMPI_SUCCESS != (ret = free_comm(MPI_FT)) || /* not #if conditional on OPAL_ENABLE_FT_MPI for ABI */
180-
OMPI_SUCCESS != (ret = free_win(MPI_WIN_BASE)) ||
181-
OMPI_SUCCESS != (ret = free_win(MPI_WIN_SIZE)) ||
182-
OMPI_SUCCESS != (ret = free_win(MPI_WIN_DISP_UNIT)) ||
183-
OMPI_SUCCESS != (ret = free_win(MPI_WIN_CREATE_FLAVOR)) ||
184-
OMPI_SUCCESS != (ret = free_win(MPI_WIN_MODEL))) {
185-
return ret;
199+
int ret = OMPI_SUCCESS, rc;
200+
201+
if (true == attrs_predefined_initialized) {
202+
203+
attrs_predefined_initialized = false;
204+
205+
if (OMPI_SUCCESS != (rc = free_comm(MPI_TAG_UB)) ||
206+
OMPI_SUCCESS != (rc = free_comm(MPI_HOST)) ||
207+
OMPI_SUCCESS != (rc = free_comm(MPI_IO)) ||
208+
OMPI_SUCCESS != (rc = free_comm(MPI_WTIME_IS_GLOBAL)) ||
209+
OMPI_SUCCESS != (rc = free_comm(MPI_APPNUM)) ||
210+
OMPI_SUCCESS != (rc = free_comm(MPI_LASTUSEDCODE)) ||
211+
OMPI_SUCCESS != (rc = free_comm(MPI_UNIVERSE_SIZE)) ||
212+
OMPI_SUCCESS != (rc = free_comm(MPI_FT)) || /* not #if conditional on OPAL_ENABLE_FT_MPI for ABI */
213+
OMPI_SUCCESS != (rc = free_win(MPI_WIN_BASE)) ||
214+
OMPI_SUCCESS != (rc = free_win(MPI_WIN_SIZE)) ||
215+
OMPI_SUCCESS != (rc = free_win(MPI_WIN_DISP_UNIT)) ||
216+
OMPI_SUCCESS != (rc = free_win(MPI_WIN_CREATE_FLAVOR)) ||
217+
OMPI_SUCCESS != (rc = free_win(MPI_WIN_MODEL))) {
218+
ret = rc;
219+
}
220+
186221
}
187-
return OMPI_SUCCESS;
222+
223+
return ret;
188224
}
189225

190226

‎ompi/communicator/comm.c‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1331,6 +1331,17 @@ int ompi_comm_create_from_group (ompi_group_t *group, const char *tag, opal_info
13311331

13321332
newcomp->instance = group->grp_instance;
13331333

1334+
/*
1335+
* setup predefined keyvals - see MPI Standard for predefined keyvals cached on
1336+
* communicators created via MPI_Comm_from_group or MPI_Intercomm_create_from_groups
1337+
*/
1338+
ompi_attr_hash_init(&newcomp->c_keyhash);
1339+
ompi_attr_set_int(COMM_ATTR,
1340+
newcomp,
1341+
&newcomp->c_keyhash,
1342+
MPI_TAG_UB, mca_pml.pml_max_tag,
1343+
true);
1344+
13341345
*newcomm = newcomp;
13351346
return MPI_SUCCESS;
13361347
}

‎ompi/communicator/comm_init.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -292,7 +292,7 @@ int ompi_comm_init_mpi3 (void)
292292
/*
293293
* finally here we set the predefined attribute keyvals
294294
*/
295-
ompi_attr_create_predefined();
295+
ompi_attr_set_predefined_keyvals_for_wm();
296296

297297
OBJ_RETAIN(&ompi_mpi_errors_are_fatal.eh);
298298
/* During dyn_init, the comm_parent error handler will be set to the same

‎ompi/instance/instance.c‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -595,6 +595,13 @@ static int ompi_mpi_instance_init_common (void)
595595
return ompi_instance_print_error ("ompi_comm_init() failed", ret);
596596
}
597597

598+
/* Construct predefined keyvals */
599+
600+
if (OMPI_SUCCESS != (ret = ompi_attr_create_predefined_keyvals())) {
601+
opal_mutex_unlock (&instance_lock);
602+
return ompi_instance_print_error ("ompi_attr_create_predefined_keyvals() failed", ret);
603+
}
604+
598605
if (mca_pml_base_requires_world ()) {
599606
/* need to set up comm world for this instance -- XXX -- FIXME -- probably won't always
600607
* be the case. */

0 commit comments

Comments
 (0)