Skip to content

Commit 5f57479

Browse files
wfaderhold21ferrol aderholdt
wfaderhold21
authored and
ferrol aderholdt
committed
Split UCC initialization and context creation
Signed-off-by: ferrol aderholdt <[email protected]>
1 parent 967e567 commit 5f57479

File tree

8 files changed

+65
-14
lines changed

8 files changed

+65
-14
lines changed

‎oshmem/mca/scoll/ucc/scoll_ucc.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,10 @@ struct mca_scoll_ucc_component_t {
4444
char * cts;
4545
int nr_modules;
4646
bool libucc_initialized;
47+
ucc_context_h ucc_context;
4748
ucc_lib_h ucc_lib;
4849
ucc_lib_attr_t ucc_lib_attr;
4950
ucc_coll_type_t cts_requested;
50-
ucc_context_h ucc_context;
5151
};
5252
typedef struct mca_scoll_ucc_component_t mca_scoll_ucc_component_t;
5353

@@ -85,6 +85,8 @@ int mca_scoll_ucc_init_query(bool enable_progress_threads, bool enable_mpi_threa
8585
int mca_scoll_ucc_team_create(mca_scoll_ucc_module_t *ucc_module,
8686
oshmem_group_t *osh_group);
8787

88+
int mca_scoll_ucc_init_ctx(oshmem_group_t *osh_group);
89+
8890
mca_scoll_base_module_t* mca_scoll_ucc_comm_query(oshmem_group_t *osh_group, int *priority);
8991

9092
int mca_scoll_ucc_barrier(struct oshmem_group_t *group, long *pSync, int alg);

‎oshmem/mca/scoll/ucc/scoll_ucc_alltoall.c‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,12 @@ static inline ucc_status_t mca_scoll_ucc_alltoall_init(const void *sbuf, void *r
4646
.global_work_buffer = ucc_module->pSync,
4747
};
4848

49+
if (NULL == mca_scoll_ucc_component.ucc_context) {
50+
if (OSHMEM_ERROR == mca_scoll_ucc_init_ctx(ucc_module->group)) {
51+
return OSHMEM_ERROR;
52+
}
53+
}
54+
4955
if (NULL == ucc_module->ucc_team) {
5056
if (OSHMEM_ERROR == mca_scoll_ucc_team_create(ucc_module, ucc_module->group)) {
5157
return OSHMEM_ERROR;

‎oshmem/mca/scoll/ucc/scoll_ucc_barrier.c‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,19 @@ static inline ucc_status_t mca_scoll_ucc_barrier_init(mca_scoll_ucc_module_t * u
1919
.mask = 0,
2020
.coll_type = UCC_COLL_TYPE_BARRIER
2121
};
22+
23+
if (NULL == mca_scoll_ucc_component.ucc_context) {
24+
if (OSHMEM_ERROR == mca_scoll_ucc_init_ctx(ucc_module->group)) {
25+
return OSHMEM_ERROR;
26+
}
27+
}
28+
2229
if (NULL == ucc_module->ucc_team) {
2330
if (OSHMEM_ERROR == mca_scoll_ucc_team_create(ucc_module, ucc_module->group)) {
2431
return OSHMEM_ERROR;
2532
}
2633
}
34+
2735
SCOLL_UCC_REQ_INIT(req, coll, ucc_module);
2836
return UCC_OK;
2937
fallback:
@@ -49,4 +57,3 @@ int mca_scoll_ucc_barrier(struct oshmem_group_t *group, long *pSync, int alg)
4957
pSync, alg);
5058
return rc;
5159
}
52-

‎oshmem/mca/scoll/ucc/scoll_ucc_broadcast.c‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,13 @@ static inline ucc_status_t mca_scoll_ucc_broadcast_init(void * buf, int count,
2828
.mem_type = UCC_MEMORY_TYPE_UNKNOWN
2929
}
3030
};
31+
32+
if (NULL == mca_scoll_ucc_component.ucc_context) {
33+
if (OSHMEM_ERROR == mca_scoll_ucc_init_ctx(ucc_module->group)) {
34+
return OSHMEM_ERROR;
35+
}
36+
}
37+
3138
if (NULL == ucc_module->ucc_team) {
3239
if (OSHMEM_ERROR == mca_scoll_ucc_team_create(ucc_module, ucc_module->group)) {
3340
return OSHMEM_ERROR;

‎oshmem/mca/scoll/ucc/scoll_ucc_collect.c‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,12 @@ static inline ucc_status_t mca_scoll_ucc_collect_init(const void * sbuf, void *
3434
},
3535
};
3636

37+
if (NULL == mca_scoll_ucc_component.ucc_context) {
38+
if (OSHMEM_ERROR == mca_scoll_ucc_init_ctx(ucc_module->group)) {
39+
return OSHMEM_ERROR;
40+
}
41+
}
42+
3743
if (NULL == ucc_module->ucc_team) {
3844
if (OSHMEM_ERROR == mca_scoll_ucc_team_create(ucc_module, ucc_module->group)) {
3945
return OSHMEM_ERROR;

‎oshmem/mca/scoll/ucc/scoll_ucc_component.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,8 @@ mca_scoll_ucc_component_t mca_scoll_ucc_component = {
6363
"basic", /* cls */
6464
SCOLL_UCC_CTS_STR, /* cts */
6565
0, /* nr_modules */
66-
false /* libucc_initialized */
66+
false, /* libucc_initialized */
67+
NULL /* ucc_context */
6768
};
6869

6970
static int ucc_register(void)

‎oshmem/mca/scoll/ucc/scoll_ucc_module.c‎

Lines changed: 26 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -57,10 +57,12 @@ static void mca_scoll_ucc_module_destruct(mca_scoll_ucc_module_t *ucc_module)
5757
}
5858

5959
if (0 == mca_scoll_ucc_component.nr_modules) {
60-
if (mca_scoll_ucc_component.libucc_initialized) {
60+
if (mca_scoll_ucc_component.libucc_initialized) {
61+
if (mca_scoll_ucc_component.ucc_context) {
62+
opal_progress_unregister(mca_scoll_ucc_progress);
63+
ucc_context_destroy(mca_scoll_ucc_component.ucc_context);
64+
}
6165
UCC_VERBOSE(1, "finalizing ucc library");
62-
opal_progress_unregister(mca_scoll_ucc_progress);
63-
ucc_context_destroy(mca_scoll_ucc_component.ucc_context);
6466
ucc_finalize(mca_scoll_ucc_component.ucc_lib);
6567
mca_scoll_ucc_component.libucc_initialized = false;
6668
}
@@ -199,17 +201,12 @@ static ucc_status_t oob_allgather_test(void *req)
199201
return oob_probe_test(oob_req);
200202
}
201203

202-
static int mca_scoll_ucc_init_ctx(oshmem_group_t *osh_group)
204+
static int mca_scoll_ucc_init(oshmem_group_t *osh_group)
203205
{
204206
mca_scoll_ucc_component_t *cm = &mca_scoll_ucc_component;
205-
ucc_mem_map_t *maps = NULL;
206-
char str_buf[256];
207207
ucc_lib_config_h lib_config;
208-
ucc_context_config_h ctx_config;
209208
ucc_thread_mode_t tm_requested;
210209
ucc_lib_params_t lib_params;
211-
ucc_context_params_t ctx_params;
212-
int segment;
213210

214211
tm_requested = oshmem_mpi_thread_multiple ? UCC_THREAD_MULTIPLE :
215212
UCC_THREAD_SINGLE;
@@ -247,6 +244,25 @@ static int mca_scoll_ucc_init_ctx(oshmem_group_t *osh_group)
247244
goto cleanup_lib;
248245
}
249246

247+
cm->libucc_initialized = true;
248+
return OSHMEM_SUCCESS;
249+
250+
cleanup_lib:
251+
ucc_finalize(cm->ucc_lib);
252+
cm->ucc_enable = 0;
253+
cm->libucc_initialized = false;
254+
return OSHMEM_ERROR;
255+
}
256+
257+
int mca_scoll_ucc_init_ctx(oshmem_group_t *osh_group)
258+
{
259+
mca_scoll_ucc_component_t *cm = &mca_scoll_ucc_component;
260+
ucc_mem_map_t *maps = NULL;
261+
char str_buf[256];
262+
ucc_context_config_h ctx_config;
263+
ucc_context_params_t ctx_params;
264+
int segment;
265+
250266
maps = (ucc_mem_map_t *)malloc(sizeof(ucc_mem_map_t) *
251267
memheap_map->n_segments);
252268
if (NULL == maps) {
@@ -398,7 +414,6 @@ static int mca_scoll_ucc_module_enable(mca_scoll_base_module_t *module,
398414
opal_show_help("help-oshmem-scoll-ucc.txt",
399415
"module_enable:fatal", true,
400416
"UCC module enable failed - aborting to prevent inconsistent application state");
401-
402417
goto err;
403418
}
404419
UCC_VERBOSE(1, "ucc enabled");
@@ -446,7 +461,7 @@ mca_scoll_ucc_comm_query(oshmem_group_t *osh_group, int *priority)
446461

447462
if (!cm->libucc_initialized) {
448463
if (memheap_map && memheap_map->n_segments > 0) {
449-
if (OSHMEM_SUCCESS != mca_scoll_ucc_init_ctx(osh_group)) {
464+
if (OSHMEM_SUCCESS != mca_scoll_ucc_init(osh_group)) {
450465
cm->ucc_enable = 0;
451466
return NULL;
452467
}

‎oshmem/mca/scoll/ucc/scoll_ucc_reduce.c‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,13 @@ static inline ucc_status_t mca_scoll_ucc_reduce_init(const void *sbuf, void *rbu
5454
coll.mask |= UCC_COLL_ARGS_FIELD_FLAGS;
5555
coll.flags = UCC_COLL_ARGS_FLAG_IN_PLACE;
5656
}
57+
58+
if (NULL == mca_scoll_ucc_component.ucc_context) {
59+
if (OSHMEM_ERROR == mca_scoll_ucc_init_ctx(ucc_module->group)) {
60+
return OSHMEM_ERROR;
61+
}
62+
}
63+
5764
if (NULL == ucc_module->ucc_team) {
5865
if (OSHMEM_ERROR == mca_scoll_ucc_team_create(ucc_module, ucc_module->group)) {
5966
return OSHMEM_ERROR;

0 commit comments

Comments
 (0)