Skip to content

Commit 50906b3

Browse files
igor-ivanovigor-ivanov
committed
oshmem: Align OSHMEM API with spec v1.3 (Add scoll/alltoall interface)
1 parent 84b79db commit 50906b3

File tree

11 files changed

+102
-13
lines changed

11 files changed

+102
-13
lines changed

‎oshmem/include/shmem.h.in‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,13 +101,15 @@ enum shmem_wait_ops {
101101
#define _SHMEM_BCAST_SYNC_SIZE (1 + _SHMEM_BARRIER_SYNC_SIZE)
102102
#define _SHMEM_COLLECT_SYNC_SIZE (1 + _SHMEM_BCAST_SYNC_SIZE)
103103
#define _SHMEM_REDUCE_SYNC_SIZE (1 + _SHMEM_BCAST_SYNC_SIZE)
104+
#define _SHMEM_ALLTOALL_SYNC_SIZE (1)
104105
#define _SHMEM_REDUCE_MIN_WRKDATA_SIZE (1)
105106
#define _SHMEM_SYNC_VALUE (-1)
106107

107108
#define SHMEM_BARRIER_SYNC_SIZE _SHMEM_BARRIER_SYNC_SIZE
108109
#define SHMEM_BCAST_SYNC_SIZE _SHMEM_BCAST_SYNC_SIZE
109110
#define SHMEM_COLLECT_SYNC_SIZE _SHMEM_COLLECT_SYNC_SIZE
110111
#define SHMEM_REDUCE_SYNC_SIZE _SHMEM_REDUCE_SYNC_SIZE
112+
#define SHMEM_ALLTOALL_SYNC_SIZE _SHMEM_ALLTOALL_SYNC_SIZE
111113
#define SHMEM_REDUCE_MIN_WRKDATA_SIZE _SHMEM_REDUCE_MIN_WRKDATA_SIZE
112114
#define SHMEM_SYNC_VALUE _SHMEM_SYNC_VALUE
113115

@@ -191,6 +193,24 @@ OSHMEM_DECLSPEC void shmem_iput32(void* target, const void* source, ptrdiff_t ts
191193
OSHMEM_DECLSPEC void shmem_iput64(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
192194
OSHMEM_DECLSPEC void shmem_iput128(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
193195

196+
/*
197+
* Nonblocking put routines
198+
*/
199+
OSHMEM_DECLSPEC void shmem_putmem_nbi(void *target, const void *source, size_t len, int pe);
200+
OSHMEM_DECLSPEC void shmem_char_put_nbi(char *target, const char *source, size_t len, int pe);
201+
OSHMEM_DECLSPEC void shmem_short_put_nbi(short *target, const short *source, size_t len, int pe);
202+
OSHMEM_DECLSPEC void shmem_int_put_nbi(int* target, const int* source, size_t len, int pe);
203+
OSHMEM_DECLSPEC void shmem_long_put_nbi(long *target, const long *source, size_t len, int pe);
204+
OSHMEM_DECLSPEC void shmem_longlong_put_nbi(long long *target, const long long *source, size_t len, int pe);
205+
OSHMEM_DECLSPEC void shmem_float_put_nbi(float *target, const float *source, size_t len, int pe);
206+
OSHMEM_DECLSPEC void shmem_double_put_nbi(double *target, const double *source, size_t len, int pe);
207+
OSHMEM_DECLSPEC void shmem_longdouble_put_nbi(long double *target, const long double *source, size_t len, int pe);
208+
OSHMEM_DECLSPEC void shmem_put8_nbi(void *target, const void *source, size_t len, int pe);
209+
OSHMEM_DECLSPEC void shmem_put16_nbi(void *target, const void *source, size_t len, int pe);
210+
OSHMEM_DECLSPEC void shmem_put32_nbi(void *target, const void *source, size_t len, int pe);
211+
OSHMEM_DECLSPEC void shmem_put64_nbi(void *target, const void *source, size_t len, int pe);
212+
OSHMEM_DECLSPEC void shmem_put128_nbi(void *target, const void *source, size_t len, int pe);
213+
194214
/*
195215
* Elemental get routines
196216
*/
@@ -232,6 +252,24 @@ OSHMEM_DECLSPEC void shmem_iget32(void* target, const void* source, ptrdiff_t ts
232252
OSHMEM_DECLSPEC void shmem_iget64(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
233253
OSHMEM_DECLSPEC void shmem_iget128(void* target, const void* source, ptrdiff_t tst, ptrdiff_t sst,size_t len, int pe);
234254

255+
/*
256+
* Nonblocking data get routines
257+
*/
258+
OSHMEM_DECLSPEC void shmem_getmem_nbi(void *target, const void *source, size_t len, int pe);
259+
OSHMEM_DECLSPEC void shmem_char_get_nbi(char *target, const char *source, size_t len, int pe);
260+
OSHMEM_DECLSPEC void shmem_short_get_nbi(short *target, const short *source, size_t len, int pe);
261+
OSHMEM_DECLSPEC void shmem_int_get_nbi(int *target, const int *source, size_t len, int pe);
262+
OSHMEM_DECLSPEC void shmem_long_get_nbi(long *target, const long *source, size_t len, int pe);
263+
OSHMEM_DECLSPEC void shmem_longlong_get_nbi(long long *target, const long long *source, size_t len, int pe);
264+
OSHMEM_DECLSPEC void shmem_float_get_nbi(float *target, const float *source, size_t len, int pe);
265+
OSHMEM_DECLSPEC void shmem_double_get_nbi(double *target, const double *source, size_t len, int pe);
266+
OSHMEM_DECLSPEC void shmem_longdouble_get_nbi(long double *target, const long double *source, size_t len, int pe);
267+
OSHMEM_DECLSPEC void shmem_get8_nbi(void *target, const void *source, size_t len, int pe);
268+
OSHMEM_DECLSPEC void shmem_get16_nbi(void *target, const void *source, size_t len, int pe);
269+
OSHMEM_DECLSPEC void shmem_get32_nbi(void *target, const void *source, size_t len, int pe);
270+
OSHMEM_DECLSPEC void shmem_get64_nbi(void *target, const void *source, size_t len, int pe);
271+
OSHMEM_DECLSPEC void shmem_get128_nbi(void *target, const void *source, size_t len, int pe);
272+
235273
/*
236274
* Atomic operations
237275
*/

‎oshmem/mca/scoll/base/scoll_base_frame.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ static void scoll_base_module_construct(mca_scoll_base_module_t *m)
4646
m->scoll_broadcast = NULL;
4747
m->scoll_collect = NULL;
4848
m->scoll_reduce = NULL;
49+
m->scoll_alltoall = NULL;
4950
m->scoll_module_enable = NULL;
5051
}
5152

‎oshmem/mca/scoll/base/scoll_base_select.c‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,22 @@ static int scoll_null_reduce(struct oshmem_group_t *group,
118118
return OSHMEM_SUCCESS;
119119
}
120120

121+
static int scoll_null_alltoall(struct oshmem_group_t *group,
122+
void *target,
123+
const void *source,
124+
ptrdiff_t dst, ptrdiff_t sst,
125+
size_t nlong,
126+
long *pSync,
127+
int alg)
128+
{
129+
if (oshmem_proc_group_is_member(group)) {
130+
SCOLL_ERROR("internal error");
131+
oshmem_shmem_abort(-1);
132+
return OSHMEM_ERROR;
133+
}
134+
return OSHMEM_SUCCESS;
135+
}
136+
121137
/*
122138
* Stuff for the OBJ interface
123139
*/
@@ -160,6 +176,7 @@ int mca_scoll_base_group_unselect(struct oshmem_group_t * group)
160176
CLOSE(group, broadcast);
161177
CLOSE(group, collect);
162178
CLOSE(group, reduce);
179+
CLOSE(group, alltoall);
163180

164181
/* All done */
165182
return OSHMEM_SUCCESS;
@@ -184,6 +201,7 @@ int mca_scoll_base_select(struct oshmem_group_t *group)
184201
group->g_scoll.scoll_broadcast = scoll_null_broadcast;
185202
group->g_scoll.scoll_collect = scoll_null_collect;
186203
group->g_scoll.scoll_reduce = scoll_null_reduce;
204+
group->g_scoll.scoll_alltoall = scoll_null_alltoall;
187205
return OSHMEM_SUCCESS;
188206
}
189207
SCOLL_VERBOSE(10,
@@ -206,10 +224,11 @@ int mca_scoll_base_select(struct oshmem_group_t *group)
206224
if (OSHMEM_SUCCESS != ret) {
207225
mca_scoll_base_group_unselect(group);
208226
} else {
227+
COPY(avail->ac_module, group, barrier);
209228
COPY(avail->ac_module, group, broadcast);
210229
COPY(avail->ac_module, group, collect);
211230
COPY(avail->ac_module, group, reduce);
212-
COPY(avail->ac_module, group, barrier);
231+
COPY(avail->ac_module, group, alltoall);
213232
}
214233
OBJ_RELEASE(avail->ac_module);
215234
OBJ_RELEASE(avail);
@@ -220,16 +239,17 @@ int mca_scoll_base_select(struct oshmem_group_t *group)
220239
if ((NULL == group->g_scoll.scoll_barrier)
221240
|| (NULL == group->g_scoll.scoll_broadcast)
222241
|| (NULL == group->g_scoll.scoll_collect)
223-
|| (NULL == group->g_scoll.scoll_reduce)) {
242+
|| (NULL == group->g_scoll.scoll_reduce)
243+
|| (NULL == group->g_scoll.scoll_alltoall)) {
224244
mca_scoll_base_group_unselect(group);
225245
return OSHMEM_ERR_NOT_FOUND;
226246
}
227247

228248
return OSHMEM_SUCCESS;
229249
}
230250

231-
static int avail_coll_compare (opal_list_item_t **a,
232-
opal_list_item_t **b)
251+
static int avail_coll_compare(opal_list_item_t **a,
252+
opal_list_item_t **b)
233253
{
234254
avail_com_t *acom = (avail_com_t *) *a;
235255
avail_com_t *bcom = (avail_com_t *) *b;

‎oshmem/mca/scoll/basic/Makefile.am‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# Copyright (c) 2013 Mellanox Technologies, Inc.
2+
# Copyright (c) 2013-2016 Mellanox Technologies, Inc.
33
# All rights reserved.
44
# $COPYRIGHT$
55
#
@@ -15,7 +15,8 @@ sources = \
1515
scoll_basic_barrier.c \
1616
scoll_basic_broadcast.c \
1717
scoll_basic_collect.c \
18-
scoll_basic_reduce.c
18+
scoll_basic_reduce.c \
19+
scoll_basic_alltoall.c
1920

2021

2122
# Make the output library in this directory, and name it either

‎oshmem/mca/scoll/basic/scoll_basic.h‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2013 Mellanox Technologies, Inc.
2+
* Copyright (c) 2013-2016 Mellanox Technologies, Inc.
33
* All rights reserved.
44
* $COPYRIGHT$
55
*
@@ -77,6 +77,13 @@ int mca_scoll_basic_reduce(struct oshmem_group_t *group,
7777
long *pSync,
7878
void *pWrk,
7979
int alg);
80+
int mca_scoll_basic_alltoall(struct oshmem_group_t *group,
81+
void *target,
82+
const void *source,
83+
ptrdiff_t dst, ptrdiff_t sst,
84+
size_t nlong,
85+
long *pSync,
86+
int alg);
8087

8188
static inline unsigned int scoll_log2(unsigned long val)
8289
{

‎oshmem/mca/scoll/basic/scoll_basic_module.c‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2013 Mellanox Technologies, Inc.
2+
* Copyright (c) 2013-2016 Mellanox Technologies, Inc.
33
* All rights reserved.
44
* $COPYRIGHT$
55
*
@@ -52,9 +52,10 @@ mca_scoll_basic_query(struct oshmem_group_t *group, int *priority)
5252
module->super.scoll_broadcast = mca_scoll_basic_broadcast;
5353
module->super.scoll_collect = mca_scoll_basic_collect;
5454
module->super.scoll_reduce = mca_scoll_basic_reduce;
55+
module->super.scoll_alltoall = mca_scoll_basic_alltoall;
5556
module->super.scoll_module_enable = mca_scoll_basic_enable;
5657
return &(module->super);
5758
}
5859

59-
return NULL ;
60+
return NULL;
6061
}

‎oshmem/mca/scoll/fca/scoll_fca.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,8 @@ struct mca_scoll_fca_module_t {
9393
mca_scoll_base_module_t *previous_collect_module;
9494
mca_scoll_base_module_reduce_fn_t previous_reduce;
9595
mca_scoll_base_module_t *previous_reduce_module;
96+
mca_scoll_base_module_alltoall_fn_t previous_alltoall;
97+
mca_scoll_base_module_t *previous_alltoall_module;
9698
};
9799
typedef struct mca_scoll_fca_module_t mca_scoll_fca_module_t;
98100
OBJ_CLASS_DECLARATION(mca_scoll_fca_module_t);

‎oshmem/mca/scoll/fca/scoll_fca_module.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -391,6 +391,7 @@ static int _save_coll_handlers(mca_scoll_fca_module_t *fca_module)
391391
FCA_SAVE_PREV_SCOLL_API(broadcast);
392392
FCA_SAVE_PREV_SCOLL_API(collect);
393393
FCA_SAVE_PREV_SCOLL_API(reduce);
394+
FCA_SAVE_PREV_SCOLL_API(alltoall);
394395

395396
return OSHMEM_SUCCESS;
396397
}
@@ -450,6 +451,7 @@ static void mca_scoll_fca_module_clear(mca_scoll_fca_module_t *fca_module)
450451
fca_module->previous_broadcast = NULL;
451452
fca_module->previous_collect = NULL;
452453
fca_module->previous_reduce = NULL;
454+
fca_module->previous_alltoall = NULL;
453455
}
454456

455457
static void mca_scoll_fca_module_construct(mca_scoll_fca_module_t *fca_module)
@@ -465,6 +467,7 @@ static void mca_scoll_fca_module_destruct(mca_scoll_fca_module_t *fca_module)
465467
OBJ_RELEASE(fca_module->previous_broadcast_module);
466468
OBJ_RELEASE(fca_module->previous_collect_module);
467469
OBJ_RELEASE(fca_module->previous_reduce_module);
470+
OBJ_RELEASE(fca_module->previous_alltoall_module);
468471
if (fca_module->fca_comm)
469472
_destroy_fca_comm(fca_module);
470473
free(fca_module->local_ranks);
@@ -541,6 +544,7 @@ mca_scoll_fca_comm_query(struct oshmem_group_t *comm, int *priority)
541544
fca_module->super.scoll_broadcast =
542545
mca_scoll_fca_component.fca_enable_bcast ? mca_scoll_fca_broadcast :
543546
NULL;
547+
fca_module->super.scoll_alltoall = NULL;
544548

545549
*priority = mca_scoll_fca_component.fca_priority;
546550
module = &fca_module->super;

‎oshmem/mca/scoll/mpi/scoll_mpi.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,8 @@ struct mca_scoll_mpi_module_t {
6969
mca_scoll_base_module_t *previous_barrier_module;
7070
mca_scoll_base_module_collect_fn_t previous_collect;
7171
mca_scoll_base_module_t *previous_collect_module;
72+
mca_scoll_base_module_alltoall_fn_t previous_alltoall;
73+
mca_scoll_base_module_t *previous_alltoall_module;
7274
};
7375
typedef struct mca_scoll_mpi_module_t mca_scoll_mpi_module_t;
7476

‎oshmem/mca/scoll/mpi/scoll_mpi_module.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ static void mca_scoll_mpi_module_clear(mca_scoll_mpi_module_t *mpi_module)
2929
mpi_module->previous_broadcast = NULL;
3030
mpi_module->previous_reduce = NULL;
3131
mpi_module->previous_collect = NULL;
32+
mpi_module->previous_alltoall = NULL;
3233
}
3334

3435
static void mca_scoll_mpi_module_construct(mca_scoll_mpi_module_t *mpi_module)
@@ -43,6 +44,7 @@ static void mca_scoll_mpi_module_destruct(mca_scoll_mpi_module_t *mpi_module)
4344
OBJ_RELEASE(mpi_module->previous_broadcast_module);
4445
OBJ_RELEASE(mpi_module->previous_reduce_module);
4546
OBJ_RELEASE(mpi_module->previous_collect_module);
47+
OBJ_RELEASE(mpi_module->previous_alltoall_module);
4648

4749
mca_scoll_mpi_module_clear(mpi_module);
4850
/* Free ompi_comm */
@@ -68,6 +70,7 @@ static int mca_scoll_mpi_save_coll_handlers(mca_scoll_base_module_t *module, osh
6870
MPI_SAVE_PREV_SCOLL_API(broadcast);
6971
MPI_SAVE_PREV_SCOLL_API(reduce);
7072
MPI_SAVE_PREV_SCOLL_API(collect);
73+
MPI_SAVE_PREV_SCOLL_API(alltoall);
7174
return OSHMEM_SUCCESS;
7275
}
7376

@@ -173,6 +176,7 @@ mca_scoll_mpi_comm_query(oshmem_group_t *osh_group, int *priority)
173176
mpi_module->super.scoll_broadcast = mca_scoll_mpi_broadcast;
174177
mpi_module->super.scoll_reduce = mca_scoll_mpi_reduce;
175178
mpi_module->super.scoll_collect = mca_scoll_mpi_collect;
179+
mpi_module->super.scoll_alltoall = NULL;
176180

177181
*priority = cm->mpi_priority;
178182
module = &mpi_module->super;

0 commit comments

Comments
 (0)