Skip to content

Commit dfea99a

Browse files
bureddybureddy
committed
HCOLL: Add alltoallv interface
1 parent b0ff2a1 commit dfea99a

File tree

3 files changed

+83
-8
lines changed

3 files changed

+83
-8
lines changed

‎ompi/mca/coll/hcoll/coll_hcoll.h‎

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,10 @@ struct mca_coll_hcoll_module_t {
141141
mca_coll_base_module_t *previous_iallreduce_module;
142142
mca_coll_base_module_igatherv_fn_t previous_igatherv;
143143
mca_coll_base_module_t *previous_igatherv_module;
144+
mca_coll_base_module_ialltoall_fn_t previous_ialltoall;
145+
mca_coll_base_module_t *previous_ialltoall_module;
146+
mca_coll_base_module_ialltoallv_fn_t previous_ialltoallv;
147+
mca_coll_base_module_t *previous_ialltoallv_module;
144148
};
145149
typedef struct mca_coll_hcoll_module_t mca_coll_hcoll_module_t;
146150

@@ -192,6 +196,15 @@ int mca_coll_hcoll_alltoall(void *sbuf, int scount,
192196
struct ompi_communicator_t *comm,
193197
mca_coll_base_module_t *module);
194198

199+
int mca_coll_hcoll_alltoallv(void *sbuf, int *scounts,
200+
int *sdisps,
201+
struct ompi_datatype_t *sdtype,
202+
void *rbuf, int *rcounts,
203+
int *rdisps,
204+
struct ompi_datatype_t *rdtype,
205+
struct ompi_communicator_t *comm,
206+
mca_coll_base_module_t *module);
207+
195208
int mca_coll_hcoll_gatherv(void* sbuf, int scount,
196209
struct ompi_datatype_t *sdtype,
197210
void* rbuf, int *rcounts, int *displs,
@@ -205,10 +218,10 @@ int mca_coll_hcoll_ibarrier(struct ompi_communicator_t *comm,
205218
mca_coll_base_module_t *module);
206219

207220
int mca_coll_hcoll_ibcast(void *buff, int count,
208-
struct ompi_datatype_t *datatype, int root,
209-
struct ompi_communicator_t *comm,
210-
ompi_request_t** request,
211-
mca_coll_base_module_t *module);
221+
struct ompi_datatype_t *datatype, int root,
222+
struct ompi_communicator_t *comm,
223+
ompi_request_t** request,
224+
mca_coll_base_module_t *module);
212225

213226
int mca_coll_hcoll_iallgather(void *sbuf, int scount,
214227
struct ompi_datatype_t *sdtype,
@@ -225,6 +238,24 @@ int mca_coll_hcoll_iallreduce(void *sbuf, void *rbuf, int count,
225238
ompi_request_t** request,
226239
mca_coll_base_module_t *module);
227240

241+
int mca_coll_hcoll_ialltoall(void *sbuf, int scount,
242+
struct ompi_datatype_t *sdtype,
243+
void* rbuf, int rcount,
244+
struct ompi_datatype_t *rdtype,
245+
struct ompi_communicator_t *comm,
246+
ompi_request_t **req,
247+
mca_coll_base_module_t *module);
248+
249+
int mca_coll_hcoll_ialltoallv(void *sbuf, int *scounts,
250+
int *sdisps,
251+
struct ompi_datatype_t *sdtype,
252+
void *rbuf, int *rcounts,
253+
int *rdisps,
254+
struct ompi_datatype_t *rdtype,
255+
struct ompi_communicator_t *comm,
256+
ompi_request_t **req,
257+
mca_coll_base_module_t *module);
258+
228259
int mca_coll_hcoll_igatherv(void* sbuf, int scount,
229260
struct ompi_datatype_t *sdtype,
230261
void* rbuf, int *rcounts, int *displs,

‎ompi/mca/coll/hcoll/coll_hcoll_module.c‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -81,19 +81,21 @@ static void mca_coll_hcoll_module_destruct(mca_coll_hcoll_module_t *hcoll_module
8181
OBJ_RELEASE(hcoll_module->previous_allreduce_module);
8282
OBJ_RELEASE(hcoll_module->previous_allgather_module);
8383
OBJ_RELEASE(hcoll_module->previous_gatherv_module);
84+
OBJ_RELEASE(hcoll_module->previous_alltoall_module);
85+
OBJ_RELEASE(hcoll_module->previous_alltoallv_module);
8486

8587
OBJ_RELEASE(hcoll_module->previous_ibarrier_module);
8688
OBJ_RELEASE(hcoll_module->previous_ibcast_module);
8789
OBJ_RELEASE(hcoll_module->previous_iallreduce_module);
8890
OBJ_RELEASE(hcoll_module->previous_iallgather_module);
8991
OBJ_RELEASE(hcoll_module->previous_igatherv_module);
92+
OBJ_RELEASE(hcoll_module->previous_ialltoall_module);
93+
OBJ_RELEASE(hcoll_module->previous_ialltoallv_module);
9094

9195
/*
9296
OBJ_RELEASE(hcoll_module->previous_allgatherv_module);
9397
OBJ_RELEASE(hcoll_module->previous_gather_module);
9498
OBJ_RELEASE(hcoll_module->previous_gatherv_module);
95-
OBJ_RELEASE(hcoll_module->previous_alltoall_module);
96-
OBJ_RELEASE(hcoll_module->previous_alltoallv_module);
9799
OBJ_RELEASE(hcoll_module->previous_alltoallw_module);
98100
OBJ_RELEASE(hcoll_module->previous_reduce_scatter_module);
99101
OBJ_RELEASE(hcoll_module->previous_reduce_module);
@@ -127,12 +129,16 @@ static int mca_coll_hcoll_save_coll_handlers(mca_coll_hcoll_module_t *hcoll_modu
127129
HCOL_SAVE_PREV_COLL_API(allreduce);
128130
HCOL_SAVE_PREV_COLL_API(allgather);
129131
HCOL_SAVE_PREV_COLL_API(gatherv);
132+
HCOL_SAVE_PREV_COLL_API(alltoall);
133+
HCOL_SAVE_PREV_COLL_API(alltoallv);
130134

131135
HCOL_SAVE_PREV_COLL_API(ibarrier);
132136
HCOL_SAVE_PREV_COLL_API(ibcast);
133137
HCOL_SAVE_PREV_COLL_API(iallreduce);
134138
HCOL_SAVE_PREV_COLL_API(iallgather);
135139
HCOL_SAVE_PREV_COLL_API(igatherv);
140+
HCOL_SAVE_PREV_COLL_API(ialltoall);
141+
HCOL_SAVE_PREV_COLL_API(ialltoallv);
136142

137143
/*
138144
These collectives are not yet part of hcoll, so
@@ -141,8 +147,6 @@ static int mca_coll_hcoll_save_coll_handlers(mca_coll_hcoll_module_t *hcoll_modu
141147
HCOL_SAVE_PREV_COLL_API(gather);
142148
HCOL_SAVE_PREV_COLL_API(reduce);
143149
HCOL_SAVE_PREV_COLL_API(allgatherv);
144-
HCOL_SAVE_PREV_COLL_API(alltoall);
145-
HCOL_SAVE_PREV_COLL_API(alltoallv);
146150
HCOL_SAVE_PREV_COLL_API(alltoallw);
147151
*/
148152
return OMPI_SUCCESS;
@@ -310,13 +314,16 @@ mca_coll_hcoll_comm_query(struct ompi_communicator_t *comm, int *priority)
310314
hcoll_module->super.coll_allgather = hcoll_collectives.coll_allgather ? mca_coll_hcoll_allgather : NULL;
311315
hcoll_module->super.coll_allreduce = hcoll_collectives.coll_allreduce ? mca_coll_hcoll_allreduce : NULL;
312316
hcoll_module->super.coll_alltoall = /*hcoll_collectives.coll_alltoall ? mca_coll_hcoll_alltoall : */ NULL;
317+
hcoll_module->super.coll_alltoallv = hcoll_collectives.coll_alltoallv ? mca_coll_hcoll_alltoallv : NULL;
313318
hcoll_module->super.coll_gatherv = hcoll_collectives.coll_gatherv ? mca_coll_hcoll_gatherv : NULL;
314319
hcoll_module->super.coll_ibarrier = hcoll_collectives.coll_ibarrier ? mca_coll_hcoll_ibarrier : NULL;
315320
hcoll_module->super.coll_ibcast = hcoll_collectives.coll_ibcast ? mca_coll_hcoll_ibcast : NULL;
316321
hcoll_module->super.coll_iallgather = hcoll_collectives.coll_iallgather ? mca_coll_hcoll_iallgather : NULL;
317322
hcoll_module->super.coll_iallreduce = hcoll_collectives.coll_iallreduce ? mca_coll_hcoll_iallreduce : NULL;
318323
hcoll_module->super.coll_gather = /*hcoll_collectives.coll_gather ? mca_coll_hcoll_gather :*/ NULL;
319324
hcoll_module->super.coll_igatherv = hcoll_collectives.coll_igatherv ? mca_coll_hcoll_igatherv : NULL;
325+
hcoll_module->super.coll_ialltoall = /*hcoll_collectives.coll_ialltoall ? mca_coll_hcoll_ialltoall : */ NULL;
326+
hcoll_module->super.coll_ialltoallv = /*hcoll_collectives.coll_ialltoallv ? mca_coll_hcoll_ialltoallv : */ NULL;
320327

321328
*priority = cm->hcoll_priority;
322329
module = &hcoll_module->super;

‎ompi/mca/coll/hcoll/coll_hcoll_ops.c‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,43 @@ int mca_coll_hcoll_alltoall(void *sbuf, int scount,
226226
return rc;
227227
}
228228

229+
int mca_coll_hcoll_alltoallv(void *sbuf, int *scounts, int *sdisps,
230+
struct ompi_datatype_t *sdtype,
231+
void *rbuf, int *rcounts, int *rdisps,
232+
struct ompi_datatype_t *rdtype,
233+
struct ompi_communicator_t *comm,
234+
mca_coll_base_module_t *module)
235+
{
236+
dte_data_representation_t stype;
237+
dte_data_representation_t rtype;
238+
int rc;
239+
HCOL_VERBOSE(20,"RUNNING HCOL ALLTOALLV");
240+
mca_coll_hcoll_module_t *hcoll_module = (mca_coll_hcoll_module_t*)module;
241+
stype = ompi_dtype_2_dte_dtype(sdtype);
242+
rtype = ompi_dtype_2_dte_dtype(rdtype);
243+
if (OPAL_UNLIKELY((HCOL_DTE_IS_ZERO(stype) || HCOL_DTE_IS_ZERO(rtype)
244+
|| HCOL_DTE_IS_COMPLEX(stype) || HCOL_DTE_IS_COMPLEX(rtype)))
245+
&& mca_coll_hcoll_component.hcoll_datatype_fallback){
246+
HCOL_VERBOSE(20,"Ompi_datatype is not supported: sdtype = %s, rdtype = %s; calling fallback alltoallv;",
247+
sdtype->super.name,
248+
rdtype->super.name);
249+
rc = hcoll_module->previous_alltoallv(sbuf, scounts, sdisps, sdtype,
250+
rbuf, rcounts, rdisps, rdtype,
251+
comm, hcoll_module->previous_alltoallv_module);
252+
return rc;
253+
}
254+
rc = hcoll_collectives.coll_alltoallv(sbuf, scounts, sdisps, stype,
255+
rbuf, rcounts, rdisps, rtype,
256+
hcoll_module->hcoll_context);
257+
if (HCOLL_SUCCESS != rc){
258+
HCOL_VERBOSE(20,"RUNNING FALLBACK ALLTOALLV");
259+
rc = hcoll_module->previous_alltoallv(sbuf, scounts, sdisps, sdtype,
260+
rbuf, rcounts, rdisps, rdtype,
261+
comm, hcoll_module->previous_alltoallv_module);
262+
}
263+
return rc;
264+
}
265+
229266
int mca_coll_hcoll_gatherv(void* sbuf, int scount,
230267
struct ompi_datatype_t *sdtype,
231268
void* rbuf, int *rcounts, int *displs,

0 commit comments

Comments
 (0)