Skip to content

Commit ded63c5

Browse files
ggouaillardetggouaillardet
committed
ompi: use ompi_coll_base_sendrecv_actual() whenever possible
Signed-off-by: Gilles Gouaillardet <[email protected]>
1 parent 52551d9 commit ded63c5

File tree

9 files changed

+56
-130
lines changed

9 files changed

+56
-130
lines changed

‎ompi/mca/coll/base/coll_base_allreduce.c‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,6 @@ ompi_coll_base_allreduce_intra_recursivedoubling(const void *sbuf, void *rbuf,
135135
int ret, line, rank, size, adjsize, remote, distance;
136136
int newrank, newremote, extra_ranks;
137137
char *tmpsend = NULL, *tmprecv = NULL, *tmpswap = NULL, *inplacebuf_free = NULL, *inplacebuf;
138-
ompi_request_t *reqs[2] = {NULL, NULL};
139138
ptrdiff_t span, gap;
140139

141140
size = ompi_comm_size(comm);
@@ -215,14 +214,11 @@ ompi_coll_base_allreduce_intra_recursivedoubling(const void *sbuf, void *rbuf,
215214
(newremote * 2 + 1):(newremote + extra_ranks);
216215

217216
/* Exchange the data */
218-
ret = MCA_PML_CALL(irecv(tmprecv, count, dtype, remote,
219-
MCA_COLL_BASE_TAG_ALLREDUCE, comm, &reqs[0]));
220-
if (MPI_SUCCESS != ret) { line = __LINE__; goto error_hndl; }
221-
ret = MCA_PML_CALL(isend(tmpsend, count, dtype, remote,
222-
MCA_COLL_BASE_TAG_ALLREDUCE,
223-
MCA_PML_BASE_SEND_STANDARD, comm, &reqs[1]));
224-
if (MPI_SUCCESS != ret) { line = __LINE__; goto error_hndl; }
225-
ret = ompi_request_wait_all(2, reqs, MPI_STATUSES_IGNORE);
217+
ret = ompi_coll_base_sendrecv_actual(tmpsend, count, dtype, remote,
218+
MCA_COLL_BASE_TAG_ALLREDUCE,
219+
tmprecv, count, dtype, remote,
220+
MCA_COLL_BASE_TAG_ALLREDUCE,
221+
comm, MPI_STATUS_IGNORE);
226222
if (MPI_SUCCESS != ret) { line = __LINE__; goto error_hndl; }
227223

228224
/* Apply operation */

‎ompi/mca/coll/base/coll_base_util.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
#include "ompi/mca/pml/pml.h"
3030
#include "coll_base_util.h"
3131

32-
int ompi_coll_base_sendrecv_actual( void* sendbuf, size_t scount,
32+
int ompi_coll_base_sendrecv_actual( const void* sendbuf, size_t scount,
3333
ompi_datatype_t* sdatatype,
3434
int dest, int stag,
3535
void* recvbuf, size_t rcount,

‎ompi/mca/coll/base/coll_base_util.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ BEGIN_C_DECLS
3636
* If one of the communications results in a zero-byte message the
3737
* communication is ignored, and no message will cross to the peer.
3838
*/
39-
int ompi_coll_base_sendrecv_actual( void* sendbuf, size_t scount,
39+
int ompi_coll_base_sendrecv_actual( const void* sendbuf, size_t scount,
4040
ompi_datatype_t* sdatatype,
4141
int dest, int stag,
4242
void* recvbuf, size_t rcount,

‎ompi/mca/coll/basic/coll_basic_allreduce.c‎

Lines changed: 12 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
* University of Stuttgart. All rights reserved.
1010
* Copyright (c) 2004-2005 The Regents of the University of California.
1111
* All rights reserved.
12-
* Copyright (c) 2015-2016 Research Organization for Information Science
12+
* Copyright (c) 2015-2017 Research Organization for Information Science
1313
* and Technology (RIST). All rights reserved.
1414
* $COPYRIGHT$
1515
*
@@ -27,6 +27,7 @@
2727
#include "ompi/op/op.h"
2828
#include "ompi/mca/coll/coll.h"
2929
#include "ompi/mca/coll/base/coll_tags.h"
30+
#include "ompi/mca/coll/base/coll_base_util.h"
3031
#include "coll_basic.h"
3132
#include "ompi/mca/pml/pml.h"
3233

@@ -83,7 +84,6 @@ mca_coll_basic_allreduce_inter(const void *sbuf, void *rbuf, int count,
8384
int err, i, rank, root = 0, rsize, line;
8485
ptrdiff_t extent, dsize, gap;
8586
char *tmpbuf = NULL, *pml_buffer = NULL;
86-
ompi_request_t *req[2];
8787
ompi_request_t **reqs = NULL;
8888

8989
rank = ompi_comm_rank(comm);
@@ -114,18 +114,11 @@ mca_coll_basic_allreduce_inter(const void *sbuf, void *rbuf, int count,
114114
}
115115

116116
/* Do a send-recv between the two root procs. to avoid deadlock */
117-
err = MCA_PML_CALL(irecv(rbuf, count, dtype, 0,
118-
MCA_COLL_BASE_TAG_ALLREDUCE, comm,
119-
&(req[0])));
120-
if (OMPI_SUCCESS != err) { line = __LINE__; goto exit; }
121-
122-
err = MCA_PML_CALL(isend(sbuf, count, dtype, 0,
123-
MCA_COLL_BASE_TAG_ALLREDUCE,
124-
MCA_PML_BASE_SEND_STANDARD,
125-
comm, &(req[1])));
126-
if (OMPI_SUCCESS != err) { line = __LINE__; goto exit; }
127-
128-
err = ompi_request_wait_all(2, req, MPI_STATUSES_IGNORE);
117+
err = ompi_coll_base_sendrecv_actual(sbuf, count, dtype, 0,
118+
MCA_COLL_BASE_TAG_ALLREDUCE,
119+
rbuf, count, dtype, 0,
120+
MCA_COLL_BASE_TAG_ALLREDUCE,
121+
comm, MPI_STATUS_IGNORE);
129122
if (OMPI_SUCCESS != err) { line = __LINE__; goto exit; }
130123

131124
/* Loop receiving and calling reduction function (C or Fortran). */
@@ -154,18 +147,11 @@ mca_coll_basic_allreduce_inter(const void *sbuf, void *rbuf, int count,
154147
/***************************************************************************/
155148
if (rank == root) {
156149
/* sendrecv between the two roots */
157-
err = MCA_PML_CALL(irecv(pml_buffer, count, dtype, 0,
158-
MCA_COLL_BASE_TAG_ALLREDUCE,
159-
comm, &(req[1])));
160-
if (OMPI_SUCCESS != err) { line = __LINE__; goto exit; }
161-
162-
err = MCA_PML_CALL(isend(rbuf, count, dtype, 0,
163-
MCA_COLL_BASE_TAG_ALLREDUCE,
164-
MCA_PML_BASE_SEND_STANDARD, comm,
165-
&(req[0])));
166-
if (OMPI_SUCCESS != err) { line = __LINE__; goto exit; }
167-
168-
err = ompi_request_wait_all(2, req, MPI_STATUSES_IGNORE);
150+
err = ompi_coll_base_sendrecv_actual(rbuf, count, dtype, 0,
151+
MCA_COLL_BASE_TAG_ALLREDUCE,
152+
pml_buffer, count, dtype, 0,
153+
MCA_COLL_BASE_TAG_ALLREDUCE,
154+
comm, MPI_STATUS_IGNORE);
169155
if (OMPI_SUCCESS != err) { line = __LINE__; goto exit; }
170156

171157
/* distribute the data to other processes in remote group.

‎ompi/mca/coll/inter/coll_inter_allgather.c‎

Lines changed: 7 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
* Copyright (c) 2004-2005 The Regents of the University of California.
1111
* All rights reserved.
1212
* Copyright (c) 2006-2010 University of Houston. All rights reserved.
13-
* Copyright (c) 2015-2016 Research Organization for Information Science
13+
* Copyright (c) 2015-2017 Research Organization for Information Science
1414
* and Technology (RIST). All rights reserved.
1515
* $COPYRIGHT$
1616
*
@@ -27,11 +27,11 @@
2727
#include "mpi.h"
2828
#include "ompi/constants.h"
2929
#include "ompi/datatype/ompi_datatype.h"
30-
#include "ompi/request/request.h"
3130
#include "ompi/communicator/communicator.h"
3231
#include "ompi/mca/coll/coll.h"
3332
#include "ompi/mca/pml/pml.h"
3433
#include "ompi/mca/coll/base/coll_tags.h"
34+
#include "ompi/mca/coll/base/coll_base_util.h"
3535

3636
/*
3737
* allgather_inter
@@ -51,7 +51,6 @@ mca_coll_inter_allgather_inter(const void *sbuf, int scount,
5151
int rank, root = 0, size, rsize, err = OMPI_SUCCESS;
5252
char *ptmp_free = NULL, *ptmp = NULL;
5353
ptrdiff_t gap, span;
54-
ompi_request_t *req[2];
5554

5655
rank = ompi_comm_rank(comm);
5756
size = ompi_comm_size(comm->c_local_comm);
@@ -77,22 +76,11 @@ mca_coll_inter_allgather_inter(const void *sbuf, int scount,
7776

7877
if (rank == root) {
7978
/* Do a send-recv between the two root procs. to avoid deadlock */
80-
err = MCA_PML_CALL(irecv(rbuf, rcount*rsize, rdtype, 0,
81-
MCA_COLL_BASE_TAG_ALLGATHER, comm,
82-
&(req[0])));
83-
if (OMPI_SUCCESS != err) {
84-
goto exit;
85-
}
86-
87-
err = MCA_PML_CALL(isend(ptmp, scount*size, sdtype, 0,
88-
MCA_COLL_BASE_TAG_ALLGATHER,
89-
MCA_PML_BASE_SEND_STANDARD,
90-
comm, &(req[1])));
91-
if (OMPI_SUCCESS != err) {
92-
goto exit;
93-
}
94-
95-
err = ompi_request_wait_all(2, req, MPI_STATUSES_IGNORE);
79+
err = ompi_coll_base_sendrecv_actual(ptmp, scount*size, sdtype, 0,
80+
MCA_COLL_BASE_TAG_ALLGATHER,
81+
rbuf, rcount*rsize, rdtype, 0,
82+
MCA_COLL_BASE_TAG_ALLGATHER,
83+
comm, MPI_STATUS_IGNORE);
9684
if (OMPI_SUCCESS != err) {
9785
goto exit;
9886
}

‎ompi/mca/coll/inter/coll_inter_allgatherv.c‎

Lines changed: 7 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
* Copyright (c) 2004-2005 The Regents of the University of California.
1111
* All rights reserved.
1212
* Copyright (c) 2006-2010 University of Houston. All rights reserved.
13-
* Copyright (c) 2015-2016 Research Organization for Information Science
13+
* Copyright (c) 2015-2017 Research Organization for Information Science
1414
* and Technology (RIST). All rights reserved.
1515
* $COPYRIGHT$
1616
*
@@ -24,11 +24,11 @@
2424

2525
#include "mpi.h"
2626
#include "ompi/datatype/ompi_datatype.h"
27-
#include "ompi/request/request.h"
2827
#include "ompi/communicator/communicator.h"
2928
#include "ompi/constants.h"
3029
#include "ompi/mca/coll/coll.h"
3130
#include "ompi/mca/coll/base/coll_tags.h"
31+
#include "ompi/mca/coll/base/coll_base_util.h"
3232
#include "ompi/mca/pml/pml.h"
3333

3434

@@ -51,7 +51,6 @@ mca_coll_inter_allgatherv_inter(const void *sbuf, int scount,
5151
int *count=NULL,*displace=NULL;
5252
char *ptmp_free=NULL, *ptmp=NULL;
5353
ompi_datatype_t *ndtype = NULL;
54-
ompi_request_t *req[2];
5554

5655
rank = ompi_comm_rank(comm);
5756
size_local = ompi_comm_size(comm->c_local_comm);
@@ -106,25 +105,14 @@ mca_coll_inter_allgatherv_inter(const void *sbuf, int scount,
106105

107106
if (0 == rank) {
108107
/* Exchange data between roots */
109-
err = MCA_PML_CALL(irecv(rbuf, 1, ndtype, 0,
110-
MCA_COLL_BASE_TAG_ALLGATHERV, comm,
111-
&(req[0])));
108+
err = ompi_coll_base_sendrecv_actual(ptmp, total, sdtype, 0,
109+
MCA_COLL_BASE_TAG_ALLGATHERV,
110+
rbuf, 1, ndtype, 0,
111+
MCA_COLL_BASE_TAG_ALLGATHERV,
112+
comm, MPI_STATUS_IGNORE);
112113
if (OMPI_SUCCESS != err) {
113114
goto exit;
114115
}
115-
116-
err = MCA_PML_CALL(isend(ptmp, total, sdtype, 0,
117-
MCA_COLL_BASE_TAG_ALLGATHERV,
118-
MCA_PML_BASE_SEND_STANDARD,
119-
comm, &(req[1])));
120-
if (OMPI_SUCCESS != err) {
121-
goto exit;
122-
}
123-
124-
err = ompi_request_wait_all(2, req, MPI_STATUSES_IGNORE);
125-
if (OMPI_SUCCESS != err) {
126-
goto exit;
127-
}
128116
}
129117

130118
/* bcast the message to all the local processes */

‎ompi/mca/coll/inter/coll_inter_allreduce.c‎

Lines changed: 7 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
* All rights reserved.
1212
* Copyright (c) 2006-2007 University of Houston. All rights reserved.
1313
* Copyright (c) 2013 Cisco Systems, Inc. All rights reserved.
14-
* Copyright (c) 2015-2016 Research Organization for Information Science
14+
* Copyright (c) 2015-2017 Research Organization for Information Science
1515
* and Technology (RIST). All rights reserved.
1616
* $COPYRIGHT$
1717
*
@@ -27,10 +27,10 @@
2727
#include "ompi/constants.h"
2828
#include "ompi/datatype/ompi_datatype.h"
2929
#include "ompi/communicator/communicator.h"
30-
#include "ompi/request/request.h"
3130
#include "ompi/op/op.h"
3231
#include "ompi/mca/coll/coll.h"
3332
#include "ompi/mca/coll/base/coll_tags.h"
33+
#include "ompi/mca/coll/base/coll_base_util.h"
3434
#include "ompi/mca/pml/pml.h"
3535

3636
/*
@@ -49,7 +49,6 @@ mca_coll_inter_allreduce_inter(const void *sbuf, void *rbuf, int count,
4949
{
5050
int err, rank, root = 0;
5151
char *tmpbuf = NULL, *pml_buffer = NULL;
52-
ompi_request_t *req[2];
5352
ptrdiff_t gap, span;
5453

5554
rank = ompi_comm_rank(comm);
@@ -73,22 +72,11 @@ mca_coll_inter_allreduce_inter(const void *sbuf, void *rbuf, int count,
7372

7473
if (rank == root) {
7574
/* Do a send-recv between the two root procs. to avoid deadlock */
76-
err = MCA_PML_CALL(irecv(rbuf, count, dtype, 0,
77-
MCA_COLL_BASE_TAG_ALLREDUCE, comm,
78-
&(req[0])));
79-
if (OMPI_SUCCESS != err) {
80-
goto exit;
81-
}
82-
83-
err = MCA_PML_CALL(isend(pml_buffer, count, dtype, 0,
84-
MCA_COLL_BASE_TAG_ALLREDUCE,
85-
MCA_PML_BASE_SEND_STANDARD,
86-
comm, &(req[1])));
87-
if (OMPI_SUCCESS != err) {
88-
goto exit;
89-
}
90-
91-
err = ompi_request_wait_all(2, req, MPI_STATUSES_IGNORE);
75+
err = ompi_coll_base_sendrecv_actual(pml_buffer, count, dtype, 0,
76+
MCA_COLL_BASE_TAG_ALLREDUCE,
77+
rbuf, count, dtype, 0,
78+
MCA_COLL_BASE_TAG_ALLREDUCE,
79+
comm, MPI_STATUS_IGNORE);
9280
if (OMPI_SUCCESS != err) {
9381
goto exit;
9482
}

‎ompi/patterns/comm/allreduce.c‎

Lines changed: 8 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include "opal/include/opal/sys/atomic.h"
2323
#include "ompi/mca/pml/pml.h"
2424
#include "ompi/patterns/net/netpatterns.h"
25+
#include "ompi/mca/coll/base/coll_base_util.h"
2526
#include "coll_ops.h"
2627
#include "commpatterns.h"
2728

@@ -42,7 +43,6 @@ OMPI_DECLSPEC int comm_allreduce_pml(void *sbuf, void *rbuf, int count,
4243
char scratch_bufers[2][MAX_TMP_BUFFER];
4344
int send_buffer=0,recv_buffer=1;
4445
char *sbuf_current, *rbuf_current;
45-
ompi_request_t *requests[2];
4646

4747
/* get size of data needed - same layout as user data, so that
4848
* we can apply the reudction routines directly on these buffers
@@ -165,32 +165,20 @@ OMPI_DECLSPEC int comm_allreduce_pml(void *sbuf, void *rbuf, int count,
165165
/* is the remote data read */
166166
pair_rank=my_exchange_node.rank_exchanges[exchange];
167167

168-
/* post non-blocking receive */
169-
rc=MCA_PML_CALL(irecv(scratch_bufers[recv_buffer],
170-
count_this_stripe,dtype,ranks_in_comm[pair_rank],
171-
-OMPI_COMMON_TAG_ALLREDUCE,
172-
comm,&(requests[0])));
168+
rc=ompi_coll_base_sendrecv_actual(scratch_bufers[send_buffer],
169+
count_this_stripe,dtype, ranks_in_comm[pair_rank],
170+
-OMPI_COMMON_TAG_ALLREDUCE,
171+
scratch_bufers[recv_buffer],
172+
count_this_stripe,dtype,ranks_in_comm[pair_rank],
173+
-OMPI_COMMON_TAG_ALLREDUCE,
174+
comm, MPI_STATUS_IGNORE);
173175
if( 0 > rc ) {
174176
fprintf(stderr," irecv failed in comm_allreduce_pml at iterations %d \n",
175177
exchange);
176178
fflush(stderr);
177179
goto Error;
178180
}
179181

180-
/* post non-blocking send */
181-
rc=MCA_PML_CALL(isend(scratch_bufers[send_buffer],
182-
count_this_stripe,dtype, ranks_in_comm[pair_rank],
183-
-OMPI_COMMON_TAG_ALLREDUCE,MCA_PML_BASE_SEND_STANDARD,
184-
comm,&(requests[1])));
185-
if( 0 > rc ) {
186-
fprintf(stderr," isend failed in comm_allreduce_pml at iterations %d \n",
187-
exchange);
188-
fflush(stderr);
189-
goto Error;
190-
}
191-
/* wait on send and receive completion */
192-
ompi_request_wait_all(2,requests,MPI_STATUSES_IGNORE);
193-
194182
/* reduce the data */
195183
if( 0 < count_this_stripe ) {
196184
ompi_op_reduce(op,

0 commit comments

Comments
 (0)