Skip to content

Commit 86acdee

Browse files
bosilcabosilca
committed
Fix the communication ordering for all cartesian neighbor collectives.
This work is rooted in the [MPI Forum issue 153](mpi-forum/mpi-issues#153). Signed-off-by: George Bosilca <[email protected]>
1 parent 196a91e commit 86acdee

File tree

6 files changed

+33
-25
lines changed

6 files changed

+33
-25
lines changed

‎ompi/mca/coll/base/coll_tags.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@
4343
#define MCA_COLL_BASE_TAG_SCATTERV -26
4444
#define MCA_COLL_BASE_TAG_NONBLOCKING_BASE -27
4545
#define MCA_COLL_BASE_TAG_NONBLOCKING_END ((-1 * INT_MAX/2) + 1)
46+
#define MCA_COLL_BASE_TAG_NEIGHBOR_BASE (MCA_COLL_BASE_TAG_NONBLOCKING_END - 1)
47+
#define MCA_COLL_BASE_TAG_NEIGHBOR_END (MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 1024)
4648
#define MCA_COLL_BASE_TAG_HCOLL_BASE (-1 * INT_MAX/2)
4749
#define MCA_COLL_BASE_TAG_HCOLL_END (-1 * INT_MAX)
4850
#endif /* MCA_COLL_BASE_TAGS_H */

‎ompi/mca/coll/basic/coll_basic_neighbor_allgather.c

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2016 The University of Tennessee and The University
6+
* Copyright (c) 2004-2019 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -70,15 +70,15 @@ mca_coll_basic_neighbor_allgather_cart(const void *sbuf, int scount,
7070
if (MPI_PROC_NULL != srank) {
7171
nreqs++;
7272
rc = MCA_PML_CALL(irecv(rbuf, rcount, rdtype, srank,
73-
MCA_COLL_BASE_TAG_ALLGATHER,
73+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim,
7474
comm, preqs++));
7575
if (OMPI_SUCCESS != rc) break;
7676

7777
nreqs++;
7878
/* remove cast from const when the pml layer is updated to take
7979
* a const for the send buffer. */
8080
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, srank,
81-
MCA_COLL_BASE_TAG_ALLGATHER,
81+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1,
8282
MCA_PML_BASE_SEND_STANDARD,
8383
comm, preqs++));
8484
if (OMPI_SUCCESS != rc) break;
@@ -89,13 +89,13 @@ mca_coll_basic_neighbor_allgather_cart(const void *sbuf, int scount,
8989
if (MPI_PROC_NULL != drank) {
9090
nreqs++;
9191
rc = MCA_PML_CALL(irecv(rbuf, rcount, rdtype, drank,
92-
MCA_COLL_BASE_TAG_ALLGATHER,
92+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1,
9393
comm, preqs++));
9494
if (OMPI_SUCCESS != rc) break;
9595

9696
nreqs++;
9797
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, drank,
98-
MCA_COLL_BASE_TAG_ALLGATHER,
98+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim,
9999
MCA_PML_BASE_SEND_STANDARD,
100100
comm, preqs++));
101101
if (OMPI_SUCCESS != rc) break;

‎ompi/mca/coll/basic/coll_basic_neighbor_allgatherv.c

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2016 The University of Tennessee and The University
6+
* Copyright (c) 2004-2019 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -69,25 +69,27 @@ mca_coll_basic_neighbor_allgatherv_cart(const void *sbuf, int scount, struct omp
6969
if (MPI_PROC_NULL != srank) {
7070
nreqs++;
7171
rc = MCA_PML_CALL(irecv((char *) rbuf + disps[i] * extent, rcounts[i], rdtype, srank,
72-
MCA_COLL_BASE_TAG_ALLGATHER, comm, preqs++));
72+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim, comm, preqs++));
7373
if (OMPI_SUCCESS != rc) break;
7474

7575
/* remove cast from const when the pml layer is updated to take
7676
* a const for the send buffer. */
7777
nreqs++;
78-
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, srank, MCA_COLL_BASE_TAG_ALLGATHER,
78+
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, srank,
79+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1,
7980
MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
8081
if (OMPI_SUCCESS != rc) break;
8182
}
8283

8384
if (MPI_PROC_NULL != drank) {
8485
nreqs++;
8586
rc = MCA_PML_CALL(irecv((char *) rbuf + disps[i+1] * extent, rcounts[i+1], rdtype, drank,
86-
MCA_COLL_BASE_TAG_ALLGATHER, comm, preqs++));
87+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1, comm, preqs++));
8788
if (OMPI_SUCCESS != rc) break;
8889

8990
nreqs++;
90-
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, drank, MCA_COLL_BASE_TAG_ALLGATHER,
91+
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, drank,
92+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim,
9193
MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
9294
if (OMPI_SUCCESS != rc) break;
9395
}

‎ompi/mca/coll/basic/coll_basic_neighbor_alltoall.c

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2016 The University of Tennessee and The University
6+
* Copyright (c) 2004-2019 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -37,6 +37,10 @@
3737
#include "coll_basic.h"
3838
#include "ompi/mca/topo/base/base.h"
3939

40+
/**
41+
* We only have 1024 tags for the neighbor collective, so for now we only support
42+
* 512 dimensions.
43+
*/
4044
static int
4145
mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_datatype_t *sdtype, void *rbuf,
4246
int rcount, struct ompi_datatype_t *rdtype, struct ompi_communicator_t *comm,
@@ -68,7 +72,7 @@ mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_
6872
if (MPI_PROC_NULL != srank) {
6973
nreqs++;
7074
rc = MCA_PML_CALL(irecv(rbuf, rcount, rdtype, srank,
71-
MCA_COLL_BASE_TAG_NONBLOCKING_BASE - 2 * dim,
75+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim,
7276
comm, preqs++));
7377
if (OMPI_SUCCESS != rc) break;
7478
}
@@ -78,7 +82,7 @@ mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_
7882
if (MPI_PROC_NULL != drank) {
7983
nreqs++;
8084
rc = MCA_PML_CALL(irecv(rbuf, rcount, rdtype, drank,
81-
MCA_COLL_BASE_TAG_NONBLOCKING_BASE - 2 * dim - 1,
85+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1,
8286
comm, preqs++));
8387
if (OMPI_SUCCESS != rc) break;
8488
}
@@ -105,7 +109,7 @@ mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_
105109
* a const for the send buffer. */
106110
nreqs++;
107111
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, srank,
108-
MCA_COLL_BASE_TAG_NONBLOCKING_BASE - 2 * dim - 1,
112+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1,
109113
MCA_PML_BASE_SEND_STANDARD,
110114
comm, preqs++));
111115
if (OMPI_SUCCESS != rc) break;
@@ -116,7 +120,7 @@ mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_
116120
if (MPI_PROC_NULL != drank) {
117121
nreqs++;
118122
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, drank,
119-
MCA_COLL_BASE_TAG_NONBLOCKING_BASE - 2 * dim,
123+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim,
120124
MCA_PML_BASE_SEND_STANDARD,
121125
comm, preqs++));
122126
if (OMPI_SUCCESS != rc) break;

‎ompi/mca/coll/basic/coll_basic_neighbor_alltoallv.c

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2016 The University of Tennessee and The University
6+
* Copyright (c) 2004-2019 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -68,14 +68,14 @@ mca_coll_basic_neighbor_alltoallv_cart(const void *sbuf, const int scounts[], co
6868
if (MPI_PROC_NULL != srank) {
6969
nreqs++;
7070
rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[i] * rdextent, rcounts[i], rdtype, srank,
71-
MCA_COLL_BASE_TAG_ALLTOALL, comm, preqs++));
71+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim, comm, preqs++));
7272
if (OMPI_SUCCESS != rc) break;
7373
}
7474

7575
if (MPI_PROC_NULL != drank) {
7676
nreqs++;
7777
rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[i+1] * rdextent, rcounts[i+1], rdtype, drank,
78-
MCA_COLL_BASE_TAG_ALLTOALL, comm, preqs++));
78+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1, comm, preqs++));
7979
if (OMPI_SUCCESS != rc) break;
8080
}
8181
}
@@ -98,14 +98,14 @@ mca_coll_basic_neighbor_alltoallv_cart(const void *sbuf, const int scounts[], co
9898
nreqs++;
9999
/* remove cast from const when the pml layer is updated to take a const for the send buffer */
100100
rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[i] * sdextent, scounts[i], sdtype, srank,
101-
MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
101+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
102102
if (OMPI_SUCCESS != rc) break;
103103
}
104104

105105
if (MPI_PROC_NULL != drank) {
106106
nreqs++;
107107
rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[i+1] * sdextent, scounts[i+1], sdtype, drank,
108-
MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
108+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
109109
if (OMPI_SUCCESS != rc) break;
110110
}
111111
}

‎ompi/mca/coll/basic/coll_basic_neighbor_alltoallw.c

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2016 The University of Tennessee and The University
6+
* Copyright (c) 2004-2019 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -65,14 +65,14 @@ mca_coll_basic_neighbor_alltoallw_cart(const void *sbuf, const int scounts[], co
6565
if (MPI_PROC_NULL != srank) {
6666
nreqs++;
6767
rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[i], rcounts[i], rdtypes[i], srank,
68-
MCA_COLL_BASE_TAG_ALLTOALL, comm, preqs++));
68+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim, comm, preqs++));
6969
if (OMPI_SUCCESS != rc) break;
7070
}
7171

7272
if (MPI_PROC_NULL != drank) {
7373
nreqs++;
7474
rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[i+1], rcounts[i+1], rdtypes[i+1], drank,
75-
MCA_COLL_BASE_TAG_ALLTOALL, comm, preqs++));
75+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1, comm, preqs++));
7676
if (OMPI_SUCCESS != rc) break;
7777
}
7878
}
@@ -95,14 +95,14 @@ mca_coll_basic_neighbor_alltoallw_cart(const void *sbuf, const int scounts[], co
9595
nreqs++;
9696
/* remove cast from const when the pml layer is updated to take a const for the send buffer */
9797
rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[i], scounts[i], sdtypes[i], srank,
98-
MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
98+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim - 1, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
9999
if (OMPI_SUCCESS != rc) break;
100100
}
101101

102102
if (MPI_PROC_NULL != drank) {
103103
nreqs++;
104104
rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[i+1], scounts[i+1], sdtypes[i+1], drank,
105-
MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
105+
MCA_COLL_BASE_TAG_NEIGHBOR_BASE - 2 * dim, MCA_PML_BASE_SEND_STANDARD, comm, preqs++));
106106
if (OMPI_SUCCESS != rc) break;
107107
}
108108
}

0 commit comments

Comments
 (0)