Skip to content

coll/sync: add support for alltoallv and alltoallw #3262

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

coll/sync: add support for alltoallv and alltoallw #3262

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
931693a
alltoallv desync fix via coll.sync component
haraldBraun-atos Mar 31, 2017
6a67820
coll/sync: add alltoallw analoguous to alltoallv
haraldBraun-atos Apr 13, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion ompi/mca/coll/sync/Makefile.am
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,9 @@ sources = \
coll_sync_reduce_scatter.c \
coll_sync_scan.c \
coll_sync_scatter.c \
coll_sync_scatterv.c
coll_sync_scatterv.c \
coll_sync_alltoallv.c \
coll_sync_alltoallw.c

if MCA_BUILD_ompi_coll_sync_DSO
component_noinst =
Expand Down
27 changes: 27 additions & 0 deletions ompi/mca/coll/sync/coll_sync.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,20 @@ mca_coll_base_module_t
int mca_coll_sync_module_enable(mca_coll_base_module_t *module,
struct ompi_communicator_t *comm);

int mca_coll_sync_alltoallv(const void *sbuf, const int *scounts, const int *sdisps,
struct ompi_datatype_t *sdtype,
void *rbuf, const int *rcounts, const int *rdisps,
struct ompi_datatype_t *rdtype,
struct ompi_communicator_t *comm,
mca_coll_base_module_t *module);

int mca_coll_sync_alltoallw(const void *sbuf, const int *scounts, const int *sdisps,
struct ompi_datatype_t * const *sdtypes,
void *rbuf, const int *rcounts, const int *rdisps,
struct ompi_datatype_t * const *rdtypes,
struct ompi_communicator_t *comm,
mca_coll_base_module_t *module);

int mca_coll_sync_barrier(struct ompi_communicator_t *comm,
mca_coll_base_module_t *module);

Expand Down Expand Up @@ -130,6 +144,14 @@ typedef struct mca_coll_sync_module_t {
/* How many ops we've executed (it's easier to have 2) */
int after_num_operations;

/* How many ops we've executed */
int before_num_operations_alltoallv;
int before_num_operations_alltoallw;

/* How many ops we've executed (it's easier to have 2) */
int after_num_operations_alltoallv;
int after_num_operations_alltoallw;

/* Avoid recursion of syncs */
bool in_operation;
} mca_coll_sync_module_t;
Expand All @@ -146,9 +168,14 @@ typedef struct mca_coll_sync_component_t {

/* Do a sync *before* each Nth collective */
int barrier_before_nops;
int barrier_before_nops_alltoallv;
int barrier_before_nops_alltoallw;

/* Do a sync *after* each Nth collective */
int barrier_after_nops;
int barrier_after_nops_alltoallv;
int barrier_after_nops_alltoallw;

} mca_coll_sync_component_t;

/* Globally exported variables */
Expand Down
73 changes: 73 additions & 0 deletions ompi/mca/coll/sync/coll_sync_alltoallv.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/

#include "ompi_config.h"

#include "coll_sync.h"


/*
* gatherv
*
* Function: - gatherv
* Accepts: - same arguments as MPI_Alltoallv()
* Returns: - MPI_SUCCESS or error code
*/
int mca_coll_sync_alltoallv(const void *sbuf, const int *scounts, const int *sdisps,
struct ompi_datatype_t *sdtype,
void *rbuf, const int *rcounts, const int *rdisps,
struct ompi_datatype_t *rdtype,
struct ompi_communicator_t *comm,
mca_coll_base_module_t *module)
{
mca_coll_sync_module_t *s = (mca_coll_sync_module_t*) module;

if (s->in_operation) {
return s->c_coll.coll_alltoallv(sbuf, scounts, sdisps, sdtype,
rbuf, rcounts, rdisps, rdtype, comm,
s->c_coll.coll_alltoallv_module);
} else {
do {
int err = MPI_SUCCESS;
s->in_operation = true;
if (OPAL_UNLIKELY(++(s->before_num_operations_alltoallv) ==
mca_coll_sync_component.barrier_before_nops_alltoallv ||
++(s->before_num_operations) ==
mca_coll_sync_component.barrier_before_nops)) {
s->before_num_operations = 0;
s->before_num_operations_alltoallv = 0;
err = s->c_coll.coll_barrier(comm, s->c_coll.coll_barrier_module);
}
if (OPAL_LIKELY(MPI_SUCCESS == err))
err = s->c_coll.coll_alltoallv(sbuf, scounts, sdisps, sdtype,
rbuf, rcounts, rdisps, rdtype, comm,
s->c_coll.coll_alltoallv_module);
if (OPAL_UNLIKELY(++(s->after_num_operations_alltoallv) ==
mca_coll_sync_component.barrier_after_nops_alltoallv ||
++(s->after_num_operations) ==
mca_coll_sync_component.barrier_after_nops)) {
s->after_num_operations = 0;
s->after_num_operations_alltoallv = 0;
err = s->c_coll.coll_barrier(comm, s->c_coll.coll_barrier_module);
}
s->in_operation = false;
return err;
} while(0);
}
}
73 changes: 73 additions & 0 deletions ompi/mca/coll/sync/coll_sync_alltoallw.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/

#include "ompi_config.h"

#include "coll_sync.h"


/*
* gatherv
*
* Function: - gatherv
* Accepts: - same arguments as MPI_Alltoallv()
* Returns: - MPI_SUCCESS or error code
*/
int mca_coll_sync_alltoallw(const void *sbuf, const int *scounts, const int *sdisps,
struct ompi_datatype_t * const *sdtypes,
void *rbuf, const int *rcounts, const int *rdisps,
struct ompi_datatype_t * const *rdtypes,
struct ompi_communicator_t *comm,
mca_coll_base_module_t *module)
{
mca_coll_sync_module_t *s = (mca_coll_sync_module_t*) module;

if (s->in_operation) {
return s->c_coll.coll_alltoallw(sbuf, scounts, sdisps, sdtypes,
rbuf, rcounts, rdisps, rdtypes, comm,
s->c_coll.coll_alltoallw_module);
} else {
do {
int err = MPI_SUCCESS;
s->in_operation = true;
if (OPAL_UNLIKELY(++(s->before_num_operations_alltoallw) ==
mca_coll_sync_component.barrier_before_nops_alltoallw ||
++(s->before_num_operations) ==
mca_coll_sync_component.barrier_before_nops)) {
s->before_num_operations = 0;
s->before_num_operations_alltoallw = 0;
err = s->c_coll.coll_barrier(comm, s->c_coll.coll_barrier_module);
}
if (OPAL_LIKELY(MPI_SUCCESS == err))
err = s->c_coll.coll_alltoallw(sbuf, scounts, sdisps, sdtypes,
rbuf, rcounts, rdisps, rdtypes, comm,
s->c_coll.coll_alltoallw_module);
if (OPAL_UNLIKELY(++(s->after_num_operations_alltoallw) ==
mca_coll_sync_component.barrier_after_nops_alltoallw ||
++(s->after_num_operations) ==
mca_coll_sync_component.barrier_after_nops)) {
s->after_num_operations = 0;
s->after_num_operations_alltoallw = 0;
err = s->c_coll.coll_barrier(comm, s->c_coll.coll_barrier_module);
}
s->in_operation = false;
return err;
} while(0);
}
}
33 changes: 33 additions & 0 deletions ompi/mca/coll/sync/coll_sync_component.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,23 @@ static int sync_register(void)
MCA_BASE_VAR_SCOPE_READONLY,
&mca_coll_sync_component.barrier_before_nops);

mca_coll_sync_component.barrier_before_nops_alltoallv = 0;
(void) mca_base_component_var_register(c, "barrier_before_alltoallv",
"Do a synchronization before each Nth alltoallv",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_coll_sync_component.barrier_before_nops_alltoallv);

mca_coll_sync_component.barrier_before_nops_alltoallw = 0;
(void) mca_base_component_var_register(c, "barrier_before_alltoallw",
"Do a synchronization before each Nth alltoallw",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_coll_sync_component.barrier_before_nops_alltoallw);


mca_coll_sync_component.barrier_after_nops = 0;
(void) mca_base_component_var_register(c, "barrier_after",
"Do a synchronization after each Nth collective",
Expand All @@ -100,5 +117,21 @@ static int sync_register(void)
MCA_BASE_VAR_SCOPE_READONLY,
&mca_coll_sync_component.barrier_after_nops);

mca_coll_sync_component.barrier_after_nops_alltoallv = 0;
(void) mca_base_component_var_register(c, "barrier_after_alltoallv",
"Do a synchronization before each Nth alltoallv",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_coll_sync_component.barrier_before_nops_alltoallv);

mca_coll_sync_component.barrier_after_nops_alltoallw = 0;
(void) mca_base_component_var_register(c, "barrier_after_alltoallw",
"Do a synchronization before each Nth alltoallw",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_coll_sync_component.barrier_before_nops_alltoallw);

return OMPI_SUCCESS;
}
Loading