Skip to content
This repository was archived by the owner on Sep 30, 2022. It is now read-only.

Commit 54f43b0

Browse files
author
Ralph Castain
committed
Restore the coll/sync module and provide a test to verify its operation
(cherry picked from commit open-mpi/ompi@9888615) Update to match v2.x definitions Adjust priority downward Set the default value of the barrier counters to zero so the coll/sync component is off by default Fix typo in the COLL_SYNC macro
1 parent baf8fd1 commit 54f43b0

18 files changed

+1014
-1
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -395,6 +395,7 @@ orte/test/mpi/pconnect
395395
orte/test/mpi/thread_init
396396
orte/test/mpi/memcached-dummy
397397
orte/test/mpi/coll_test
398+
orte/test/mpi/badcoll
398399

399400
orte/test/system/radix
400401
orte/test/system/sigusr_trap

‎ompi/mca/coll/sync/Makefile.am‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
#
2+
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
3+
# University Research and Technology
4+
# Corporation. All rights reserved.
5+
# Copyright (c) 2004-2005 The University of Tennessee and The University
6+
# of Tennessee Research Foundation. All rights
7+
# reserved.
8+
# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
9+
# University of Stuttgart. All rights reserved.
10+
# Copyright (c) 2004-2005 The Regents of the University of California.
11+
# All rights reserved.
12+
# Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
13+
# Copyright (c) 2016 Intel, Inc. All rights reserved
14+
# $COPYRIGHT$
15+
#
16+
# Additional copyrights may follow
17+
#
18+
# $HEADER$
19+
#
20+
21+
dist_ompidata_DATA = help-coll-sync.txt
22+
23+
sources = \
24+
coll_sync.h \
25+
coll_sync_component.c \
26+
coll_sync_module.c \
27+
coll_sync_bcast.c \
28+
coll_sync_exscan.c \
29+
coll_sync_gather.c \
30+
coll_sync_gatherv.c \
31+
coll_sync_reduce.c \
32+
coll_sync_reduce_scatter.c \
33+
coll_sync_scan.c \
34+
coll_sync_scatter.c \
35+
coll_sync_scatterv.c
36+
37+
if MCA_BUILD_ompi_coll_sync_DSO
38+
component_noinst =
39+
component_install = mca_coll_sync.la
40+
else
41+
component_noinst = libmca_coll_sync.la
42+
component_install =
43+
endif
44+
45+
mcacomponentdir = $(ompilibdir)
46+
mcacomponent_LTLIBRARIES = $(component_install)
47+
mca_coll_sync_la_SOURCES = $(sources)
48+
mca_coll_sync_la_LDFLAGS = -module -avoid-version
49+
50+
noinst_LTLIBRARIES = $(component_noinst)
51+
libmca_coll_sync_la_SOURCES =$(sources)
52+
libmca_coll_sync_la_LDFLAGS = -module -avoid-version

‎ompi/mca/coll/sync/coll_sync.h‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
/*
2+
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
3+
* University Research and Technology
4+
* Corporation. All rights reserved.
5+
* Copyright (c) 2004-2006 The University of Tennessee and The University
6+
* of Tennessee Research Foundation. All rights
7+
* reserved.
8+
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9+
* University of Stuttgart. All rights reserved.
10+
* Copyright (c) 2004-2005 The Regents of the University of California.
11+
* All rights reserved.
12+
* Copyright (c) 2008-2009 Cisco Systems, Inc. All rights reserved.
13+
* $COPYRIGHT$
14+
*
15+
* Additional copyrights may follow
16+
*
17+
* $HEADER$
18+
*/
19+
20+
#ifndef MCA_COLL_SYNC_EXPORT_H
21+
#define MCA_COLL_SYNC_EXPORT_H
22+
23+
#include "ompi_config.h"
24+
25+
#include "mpi.h"
26+
27+
#include "opal/class/opal_object.h"
28+
#include "opal/mca/mca.h"
29+
#include "opal/util/output.h"
30+
31+
#include "ompi/constants.h"
32+
#include "ompi/mca/coll/coll.h"
33+
#include "ompi/mca/coll/base/base.h"
34+
#include "ompi/communicator/communicator.h"
35+
36+
BEGIN_C_DECLS
37+
38+
/* API functions */
39+
40+
int mca_coll_sync_init_query(bool enable_progress_threads,
41+
bool enable_mpi_threads);
42+
mca_coll_base_module_t
43+
*mca_coll_sync_comm_query(struct ompi_communicator_t *comm,
44+
int *priority);
45+
46+
int mca_coll_sync_module_enable(mca_coll_base_module_t *module,
47+
struct ompi_communicator_t *comm);
48+
49+
int mca_coll_sync_barrier(struct ompi_communicator_t *comm,
50+
mca_coll_base_module_t *module);
51+
52+
int mca_coll_sync_bcast(void *buff, int count,
53+
struct ompi_datatype_t *datatype,
54+
int root,
55+
struct ompi_communicator_t *comm,
56+
mca_coll_base_module_t *module);
57+
58+
int mca_coll_sync_exscan(const void *sbuf, void *rbuf, int count,
59+
struct ompi_datatype_t *dtype,
60+
struct ompi_op_t *op,
61+
struct ompi_communicator_t *comm,
62+
mca_coll_base_module_t *module);
63+
64+
int mca_coll_sync_gather(const void *sbuf, int scount,
65+
struct ompi_datatype_t *sdtype,
66+
void *rbuf, int rcount,
67+
struct ompi_datatype_t *rdtype,
68+
int root,
69+
struct ompi_communicator_t *comm,
70+
mca_coll_base_module_t *module);
71+
72+
int mca_coll_sync_gatherv(const void *sbuf, int scount,
73+
struct ompi_datatype_t *sdtype,
74+
void *rbuf, int *rcounts, int *disps,
75+
struct ompi_datatype_t *rdtype,
76+
int root,
77+
struct ompi_communicator_t *comm,
78+
mca_coll_base_module_t *module);
79+
80+
int mca_coll_sync_reduce(const void *sbuf, void *rbuf, int count,
81+
struct ompi_datatype_t *dtype,
82+
struct ompi_op_t *op,
83+
int root,
84+
struct ompi_communicator_t *comm,
85+
mca_coll_base_module_t *module);
86+
87+
int mca_coll_sync_reduce_scatter(const void *sbuf, void *rbuf,
88+
int *rcounts,
89+
struct ompi_datatype_t *dtype,
90+
struct ompi_op_t *op,
91+
struct ompi_communicator_t *comm,
92+
mca_coll_base_module_t *module);
93+
94+
int mca_coll_sync_scan(const void *sbuf, void *rbuf, int count,
95+
struct ompi_datatype_t *dtype,
96+
struct ompi_op_t *op,
97+
struct ompi_communicator_t *comm,
98+
mca_coll_base_module_t *module);
99+
100+
int mca_coll_sync_scatter(const void *sbuf, int scount,
101+
struct ompi_datatype_t *sdtype,
102+
void *rbuf, int rcount,
103+
struct ompi_datatype_t *rdtype,
104+
int root,
105+
struct ompi_communicator_t *comm,
106+
mca_coll_base_module_t *module);
107+
108+
int mca_coll_sync_scatterv(const void *sbuf, int *scounts, int *disps,
109+
struct ompi_datatype_t *sdtype,
110+
void *rbuf, int rcount,
111+
struct ompi_datatype_t *rdtype,
112+
int root,
113+
struct ompi_communicator_t *comm,
114+
mca_coll_base_module_t *module);
115+
116+
117+
/* Types */
118+
/* Module */
119+
120+
typedef struct mca_coll_sync_module_t {
121+
mca_coll_base_module_t super;
122+
123+
/* Pointers to all the "real" collective functions */
124+
mca_coll_base_comm_coll_t c_coll;
125+
126+
/* How many ops we've executed */
127+
int before_num_operations;
128+
129+
/* How many ops we've executed (it's easier to have 2) */
130+
int after_num_operations;
131+
132+
/* Avoid recursion of syncs */
133+
bool in_operation;
134+
} mca_coll_sync_module_t;
135+
136+
OBJ_CLASS_DECLARATION(mca_coll_sync_module_t);
137+
138+
/* Component */
139+
140+
typedef struct mca_coll_sync_component_t {
141+
mca_coll_base_component_2_0_0_t super;
142+
143+
/* Priority of this component */
144+
int priority;
145+
146+
/* Do a sync *before* each Nth collective */
147+
int barrier_before_nops;
148+
149+
/* Do a sync *after* each Nth collective */
150+
int barrier_after_nops;
151+
} mca_coll_sync_component_t;
152+
153+
/* Globally exported variables */
154+
155+
OMPI_MODULE_DECLSPEC extern mca_coll_sync_component_t mca_coll_sync_component;
156+
157+
/* Macro used in most of the collectives */
158+
159+
#define COLL_SYNC(m, op) \
160+
do { \
161+
int err = MPI_SUCCESS; \
162+
(m)->in_operation = true; \
163+
if (OPAL_UNLIKELY(++((m)->before_num_operations) == \
164+
mca_coll_sync_component.barrier_before_nops)) { \
165+
(m)->before_num_operations = 0; \
166+
err = (m)->c_coll.coll_barrier(comm, (m)->c_coll.coll_barrier_module); \
167+
} \
168+
if (OPAL_LIKELY(MPI_SUCCESS == err)) { \
169+
err = op; \
170+
} \
171+
if (OPAL_UNLIKELY(++((m)->after_num_operations) == \
172+
mca_coll_sync_component.barrier_after_nops) && \
173+
OPAL_LIKELY(MPI_SUCCESS == err)) { \
174+
(m)->after_num_operations = 0; \
175+
err = (m)->c_coll.coll_barrier(comm, (m)->c_coll.coll_barrier_module); \
176+
} \
177+
(m)->in_operation = false; \
178+
return err; \
179+
} while(0)
180+
181+
END_C_DECLS
182+
183+
#endif /* MCA_COLL_SYNC_EXPORT_H */

‎ompi/mca/coll/sync/coll_sync_bcast.c‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
3+
* University Research and Technology
4+
* Corporation. All rights reserved.
5+
* Copyright (c) 2004-2005 The University of Tennessee and The University
6+
* of Tennessee Research Foundation. All rights
7+
* reserved.
8+
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9+
* University of Stuttgart. All rights reserved.
10+
* Copyright (c) 2004-2005 The Regents of the University of California.
11+
* All rights reserved.
12+
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
13+
* $COPYRIGHT$
14+
*
15+
* Additional copyrights may follow
16+
*
17+
* $HEADER$
18+
*/
19+
20+
#include "ompi_config.h"
21+
22+
#include "mpi.h"
23+
#include "coll_sync.h"
24+
25+
26+
/*
27+
* bcast
28+
*
29+
* Function: - broadcast
30+
* Accepts: - same arguments as MPI_Bcast()
31+
* Returns: - MPI_SUCCESS or error code
32+
*/
33+
int mca_coll_sync_bcast(void *buff, int count,
34+
struct ompi_datatype_t *datatype, int root,
35+
struct ompi_communicator_t *comm,
36+
mca_coll_base_module_t *module)
37+
{
38+
mca_coll_sync_module_t *s = (mca_coll_sync_module_t*) module;
39+
40+
if (s->in_operation) {
41+
return s->c_coll.coll_bcast(buff, count, datatype, root, comm,
42+
s->c_coll.coll_bcast_module);
43+
} else {
44+
COLL_SYNC(s, s->c_coll.coll_bcast(buff, count, datatype, root, comm,
45+
s->c_coll.coll_bcast_module));
46+
}
47+
}

‎ompi/mca/coll/sync/coll_sync_component.c‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
/*
2+
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
3+
* University Research and Technology
4+
* Corporation. All rights reserved.
5+
* Copyright (c) 2004-2005 The University of Tennessee and The University
6+
* of Tennessee Research Foundation. All rights
7+
* reserved.
8+
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9+
* University of Stuttgart. All rights reserved.
10+
* Copyright (c) 2004-2005 The Regents of the University of California.
11+
* All rights reserved.
12+
* Copyright (c) 2008-2009 Cisco Systems, Inc. All rights reserved.
13+
* $COPYRIGHT$
14+
*
15+
* Additional copyrights may follow
16+
*
17+
* $HEADER$
18+
*/
19+
20+
#include "ompi_config.h"
21+
22+
#include <string.h>
23+
24+
#include "opal/util/output.h"
25+
26+
#include "mpi.h"
27+
#include "ompi/constants.h"
28+
#include "coll_sync.h"
29+
30+
/*
31+
* Public string showing the coll ompi_sync component version number
32+
*/
33+
const char *mca_coll_sync_component_version_string =
34+
"Open MPI sync collective MCA component version " OMPI_VERSION;
35+
36+
/*
37+
* Local function
38+
*/
39+
static int sync_register(void);
40+
41+
/*
42+
* Instantiate the public struct with all of our public information
43+
* and pointers to our public functions in it
44+
*/
45+
46+
mca_coll_sync_component_t mca_coll_sync_component = {
47+
{
48+
/* First, the mca_component_t struct containing meta information
49+
* about the component itself */
50+
51+
.collm_version = {
52+
MCA_COLL_BASE_VERSION_2_0_0,
53+
54+
/* Component name and version */
55+
.mca_component_name = "sync",
56+
MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
57+
OMPI_RELEASE_VERSION),
58+
59+
/* Component open and close functions */
60+
.mca_register_component_params = sync_register
61+
},
62+
.collm_data = {
63+
/* The component is checkpoint ready */
64+
MCA_BASE_METADATA_PARAM_CHECKPOINT
65+
},
66+
67+
/* Initialization / querying functions */
68+
69+
.collm_init_query = mca_coll_sync_init_query,
70+
.collm_comm_query = mca_coll_sync_comm_query
71+
},
72+
};
73+
74+
75+
static int sync_register(void)
76+
{
77+
mca_base_component_t *c = &mca_coll_sync_component.super.collm_version;
78+
79+
mca_coll_sync_component.priority = 50;
80+
(void) mca_base_component_var_register(c, "priority",
81+
"Priority of the sync coll component; only relevant if barrier_before or barrier_after is > 0",
82+
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
83+
OPAL_INFO_LVL_9,
84+
MCA_BASE_VAR_SCOPE_READONLY,
85+
&mca_coll_sync_component.priority);
86+
87+
mca_coll_sync_component.barrier_before_nops = 0;
88+
(void) mca_base_component_var_register(c, "barrier_before",
89+
"Do a synchronization before each Nth collective",
90+
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
91+
OPAL_INFO_LVL_9,
92+
MCA_BASE_VAR_SCOPE_READONLY,
93+
&mca_coll_sync_component.barrier_before_nops);
94+
95+
mca_coll_sync_component.barrier_after_nops = 0;
96+
(void) mca_base_component_var_register(c, "barrier_after",
97+
"Do a synchronization after each Nth collective",
98+
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
99+
OPAL_INFO_LVL_9,
100+
MCA_BASE_VAR_SCOPE_READONLY,
101+
&mca_coll_sync_component.barrier_after_nops);
102+
103+
return OMPI_SUCCESS;
104+
}

0 commit comments

Comments
 (0)