Skip to content

Commit b1c4e50

Browse files
jjhurseyjjhursey
authored
Merge pull request #2934 from jjhursey/topic/coll-comm-restructure
Move coll structure outside of the communicator
2 parents c320b36 + 366d64b commit b1c4e50

File tree

156 files changed

+761
-752
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

156 files changed

+761
-752
lines changed

‎ompi/communicator/comm.c

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2013 The University of Tennessee and The University
6+
* Copyright (c) 2004-2017 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -292,10 +292,10 @@ int ompi_comm_create ( ompi_communicator_t *comm, ompi_group_t *group,
292292
goto exit;
293293
}
294294

295-
rc = comm->c_coll.coll_allgather ( &(group->grp_my_rank),
295+
rc = comm->c_coll->coll_allgather ( &(group->grp_my_rank),
296296
1, MPI_INT, allranks,
297297
1, MPI_INT, comm,
298-
comm->c_coll.coll_allgather_module);
298+
comm->c_coll->coll_allgather_module);
299299
if ( OMPI_SUCCESS != rc ) {
300300
goto exit;
301301
}
@@ -434,15 +434,15 @@ int ompi_comm_split( ompi_communicator_t* comm, int color, int key,
434434
if ( inter ) {
435435
allgatherfct = (ompi_comm_allgatherfct *)ompi_comm_allgather_emulate_intra;
436436
} else {
437-
allgatherfct = (ompi_comm_allgatherfct *)comm->c_coll.coll_allgather;
437+
allgatherfct = (ompi_comm_allgatherfct *)comm->c_coll->coll_allgather;
438438
}
439439

440440
results = (int*) malloc ( 2 * size * sizeof(int));
441441
if ( NULL == results ) {
442442
return OMPI_ERR_OUT_OF_RESOURCE;
443443
}
444444

445-
rc = allgatherfct( myinfo, 2, MPI_INT, results, 2, MPI_INT, comm, comm->c_coll.coll_allgather_module );
445+
rc = allgatherfct( myinfo, 2, MPI_INT, results, 2, MPI_INT, comm, comm->c_coll->coll_allgather_module );
446446
if ( OMPI_SUCCESS != rc ) {
447447
goto exit;
448448
}
@@ -503,9 +503,9 @@ int ompi_comm_split( ompi_communicator_t* comm, int color, int key,
503503
}
504504

505505
/* this is an allgather on an inter-communicator */
506-
rc = comm->c_coll.coll_allgather( myinfo, 2, MPI_INT, rresults, 2,
506+
rc = comm->c_coll->coll_allgather( myinfo, 2, MPI_INT, rresults, 2,
507507
MPI_INT, comm,
508-
comm->c_coll.coll_allgather_module);
508+
comm->c_coll->coll_allgather_module);
509509
if ( OMPI_SUCCESS != rc ) {
510510
goto exit;
511511
}
@@ -767,8 +767,8 @@ static int ompi_comm_split_verify (ompi_communicator_t *comm, int split_type, in
767767
results[rank * 2] = split_type;
768768
results[rank * 2 + 1] = key;
769769

770-
rc = comm->c_coll.coll_allgather (MPI_IN_PLACE, 2, MPI_INT, results, 2, MPI_INT, comm,
771-
comm->c_coll.coll_allgather_module);
770+
rc = comm->c_coll->coll_allgather (MPI_IN_PLACE, 2, MPI_INT, results, 2, MPI_INT, comm,
771+
comm->c_coll->coll_allgather_module);
772772
if (OMPI_SUCCESS != rc) {
773773
free (results);
774774
return rc;
@@ -810,8 +810,8 @@ int ompi_comm_split_type (ompi_communicator_t *comm, int split_type, int key,
810810
tmp[2] = key;
811811
tmp[3] = -key;
812812

813-
rc = comm->c_coll.coll_allreduce (MPI_IN_PLACE, &tmp, 4, MPI_INT, MPI_MAX, comm,
814-
comm->c_coll.coll_allreduce_module);
813+
rc = comm->c_coll->coll_allreduce (MPI_IN_PLACE, &tmp, 4, MPI_INT, MPI_MAX, comm,
814+
comm->c_coll->coll_allreduce_module);
815815
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
816816
return rc;
817817
}
@@ -822,16 +822,16 @@ int ompi_comm_split_type (ompi_communicator_t *comm, int split_type, int key,
822822
/* at least one rank supplied a different split type check if our split_type is ok */
823823
ok = (MPI_UNDEFINED == split_type) || global_split_type == split_type;
824824

825-
rc = comm->c_coll.coll_allreduce (MPI_IN_PLACE, &ok, 1, MPI_INT, MPI_MIN, comm,
826-
comm->c_coll.coll_allreduce_module);
825+
rc = comm->c_coll->coll_allreduce (MPI_IN_PLACE, &ok, 1, MPI_INT, MPI_MIN, comm,
826+
comm->c_coll->coll_allreduce_module);
827827
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
828828
return rc;
829829
}
830830

831831
if (inter) {
832832
/* need an extra allreduce to ensure that all ranks have the same result */
833-
rc = comm->c_coll.coll_allreduce (MPI_IN_PLACE, &ok, 1, MPI_INT, MPI_MIN, comm,
834-
comm->c_coll.coll_allreduce_module);
833+
rc = comm->c_coll->coll_allreduce (MPI_IN_PLACE, &ok, 1, MPI_INT, MPI_MIN, comm,
834+
comm->c_coll->coll_allreduce_module);
835835
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
836836
return rc;
837837
}
@@ -1575,9 +1575,9 @@ ompi_proc_t **ompi_comm_get_rprocs ( ompi_communicator_t *local_comm,
15751575
}
15761576

15771577
/* broadcast buffer length to all processes in local_comm */
1578-
rc = local_comm->c_coll.coll_bcast( &rlen, 1, MPI_INT,
1578+
rc = local_comm->c_coll->coll_bcast( &rlen, 1, MPI_INT,
15791579
local_leader, local_comm,
1580-
local_comm->c_coll.coll_bcast_module );
1580+
local_comm->c_coll->coll_bcast_module );
15811581
if ( OMPI_SUCCESS != rc ) {
15821582
goto err_exit;
15831583
}
@@ -1607,9 +1607,9 @@ ompi_proc_t **ompi_comm_get_rprocs ( ompi_communicator_t *local_comm,
16071607
}
16081608

16091609
/* broadcast name list to all proceses in local_comm */
1610-
rc = local_comm->c_coll.coll_bcast( recvbuf, rlen, MPI_BYTE,
1610+
rc = local_comm->c_coll->coll_bcast( recvbuf, rlen, MPI_BYTE,
16111611
local_leader, local_comm,
1612-
local_comm->c_coll.coll_bcast_module);
1612+
local_comm->c_coll->coll_bcast_module);
16131613
if ( OMPI_SUCCESS != rc ) {
16141614
goto err_exit;
16151615
}
@@ -1741,10 +1741,10 @@ int ompi_comm_determine_first ( ompi_communicator_t *intercomm, int high )
17411741
scount = 1;
17421742
}
17431743

1744-
rc = intercomm->c_coll.coll_allgatherv(&high, scount, MPI_INT,
1744+
rc = intercomm->c_coll->coll_allgatherv(&high, scount, MPI_INT,
17451745
&rhigh, rcounts, rdisps,
17461746
MPI_INT, intercomm,
1747-
intercomm->c_coll.coll_allgatherv_module);
1747+
intercomm->c_coll->coll_allgatherv_module);
17481748
if ( NULL != rdisps ) {
17491749
free ( rdisps );
17501750
}

‎ompi/communicator/comm_cid.c

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2011 The University of Tennessee and The University
6+
* Copyright (c) 2004-2017 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -591,8 +591,8 @@ static int ompi_comm_allreduce_intra_nb (int *inbuf, int *outbuf, int count, str
591591
{
592592
ompi_communicator_t *comm = context->comm;
593593

594-
return comm->c_coll.coll_iallreduce (inbuf, outbuf, count, MPI_INT, op, comm,
595-
req, comm->c_coll.coll_iallreduce_module);
594+
return comm->c_coll->coll_iallreduce (inbuf, outbuf, count, MPI_INT, op, comm,
595+
req, comm->c_coll->coll_iallreduce_module);
596596
}
597597

598598
/* Non-blocking version of ompi_comm_allreduce_inter */
@@ -641,9 +641,9 @@ static int ompi_comm_allreduce_inter_nb (int *inbuf, int *outbuf,
641641

642642
/* Execute the inter-allreduce: the result from the local will be in the buffer of the remote group
643643
* and vise-versa. */
644-
rc = intercomm->c_local_comm->c_coll.coll_ireduce (inbuf, context->tmpbuf, count, MPI_INT, op, 0,
644+
rc = intercomm->c_local_comm->c_coll->coll_ireduce (inbuf, context->tmpbuf, count, MPI_INT, op, 0,
645645
intercomm->c_local_comm, &subreq,
646-
intercomm->c_local_comm->c_coll.coll_ireduce_module);
646+
intercomm->c_local_comm->c_coll->coll_ireduce_module);
647647
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
648648
ompi_comm_request_return (request);
649649
return rc;
@@ -704,8 +704,8 @@ static int ompi_comm_allreduce_inter_bcast (ompi_comm_request_t *request)
704704
int rc;
705705

706706
/* both roots have the same result. broadcast to the local group */
707-
rc = comm->c_coll.coll_ibcast (context->outbuf, context->count, MPI_INT, 0, comm,
708-
&subreq, comm->c_coll.coll_ibcast_module);
707+
rc = comm->c_coll->coll_ibcast (context->outbuf, context->count, MPI_INT, 0, comm,
708+
&subreq, comm->c_coll->coll_ibcast_module);
709709
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
710710
return rc;
711711
}
@@ -720,9 +720,9 @@ static int ompi_comm_allreduce_bridged_schedule_bcast (ompi_comm_request_t *requ
720720
ompi_request_t *subreq;
721721
int rc;
722722

723-
rc = comm->c_coll.coll_ibcast (context->outbuf, context->count, MPI_INT,
723+
rc = comm->c_coll->coll_ibcast (context->outbuf, context->count, MPI_INT,
724724
context->cid_context->local_leader, comm,
725-
&subreq, comm->c_coll.coll_ibcast_module);
725+
&subreq, comm->c_coll->coll_ibcast_module);
726726
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
727727
return rc;
728728
}
@@ -803,9 +803,9 @@ static int ompi_comm_allreduce_intra_bridge_nb (int *inbuf, int *outbuf,
803803
}
804804

805805
/* step 1: reduce to the local leader */
806-
rc = comm->c_coll.coll_ireduce (inbuf, context->tmpbuf, count, MPI_INT, op,
806+
rc = comm->c_coll->coll_ireduce (inbuf, context->tmpbuf, count, MPI_INT, op,
807807
cid_context->local_leader, comm, &subreq,
808-
comm->c_coll.coll_ireduce_module);
808+
comm->c_coll->coll_ireduce_module);
809809
if ( OMPI_SUCCESS != rc ) {
810810
ompi_comm_request_return (request);
811811
return rc;
@@ -955,9 +955,9 @@ static int ompi_comm_allreduce_intra_pmix_nb (int *inbuf, int *outbuf,
955955
request->context = &context->super;
956956

957957
/* comm is an intra-communicator */
958-
rc = comm->c_coll.coll_ireduce (inbuf, context->tmpbuf, count, MPI_INT, op,
958+
rc = comm->c_coll->coll_ireduce (inbuf, context->tmpbuf, count, MPI_INT, op,
959959
cid_context->local_leader, comm,
960-
&subreq, comm->c_coll.coll_ireduce_module);
960+
&subreq, comm->c_coll->coll_ireduce_module);
961961
if ( OMPI_SUCCESS != rc ) {
962962
ompi_comm_request_return (request);
963963
return rc;

‎ompi/communicator/comm_init.c

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2013 The University of Tennessee and The University
6+
* Copyright (c) 2004-2017 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -353,6 +353,7 @@ static void ompi_comm_construct(ompi_communicator_t* comm)
353353
comm->error_handler = NULL;
354354
comm->c_pml_comm = NULL;
355355
comm->c_topo = NULL;
356+
comm->c_coll = NULL;
356357

357358
/* A keyhash will be created if/when an attribute is cached on
358359
this communicator */
@@ -362,12 +363,6 @@ static void ompi_comm_construct(ompi_communicator_t* comm)
362363
#ifdef OMPI_WANT_PERUSE
363364
comm->c_peruse_handles = NULL;
364365
#endif
365-
366-
/* Need to zero out the collectives module because we sometimes
367-
call coll_unselect without a matching call to coll_select, and
368-
we need an easy way for the coll base code to realize we've
369-
done this. */
370-
memset(&comm->c_coll, 0, sizeof(mca_coll_base_comm_coll_t));
371366
}
372367

373368
static void ompi_comm_destruct(ompi_communicator_t* comm)
@@ -379,7 +374,7 @@ static void ompi_comm_destruct(ompi_communicator_t* comm)
379374

380375
/* Release the collective module */
381376

382-
if ( MPI_COMM_NULL != comm ) {
377+
if ( NULL != comm->c_coll ) {
383378
mca_coll_base_comm_unselect(comm);
384379
}
385380

‎ompi/communicator/communicator.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2013 The University of Tennessee and The University
6+
* Copyright (c) 2004-2017 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -169,7 +169,7 @@ struct ompi_communicator_t {
169169
struct mca_pml_comm_t *c_pml_comm;
170170

171171
/* Collectives module interface and data */
172-
mca_coll_base_comm_coll_t c_coll;
172+
mca_coll_base_comm_coll_t *c_coll;
173173
};
174174
typedef struct ompi_communicator_t ompi_communicator_t;
175175

@@ -256,7 +256,7 @@ typedef struct ompi_communicator_t ompi_communicator_t;
256256
* the PREDEFINED_COMMUNICATOR_PAD macro?
257257
* A: Most likely not, but it would be good to check.
258258
*/
259-
#define PREDEFINED_COMMUNICATOR_PAD (sizeof(void*) * 192)
259+
#define PREDEFINED_COMMUNICATOR_PAD (sizeof(void*) * 64)
260260

261261
struct ompi_predefined_communicator_t {
262262
struct ompi_communicator_t comm;

‎ompi/dpm/dpm.c

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2011 The University of Tennessee and The University
6+
* Copyright (c) 2004-2017 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -229,8 +229,8 @@ int ompi_dpm_connect_accept(ompi_communicator_t *comm, int root,
229229
* side's participants */
230230

231231
/* bcast the list-length to all processes in the local comm */
232-
rc = comm->c_coll.coll_bcast(&rportlen, 1, MPI_INT, root, comm,
233-
comm->c_coll.coll_bcast_module);
232+
rc = comm->c_coll->coll_bcast(&rportlen, 1, MPI_INT, root, comm,
233+
comm->c_coll->coll_bcast_module);
234234
if (OMPI_SUCCESS != rc) {
235235
free(rport);
236236
goto exit;
@@ -245,8 +245,8 @@ int ompi_dpm_connect_accept(ompi_communicator_t *comm, int root,
245245
}
246246
}
247247
/* now share the list of remote participants */
248-
rc = comm->c_coll.coll_bcast(rport, rportlen, MPI_BYTE, root, comm,
249-
comm->c_coll.coll_bcast_module);
248+
rc = comm->c_coll->coll_bcast(rport, rportlen, MPI_BYTE, root, comm,
249+
comm->c_coll->coll_bcast_module);
250250
if (OMPI_SUCCESS != rc) {
251251
free(rport);
252252
goto exit;

0 commit comments

Comments
 (0)