Merge pull request #2808 from jjhursey/fix/ibm/reduce-local-to-coll

coll: Move reduce_local into the coll framework

Author: jjhursey

ompi/communicator/communicator.h

@@ -20,6 +20,7 @@
  * Copyright (c) 2014-2015 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -232,6 +233,28 @@ typedef struct ompi_communicator_t ompi_communicator_t;
  * size so when the bitness changes the size of the handle changes.
  * This is done so we don't end up needing a structure that is
  * incredibly larger than necessary because of the bitness.
+ *
+ * This padding mechanism works as a (likely) compile time check for when the
+ * size of the ompi_communicator_t exceeds the predetermined size of the
+ * ompi_predefined_communicator_t. It also allows us to change the size of
+ * the ompi_communicator_t without impacting the size of the
+ * ompi_predefined_communicator_t structure for some number of additions.
+ *
+ * As an example:
+ * If the size of ompi_communicator_t is less than the size of the _PAD then
+ * the _PAD ensures that the size of the ompi_predefined_communicator_t is
+ * whatever size is defined below in the _PAD macro.
+ * However, if the size of the ompi_communicator_t grows larger than the _PAD
+ * (say by adding a few more function pointers to the structure) then the
+ * 'padding' variable will be initialized to a large number often triggering
+ * a 'array is too large' compile time error. This signals two things:
+ * 1) That the _PAD should be increased.
+ * 2) That users need to be made aware of the size change for the
+ *    ompi_predefined_communicator_t structure.
+ *
+ * Q: So you just made a change to communicator structure, do you need to adjust
+ * the PREDEFINED_COMMUNICATOR_PAD macro?
+ * A: Most likely not, but it would be good to check.
  */
 #define PREDEFINED_COMMUNICATOR_PAD (sizeof(void*) * 192)
 

ompi/mca/coll/base/coll_base_comm_select.c

@@ -19,7 +19,7 @@
  *                         reserved.
  * Copyright (c) 2014      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016      IBM Corporation.  All rights reserved.
+ * Copyright (c) 2016-2017 IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -52,7 +52,7 @@ struct avail_coll_t {
     opal_list_item_t super;
 
     int ac_priority;
-    mca_coll_base_module_2_1_0_t *ac_module;
+    mca_coll_base_module_2_2_0_t *ac_module;
     const char * ac_component_name;
 };
 typedef struct avail_coll_t avail_coll_t;
@@ -65,16 +65,16 @@ static opal_list_t *check_components(opal_list_t * components,
                                      ompi_communicator_t * comm);
 static int check_one_component(ompi_communicator_t * comm,
                                const mca_base_component_t * component,
-                               mca_coll_base_module_2_1_0_t ** module);
+                               mca_coll_base_module_2_2_0_t ** module);
 
 static int query(const mca_base_component_t * component,
                  ompi_communicator_t * comm, int *priority,
-                 mca_coll_base_module_2_1_0_t ** module);
+                 mca_coll_base_module_2_2_0_t ** module);
 
 static int query_2_0_0(const mca_coll_base_component_2_0_0_t *
                        coll_component, ompi_communicator_t * comm,
                        int *priority,
-                       mca_coll_base_module_2_1_0_t ** module);
+                       mca_coll_base_module_2_2_0_t ** module);
 
 /*
  * Stuff for the OBJ interface
@@ -203,6 +203,8 @@ int mca_coll_base_comm_select(ompi_communicator_t * comm)
             COPY(avail->ac_module, comm, ineighbor_alltoall);
             COPY(avail->ac_module, comm, ineighbor_alltoallv);
             COPY(avail->ac_module, comm, ineighbor_alltoallw);
+
+            COPY(avail->ac_module, comm, reduce_local);
         }
         /* release the original module reference and the list item */
         OBJ_RELEASE(avail->ac_module);
@@ -246,7 +248,8 @@ int mca_coll_base_comm_select(ompi_communicator_t * comm)
         CHECK_NULL(which_func, comm, ireduce_scatter) ||
         ((OMPI_COMM_IS_INTRA(comm)) && CHECK_NULL(which_func, comm, iscan)) ||
         CHECK_NULL(which_func, comm, iscatter) ||
-        CHECK_NULL(which_func, comm, iscatterv)) {
+        CHECK_NULL(which_func, comm, iscatterv) ||
+        CHECK_NULL(which_func, comm, reduce_local) ) {
         /* TODO -- Once the topology flags are set before coll_select then
          * check if neighborhood collectives have been set. */
 
@@ -285,7 +288,7 @@ static opal_list_t *check_components(opal_list_t * components,
     int priority;
     const mca_base_component_t *component;
     mca_base_component_list_item_t *cli;
-    mca_coll_base_module_2_1_0_t *module;
+    mca_coll_base_module_2_2_0_t *module;
     opal_list_t *selectable;
     avail_coll_t *avail;
 
@@ -341,7 +344,7 @@ static opal_list_t *check_components(opal_list_t * components,
  */
 static int check_one_component(ompi_communicator_t * comm,
                                const mca_base_component_t * component,
-                               mca_coll_base_module_2_1_0_t ** module)
+                               mca_coll_base_module_2_2_0_t ** module)
 {
     int err;
     int priority = -1;
@@ -375,7 +378,7 @@ static int check_one_component(ompi_communicator_t * comm,
  */
 static int query(const mca_base_component_t * component,
                  ompi_communicator_t * comm,
-                 int *priority, mca_coll_base_module_2_1_0_t ** module)
+                 int *priority, mca_coll_base_module_2_2_0_t ** module)
 {
     *module = NULL;
     if (2 == component->mca_type_major_version &&
@@ -395,9 +398,9 @@ static int query(const mca_base_component_t * component,
 
 static int query_2_0_0(const mca_coll_base_component_2_0_0_t * component,
                        ompi_communicator_t * comm, int *priority,
-                       mca_coll_base_module_2_1_0_t ** module)
+                       mca_coll_base_module_2_2_0_t ** module)
 {
-    mca_coll_base_module_2_1_0_t *ret;
+    mca_coll_base_module_2_2_0_t *ret;
 
     /* There's currently no need for conversion */
 

ompi/mca/coll/base/coll_base_comm_unselect.c

@@ -14,6 +14,7 @@
  * Copyright (c) 2013 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2014      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -98,6 +99,8 @@ int mca_coll_base_comm_unselect(ompi_communicator_t * comm)
     CLOSE(comm, ineighbor_alltoallv);
     CLOSE(comm, ineighbor_alltoallw);
 
+    CLOSE(comm, reduce_local);
+
     /* All done */
     return OMPI_SUCCESS;
 }

ompi/mca/coll/base/coll_base_functions.h

@@ -16,6 +16,7 @@
  *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2017 IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -169,6 +170,11 @@ int ompi_coll_base_scatter_intra_binomial(SCATTER_ARGS);
 
 /* ScatterV */
 
+/* Reduce_local */
+int mca_coll_base_reduce_local(const void *inbuf, void *inoutbuf, int count,
+                               struct ompi_datatype_t * dtype, struct ompi_op_t * op,
+                               mca_coll_base_module_t *module);
+
 END_C_DECLS
 
 #define COLL_BASE_UPDATE_BINTREE( OMPI_COMM, BASE_MODULE, ROOT )	\

ompi/mca/coll/base/coll_base_reduce.c

@@ -14,6 +14,7 @@
  *                         reserved.
  * Copyright (c) 2015-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2017 IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -34,6 +35,15 @@
 #include "ompi/mca/coll/base/coll_base_functions.h"
 #include "coll_base_topo.h"
 
+int mca_coll_base_reduce_local(const void *inbuf, void *inoutbuf, int count,
+                               struct ompi_datatype_t * dtype, struct ompi_op_t * op,
+                               mca_coll_base_module_t *module)
+{
+    /* XXX -- CONST -- do not cast away const -- update ompi/op/op.h */
+    ompi_op_reduce(op, (void *)inbuf, inoutbuf, count, dtype);
+    return OMPI_SUCCESS;
+}
+
 /**
  * This is a generic implementation of the reduce protocol. It used the tree
  * provided as an argument and execute all operations using a segment of

ompi/mca/coll/basic/coll_basic_module.c

@@ -15,6 +15,7 @@
  *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2017 IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -132,6 +133,8 @@ mca_coll_basic_comm_query(struct ompi_communicator_t *comm,
     basic_module->super.coll_neighbor_alltoallv = mca_coll_basic_neighbor_alltoallv;
     basic_module->super.coll_neighbor_alltoallw = mca_coll_basic_neighbor_alltoallw;
 
+    basic_module->super.coll_reduce_local = mca_coll_base_reduce_local;
+
     return &(basic_module->super);
 }