Merge pull request #2215 from hjelmn/v2.x_comm

jsquyres · web-flow · commit 87a79fa37654 · 2016-10-31T15:43:01.000-04:00
v2.x communicator code updates
diff --git a/ompi/communicator/comm.c b/ompi/communicator/comm.c
diff --git a/ompi/communicator/comm_cid.c b/ompi/communicator/comm_cid.c
diff --git a/ompi/communicator/comm_init.c b/ompi/communicator/comm_init.c
@@ -206,10 +206,6 @@ int ompi_comm_init(void)
     OBJ_RETAIN(&ompi_mpi_group_null.group);
     OBJ_RETAIN(&ompi_mpi_errors_are_fatal.eh);
 
-    /* initialize the comm_reg stuff for multi-threaded comm_cid
-       allocation */
-    ompi_comm_reg_init();
-
     /* initialize communicator requests (for ompi_comm_idup) */
     ompi_comm_request_init ();
 
@@ -328,27 +324,15 @@ int ompi_comm_finalize(void)
         }
     }
 
-
     OBJ_DESTRUCT (&ompi_mpi_communicators);
     OBJ_DESTRUCT (&ompi_comm_f_to_c_table);
 
-    /* finalize the comm_reg stuff */
-    ompi_comm_reg_finalize();
-
     /* finalize communicator requests */
     ompi_comm_request_fini ();
 
     return OMPI_SUCCESS;
 }
 
-/*
- * For linking only. To be checked.
- */
-int ompi_comm_link_function(void)
-{
-    return OMPI_SUCCESS;
-}
-
 /********************************************************************************/
 /********************************************************************************/
 /********************************************************************************/
diff --git a/ompi/communicator/comm_request.c b/ompi/communicator/comm_request.c
@@ -234,6 +234,7 @@ static void ompi_comm_request_destruct (ompi_comm_request_t *request)
 {
     OBJ_DESTRUCT(&request->schedule);
 }
+
 OBJ_CLASS_INSTANCE(ompi_comm_request_t, ompi_request_t,
                    ompi_comm_request_construct,
                    ompi_comm_request_destruct);
@@ -257,10 +258,10 @@ ompi_comm_request_t *ompi_comm_request_get (void)
 void ompi_comm_request_return (ompi_comm_request_t *request)
 {
     if (request->context) {
-        free (request->context);
-        request->context = NULL;
+        OBJ_RELEASE (request->context);
     }
 
+    OMPI_REQUEST_FINI(&request->super);
     opal_free_list_return (&ompi_comm_requests, (opal_free_list_item_t *) request);
 }
 
diff --git a/ompi/communicator/comm_request.h b/ompi/communicator/comm_request.h
@@ -22,7 +22,7 @@
 typedef struct ompi_comm_request_t {
     ompi_request_t super;
 
-    void *context;
+    opal_object_t *context;
     opal_list_t schedule;
 } ompi_comm_request_t;
 OBJ_CLASS_DECLARATION(ompi_comm_request_t);
diff --git a/ompi/communicator/communicator.h b/ompi/communicator/communicator.h
@@ -351,7 +351,6 @@ static inline bool ompi_comm_peer_invalid(ompi_communicator_t* comm, int peer_id
  * Initialise MPI_COMM_WORLD and MPI_COMM_SELF
  */
 int ompi_comm_init(void);
-OMPI_DECLSPEC int ompi_comm_link_function(void);
 
 /**
  * extract the local group from a communicator
@@ -493,24 +492,27 @@ ompi_communicator_t* ompi_comm_allocate (int local_group_size,
  * @param mode: combination of input
  *              OMPI_COMM_CID_INTRA:        intra-comm
  *              OMPI_COMM_CID_INTER:        inter-comm
+ *              OMPI_COMM_CID_GROUP:        only decide CID within the ompi_group_t
+ *                                          associated with the communicator. arg0
+ *                                          must point to an int which will be used
+ *                                          as the pml tag for communication.
  *              OMPI_COMM_CID_INTRA_BRIDGE: 2 intracomms connected by
- *                                          a bridge comm. local_leader
- *                                          and remote leader are in this
- *                                          case an int (rank in bridge-comm).
+ *                                          a bridge comm. arg0 and arg1 must point
+ *                                          to integers representing the local and
+ *                                          remote leader ranks. the remote leader rank
+ *                                          is a rank in the bridgecomm.
  *              OMPI_COMM_CID_INTRA_PMIX:   2 intracomms, leaders talk
- *                                          through PMIx. lleader and rleader
- *                                          are the required contact information.
+ *                                          through PMIx. arg0 must point to an integer
+ *                                          representing the local leader rank. arg1
+ *                                          must point to a string representing the
+ *                                          port of the remote leader.
  * @param send_first: to avoid a potential deadlock for
  *                    the OOB version.
  * This routine has to be thread safe in the final version.
  */
-OMPI_DECLSPEC int ompi_comm_nextcid ( ompi_communicator_t* newcomm,
-                                      ompi_communicator_t* oldcomm,
-                                      ompi_communicator_t* bridgecomm,
-                                      void* local_leader,
-                                      void* remote_leader,
-                                      int mode,
-                                      int send_first);
+OMPI_DECLSPEC int ompi_comm_nextcid (ompi_communicator_t *newcomm, ompi_communicator_t *comm,
+                                     ompi_communicator_t *bridgecomm, const void *arg0, const void *arg1,
+                                     bool send_first, int mode);
 
 /**
  * allocate new communicator ID (non-blocking)
@@ -522,10 +524,9 @@ OMPI_DECLSPEC int ompi_comm_nextcid ( ompi_communicator_t* newcomm,
  *              OMPI_COMM_CID_INTER:        inter-comm
  * This routine has to be thread safe in the final version.
  */
-OMPI_DECLSPEC int ompi_comm_nextcid_nb (ompi_communicator_t* newcomm,
-                                        ompi_communicator_t* comm,
-                                        ompi_communicator_t* bridgecomm,
-                                        int mode, ompi_request_t **req);
+OMPI_DECLSPEC int ompi_comm_nextcid_nb (ompi_communicator_t *newcomm, ompi_communicator_t *comm,
+                                        ompi_communicator_t *bridgecomm, const void *arg0, const void *arg1,
+                                        bool send_first, int mode, ompi_request_t **req);
 
 /**
  * shut down the communicator infrastructure.
@@ -618,18 +619,25 @@ int ompi_comm_determine_first ( ompi_communicator_t *intercomm,
                                 int high );
 
 
-OMPI_DECLSPEC int ompi_comm_activate ( ompi_communicator_t** newcomm,
-                                       ompi_communicator_t* comm,
-                                       ompi_communicator_t* bridgecomm,
-                                       void* local_leader,
-                                       void* remote_leader,
-                                       int mode,
-                                       int send_first );
+OMPI_DECLSPEC int ompi_comm_activate (ompi_communicator_t **newcomm, ompi_communicator_t *comm,
+                                      ompi_communicator_t *bridgecomm, const void *arg0,
+                                      const void *arg1, bool send_first, int mode);
 
-OMPI_DECLSPEC int ompi_comm_activate_nb (ompi_communicator_t **newcomm,
-                                         ompi_communicator_t *comm,
-                                         ompi_communicator_t *bridgecomm,
-                                         int mode, ompi_request_t **req);
+/**
+ * Non-blocking variant of comm_activate.
+ *
+ * @param[inout] newcomm    New communicator
+ * @param[in]    comm       Parent communicator
+ * @param[in]    bridgecomm Bridge communicator (used for PMIX and bridge modes)
+ * @param[in]    arg0       Mode argument 0
+ * @param[in]    arg1       Mode argument 1
+ * @param[in]    send_first Send first from this process (PMIX mode only)
+ * @param[in]    mode       Collective mode
+ * @param[out]   req        New request object to track this operation
+ */
+OMPI_DECLSPEC int ompi_comm_activate_nb (ompi_communicator_t **newcomm, ompi_communicator_t *comm,
+                                         ompi_communicator_t *bridgecomm, const void *arg0,
+                                         const void *arg1, bool send_first, int mode, ompi_request_t **req);
 
 /**
  * a simple function to dump the structure
@@ -639,14 +647,6 @@ int ompi_comm_dump ( ompi_communicator_t *comm );
 /* setting name */
 int ompi_comm_set_name (ompi_communicator_t *comm, const char *name );
 
-/*
- * these are the init and finalize functions for the comm_reg
- * stuff. These routines are necessary for handling multi-threading
- * scenarious in the communicator_cid allocation
- */
-void ompi_comm_reg_init(void);
-void ompi_comm_reg_finalize(void);
-
 /* global variable to save the number od dynamic communicators */
 extern int ompi_comm_num_dyncomm;
 
diff --git a/ompi/dpm/dpm.c b/ompi/dpm/dpm.c
@@ -469,25 +469,25 @@ int ompi_dpm_connect_accept(ompi_communicator_t *comm, int root,
     new_group_pointer = MPI_GROUP_NULL;
 
     /* allocate comm_cid */
-    rc = ompi_comm_nextcid ( newcomp,                  /* new communicator */
-                             comm,                     /* old communicator */
-                             NULL,                     /* bridge comm */
-                             &root,                    /* local leader */
-                             (void*)port_string,       /* rendezvous point */
-                             OMPI_COMM_CID_INTRA_PMIX, /* mode */
-                             send_first );             /* send or recv first */
+    rc = ompi_comm_nextcid ( newcomp,                   /* new communicator */
+                             comm,                      /* old communicator */
+                             NULL,                      /* bridge comm */
+                             &root,                     /* local leader */
+                             (void*)port_string,        /* rendezvous point */
+                             send_first,                /* send or recv first */
+                             OMPI_COMM_CID_INTRA_PMIX); /* mode */
     if (OMPI_SUCCESS != rc) {
         goto exit;
     }
 
     /* activate comm and init coll-component */
-    rc = ompi_comm_activate ( &newcomp,                 /* new communicator */
-                              comm,                     /* old communicator */
-                              NULL,                     /* bridge comm */
-                              &root,                    /* local leader */
-                              (void*)port_string,       /* rendezvous point */
-                              OMPI_COMM_CID_INTRA_PMIX, /* mode */
-                              send_first );             /* send or recv first */
+    rc = ompi_comm_activate ( &newcomp,                  /* new communicator */
+                              comm,                      /* old communicator */
+                              NULL,                      /* bridge comm */
+                              &root,                     /* local leader */
+                              (void*)port_string,        /* rendezvous point */
+                              send_first,                /* send or recv first */
+                              OMPI_COMM_CID_INTRA_PMIX); /* mode */
     if (OMPI_SUCCESS != rc) {
         goto exit;
     }
@@ -500,7 +500,7 @@ int ompi_dpm_connect_accept(ompi_communicator_t *comm, int root,
  exit:
     if (OMPI_SUCCESS != rc) {
         if (MPI_COMM_NULL != newcomp && NULL != newcomp) {
-            OBJ_RETAIN(newcomp);
+            OBJ_RELEASE(newcomp);
             newcomp = MPI_COMM_NULL;
         }
     }
diff --git a/ompi/mpi/c/intercomm_create.c b/ompi/mpi/c/intercomm_create.c
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -14,6 +15,8 @@
  * Copyright (c) 2012-2013 Inria.  All rights reserved.
  * Copyright (c) 2014-2015 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -196,26 +199,15 @@ int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader,
     new_group_pointer = MPI_GROUP_NULL;
 
     /* Determine context id. It is identical to f_2_c_handle */
-    rc = ompi_comm_nextcid ( newcomp,                     /* new comm */
-                             local_comm,                  /* old comm */
-                             bridge_comm,                 /* bridge comm */
-                             &lleader,                    /* local leader */
-                             &rleader,                    /* remote_leader */
-                             OMPI_COMM_CID_INTRA_BRIDGE,  /* mode */
-                             -1 );                        /* send_first */
-
+    rc = ompi_comm_nextcid (newcomp, local_comm, bridge_comm, &lleader,
+                            &rleader, false, OMPI_COMM_CID_INTRA_BRIDGE);
     if ( MPI_SUCCESS != rc ) {
         goto err_exit;
     }
 
     /* activate comm and init coll-module */
-    rc = ompi_comm_activate ( &newcomp,
-                             local_comm,                  /* old comm */
-                             bridge_comm,                 /* bridge comm */
-                             &lleader,                    /* local leader */
-                             &rleader,                    /* remote_leader */
-                             OMPI_COMM_CID_INTRA_BRIDGE,  /* mode */
-                             -1 );                        /* send_first */
+    rc = ompi_comm_activate (&newcomp, local_comm, bridge_comm, &lleader, &rleader,
+                             false, OMPI_COMM_CID_INTRA_BRIDGE);
     if ( MPI_SUCCESS != rc ) {
         goto err_exit;
     }
diff --git a/ompi/mpi/c/intercomm_merge.c b/ompi/mpi/c/intercomm_merge.c
@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -14,6 +15,8 @@
  * Copyright (c) 2012-2013 Inria.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -115,26 +118,16 @@ int MPI_Intercomm_merge(MPI_Comm intercomm, int high,
     OBJ_RELEASE(new_group_pointer);
     new_group_pointer = MPI_GROUP_NULL;
 
-    /* Determine context id. It is identical to f_2_c_handle */
-    rc = ompi_comm_nextcid ( newcomp,              /* new comm */
-                             intercomm,            /* old comm */
-                             NULL,                 /* bridge comm */
-                             NULL,                 /* local leader */
-                             NULL,                 /* remote_leader */
-                             OMPI_COMM_CID_INTER,  /* mode */
-                             -1 );                 /* send_first */
+    /* Determine context id */
+    rc = ompi_comm_nextcid (newcomp, intercomm, NULL, NULL, NULL, false,
+                            OMPI_COMM_CID_INTER);
     if ( OMPI_SUCCESS != rc ) {
         goto exit;
     }
 
     /* activate communicator and init coll-module */
-    rc = ompi_comm_activate( &newcomp,             /* new comm */
-                             intercomm,            /* old comm */
-                             NULL,                 /* bridge comm */
-                             NULL,                 /* local leader */
-                             NULL,                 /* remote_leader */
-                             OMPI_COMM_CID_INTER,  /* mode */
-                             -1 );                 /* send_first */
+    rc = ompi_comm_activate (&newcomp, intercomm, NULL, NULL, NULL, false,
+                             OMPI_COMM_CID_INTER);
     if ( OMPI_SUCCESS != rc ) {
         goto exit;
     }
diff --git a/ompi/request/req_wait.c b/ompi/request/req_wait.c
@@ -196,7 +196,7 @@ int ompi_request_default_wait_all( size_t count,
     rptr = requests;
     for (i = 0; i < count; i++) {
         request = *rptr++;
-        
+
         if( request->req_state == OMPI_REQUEST_INACTIVE ) {
             completed++;
             continue;

Original file line number	Diff line number	Diff line change
`@@ -234,6 +234,7 @@ static void ompi_comm_request_destruct (ompi_comm_request_t *request)`
`234`	`234`	`{`
`235`	`235`	`OBJ_DESTRUCT(&request->schedule);`
`236`	`236`	`}`
	`237`	`+`
`237`	`238`	`OBJ_CLASS_INSTANCE(ompi_comm_request_t, ompi_request_t,`
`238`	`239`	`ompi_comm_request_construct,`
`239`	`240`	`ompi_comm_request_destruct);`
`@@ -257,10 +258,10 @@ ompi_comm_request_t *ompi_comm_request_get (void)`
`257`	`258`	`void ompi_comm_request_return (ompi_comm_request_t *request)`
`258`	`259`	`{`
`259`	`260`	`if (request->context) {`
`260`		`- free (request->context);`
`261`		`- request->context = NULL;`
	`261`	`+ OBJ_RELEASE (request->context);`
`262`	`262`	`}`
`263`	`263`
	`264`	`+ OMPI_REQUEST_FINI(&request->super);`
`264`	`265`	`opal_free_list_return (&ompi_comm_requests, (opal_free_list_item_t *) request);`
`265`	`266`	`}`
`266`	`267`