Merge pull request open-mpi#626 from hjelmn/v2.x_add_procs

v2.x dynamic add_procs

Author: jsquyres

ompi/communicator/comm.c

@@ -139,54 +139,8 @@ static opal_mutex_t ompi_cid_lock;
 static opal_list_t ompi_registered_comms;
 
 
-/* This variable is zero (false) if all processes in MPI_COMM_WORLD
- * did not require MPI_THREAD_MULTIPLE support, and is 1 (true) as
- * soon as at least one process requested support for THREAD_MULTIPLE */
-static int ompi_comm_world_thread_level_mult=0;
-
-
 int ompi_comm_cid_init (void)
 {
-#if OMPI_ENABLE_THREAD_MULTIPLE
-    ompi_proc_t **procs, *thisproc;
-    uint8_t thread_level;
-    uint8_t *tlpointer;
-    int ret;
-    size_t i, size, numprocs;
-
-    /** Note that the following call only returns processes
-     * with the same jobid. This is on purpose, since
-     * we switch for the dynamic communicators anyway
-     * to the original (slower) cid allocation algorithm.
-     */
-    procs = ompi_proc_world ( &numprocs );
-
-    for ( i=0; i<numprocs; i++ ) {
-        thisproc = procs[i];
-
-        OPAL_MODEX_RECV_STRING(ret, "MPI_THREAD_LEVEL",
-                               &thisproc->super.proc_name,
-                               (uint8_t**)&tlpointer, &size);
-        if (OMPI_SUCCESS == ret) {
-            thread_level = *((uint8_t *) tlpointer);
-            if ( OMPI_THREADLEVEL_IS_MULTIPLE (thread_level) ) {
-                ompi_comm_world_thread_level_mult = 1;
-                break;
-            }
-        } else if (OMPI_ERR_NOT_IMPLEMENTED == ret) {
-            if (ompi_mpi_thread_multiple) {
-                ompi_comm_world_thread_level_mult = 1;
-            }
-            break;
-        } else {
-            return ret;
-        }
-    }
-    free(procs);
-#else
-    ompi_comm_world_thread_level_mult = 0; // silence compiler warning if not used
-#endif
-
     return OMPI_SUCCESS;
 }
 

ompi/communicator/comm_cid.c

@@ -13,7 +13,7 @@
  * Copyright (c) 2006-2010 University of Houston. All rights reserved.
  * Copyright (c) 2007-2012 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2009      Sun Microsystems, Inc. All rights reserved.
- * Copyright (c) 2012-2014 Los Alamos National Security, LLC.
+ * Copyright (c) 2012-2015 Los Alamos National Security, LLC.
  *                         All rights reserved.
  * Copyright (c) 2011-2013 Inria.  All rights reserved.
  * Copyright (c) 2011-2013 Universite Bordeaux 1
@@ -102,12 +102,26 @@ int ompi_comm_init(void)
     OBJ_CONSTRUCT(&ompi_mpi_comm_world, ompi_communicator_t);
     assert(ompi_mpi_comm_world.comm.c_f_to_c_index == 0);
     group = OBJ_NEW(ompi_group_t);
-    group->grp_proc_pointers = ompi_proc_world(&size);
-    group->grp_proc_count    = (int)size;
+
+    size = ompi_process_info.num_procs;
+    group->grp_proc_pointers = (ompi_proc_t **) calloc (size, sizeof (ompi_proc_t *));
+    group->grp_proc_count = size;
+
+    for (size_t i = 0 ; i < size ; ++i) {
+        opal_process_name_t name = {.vpid = i, .jobid = OMPI_PROC_MY_NAME->jobid};
+        /* look for existing ompi_proc_t that matches this name */
+        group->grp_proc_pointers[i] = (ompi_proc_t *) ompi_proc_lookup (name);
+        if (NULL == group->grp_proc_pointers[i]) {
+            /* set sentinel value */
+            group->grp_proc_pointers[i] = (ompi_proc_t *) ompi_proc_name_to_sentinel (name);
+        } else {
+            OBJ_RETAIN (group->grp_proc_pointers[i]);
+        }
+    }
+
     OMPI_GROUP_SET_INTRINSIC (group);
     OMPI_GROUP_SET_DENSE (group);
     ompi_set_group_rank(group, ompi_proc_local());
-    ompi_group_increment_proc_count (group);
 
     ompi_mpi_comm_world.comm.c_contextid    = 0;
     ompi_mpi_comm_world.comm.c_id_start_index = 4;

ompi/communicator/comm_init.c

@@ -535,6 +535,18 @@ int ompi_comm_finalize (void);
 /**
  * This is THE routine, where all the communicator stuff
  * is really set.
+ *
+ * @param[out] newcomm            new ompi communicator object
+ * @param[in]  oldcomm            old communicator
+ * @param[in]  local_size         size of local_ranks array
+ * @param[in]  local_ranks        local ranks (not used if local_group != NULL)
+ * @param[in]  remote_size        size of remote_ranks array
+ * @param[in]  remote_ranks       remote ranks (intercomm) (not used if remote_group != NULL)
+ * @param[in]  attr               attributes (can be NULL)
+ * @param[in]  errh               error handler
+ * @param[in]  copy_topocomponent whether to copy the topology
+ * @param[in]  local_group        local process group (may be NULL if local_ranks array supplied)
+ * @param[in]  remote_group       remote process group (may be NULL)
  */
 OMPI_DECLSPEC int ompi_comm_set ( ompi_communicator_t** newcomm,
                                   ompi_communicator_t* oldcomm,
@@ -548,6 +560,23 @@ OMPI_DECLSPEC int ompi_comm_set ( ompi_communicator_t** newcomm,
                                   ompi_group_t *local_group,
                                   ompi_group_t *remote_group   );
 
+/**
+ * This is THE routine, where all the communicator stuff
+ * is really set. Non-blocking version.
+ *
+ * @param[out] newcomm            new ompi communicator object
+ * @param[in]  oldcomm            old communicator
+ * @param[in]  local_size         size of local_ranks array
+ * @param[in]  local_ranks        local ranks (not used if local_group != NULL)
+ * @param[in]  remote_size        size of remote_ranks array
+ * @param[in]  remote_ranks       remote ranks (intercomm) (not used if remote_group != NULL)
+ * @param[in]  attr               attributes (can be NULL)
+ * @param[in]  errh               error handler
+ * @param[in]  copy_topocomponent whether to copy the topology
+ * @param[in]  local_group        local process group (may be NULL if local_ranks array supplied)
+ * @param[in]  remote_group       remote process group (may be NULL)
+ * @param[out] req                ompi_request_t object for tracking completion
+ */
 OMPI_DECLSPEC int ompi_comm_set_nb ( ompi_communicator_t **ncomm,
                                      ompi_communicator_t *oldcomm,
                                      int local_size,

ompi/communicator/communicator.h

@@ -13,7 +13,7 @@
  * Copyright (c) 2007-2015 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2006-2009 University of Houston.  All rights reserved.
  * Copyright (c) 2009      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2011-2013 Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2011-2015 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2013-2015 Intel, Inc. All rights reserved
  * Copyright (c) 2014-2015 Research Organization for Information Science
@@ -1210,63 +1210,53 @@ static int disconnect_waitall (int count, ompi_dpm_disconnect_obj **objs)
 /**********************************************************************/
 /**********************************************************************/
 /**********************************************************************/
+static bool ompi_dpm_group_is_dyn (ompi_group_t *group, ompi_jobid_t thisjobid)
+{
+    int size = group ? ompi_group_size (group) : 0;
+
+    for (int i = 1 ; i < size ; ++i) {
+        opal_process_name_t name = ompi_group_get_proc_name (group, i);
+
+        if (thisjobid != ((ompi_process_name_t *) &name)->jobid) {
+            /* at least one is different */
+            return true;
+        }
+    }
+
+    return false;
+}
+
 /* All we want to do in this function is determine if the number of
  * jobids in the local and/or remote group is > 1. This tells us to
  * set the disconnect flag. We don't actually care what the true
  * number -is-, only that it is > 1
  */
 void ompi_dpm_mark_dyncomm(ompi_communicator_t *comm)
 {
-    int i;
-    int size, rsize;
-    bool found=false;
+    bool found;
     ompi_jobid_t thisjobid;
-    ompi_group_t *grp=NULL;
-    ompi_proc_t *proc = NULL;
 
     /* special case for MPI_COMM_NULL */
     if (comm == MPI_COMM_NULL) {
         return;
     }
 
-    size  = ompi_comm_size(comm);
-    rsize = ompi_comm_remote_size(comm);
+    thisjobid = ompi_group_get_proc_name (comm->c_local_group, 0).jobid;
 
     /* loop over all processes in local group and check for
      * a different jobid
      */
-    grp = comm->c_local_group;
-    proc = ompi_group_peer_lookup(grp,0);
-    thisjobid = ((ompi_process_name_t*)&proc->super.proc_name)->jobid;
-
-    for (i=1; i< size; i++) {
-        proc = ompi_group_peer_lookup(grp,i);
-        if (thisjobid != ((ompi_process_name_t*)&proc->super.proc_name)->jobid) {
-            /* at least one is different */
-            found = true;
-            goto complete;
-        }
-    }
-
-    /* if inter-comm, loop over all processes in remote_group
-     * and see if any are different from thisjobid
-     */
-    grp = comm->c_remote_group;
-    for (i=0; i< rsize; i++) {
-        proc = ompi_group_peer_lookup(grp,i);
-        if (thisjobid != ((ompi_process_name_t*)&proc->super.proc_name)->jobid) {
-            /* at least one is different */
-            found = true;
-            break;
-        }
+    found = ompi_dpm_group_is_dyn (comm->c_local_group, thisjobid);
+    if (!found) {
+        /* if inter-comm, loop over all processes in remote_group
+         * and see if any are different from thisjobid
+         */
+        found = ompi_dpm_group_is_dyn (comm->c_remote_group, thisjobid);
     }
 
- complete:
     /* if a different jobid was found, set the disconnect flag*/
     if (found) {
         ompi_comm_num_dyncomm++;
         OMPI_COMM_SET_DYNAMIC(comm);
     }
-
-    return;
 }

ompi/dpm/dpm.c

@@ -14,7 +14,7 @@
  * Copyright (c) 2007      Cisco Systems, Inc. All rights reserved.
  * Copyright (c) 2012      Oak Ridge National Labs.  All rights reserved.
  * Copyright (c) 2012-2013 Inria.  All rights reserved.
- * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2013-2015 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -49,16 +49,14 @@ int ompi_group_translate_ranks ( ompi_group_t *group1,
                                  ompi_group_t *group2,
                                  int *ranks2)
 {
-    int rank, proc, proc2;
-    struct ompi_proc_t *proc1_pointer, *proc2_pointer;
-
     if ( MPI_GROUP_EMPTY == group1 || MPI_GROUP_EMPTY == group2 ) {
-        for (proc = 0; proc < n_ranks ; proc++) {
+        for (int proc = 0; proc < n_ranks ; ++proc) {
             ranks2[proc] = MPI_UNDEFINED;
         }
         return MPI_SUCCESS;
     }
 
+#if OMPI_GROUP_SPARSE
     /*
      * If we are translating from a parent to a child that uses the sparse format
      * or vice versa, we use the translate ranks function corresponding to the
@@ -80,8 +78,11 @@ int ompi_group_translate_ranks ( ompi_group_t *group1,
                 (group1,n_ranks,ranks1,group2,ranks2);
         }
 
+        /* unknown sparse group type */
+        assert (0);
     }
-    else if( group2->grp_parent_group_ptr == group1 ) { /* from parent to child*/
+
+    if( group2->grp_parent_group_ptr == group1 ) { /* from parent to child*/
         if(OMPI_GROUP_IS_SPORADIC(group2)) {
             return ompi_group_translate_ranks_sporadic
                 (group1,n_ranks,ranks1,group2,ranks2);
@@ -95,28 +96,32 @@ int ompi_group_translate_ranks ( ompi_group_t *group1,
                 (group1,n_ranks,ranks1,group2,ranks2);
         }
 
+        /* unknown sparse group type */
+        assert (0);
     }
-    else {
-        /* loop over all ranks */
-        for (proc = 0; proc < n_ranks; proc++) {
-            rank=ranks1[proc];
-            if ( MPI_PROC_NULL == rank) {
-                ranks2[proc] = MPI_PROC_NULL;
-            }
-            else {
-                proc1_pointer = ompi_group_peer_lookup(group1 ,rank);
-                /* initialize to no "match" */
-                ranks2[proc] = MPI_UNDEFINED;
-                for (proc2 = 0; proc2 < group2->grp_proc_count; proc2++) {
-                    proc2_pointer= ompi_group_peer_lookup(group2, proc2);
-                    if ( proc1_pointer == proc2_pointer) {
-                        ranks2[proc] = proc2;
-                        break;
-                    }
-                }  /* end proc2 loop */
-            } /* end proc loop */
+#endif
+
+    /* loop over all ranks */
+    for (int proc = 0; proc < n_ranks; ++proc) {
+        struct ompi_proc_t *proc1_pointer, *proc2_pointer;
+        int rank = ranks1[proc];
+
+        if ( MPI_PROC_NULL == rank) {
+            ranks2[proc] = MPI_PROC_NULL;
+            continue;
         }
-    }
+
+        proc1_pointer = ompi_group_get_proc_ptr_raw (group1, rank);
+        /* initialize to no "match" */
+        ranks2[proc] = MPI_UNDEFINED;
+        for (int proc2 = 0; proc2 < group2->grp_proc_count; ++proc2) {
+            proc2_pointer = ompi_group_get_proc_ptr_raw (group2, proc2);
+            if ( proc1_pointer == proc2_pointer) {
+                ranks2[proc] = proc2;
+                break;
+            }
+        }  /* end proc2 loop */
+    } /* end proc loop */
 
     return MPI_SUCCESS;
 }
@@ -168,25 +173,6 @@ int ompi_group_dump (ompi_group_t* group)
     return OMPI_SUCCESS;
 }
 
-/*
- * This is the function that iterates through the sparse groups to the dense group
- * to reach the process pointer
- */
-ompi_proc_t* ompi_group_get_proc_ptr (ompi_group_t* group , int rank)
-{
-    int ranks1,ranks2;
-    do {
-        if(OMPI_GROUP_IS_DENSE(group)) {
-            return group->grp_proc_pointers[rank];
-        }
-        ranks1 = rank;
-        ompi_group_translate_ranks( group, 1, &ranks1,
-                                    group->grp_parent_group_ptr,&ranks2);
-        rank = ranks2;
-        group = group->grp_parent_group_ptr;
-    } while (1);
-}
-
 int ompi_group_minloc ( int list[] , int length )
 {
     int i,index,min;
@@ -568,3 +554,23 @@ int ompi_group_compare(ompi_group_t *group1,
 
     return return_value;
 }
+
+bool ompi_group_have_remote_peers (ompi_group_t *group)
+{
+    for (int i = 0 ; i < group->grp_proc_count ; ++i) {
+        ompi_proc_t *proc = NULL;
+#if OMPI_GROUP_SPARSE
+        proc = ompi_group_peer_lookup (group, i);
+#else
+        if (ompi_proc_is_sentinel (group->grp_proc_pointers[i])) {
+            return true;
+        }
+        proc = group->grp_proc_pointers[i];
+#endif
+        if (!OPAL_PROC_ON_LOCAL_NODE(proc->super.proc_flags)) {
+            return true;
+        }
+    }
+
+    return false;
+}