Create an alternative mapping method that pushes responsibility

onto the backend daemons. By default, let mpirun only pack the app_context
info and send that to the backend daemons where the mapping will
be done. This significantly reduces the computational time on mpirun as it isn't
running up/down the topology tree computing thousands of binding
locations, and it reduces the launch message to a very small number of
bytes.

When running -novm, fall back to the old way of doing things
where mpirun computes the entire map and binding, and then sends
the full info to the backend daemon.

Add a new cmd line option/mca param --fwd-mpirun-port that allows
mpirun to dynamically select a port, but then passes that back to
all the other daemons so they will use that port as a static port
for their own wireup. In this mode, we no longer "phone home" directly
to mpirun, but instead use the static port to wireup at daemon
start. We then use the routing tree to rollup the initial
launch report, and limit the number of open sockets on mpirun's node.

Update ras simulator to track the new nidmap code

Cleanup some bugs in the nidmap regex code, and enhance the error message for not enough slots to include the host on which the problem is found.

Update gadget platform file

Initialize the range count when starting a new range

Fix the no-np case in managed allocation

Ensure DVM node usage gets cleaned up after each job

Update scaling.pl script to use --fwd-mpirun-port. Pre-connect the daemon to its parent during launch while we are otherwise waiting for the daemon's children to send their "phone home" rollup messages

Signed-off-by: Ralph Castain <[email protected]>

Author: Ralph Castain

contrib/platform/intel/bend/gadget

@@ -8,7 +8,7 @@ enable_heterogeneous=no
 enable_picky=yes
 enable_debug=yes
 enable_shared=yes
-enable_static=no
+enable_static=yes
 enable_memchecker=no
 enable_ipv6=no
 enable_mpi_fortran=no
@@ -18,7 +18,10 @@ enable_cxx_exceptions=no
 enable_oshmem=no
 enable_mpi_java=no
 enable_io_romio=no
+enable_builtin_atomics=no
 enable_contrib_no_build=libnbc
+enable_mca_no_build=btl-tcp,btl-sm,rcache-udreg
+enable_mca_direct=pml-ob1
 with_memory_manager=no
 with_tm=no
 with_verbs=no

contrib/scaling/scaling.pl

@@ -24,7 +24,7 @@
 my @csvrow;
 
 my @tests = qw(/bin/true ./orte_no_op ./mpi_no_op ./mpi_no_op ./mpi_no_op);
-my @options = ("", "", "", "-mca mpi_add_procs_cutoff 0 -mca pmix_base_async_modex 1", "-mca mpi_add_procs_cutoff 0 -mca pmix_base_async_modex 1 -mca async_mpi_init 1 -mca async_mpi_finalize 1");
+my @options = ("", "", "", "--fwd-mpirun-port -mca mpi_add_procs_cutoff 0 -mca pmix_base_async_modex 1", "--fwd-mpirun-port -mca mpi_add_procs_cutoff 0 -mca pmix_base_async_modex 1 -mca async_mpi_init 1 -mca async_mpi_finalize 1");
 my @starterlist = qw(mpirun orterun srun aprun);
 my @starteroptionlist = ("--novm",
                          "--hnp file:dvm_uri",

orte/mca/ess/base/ess_base_std_orted.c

@@ -58,6 +58,7 @@
 #include "orte/mca/plm/base/base.h"
 #include "orte/mca/odls/base/base.h"
 #include "orte/mca/errmgr/errmgr.h"
+#include "orte/mca/rmaps/base/base.h"
 #if OPAL_ENABLE_FT_CR == 1
 #include "orte/mca/snapc/base/base.h"
 #include "orte/mca/sstore/base/base.h"
@@ -116,6 +117,7 @@ int orte_ess_base_orted_setup(char **hosts)
     char *param;
     hwloc_obj_t obj;
     unsigned i, j;
+    orte_topology_t *t;
     opal_list_t transports;
 
     /* my name is set, xfer it to the OPAL layer */
@@ -333,13 +335,8 @@ int orte_ess_base_orted_setup(char **hosts)
     /* create and store a node object where we are */
     node = OBJ_NEW(orte_node_t);
     node->name = strdup(orte_process_info.nodename);
-    node->index = opal_pointer_array_set_item(orte_node_pool, ORTE_PROC_MY_NAME->vpid, node);
-    /* point our topology to the one detected locally */
-    node->topology = OBJ_NEW(orte_topology_t);
-    node->topology->sig = strdup(orte_topo_signature);
-    node->topology->topo = opal_hwloc_topology;
-    /* add it to the array of known ones */
-    opal_pointer_array_add(orte_node_topologies, node->topology);
+    node->index = ORTE_PROC_MY_NAME->vpid;
+    opal_pointer_array_set_item(orte_node_pool, ORTE_PROC_MY_NAME->vpid, node);
 
     /* create and store a proc object for us */
     proc = OBJ_NEW(orte_proc_t);
@@ -496,14 +493,40 @@ int orte_ess_base_orted_setup(char **hosts)
         error = "orte_rtc_base_select";
         goto error;
     }
+    if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_rmaps_base_framework, 0))) {
+        ORTE_ERROR_LOG(ret);
+        error = "orte_rmaps_base_open";
+        goto error;
+    }
+    if (ORTE_SUCCESS != (ret = orte_rmaps_base_select())) {
+        ORTE_ERROR_LOG(ret);
+        error = "orte_rmaps_base_find_available";
+        goto error;
+    }
+
+    /* if a topology file was given, then the rmaps framework open
+     * will have reset our topology. Ensure we always get the right
+     * one by setting our node topology afterwards
+     */
+    t = OBJ_NEW(orte_topology_t);
+    t->topo = opal_hwloc_topology;
+    /* generate the signature */
+    orte_topo_signature = opal_hwloc_base_get_topo_signature(opal_hwloc_topology);
+    t->sig = strdup(orte_topo_signature);
+    opal_pointer_array_add(orte_node_topologies, t);
+    node->topology = t;
+    if (15 < opal_output_get_verbosity(orte_ess_base_framework.framework_output)) {
+        opal_output(0, "%s Topology Info:", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
+        opal_dss.dump(0, opal_hwloc_topology, OPAL_HWLOC_TOPO);
+    }
 
-    /* if we are using static ports, then we need to setup
+    /* if we were given the host list, then we need to setup
      * the daemon info so the RML can function properly
      * without requiring a wireup stage. This must be done
      * after we enable_comm as that function determines our
      * own port, which we need in order to construct the nidmap
      */
-    if (orte_static_ports) {
+    if (NULL != hosts) {
         /* extract the node info from the environment and
          * build a nidmap from it - this will update the
          * routing plan as well

orte/mca/ess/hnp/ess_hnp_module.c

@@ -427,7 +427,8 @@ static int rte_init(void)
     /* create and store a node object where we are */
     node = OBJ_NEW(orte_node_t);
     node->name = strdup(orte_process_info.nodename);
-    node->index = opal_pointer_array_set_item(orte_node_pool, 0, node);
+    node->index = ORTE_PROC_MY_NAME->vpid;
+    opal_pointer_array_set_item(orte_node_pool, 0, node);
 
     /* create and store a proc object for us */
     proc = OBJ_NEW(orte_proc_t);

orte/mca/grpcomm/base/grpcomm_base_stubs.c

@@ -545,6 +545,8 @@ static int pack_xcast(orte_grpcomm_signature_t *sig,
         OBJ_DESTRUCT(&data);
     }
 
+    OPAL_OUTPUT_VERBOSE((1, orte_grpcomm_base_framework.framework_output,
+                         "MSG SIZE: %lu", buffer->bytes_used));
     return ORTE_SUCCESS;
 }