Merge pull request #2916 from rhc54/topic/sim

Create an alternative mapping method

Author: Ralph Castain

contrib/platform/intel/bend/gadget

@@ -8,7 +8,7 @@ enable_heterogeneous=no
 enable_picky=yes
 enable_debug=yes
 enable_shared=yes
-enable_static=no
+enable_static=yes
 enable_memchecker=no
 enable_ipv6=no
 enable_mpi_fortran=no
@@ -18,7 +18,10 @@ enable_cxx_exceptions=no
 enable_oshmem=no
 enable_mpi_java=no
 enable_io_romio=no
+enable_builtin_atomics=no
 enable_contrib_no_build=libnbc
+enable_mca_no_build=btl-tcp,btl-sm,rcache-udreg
+enable_mca_direct=pml-ob1
 with_memory_manager=no
 with_tm=no
 with_verbs=no

contrib/scaling/scaling.pl

@@ -24,7 +24,7 @@
 my @csvrow;
 
 my @tests = qw(/bin/true ./orte_no_op ./mpi_no_op ./mpi_no_op ./mpi_no_op);
-my @options = ("", "", "", "-mca mpi_add_procs_cutoff 0 -mca pmix_base_async_modex 1", "-mca mpi_add_procs_cutoff 0 -mca pmix_base_async_modex 1 -mca async_mpi_init 1 -mca async_mpi_finalize 1");
+my @options = ("", "", "", "--fwd-mpirun-port -mca mpi_add_procs_cutoff 0 -mca pmix_base_async_modex 1", "--fwd-mpirun-port -mca mpi_add_procs_cutoff 0 -mca pmix_base_async_modex 1 -mca async_mpi_init 1 -mca async_mpi_finalize 1");
 my @starterlist = qw(mpirun orterun srun aprun);
 my @starteroptionlist = ("--novm",
                          "--hnp file:dvm_uri",

orte/mca/ess/base/ess_base_std_orted.c

@@ -58,6 +58,7 @@
 #include "orte/mca/plm/base/base.h"
 #include "orte/mca/odls/base/base.h"
 #include "orte/mca/errmgr/errmgr.h"
+#include "orte/mca/rmaps/base/base.h"
 #if OPAL_ENABLE_FT_CR == 1
 #include "orte/mca/snapc/base/base.h"
 #include "orte/mca/sstore/base/base.h"
@@ -116,6 +117,7 @@ int orte_ess_base_orted_setup(char **hosts)
     char *param;
     hwloc_obj_t obj;
     unsigned i, j;
+    orte_topology_t *t;
     opal_list_t transports;
 
     /* my name is set, xfer it to the OPAL layer */
@@ -333,13 +335,8 @@ int orte_ess_base_orted_setup(char **hosts)
     /* create and store a node object where we are */
     node = OBJ_NEW(orte_node_t);
     node->name = strdup(orte_process_info.nodename);
-    node->index = opal_pointer_array_set_item(orte_node_pool, ORTE_PROC_MY_NAME->vpid, node);
-    /* point our topology to the one detected locally */
-    node->topology = OBJ_NEW(orte_topology_t);
-    node->topology->sig = strdup(orte_topo_signature);
-    node->topology->topo = opal_hwloc_topology;
-    /* add it to the array of known ones */
-    opal_pointer_array_add(orte_node_topologies, node->topology);
+    node->index = ORTE_PROC_MY_NAME->vpid;
+    opal_pointer_array_set_item(orte_node_pool, ORTE_PROC_MY_NAME->vpid, node);
 
     /* create and store a proc object for us */
     proc = OBJ_NEW(orte_proc_t);
@@ -496,14 +493,40 @@ int orte_ess_base_orted_setup(char **hosts)
         error = "orte_rtc_base_select";
         goto error;
     }
+    if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_rmaps_base_framework, 0))) {
+        ORTE_ERROR_LOG(ret);
+        error = "orte_rmaps_base_open";
+        goto error;
+    }
+    if (ORTE_SUCCESS != (ret = orte_rmaps_base_select())) {
+        ORTE_ERROR_LOG(ret);
+        error = "orte_rmaps_base_find_available";
+        goto error;
+    }
+
+    /* if a topology file was given, then the rmaps framework open
+     * will have reset our topology. Ensure we always get the right
+     * one by setting our node topology afterwards
+     */
+    t = OBJ_NEW(orte_topology_t);
+    t->topo = opal_hwloc_topology;
+    /* generate the signature */
+    orte_topo_signature = opal_hwloc_base_get_topo_signature(opal_hwloc_topology);
+    t->sig = strdup(orte_topo_signature);
+    opal_pointer_array_add(orte_node_topologies, t);
+    node->topology = t;
+    if (15 < opal_output_get_verbosity(orte_ess_base_framework.framework_output)) {
+        opal_output(0, "%s Topology Info:", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
+        opal_dss.dump(0, opal_hwloc_topology, OPAL_HWLOC_TOPO);
+    }
 
-    /* if we are using static ports, then we need to setup
+    /* if we were given the host list, then we need to setup
      * the daemon info so the RML can function properly
      * without requiring a wireup stage. This must be done
      * after we enable_comm as that function determines our
      * own port, which we need in order to construct the nidmap
      */
-    if (orte_static_ports) {
+    if (NULL != hosts) {
         /* extract the node info from the environment and
          * build a nidmap from it - this will update the
          * routing plan as well

orte/mca/ess/hnp/ess_hnp_module.c

@@ -427,7 +427,8 @@ static int rte_init(void)
     /* create and store a node object where we are */
     node = OBJ_NEW(orte_node_t);
     node->name = strdup(orte_process_info.nodename);
-    node->index = opal_pointer_array_set_item(orte_node_pool, 0, node);
+    node->index = ORTE_PROC_MY_NAME->vpid;
+    opal_pointer_array_set_item(orte_node_pool, 0, node);
 
     /* create and store a proc object for us */
     proc = OBJ_NEW(orte_proc_t);

orte/mca/grpcomm/base/grpcomm_base_stubs.c

@@ -545,6 +545,8 @@ static int pack_xcast(orte_grpcomm_signature_t *sig,
         OBJ_DESTRUCT(&data);
     }
 
+    OPAL_OUTPUT_VERBOSE((1, orte_grpcomm_base_framework.framework_output,
+                         "MSG SIZE: %lu", buffer->bytes_used));
     return ORTE_SUCCESS;
 }