If the HNP is going to request topology info, it cannot do so via a routed OOB message as the intervening daemons may not be ready. So disable routing until the VM is ready, and have daemons start routing as they receive the xcast launch msg (which includes the data they need to talk to their peers).

Do a little optimization and minimize recomputation of the routing plan.

Signed-off-by: Ralph Castain <[email protected]>

Author: Ralph Castain

orte/mca/ess/base/ess_base_std_orted.c

@@ -497,12 +497,6 @@ int orte_ess_base_orted_setup(char **hosts)
         goto error;
     }
 
-    /* be sure to update the routing tree so the initial "phone home"
-     * to mpirun goes through the tree if static ports were enabled - still
-     * need to do it anyway just to initialize things
-     */
-    orte_routed.update_routing_plan(NULL);
-
     /* if we are using static ports, then we need to setup
      * the daemon info so the RML can function properly
      * without requiring a wireup stage. This must be done
@@ -519,6 +513,12 @@ int orte_ess_base_orted_setup(char **hosts)
             error = "construct daemon map from static ports";
             goto error;
         }
+        /* be sure to update the routing tree so the initial "phone home"
+         * to mpirun goes through the tree if static ports were enabled
+         */
+        orte_routed.update_routing_plan(NULL);
+        /* routing can be enabled */
+        orte_routed_base.routing_enabled = true;
     }
 
     /* Now provide a chance for the PLM

orte/mca/grpcomm/direct/grpcomm_direct.c

@@ -27,7 +27,7 @@
 #include "orte/mca/errmgr/errmgr.h"
 #include "orte/mca/rml/base/base.h"
 #include "orte/mca/rml/base/rml_contact.h"
-#include "orte/mca/routed/routed.h"
+#include "orte/mca/routed/base/base.h"
 #include "orte/mca/state/state.h"
 #include "orte/util/compress.h"
 #include "orte/util/name_fns.h"
@@ -386,8 +386,14 @@ static void xcast_recv(int status, orte_process_name_t* sender,
                     goto relay;
                 }
 
-                /* update the routing plan */
-                orte_routed.update_routing_plan(rtmod);
+                if (!ORTE_PROC_IS_HNP) {
+                    /* update the routing plan - the HNP already did
+                     * it when it computed the VM, so don't waste time
+                     * re-doing it here */
+                    orte_routed.update_routing_plan(rtmod);
+                }
+                /* routing is now possible */
+                orte_routed_base.routing_enabled = true;
 
                 /* see if we have wiring info as well */
                 cnt=1;

orte/mca/plm/base/plm_base_launch_support.c

@@ -410,15 +410,6 @@ void orte_plm_base_complete_setup(int fd, short args, void *cbdata)
         return;
     }
 
-    orte_process_info.num_procs = jdatorted->num_procs;
-
-    if (orte_process_info.max_procs < orte_process_info.num_procs) {
-        orte_process_info.max_procs = orte_process_info.num_procs;
-    }
-
-    /* ensure all routing plans are up-to-date */
-    orte_routed.update_routing_plan(NULL);
-
     /* If this job is being started by me, then there is nothing
      * further we need to do as any user directives (e.g., to tie
      * off IO to /dev/null) will have been included in the launch
@@ -2158,7 +2149,8 @@ int orte_plm_base_setup_virtual_machine(orte_job_t *jdata)
             orte_process_info.max_procs = orte_process_info.num_procs;
         }
 
-        /* ensure all routing plans are up-to-date */
+        /* ensure all routing plans are up-to-date - we need this
+         * so we know how to tree-spawn and/or xcast info */
         orte_routed.update_routing_plan(NULL);
     }
 

orte/mca/plm/rsh/plm_rsh_module.c

@@ -801,17 +801,20 @@ static int remote_spawn(opal_buffer_t *launch)
         goto cleanup;
     }
 
-    /* get the updated routing list */
-    rtmod = orte_rml.get_routed(orte_coll_conduit);
-    OBJ_CONSTRUCT(&coll, opal_list_t);
-    orte_routed.get_routing_list(rtmod, &coll);
-
     /* extract and update the daemon map */
     if (ORTE_SUCCESS != (rc = orte_util_decode_daemon_nodemap(launch))) {
         ORTE_ERROR_LOG(rc);
         goto cleanup;
     }
 
+    /* since we are tree-spawning, we need to update the routing plan */
+    orte_routed.update_routing_plan(NULL);
+
+    /* get the updated routing list */
+    rtmod = orte_rml.get_routed(orte_coll_conduit);
+    OBJ_CONSTRUCT(&coll, opal_list_t);
+    orte_routed.get_routing_list(rtmod, &coll);
+
     /* if I have no children, just return */
     if (0 == opal_list_get_size(&coll)) {
         OPAL_OUTPUT_VERBOSE((1, orte_plm_base_framework.framework_output,

orte/mca/routed/base/base.h

@@ -1,7 +1,7 @@
 /*
  * Copyright (c) 2007-2013 Los Alamos National Security, LLC.
  *                         All rights reserved.
- * Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -41,6 +41,7 @@ OBJ_CLASS_DECLARATION(orte_routed_base_active_t);
 
 typedef struct {
     opal_list_t actives;
+    bool routing_enabled;
 } orte_routed_base_t;
 ORTE_DECLSPEC extern orte_routed_base_t orte_routed_base;
 

orte/mca/routed/base/routed_base_fns.c

@@ -12,7 +12,7 @@
  * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2011-2012 Los Alamos National Security, LLC.  All rights
  *                         reserved.
- * Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -110,7 +110,7 @@ orte_process_name_t orte_routed_base_get_route(char *module, orte_process_name_t
     orte_routed_base_active_t *active;
 
     /* a NULL module corresponds to direct */
-    if (NULL == module) {
+    if (!orte_routed_base.routing_enabled || NULL == module) {
         return *target;
     }
 
@@ -178,6 +178,7 @@ void orte_routed_base_update_routing_plan(char *module)
             }
         }
     }
+
     return;
 }
 

orte/mca/routed/base/routed_base_frame.c

@@ -10,7 +10,7 @@
  *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2016      Intel, Inc.  All rights reserved.
+ * Copyright (c) 2016-2017 Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -58,6 +58,8 @@ static int orte_routed_base_open(mca_base_open_flag_t flags)
 {
     /* setup our list of actives */
     OBJ_CONSTRUCT(&orte_routed_base.actives, opal_list_t);
+    /* start with routing DISABLED */
+    orte_routed_base.routing_enabled = false;
 
     /* Open up all available components */
     return mca_base_framework_components_open(&orte_routed_base_framework, flags);

orte/mca/routed/debruijn/routed_debruijn_component.c

@@ -49,7 +49,7 @@ static int orte_routed_debruijn_component_query(mca_base_module_t **module, int
      * systems, we will allow other options that have even fewer hops to
      * support wireup
      */
-    *priority = 70;
+    *priority = 10;
     *module = (mca_base_module_t *) &orte_routed_debruijn_module;
     return ORTE_SUCCESS;
 }

orte/mca/routed/radix/routed_radix.c

@@ -6,7 +6,7 @@
  *                         reserved.
  * Copyright (c) 2011-2012 Los Alamos National Security, LLC.  All rights
  *                         reserved.
- * Copyright (c) 2013-2016 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2013-2017 Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -538,4 +538,3 @@ static int radix_ft_event(int state)
     return exit_status;
 }
 #endif
-

orte/mca/routed/radix/routed_radix_component.c

@@ -69,7 +69,7 @@ static int orte_routed_radix_component_query(mca_base_module_t **module, int *pr
         return ORTE_ERR_BAD_PARAM;
     }
 
-    *priority = 50;
+    *priority = 70;
     *module = (mca_base_module_t *) &orte_routed_radix_module;
     return ORTE_SUCCESS;
 }