Only instantiate the HWLOC topology in an MPI process if it actually will be used.

There are only five places in the non-daemon code paths where opal_hwloc_topology is currently referenced:

* shared memory BTLs (sm, smcuda). I have added a code path to those components that uses the location string
  instead of the topology itself, if available, thus avoiding instantiating the topology

* openib BTL. This uses the distance matrix. At present, I haven't developed a method
  for replacing that reference. Thus, this component will instantiate the topology

* usnic BTL. Uses the distance matrix.

* treematch TOPO component. Does some complex tree-based algorithm, so it will instantiate
  the topology

* ess base functions. If a process is direct launched and not bound at launch, this
  code attempts to bind it. Thus, procs in this scenario will instantiate the
  topology

Note that instantiating the topology on complex chips such as KNL can consume
megabytes of memory.

Fix pernode binding policy

Properly handle the unbound case

Correct pointer usage

Do not free static error messages!

Signed-off-by: Ralph Castain <[email protected]>

Author: Ralph Castain

ompi/errhandler/errhandler_predefined.c

@@ -15,6 +15,7 @@
  * Copyright (c) 2010-2011 Oak Ridge National Labs.  All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, LLC.
  *                         All rights reserved.
+ * Copyright (c) 2016      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -51,7 +52,7 @@ static void out(char *str, char *arg);
 
 
 void ompi_mpi_errors_are_fatal_comm_handler(struct ompi_communicator_t **comm,
-					    int *error_code, ...)
+                                            int *error_code, ...)
 {
   char *name;
   struct ompi_communicator_t *abort_comm;
@@ -72,7 +73,7 @@ void ompi_mpi_errors_are_fatal_comm_handler(struct ompi_communicator_t **comm,
 
 
 void ompi_mpi_errors_are_fatal_file_handler(struct ompi_file_t **file,
-					    int *error_code, ...)
+                                            int *error_code, ...)
 {
   char *name;
   struct ompi_communicator_t *abort_comm;
@@ -93,7 +94,7 @@ void ompi_mpi_errors_are_fatal_file_handler(struct ompi_file_t **file,
 
 
 void ompi_mpi_errors_are_fatal_win_handler(struct ompi_win_t **win,
-					   int *error_code, ...)
+                                           int *error_code, ...)
 {
   char *name;
   struct ompi_communicator_t *abort_comm = NULL;
@@ -111,7 +112,7 @@ void ompi_mpi_errors_are_fatal_win_handler(struct ompi_win_t **win,
 }
 
 void ompi_mpi_errors_return_comm_handler(struct ompi_communicator_t **comm,
-					 int *error_code, ...)
+                                         int *error_code, ...)
 {
     /* Don't need anything more -- just need this function to exist */
     /* Silence some compiler warnings */
@@ -123,7 +124,7 @@ void ompi_mpi_errors_return_comm_handler(struct ompi_communicator_t **comm,
 
 
 void ompi_mpi_errors_return_file_handler(struct ompi_file_t **file,
-					 int *error_code, ...)
+                                         int *error_code, ...)
 {
     /* Don't need anything more -- just need this function to exist */
     /* Silence some compiler warnings */
@@ -135,7 +136,7 @@ void ompi_mpi_errors_return_file_handler(struct ompi_file_t **file,
 
 
 void ompi_mpi_errors_return_win_handler(struct ompi_win_t **win,
-					int *error_code, ...)
+                                        int *error_code, ...)
 {
     /* Don't need anything more -- just need this function to exist */
     /* Silence some compiler warnings */
@@ -181,6 +182,7 @@ static void backend_fatal_aggregate(char *type,
     const char* const unknown_error_code = "Error code: %d (no associated error message)";
     const char* const unknown_error = "Unknown error";
     const char* const unknown_prefix = "[?:?]";
+    bool generated = false;
 
     // these do not own what they point to; they're
     // here to avoid repeating expressions such as
@@ -209,6 +211,8 @@ static void backend_fatal_aggregate(char *type,
                 err_msg = NULL;
                 opal_output(0, "%s", "Could not write to err_msg");
                 opal_output(0, unknown_error_code, *error_code);
+            } else {
+                generated = true;
             }
         }
     }
@@ -254,7 +258,9 @@ static void backend_fatal_aggregate(char *type,
     }
 
     free(prefix);
-    free(err_msg);
+    if (generated) {
+        free(err_msg);
+    }
 }
 
 /*

ompi/mca/topo/treematch/topo_treematch_component.c

@@ -4,6 +4,7 @@
  *                         reserved.
  * Copyright (c) 2011-2015 INRIA.  All rights reserved.
  * Copyright (c) 2011-2015 Université Bordeaux 1
+ * Copyright (c) 2016      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -61,9 +62,6 @@ mca_topo_treematch_component_2_2_0_t mca_topo_treematch_component =
 
 static int init_query(bool enable_progress_threads, bool enable_mpi_threads)
 {
-    if(NULL == opal_hwloc_topology) {
-        return OPAL_ERR_NOT_SUPPORTED;
-    }
     return OMPI_SUCCESS;
 }
 
@@ -97,4 +95,3 @@ static int mca_topo_treematch_component_register(void)
                                           MCA_BASE_VAR_SCOPE_READONLY, &mca_topo_treematch_component.reorder_mode);
     return OMPI_SUCCESS;
 }
-

ompi/mca/topo/treematch/topo_treematch_dist_graph_create.c

@@ -5,7 +5,7 @@
  *                         reserved.
  * Copyright (c) 2011-2015 INRIA.  All rights reserved.
  * Copyright (c) 2012-2015 Bordeaux Poytechnic Institute
- * Copyright (c) 2015      Intel, Inc. All rights reserved
+ * Copyright (c) 2015-2016 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
@@ -256,7 +256,9 @@ int mca_topo_treematch_dist_graph_create(mca_topo_base_module_t* topo_module,
     /* Then, we need to know if the processes are bound */
     /* We make the hypothesis that all processes are in  */
     /* the same state : all bound or none bound */
-    assert(NULL != opal_hwloc_topology);
+    if (OPAL_SUCCESS != opal_hwloc_base_get_topology()) {
+        goto fallback;
+    }
     root_obj = hwloc_get_root_obj(opal_hwloc_topology);
     if (NULL == root_obj) goto fallback;
 
@@ -873,7 +875,7 @@ int mca_topo_treematch_dist_graph_create(mca_topo_base_module_t* topo_module,
         if( -1 == hwloc_err) goto fallback;
 
         /* Report new binding to ORTE/OPAL */
-        /*	hwloc_bitmap_list_asprintf(&orte_process_info.cpuset,set);   */
+        /*      hwloc_bitmap_list_asprintf(&orte_process_info.cpuset,set);   */
         err = hwloc_bitmap_snprintf (set_as_string,64,set);
 
 #ifdef __DEBUG__

ompi/runtime/ompi_mpi_init.c

@@ -508,16 +508,6 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
     /* check for timing request - get stop time and report elapsed time if so */
     OPAL_TIMING_MNEXT((&tm,"time from completion of rte_init to modex"));
 
-    /* if hwloc is available but didn't get setup for some
-     * reason, do so now
-     */
-    if (NULL == opal_hwloc_topology) {
-        if (OPAL_SUCCESS != (ret = opal_hwloc_base_get_topology())) {
-            error = "Topology init";
-            goto error;
-        }
-    }
-
     /* Register the default errhandler callback  */
     errtrk.status = OPAL_ERROR;
     errtrk.active = true;

opal/mca/btl/openib/btl_openib_component.c

@@ -18,7 +18,7 @@
  * Copyright (c) 2009-2012 Oracle and/or its affiliates.  All rights reserved.
  * Copyright (c) 2011-2015 NVIDIA Corporation.  All rights reserved.
  * Copyright (c) 2012      Oak Ridge National Laboratory.  All rights reserved
- * Copyright (c) 2013-2015 Intel, Inc. All rights reserved
+ * Copyright (c) 2013-2016 Intel, Inc.  All rights reserved.
  * Copyright (c) 2014-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014      Bull SAS.  All rights reserved.
@@ -1502,13 +1502,33 @@ static uint64_t read_module_param(char *file, uint64_t value, uint64_t max)
 static uint64_t calculate_total_mem (void)
 {
     hwloc_obj_t machine;
+    int rc;
+    uint64_t mem, *mptr;
+    opal_process_name_t wildcard_rank;
 
-    machine = hwloc_get_next_obj_by_type (opal_hwloc_topology, HWLOC_OBJ_MACHINE, NULL);
-    if (NULL == machine) {
-        return 0;
+    /* first try to retrieve it from PMIx as it may have
+     * been provided */
+    wildcard_rank.jobid = OPAL_PROC_MY_NAME.jobid;
+    wildcard_rank.vpid = OPAL_VPID_WILDCARD;
+    mptr = &mem;
+    OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, OPAL_PMIX_AVAIL_PHYS_MEMORY,
+                                   &wildcard_rank, &mptr, OPAL_UINT64);
+    if (OPAL_SUCCESS == rc) {
+        return mem;
+    }
+
+    /* if not available, then ensure that the topology has been
+     * loaded and try to get it from there */
+    if (OPAL_SUCCESS == opal_hwloc_base_get_topology()) {
+        machine = hwloc_get_next_obj_by_type (opal_hwloc_topology, HWLOC_OBJ_MACHINE, NULL);
+        if (NULL == machine) {
+            return 0;
+        }
+        return machine->memory.total_memory;
     }
 
-    return machine->memory.total_memory;
+    /* couldn't find it */
+    return 0;
 }
 
 
@@ -2312,7 +2332,8 @@ static float get_ib_dev_distance(struct ibv_device *dev)
     float distance = 0;
 
     /* Override any distance logic so all devices are used */
-    if (0 != mca_btl_openib_component.ignore_locality) {
+    if (0 != mca_btl_openib_component.ignore_locality ||
+        OPAL_SUCCESS != opal_hwloc_base_get_topology()) {
         return distance;
     }
 

opal/mca/btl/sm/btl_sm.c

@@ -52,7 +52,7 @@
 #include "opal/util/show_help.h"
 #include "opal/util/printf.h"
 #include "opal/mca/hwloc/base/base.h"
-#include "opal/mca/pmix/pmix.h"
+#include "opal/mca/pmix/base/base.h"
 #include "opal/mca/shmem/base/base.h"
 #include "opal/mca/shmem/shmem.h"
 
@@ -242,7 +242,7 @@ sm_btl_first_time_init(mca_btl_sm_t *sm_btl,
         free(loc);
     } else {
         /* If we have hwloc support, then get accurate information */
-        if (NULL != opal_hwloc_topology) {
+        if (OPAL_SUCCESS == opal_hwloc_base_get_topology()) {
             i = opal_hwloc_base_get_nbobjs_by_type(opal_hwloc_topology,
                                                    HWLOC_OBJ_NODE, 0,
                                                    OPAL_HWLOC_AVAILABLE);
@@ -257,6 +257,7 @@ sm_btl_first_time_init(mca_btl_sm_t *sm_btl,
         }
     }
     /* see if we were given our location */
+    loc = NULL;
     OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, OPAL_PMIX_LOCALITY_STRING,
                                    &OPAL_PROC_MY_NAME, &loc, OPAL_STRING);
     if (OPAL_SUCCESS == rc) {
@@ -283,8 +284,7 @@ sm_btl_first_time_init(mca_btl_sm_t *sm_btl,
         }
     } else {
         /* If we have hwloc support, then get accurate information */
-        if (NULL != opal_hwloc_topology && num_mem_nodes > 0 &&
-            NULL != opal_process_info.cpuset) {
+        if (OPAL_SUCCESS == opal_hwloc_base_get_topology() && num_mem_nodes > 0) {
             int numa=0, w;
             unsigned n_bound=0;
             hwloc_cpuset_t avail;