Merge pull request #2648 from rhc54/topic/topo

Only instantiate the HWLOC topology in an MPI process if it actually will be used.

Author: rhc54

ompi/errhandler/errhandler_predefined.c

@@ -15,6 +15,7 @@
  * Copyright (c) 2010-2011 Oak Ridge National Labs.  All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, LLC.
  *                         All rights reserved.
+ * Copyright (c) 2016      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -51,7 +52,7 @@ static void out(char *str, char *arg);
 
 
 void ompi_mpi_errors_are_fatal_comm_handler(struct ompi_communicator_t **comm,
-					    int *error_code, ...)
+                                            int *error_code, ...)
 {
   char *name;
   struct ompi_communicator_t *abort_comm;
@@ -72,7 +73,7 @@ void ompi_mpi_errors_are_fatal_comm_handler(struct ompi_communicator_t **comm,
 
 
 void ompi_mpi_errors_are_fatal_file_handler(struct ompi_file_t **file,
-					    int *error_code, ...)
+                                            int *error_code, ...)
 {
   char *name;
   struct ompi_communicator_t *abort_comm;
@@ -93,7 +94,7 @@ void ompi_mpi_errors_are_fatal_file_handler(struct ompi_file_t **file,
 
 
 void ompi_mpi_errors_are_fatal_win_handler(struct ompi_win_t **win,
-					   int *error_code, ...)
+                                           int *error_code, ...)
 {
   char *name;
   struct ompi_communicator_t *abort_comm = NULL;
@@ -111,7 +112,7 @@ void ompi_mpi_errors_are_fatal_win_handler(struct ompi_win_t **win,
 }
 
 void ompi_mpi_errors_return_comm_handler(struct ompi_communicator_t **comm,
-					 int *error_code, ...)
+                                         int *error_code, ...)
 {
     /* Don't need anything more -- just need this function to exist */
     /* Silence some compiler warnings */
@@ -123,7 +124,7 @@ void ompi_mpi_errors_return_comm_handler(struct ompi_communicator_t **comm,
 
 
 void ompi_mpi_errors_return_file_handler(struct ompi_file_t **file,
-					 int *error_code, ...)
+                                         int *error_code, ...)
 {
     /* Don't need anything more -- just need this function to exist */
     /* Silence some compiler warnings */
@@ -135,7 +136,7 @@ void ompi_mpi_errors_return_file_handler(struct ompi_file_t **file,
 
 
 void ompi_mpi_errors_return_win_handler(struct ompi_win_t **win,
-					int *error_code, ...)
+                                        int *error_code, ...)
 {
     /* Don't need anything more -- just need this function to exist */
     /* Silence some compiler warnings */
@@ -181,6 +182,7 @@ static void backend_fatal_aggregate(char *type,
     const char* const unknown_error_code = "Error code: %d (no associated error message)";
     const char* const unknown_error = "Unknown error";
     const char* const unknown_prefix = "[?:?]";
+    bool generated = false;
 
     // these do not own what they point to; they're
     // here to avoid repeating expressions such as
@@ -209,6 +211,8 @@ static void backend_fatal_aggregate(char *type,
                 err_msg = NULL;
                 opal_output(0, "%s", "Could not write to err_msg");
                 opal_output(0, unknown_error_code, *error_code);
+            } else {
+                generated = true;
             }
         }
     }
@@ -254,7 +258,9 @@ static void backend_fatal_aggregate(char *type,
     }
 
     free(prefix);
-    free(err_msg);
+    if (generated) {
+        free(err_msg);
+    }
 }
 
 /*

ompi/mca/topo/treematch/topo_treematch_component.c

@@ -4,6 +4,7 @@
  *                         reserved.
  * Copyright (c) 2011-2015 INRIA.  All rights reserved.
  * Copyright (c) 2011-2015 Université Bordeaux 1
+ * Copyright (c) 2016      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -61,9 +62,6 @@ mca_topo_treematch_component_2_2_0_t mca_topo_treematch_component =
 
 static int init_query(bool enable_progress_threads, bool enable_mpi_threads)
 {
-    if(NULL == opal_hwloc_topology) {
-        return OPAL_ERR_NOT_SUPPORTED;
-    }
     return OMPI_SUCCESS;
 }
 
@@ -97,4 +95,3 @@ static int mca_topo_treematch_component_register(void)
                                           MCA_BASE_VAR_SCOPE_READONLY, &mca_topo_treematch_component.reorder_mode);
     return OMPI_SUCCESS;
 }
-

ompi/mca/topo/treematch/topo_treematch_dist_graph_create.c

@@ -5,7 +5,7 @@
  *                         reserved.
  * Copyright (c) 2011-2015 INRIA.  All rights reserved.
  * Copyright (c) 2012-2015 Bordeaux Poytechnic Institute
- * Copyright (c) 2015      Intel, Inc. All rights reserved
+ * Copyright (c) 2015-2016 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
@@ -256,7 +256,9 @@ int mca_topo_treematch_dist_graph_create(mca_topo_base_module_t* topo_module,
     /* Then, we need to know if the processes are bound */
     /* We make the hypothesis that all processes are in  */
     /* the same state : all bound or none bound */
-    assert(NULL != opal_hwloc_topology);
+    if (OPAL_SUCCESS != opal_hwloc_base_get_topology()) {
+        goto fallback;
+    }
     root_obj = hwloc_get_root_obj(opal_hwloc_topology);
     if (NULL == root_obj) goto fallback;
 
@@ -873,7 +875,7 @@ int mca_topo_treematch_dist_graph_create(mca_topo_base_module_t* topo_module,
         if( -1 == hwloc_err) goto fallback;
 
         /* Report new binding to ORTE/OPAL */
-        /*	hwloc_bitmap_list_asprintf(&orte_process_info.cpuset,set);   */
+        /*      hwloc_bitmap_list_asprintf(&orte_process_info.cpuset,set);   */
         err = hwloc_bitmap_snprintf (set_as_string,64,set);
 
 #ifdef __DEBUG__

ompi/runtime/ompi_mpi_init.c

@@ -508,16 +508,6 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
     /* check for timing request - get stop time and report elapsed time if so */
     OPAL_TIMING_MNEXT((&tm,"time from completion of rte_init to modex"));
 
-    /* if hwloc is available but didn't get setup for some
-     * reason, do so now
-     */
-    if (NULL == opal_hwloc_topology) {
-        if (OPAL_SUCCESS != (ret = opal_hwloc_base_get_topology())) {
-            error = "Topology init";
-            goto error;
-        }
-    }
-
     /* Register the default errhandler callback  */
     errtrk.status = OPAL_ERROR;
     errtrk.active = true;

opal/mca/btl/openib/btl_openib_component.c

@@ -18,7 +18,7 @@
  * Copyright (c) 2009-2012 Oracle and/or its affiliates.  All rights reserved.
  * Copyright (c) 2011-2015 NVIDIA Corporation.  All rights reserved.
  * Copyright (c) 2012      Oak Ridge National Laboratory.  All rights reserved
- * Copyright (c) 2013-2015 Intel, Inc. All rights reserved
+ * Copyright (c) 2013-2016 Intel, Inc.  All rights reserved.
  * Copyright (c) 2014-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014      Bull SAS.  All rights reserved.
@@ -1502,13 +1502,33 @@ static uint64_t read_module_param(char *file, uint64_t value, uint64_t max)
 static uint64_t calculate_total_mem (void)
 {
     hwloc_obj_t machine;
+    int rc;
+    uint64_t mem, *mptr;
+    opal_process_name_t wildcard_rank;
 
-    machine = hwloc_get_next_obj_by_type (opal_hwloc_topology, HWLOC_OBJ_MACHINE, NULL);
-    if (NULL == machine) {
-        return 0;
+    /* first try to retrieve it from PMIx as it may have
+     * been provided */
+    wildcard_rank.jobid = OPAL_PROC_MY_NAME.jobid;
+    wildcard_rank.vpid = OPAL_VPID_WILDCARD;
+    mptr = &mem;
+    OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, OPAL_PMIX_AVAIL_PHYS_MEMORY,
+                                   &wildcard_rank, &mptr, OPAL_UINT64);
+    if (OPAL_SUCCESS == rc) {
+        return mem;
+    }
+
+    /* if not available, then ensure that the topology has been
+     * loaded and try to get it from there */
+    if (OPAL_SUCCESS == opal_hwloc_base_get_topology()) {
+        machine = hwloc_get_next_obj_by_type (opal_hwloc_topology, HWLOC_OBJ_MACHINE, NULL);
+        if (NULL == machine) {
+            return 0;
+        }
+        return machine->memory.total_memory;
     }
 
-    return machine->memory.total_memory;
+    /* couldn't find it */
+    return 0;
 }
 
 
@@ -2312,7 +2332,8 @@ static float get_ib_dev_distance(struct ibv_device *dev)
     float distance = 0;
 
     /* Override any distance logic so all devices are used */
-    if (0 != mca_btl_openib_component.ignore_locality) {
+    if (0 != mca_btl_openib_component.ignore_locality ||
+        OPAL_SUCCESS != opal_hwloc_base_get_topology()) {
         return distance;
     }
 

opal/mca/btl/sm/btl_sm.c

@@ -52,7 +52,7 @@
 #include "opal/util/show_help.h"
 #include "opal/util/printf.h"
 #include "opal/mca/hwloc/base/base.h"
-#include "opal/mca/pmix/pmix.h"
+#include "opal/mca/pmix/base/base.h"
 #include "opal/mca/shmem/base/base.h"
 #include "opal/mca/shmem/shmem.h"
 
@@ -242,7 +242,7 @@ sm_btl_first_time_init(mca_btl_sm_t *sm_btl,
         free(loc);
     } else {
         /* If we have hwloc support, then get accurate information */
-        if (NULL != opal_hwloc_topology) {
+        if (OPAL_SUCCESS == opal_hwloc_base_get_topology()) {
             i = opal_hwloc_base_get_nbobjs_by_type(opal_hwloc_topology,
                                                    HWLOC_OBJ_NODE, 0,
                                                    OPAL_HWLOC_AVAILABLE);
@@ -257,6 +257,7 @@ sm_btl_first_time_init(mca_btl_sm_t *sm_btl,
         }
     }
     /* see if we were given our location */
+    loc = NULL;
     OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, OPAL_PMIX_LOCALITY_STRING,
                                    &OPAL_PROC_MY_NAME, &loc, OPAL_STRING);
     if (OPAL_SUCCESS == rc) {
@@ -283,8 +284,7 @@ sm_btl_first_time_init(mca_btl_sm_t *sm_btl,
         }
     } else {
         /* If we have hwloc support, then get accurate information */
-        if (NULL != opal_hwloc_topology && num_mem_nodes > 0 &&
-            NULL != opal_process_info.cpuset) {
+        if (OPAL_SUCCESS == opal_hwloc_base_get_topology() && num_mem_nodes > 0) {
             int numa=0, w;
             unsigned n_bound=0;
             hwloc_cpuset_t avail;