Merge pull request open-mpi#703 from hjelmn/v2.x_cm_priority

Adjust the way cm's priority is set

Author: hppritcha

ompi/mca/crcp/base/crcp_base_select.c

@@ -162,7 +162,7 @@ int ompi_crcp_base_select(void)
     if( OPAL_SUCCESS != mca_base_select("crcp", ompi_crcp_base_framework.framework_output,
                                         &ompi_crcp_base_framework.framework_components,
                                         (mca_base_module_t **) &best_module,
-                                        (mca_base_component_t **) &best_component) ) {
+                                        (mca_base_component_t **) &best_component, NULL) ) {
         /* This will only happen if no component was selected */
         return OMPI_ERROR;
     }

ompi/mca/mtl/base/base.h

@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -9,6 +10,8 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2006 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -34,7 +37,8 @@ BEGIN_C_DECLS
 OMPI_DECLSPEC extern mca_mtl_base_component_t* ompi_mtl_base_selected_component;
 
 OMPI_DECLSPEC int ompi_mtl_base_select(bool enable_progress_threads,
-                                       bool enable_mpi_threads);
+                                       bool enable_mpi_threads,
+                                       int *priority);
 
 OMPI_DECLSPEC extern mca_base_framework_t ompi_mtl_base_framework;
 

ompi/mca/mtl/base/mtl_base_frame.c

@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -48,20 +49,23 @@ mca_mtl_base_module_t *ompi_mtl = NULL;
  * need to reexamine this at a later time.
  */
 int
-ompi_mtl_base_select(bool enable_progress_threads,
-                     bool enable_mpi_threads)
+ompi_mtl_base_select (bool enable_progress_threads,
+                      bool enable_mpi_threads,
+                      int *priority)
 {
     int ret = OMPI_ERR_NOT_FOUND;
     mca_mtl_base_component_t *best_component = NULL;
     mca_mtl_base_module_t *best_module = NULL;
+    int best_priority;
 
     /*
      * Select the best component
      */
     if( OPAL_SUCCESS != mca_base_select("mtl", ompi_mtl_base_framework.framework_output,
                                         &ompi_mtl_base_framework.framework_components,
                                         (mca_base_module_t **) &best_module,
-                                        (mca_base_component_t **) &best_component) ) {
+                                        (mca_base_component_t **) &best_component,
+                                        &best_priority) ) {
         /* notify caller that no available component found */
         return ret;
     }
@@ -81,6 +85,7 @@ ompi_mtl_base_select(bool enable_progress_threads,
                              "select: init returned success");
         ompi_mtl_base_selected_component = best_component;
         ompi_mtl = best_module;
+        *priority = best_priority;
         ret = OMPI_SUCCESS;
     }
 

ompi/mca/mtl/mxm/mtl_mxm_component.c

@@ -130,7 +130,8 @@ static int ompi_mtl_mxm_component_register(void)
     free(runtime_version);
 #endif
 
-    param_priority = 100;
+    /* set high enought to defeat ob1's default */
+    param_priority = 30;
     (void) mca_base_component_var_register (c,
                                             "priority", "Priority of the MXM MTL component",
                                             MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,

ompi/mca/mtl/ofi/mtl_ofi_component.c

@@ -56,7 +56,7 @@ mca_mtl_ofi_component_t mca_mtl_ofi_component = {
 static int
 ompi_mtl_ofi_component_register(void)
 {
-    param_priority = 10;   /* for now give a lower priority than the psm mtl */
+    param_priority = 25;   /* for now give a lower priority than the psm mtl */
     mca_base_component_var_register(&mca_mtl_ofi_component.super.mtl_version,
                                     "priority", "Priority of the OFI MTL component",
                                     MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,

ompi/mca/mtl/psm/mtl_psm_component.c

@@ -90,7 +90,8 @@ ompi_mtl_psm_component_register(void)
 #endif
 
 
-    param_priority = 100;
+    /* set priority high enough to beat ob1's default */
+    param_priority = 30;
     (void) mca_base_component_var_register (&mca_mtl_psm_component.super.mtl_version,
                                             "priority", "Priority of the PSM MTL component",
                                             MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,

ompi/mca/mtl/psm2/mtl_psm2_component.c

@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -10,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2006-2010 QLogic Corporation. All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.
+ * Copyright (c) 2012-2015 Los Alamos National Security, LLC.
  *                         All rights reserved.
  * Copyright (c) 2013-2015 Intel, Inc. All rights reserved
  * $COPYRIGHT$
@@ -85,7 +86,8 @@ ompi_mtl_psm2_component_register(void)
                                            MCA_BASE_VAR_SCOPE_READONLY,
                                            &ompi_mtl_psm2.connect_timeout);
 
-    param_priority = 120;
+    /* set priority high enough to beat ob1's default (also set higher than psm) */
+    param_priority = 40;
     (void) mca_base_component_var_register (&mca_mtl_psm2_component.super.mtl_version,
                                             "priority", "Priority of the PSM2 MTL component",
                                             MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,

ompi/mca/pml/cm/pml_cm.h

@@ -50,7 +50,6 @@ struct ompi_pml_cm_t {
     int                   free_list_num;
     int                   free_list_max;
     int                   free_list_inc;
-    int                   default_priority;
 };
 typedef struct ompi_pml_cm_t ompi_pml_cm_t;
 extern ompi_pml_cm_t ompi_pml_cm;

ompi/mca/pml/cm/pml_cm_component.c

@@ -101,14 +101,6 @@ mca_pml_cm_component_register(void)
                                            MCA_BASE_VAR_SCOPE_READONLY,
                                            &ompi_pml_cm.free_list_inc);
 
-    ompi_pml_cm.default_priority = 10;
-    (void) mca_base_component_var_register(&mca_pml_cm_component.pmlm_version, "priority",
-                                           "CM PML selection priority",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &ompi_pml_cm.default_priority);
-
     return OPAL_SUCCESS;
 }
 
@@ -143,32 +135,16 @@ mca_pml_cm_component_init(int* priority,
 {
     int ret;
 
-    if((*priority) > ompi_pml_cm.default_priority) {
-        *priority = ompi_pml_cm.default_priority;
-        return NULL;
-    }
-    *priority = ompi_pml_cm.default_priority;
+    *priority = -1;
+
     opal_output_verbose( 10, 0,
                          "in cm pml priority is %d\n", *priority);
     /* find a useable MTL */
-    ret = ompi_mtl_base_select(enable_progress_threads, enable_mpi_threads);
+    ret = ompi_mtl_base_select(enable_progress_threads, enable_mpi_threads, priority);
     if (OMPI_SUCCESS != ret) {
-        *priority = -1;
         return NULL;
-    } else if((strcmp(ompi_mtl_base_selected_component->mtl_version.mca_component_name, "psm") == 0) ||
-              (strcmp(ompi_mtl_base_selected_component->mtl_version.mca_component_name, "psm2") == 0) ||
-              (strcmp(ompi_mtl_base_selected_component->mtl_version.mca_component_name, "mxm") == 0) ||
-              (strcmp(ompi_mtl_base_selected_component->mtl_version.mca_component_name, "ofi") == 0) ||
-              (strcmp(ompi_mtl_base_selected_component->mtl_version.mca_component_name, "portals4") == 0)) {
-        /*
-         * If MTL is MXM or PSM then up our priority
-         * For every other communication layer having MTLs and BTLs, the user/admin
-         * may still select PML/ob1 (BTLs) or PML/cm (MTLs) if preferable for the app/site.
-         */
-        *priority = 30;
     }
 
-
     if (ompi_mtl->mtl_flags & MCA_MTL_BASE_FLAG_REQUIRE_WORLD) {
         ompi_pml_cm.super.pml_flags |= MCA_PML_BASE_FLAG_REQUIRE_WORLD;
     }

ompi/mca/pml/ob1/pml_ob1_component.c

@@ -254,10 +254,6 @@ mca_pml_ob1_component_init( int* priority,
     opal_output_verbose( 10, mca_pml_ob1_output,
                          "in ob1, my priority is %d\n", mca_pml_ob1.priority);
 
-    if((*priority) > mca_pml_ob1.priority) {
-        *priority = mca_pml_ob1.priority;
-        return NULL;
-    }
     *priority = mca_pml_ob1.priority;
 
     allocator_component = mca_allocator_component_lookup( mca_pml_ob1.allocator_name );