pml/cm: use the priority of the mtl component

This commit changes the priority of mtl components to be relative to
pml/ob1 and updates the mtl interface to expose this priority. cm now
sets its own priority based on the priority of the selected mtl
component.

(cherry picked from open-mpi/ompi@53f6b57)

Signed-off-by: Nathan Hjelm <[email protected]>

Author: hjelmn

ompi/mca/mtl/base/base.h

@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -9,6 +10,8 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2006 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -34,7 +37,8 @@ BEGIN_C_DECLS
 OMPI_DECLSPEC extern mca_mtl_base_component_t* ompi_mtl_base_selected_component;
 
 OMPI_DECLSPEC int ompi_mtl_base_select(bool enable_progress_threads,
-                                       bool enable_mpi_threads);
+                                       bool enable_mpi_threads,
+                                       int *priority);
 
 OMPI_DECLSPEC extern mca_base_framework_t ompi_mtl_base_framework;
 

ompi/mca/mtl/base/mtl_base_frame.c

@@ -47,8 +47,9 @@ mca_mtl_base_module_t *ompi_mtl = NULL;
  * need to reexamine this at a later time.
  */
 int
-ompi_mtl_base_select(bool enable_progress_threads,
-                     bool enable_mpi_threads)
+ompi_mtl_base_select (bool enable_progress_threads,
+                      bool enable_mpi_threads,
+                      int *priority)
 {
     int ret = OMPI_ERR_NOT_FOUND;
     mca_mtl_base_component_t *best_component = NULL;
@@ -82,6 +83,7 @@ ompi_mtl_base_select(bool enable_progress_threads,
                              "select: init returned success");
         ompi_mtl_base_selected_component = best_component;
         ompi_mtl = best_module;
+        *priority = best_priority;
         ret = OMPI_SUCCESS;
     }
 

ompi/mca/mtl/mxm/mtl_mxm_component.c

@@ -1,5 +1,9 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (C) Mellanox Technologies Ltd. 2001-2011.  ALL RIGHTS RESERVED.
+ * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -89,7 +93,7 @@ static int ompi_mtl_mxm_component_register(void)
                                            MCA_BASE_VAR_SCOPE_READONLY,
                                            &ompi_mtl_mxm.mxm_np);
 
-    param_priority = 100;
+    param_priority = 30;
     (void) mca_base_component_var_register (c,
                                             "priority", "Priority of the MXM MTL component",
                                             MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,

ompi/mca/mtl/ofi/mtl_ofi_component.c

@@ -56,7 +56,7 @@ mca_mtl_ofi_component_t mca_mtl_ofi_component = {
 static int
 ompi_mtl_ofi_component_register(void)
 {
-    param_priority = 10;   /* for now give a lower priority than the psm mtl */
+    param_priority = 10;   /* for now give a lower priority than the psm mtl and ob1 */
     mca_base_component_var_register(&mca_mtl_ofi_component.super.mtl_version,
                                     "priority", "Priority of the OFI MTL component",
                                     MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,

ompi/mca/mtl/psm/mtl_psm_component.c

@@ -90,7 +90,8 @@ ompi_mtl_psm_component_register(void)
 #endif
 
 
-    param_priority = 100;
+    /* set priority high enough to beat ob1's default */
+    param_priority = 30;
     (void) mca_base_component_var_register (&mca_mtl_psm_component.super.mtl_version,
                                             "priority", "Priority of the PSM MTL component",
                                             MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,

ompi/mca/mtl/psm2/mtl_psm2_component.c

@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -10,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2006-2010 QLogic Corporation. All rights reserved.
- * Copyright (c) 2012-2013 Los Alamos National Security, LLC.
+ * Copyright (c) 2012-2015 Los Alamos National Security, LLC.
  *                         All rights reserved.
  * Copyright (c) 2013-2015 Intel, Inc. All rights reserved
  * $COPYRIGHT$
@@ -86,7 +87,8 @@ ompi_mtl_psm2_component_register(void)
                                            MCA_BASE_VAR_SCOPE_READONLY,
                                            &ompi_mtl_psm2.connect_timeout);
 
-    param_priority = 120;
+    /* set priority high enough to beat ob1's default (also set higher than psm) */
+    param_priority = 40;
     (void) mca_base_component_var_register (&mca_mtl_psm2_component.super.mtl_version,
                                             "priority", "Priority of the PSM2 MTL component",
                                             MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,

ompi/mca/pml/cm/pml_cm.h

@@ -49,7 +49,6 @@ struct ompi_pml_cm_t {
     int                   free_list_num;
     int                   free_list_max;
     int                   free_list_inc;
-    int                   default_priority;
 };
 typedef struct ompi_pml_cm_t ompi_pml_cm_t;
 extern ompi_pml_cm_t ompi_pml_cm;

ompi/mca/pml/cm/pml_cm_component.c

@@ -104,14 +104,6 @@ mca_pml_cm_component_register(void)
                                            MCA_BASE_VAR_SCOPE_READONLY,
                                            &ompi_pml_cm.free_list_inc);
 
-    ompi_pml_cm.default_priority = 10;
-    (void) mca_base_component_var_register(&mca_pml_cm_component.pmlm_version, "priority",
-                                           "CM PML selection priority",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &ompi_pml_cm.default_priority);
-
     return OPAL_SUCCESS;
 }
 
@@ -146,26 +138,14 @@ mca_pml_cm_component_init(int* priority,
 {
     int ret;
 
+    *priority = -1;
+
     opal_output_verbose( 10, 0,
                          "in cm pml priority is %d\n", *priority);
     /* find a useable MTL */
-    ret = ompi_mtl_base_select(enable_progress_threads, enable_mpi_threads);
+    ret = ompi_mtl_base_select(enable_progress_threads, enable_mpi_threads, priority);
     if (OMPI_SUCCESS != ret) {
-        *priority = -1;
         return NULL;
-    } else if((strcmp(ompi_mtl_base_selected_component->mtl_version.mca_component_name, "psm") == 0) ||
-              (strcmp(ompi_mtl_base_selected_component->mtl_version.mca_component_name, "psm2") == 0) ||
-              (strcmp(ompi_mtl_base_selected_component->mtl_version.mca_component_name, "mxm") == 0) ||
-              (strcmp(ompi_mtl_base_selected_component->mtl_version.mca_component_name, "ofi") == 0) ||
-              (strcmp(ompi_mtl_base_selected_component->mtl_version.mca_component_name, "portals4") == 0)) {
-        /*
-         * If MTL is MXM or PSM then up our priority
-         * For every other communication layer having MTLs and BTLs, the user/admin
-         * may still select PML/ob1 (BTLs) or PML/cm (MTLs) if preferable for the app/site.
-         */
-        *priority = 30;
-    } else {
-        *priority = ompi_pml_cm.default_priority;
     }