Allow individual jobs to set their map/rank/bind policies

Override the defaults when provided. Ignore LSF binding file if user
overrides by specifying a policy.

Fixes open-mpi#6631

Signed-off-by: Ralph Castain <[email protected]>

Author: rhc54

orte/mca/ras/lsf/ras_lsf_module.c

@@ -10,7 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2007-2017 Cisco Systems, Inc.  All rights reserved
- * Copyright (c) 2014      Intel, Inc. All rights reserved
+ * Copyright (c) 2014-2019 Intel, Inc.  All rights reserved.
  * Copyright (c) 2016      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
@@ -70,6 +70,7 @@ static int allocate(orte_job_t *jdata, opal_list_t *nodes)
     char *affinity_file;
     struct stat buf;
     char *ptr;
+    bool directives_given = false;
 
     /* get the list of allocated nodes */
     if ((num_nodes = lsb_getalloc(&nodelist)) < 0) {
@@ -112,8 +113,19 @@ static int allocate(orte_job_t *jdata, opal_list_t *nodes)
     /* release the nodelist from lsf */
     opal_argv_free(nodelist);
 
+    /* check to see if any mapping or binding directives were given */
+    if (NULL != jdata && NULL != jdata->map) {
+        if ((ORTE_MAPPING_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) ||
+            OPAL_BINDING_POLICY_IS_SET(jdata->map->binding)) {
+            directives_given = true;
+        }
+    } else if ((ORTE_MAPPING_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping)) ||
+               OPAL_BINDING_POLICY_IS_SET(opal_hwloc_binding_policy) {
+            directives_given = true;
+    }
+
     /* check for an affinity file */
-    if (NULL != (affinity_file = getenv("LSB_AFFINITY_HOSTFILE"))) {
+    if (!directives_given && NULL != (affinity_file = getenv("LSB_AFFINITY_HOSTFILE"))) {
         /* check to see if the file is empty - if it is,
          * then affinity wasn't actually set for this job */
         if (0 != stat(affinity_file, &buf)) {

orte/mca/rmaps/base/rmaps_base_frame.c

@@ -557,6 +557,7 @@ static int check_modifiers(char *ck, orte_mapping_policy_t *tmp)
     for (i=0; NULL != ck2[i]; i++) {
         if (0 == strncasecmp(ck2[i], "span", strlen(ck2[i]))) {
             ORTE_SET_MAPPING_DIRECTIVE(*tmp, ORTE_MAPPING_SPAN);
+            ORTE_SET_MAPPING_DIRECTIVE(*tmp, ORTE_MAPPING_GIVEN);
             found = true;
         } else if (0 == strncasecmp(ck2[i], "pe", strlen("pe"))) {
             /* break this at the = sign to get the number */

orte/orted/pmix/pmix_server_dyn.c

@@ -304,13 +304,6 @@ int pmix_server_spawn_fn(opal_process_name_t *requestor,
 
         /***   MAP-BY   ***/
         } else if (0 == strcmp(info->key, OPAL_PMIX_MAPBY)) {
-            if (ORTE_MAPPING_POLICY_IS_SET(jdata->map->mapping)) {
-                /* not allowed to provide multiple mapping policies */
-                orte_show_help("help-orte-rmaps-base.txt", "redefining-policy",
-                               true, "mapping", info->data.string,
-                               orte_rmaps_base_print_mapping(orte_rmaps_base.mapping));
-                return ORTE_ERR_BAD_PARAM;
-            }
             rc = orte_rmaps_base_set_mapping_policy(jdata, &jdata->map->mapping,
                                                     NULL, info->data.string);
             if (ORTE_SUCCESS != rc) {
@@ -319,13 +312,6 @@ int pmix_server_spawn_fn(opal_process_name_t *requestor,
 
         /***   RANK-BY   ***/
         } else if (0 == strcmp(info->key, OPAL_PMIX_RANKBY)) {
-            if (ORTE_RANKING_POLICY_IS_SET(jdata->map->ranking)) {
-                /* not allowed to provide multiple ranking policies */
-                orte_show_help("help-orte-rmaps-base.txt", "redefining-policy",
-                               true, "ranking", info->data.string,
-                               orte_rmaps_base_print_ranking(orte_rmaps_base.ranking));
-                return ORTE_ERR_BAD_PARAM;
-            }
             rc = orte_rmaps_base_set_ranking_policy(&jdata->map->ranking,
                                                     jdata->map->mapping,
                                                     info->data.string);
@@ -335,13 +321,6 @@ int pmix_server_spawn_fn(opal_process_name_t *requestor,
 
         /***   BIND-TO   ***/
         } else if (0 == strcmp(info->key, OPAL_PMIX_BINDTO)) {
-            if (OPAL_BINDING_POLICY_IS_SET(jdata->map->binding)) {
-                /* not allowed to provide multiple mapping policies */
-                orte_show_help("help-opal-hwloc-base.txt", "redefining-policy", true,
-                               info->data.string,
-                               opal_hwloc_base_print_binding(opal_hwloc_binding_policy));
-                return ORTE_ERR_BAD_PARAM;
-            }
             rc = opal_hwloc_base_set_binding_policy(&jdata->map->binding,
                                                     info->data.string);
             if (ORTE_SUCCESS != rc) {