From 40903140d0ef5a65e7693764297458366aa95613 Mon Sep 17 00:00:00 2001 From: Joshua Hursey Date: Thu, 25 Aug 2016 13:57:00 -0400 Subject: [PATCH 1/3] orte: Expand the application of !orte_keep_fqdn_hostnames * Expand the use of the `orte_keep_fqdn_hostnames` MCA parameter when it is set to false. * If that parameter is set to false (default) then short hostnames (e.g., `node01`) will match with the long hostnames (e.g., `node01.mycluster.org`). This allows a user (or resource manager) to mix the use of short and long hostnames. - Note that this mechanism does _not_ perform a DNS lookup, but instead strips off the FQDN by truncating the hostname string at the first `.` character (when not an IP address). - By default (`false`) the following is true: `node01 == node01.mycluster.org == node01.bogus.com` since we use `node01` as the hostname. (cherry picked from commit open-mpi/ompi@d26dd2c20e19be864bd5a2e20770d1773fd75406) --- orte/mca/plm/base/plm_base_launch_support.c | 3 +- orte/mca/rmaps/base/rmaps_base_support_fns.c | 3 +- orte/mca/rmaps/rank_file/rmaps_rank_file.c | 18 +++++++ orte/mca/rmaps/seq/rmaps_seq.c | 51 +++++++++++++++++++- orte/util/dash_host/dash_host.c | 17 +++++++ orte/util/hostfile/hostfile.c | 31 ++++++++++++ 6 files changed, 119 insertions(+), 4 deletions(-) diff --git a/orte/mca/plm/base/plm_base_launch_support.c b/orte/mca/plm/base/plm_base_launch_support.c index 2c827fb24a..c159fe5757 100644 --- a/orte/mca/plm/base/plm_base_launch_support.c +++ b/orte/mca/plm/base/plm_base_launch_support.c @@ -16,6 +16,7 @@ * Copyright (c) 2013-2015 Intel, Inc. All rights reserved. * Copyright (c) 2014-2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -1491,7 +1492,7 @@ int orte_plm_base_setup_virtual_machine(orte_job_t *jdata) bool one_filter = false; int num_nodes; bool default_hostfile_used; - char *hosts; + char *hosts = NULL; bool singleton=false; OPAL_OUTPUT_VERBOSE((5, orte_plm_base_framework.framework_output, diff --git a/orte/mca/rmaps/base/rmaps_base_support_fns.c b/orte/mca/rmaps/base/rmaps_base_support_fns.c index 5c41004541..96997481de 100644 --- a/orte/mca/rmaps/base/rmaps_base_support_fns.c +++ b/orte/mca/rmaps/base/rmaps_base_support_fns.c @@ -13,6 +13,7 @@ * Copyright (c) 2011-2012 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2014 Intel, Inc. All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -147,7 +148,7 @@ int orte_rmaps_base_get_target_nodes(opal_list_t *allocated_nodes, orte_std_cntr orte_job_t *daemons; bool novm; opal_list_t nodes; - char *hosts; + char *hosts = NULL; /** set default answer */ *total_num_slots = 0; diff --git a/orte/mca/rmaps/rank_file/rmaps_rank_file.c b/orte/mca/rmaps/rank_file/rmaps_rank_file.c index 48439551b1..a00d5a7a6e 100644 --- a/orte/mca/rmaps/rank_file/rmaps_rank_file.c +++ b/orte/mca/rmaps/rank_file/rmaps_rank_file.c @@ -17,6 +17,7 @@ * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. * * $COPYRIGHT$ * @@ -33,6 +34,9 @@ #ifdef HAVE_UNISTD_H #include #endif /* HAVE_UNISTD_H */ +#if HAVE_ARPA_INET_H +#include +#endif #include #include "opal/util/argv.h" @@ -489,6 +493,20 @@ static int orte_rmaps_rank_file_parse(const char *rankfile) goto unlock; } opal_argv_free (argv); + + // Strip off the FQDN if present + if( !orte_keep_fqdn_hostnames ) { + char *ptr; + struct in_addr buf; + /* if the nodename is an IP address, do not mess with it! */ + if (0 == inet_pton(AF_INET, node_name, &buf) && + 0 == inet_pton(AF_INET6, node_name, &buf)) { + if (NULL != (ptr = strchr(node_name, '.'))) { + *ptr = '\0'; + } + } + } + /* check the rank item */ if (NULL == rfmap) { orte_show_help("help-rmaps_rank_file.txt", "bad-syntax", true, rankfile); diff --git a/orte/mca/rmaps/seq/rmaps_seq.c b/orte/mca/rmaps/seq/rmaps_seq.c index 0f7f5d24bf..d2da3c5035 100644 --- a/orte/mca/rmaps/seq/rmaps_seq.c +++ b/orte/mca/rmaps/seq/rmaps_seq.c @@ -15,6 +15,7 @@ * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -70,9 +71,11 @@ static void sn_des(seq_node_t *p) { if (NULL != p->hostname) { free(p->hostname); + p->hostname = NULL; } if (NULL != p->cpuset) { free(p->cpuset); + p->cpuset = NULL; } } OBJ_CLASS_INSTANCE(seq_node_t, @@ -101,7 +104,7 @@ static int orte_rmaps_seq_map(orte_job_t *jdata) opal_list_t node_list, *seq_list, sq_list; orte_proc_t *proc; mca_base_component_t *c = &mca_rmaps_seq_component.base_version; - char *hosts, *sep, *eptr; + char *hosts = NULL, *sep, *eptr; FILE *fp; opal_hwloc_resource_type_t rtype; @@ -156,7 +159,7 @@ static int orte_rmaps_seq_map(orte_job_t *jdata) /* if there is a default hostfile, go and get its ordered list of nodes */ OBJ_CONSTRUCT(&default_seq_list, opal_list_t); if (NULL != orte_default_hostfile) { - char *hstname; + char *hstname = NULL; /* open the file */ fp = fopen(orte_default_hostfile, "r"); if (NULL == fp) { @@ -170,6 +173,11 @@ static int orte_rmaps_seq_map(orte_job_t *jdata) /* blank line - ignore */ continue; } + if( '#' == hstname[0] ) { + free(hstname); + /* Comment line - ignore */ + continue; + } sq = OBJ_NEW(seq_node_t); if (NULL != (sep = strchr(hstname, ' '))) { *sep = '\0'; @@ -182,6 +190,21 @@ static int orte_rmaps_seq_map(orte_job_t *jdata) *(eptr+1) = 0; sq->cpuset = strdup(sep); } + + // Strip off the FQDN if present + if( !orte_keep_fqdn_hostnames ) { + char *ptr; + struct in_addr buf; + + /* if the nodename is an IP address, do not mess with it! */ + if (0 == inet_pton(AF_INET, hstname, &buf) && + 0 == inet_pton(AF_INET6, hstname, &buf)) { + if (NULL != (ptr = strchr(hstname, '.'))) { + *ptr = '\0'; + } + } + } + sq->hostname = hstname; opal_list_append(&default_seq_list, &sq->super); } @@ -255,6 +278,16 @@ static int orte_rmaps_seq_map(orte_job_t *jdata) goto error; } while (NULL != (hstname = orte_getline(fp))) { + if (0 == strlen(hstname)) { + free(hstname); + /* blank line - ignore */ + continue; + } + if( '#' == hstname[0] ) { + free(hstname); + /* Comment line - ignore */ + continue; + } sq = OBJ_NEW(seq_node_t); if (NULL != (sep = strchr(hstname, ' '))) { *sep = '\0'; @@ -267,6 +300,20 @@ static int orte_rmaps_seq_map(orte_job_t *jdata) *(eptr+1) = 0; sq->cpuset = strdup(sep); } + + // Strip off the FQDN if present + if( !orte_keep_fqdn_hostnames ) { + char *ptr; + struct in_addr buf; + /* if the nodename is an IP address, do not mess with it! */ + if (0 == inet_pton(AF_INET, hstname, &buf) && + 0 == inet_pton(AF_INET6, hstname, &buf)) { + if (NULL != (ptr = strchr(hstname, '.'))) { + (*ptr) = '\0'; + } + } + } + sq->hostname = hstname; opal_list_append(&sq_list, &sq->super); } diff --git a/orte/util/dash_host/dash_host.c b/orte/util/dash_host/dash_host.c index b81d791a62..d38ede035a 100644 --- a/orte/util/dash_host/dash_host.c +++ b/orte/util/dash_host/dash_host.c @@ -13,6 +13,7 @@ * Copyright (c) 2014-2015 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -23,6 +24,9 @@ #include "orte_config.h" #include +#if HAVE_ARPA_INET_H +#include +#endif #include "orte/constants.h" #include "orte/types.h" @@ -207,6 +211,19 @@ int orte_util_add_dash_host_nodes(opal_list_t *nodes, ndname = mini_map[i]; } + // Strip off the FQDN if present + if( !orte_keep_fqdn_hostnames ) { + char *ptr; + struct in_addr buf; + /* if the nodename is an IP address, do not mess with it! */ + if (0 == inet_pton(AF_INET, ndname, &buf) && + 0 == inet_pton(AF_INET6, ndname, &buf)) { + if (NULL != (ptr = strchr(ndname, '.'))) { + *ptr = '\0'; + } + } + } + /* see if the node is already on the list */ found = false; OPAL_LIST_FOREACH(node, &adds, orte_node_t) { diff --git a/orte/util/hostfile/hostfile.c b/orte/util/hostfile/hostfile.c index 9a68791f31..d3a68b3211 100644 --- a/orte/util/hostfile/hostfile.c +++ b/orte/util/hostfile/hostfile.c @@ -15,6 +15,7 @@ * Copyright (c) 2013-2014 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -27,6 +28,9 @@ #ifdef HAVE_UNISTD_H #include #endif +#if HAVE_ARPA_INET_H +#include +#endif #include #include #include @@ -164,6 +168,19 @@ static int hostfile_parse_line(int token, opal_list_t* updates, } opal_argv_free (argv); + // Strip off the FQDN if present + if( !orte_keep_fqdn_hostnames ) { + char *ptr; + struct in_addr buf; + /* if the nodename is an IP address, do not mess with it! */ + if (0 == inet_pton(AF_INET, node_name, &buf) && + 0 == inet_pton(AF_INET6, node_name, &buf)) { + if (NULL != (ptr = strchr(node_name, '.'))) { + *ptr = '\0'; + } + } + } + /* if the first letter of the name is '^', then this is a node * to be excluded. Remove the ^ character so the nodename is * usable, and put it on the exclude list @@ -270,6 +287,20 @@ static int hostfile_parse_line(int token, opal_list_t* updates, opal_output(0, "WARNING: Unhandled user@host-combination\n"); /* XXX */ } opal_argv_free (argv); + + // Strip off the FQDN if present + if( !orte_keep_fqdn_hostnames ) { + char *ptr; + struct in_addr buf; + /* if the nodename is an IP address, do not mess with it! */ + if (0 == inet_pton(AF_INET, node_name, &buf) && + 0 == inet_pton(AF_INET6, node_name, &buf)) { + if (NULL != (ptr = strchr(node_name, '.'))) { + *ptr = '\0'; + } + } + } + /* Do we need to make a new node object? */ if (NULL == (node = hostfile_lookup(updates, node_name))) { node = OBJ_NEW(orte_node_t); From cd6b7162ea5e95d1e1d0613e76d74385ccbe5f4f Mon Sep 17 00:00:00 2001 From: Joshua Hursey Date: Fri, 2 Sep 2016 12:33:34 -0400 Subject: [PATCH 2/3] orte: !FQDN implementation to use opal_net_isaddr * Switch to use opal_net_isaddr() for checking if a name is an IP address - as it is a bit cleaner, and uses common functionality. (cherry picked from commit open-mpi/ompi@fe937d1e823af9a5916d79955ac62a45cb504096) --- orte/mca/rmaps/rank_file/rmaps_rank_file.c | 17 ++++-------- orte/mca/rmaps/seq/rmaps_seq.c | 28 +++++++------------- orte/util/dash_host/dash_host.c | 17 ++++-------- orte/util/hostfile/hostfile.c | 30 +++++++--------------- orte/util/proc_info.c | 20 +++++---------- 5 files changed, 35 insertions(+), 77 deletions(-) diff --git a/orte/mca/rmaps/rank_file/rmaps_rank_file.c b/orte/mca/rmaps/rank_file/rmaps_rank_file.c index a00d5a7a6e..a3195a6147 100644 --- a/orte/mca/rmaps/rank_file/rmaps_rank_file.c +++ b/orte/mca/rmaps/rank_file/rmaps_rank_file.c @@ -34,13 +34,11 @@ #ifdef HAVE_UNISTD_H #include #endif /* HAVE_UNISTD_H */ -#if HAVE_ARPA_INET_H -#include -#endif #include #include "opal/util/argv.h" #include "opal/util/if.h" +#include "opal/util/net.h" #include "opal/class/opal_pointer_array.h" #include "opal/mca/hwloc/base/base.h" @@ -494,16 +492,11 @@ static int orte_rmaps_rank_file_parse(const char *rankfile) } opal_argv_free (argv); - // Strip off the FQDN if present - if( !orte_keep_fqdn_hostnames ) { + // Strip off the FQDN if present, ignore IP addresses + if( !orte_keep_fqdn_hostnames && !opal_net_isaddr(node_name) ) { char *ptr; - struct in_addr buf; - /* if the nodename is an IP address, do not mess with it! */ - if (0 == inet_pton(AF_INET, node_name, &buf) && - 0 == inet_pton(AF_INET6, node_name, &buf)) { - if (NULL != (ptr = strchr(node_name, '.'))) { - *ptr = '\0'; - } + if (NULL != (ptr = strchr(node_name, '.'))) { + *ptr = '\0'; } } diff --git a/orte/mca/rmaps/seq/rmaps_seq.c b/orte/mca/rmaps/seq/rmaps_seq.c index d2da3c5035..13fc21c451 100644 --- a/orte/mca/rmaps/seq/rmaps_seq.c +++ b/orte/mca/rmaps/seq/rmaps_seq.c @@ -35,6 +35,7 @@ #include #include "opal/util/if.h" +#include "opal/util/net.h" #include "opal/mca/hwloc/hwloc.h" #include "orte/util/show_help.h" @@ -191,17 +192,11 @@ static int orte_rmaps_seq_map(orte_job_t *jdata) sq->cpuset = strdup(sep); } - // Strip off the FQDN if present - if( !orte_keep_fqdn_hostnames ) { + // Strip off the FQDN if present, ignore IP addresses + if( !orte_keep_fqdn_hostnames && !opal_net_isaddr(hstname) ) { char *ptr; - struct in_addr buf; - - /* if the nodename is an IP address, do not mess with it! */ - if (0 == inet_pton(AF_INET, hstname, &buf) && - 0 == inet_pton(AF_INET6, hstname, &buf)) { - if (NULL != (ptr = strchr(hstname, '.'))) { - *ptr = '\0'; - } + if (NULL != (ptr = strchr(hstname, '.'))) { + *ptr = '\0'; } } @@ -301,16 +296,11 @@ static int orte_rmaps_seq_map(orte_job_t *jdata) sq->cpuset = strdup(sep); } - // Strip off the FQDN if present - if( !orte_keep_fqdn_hostnames ) { + // Strip off the FQDN if present, ignore IP addresses + if( !orte_keep_fqdn_hostnames && !opal_net_isaddr(hstname) ) { char *ptr; - struct in_addr buf; - /* if the nodename is an IP address, do not mess with it! */ - if (0 == inet_pton(AF_INET, hstname, &buf) && - 0 == inet_pton(AF_INET6, hstname, &buf)) { - if (NULL != (ptr = strchr(hstname, '.'))) { - (*ptr) = '\0'; - } + if (NULL != (ptr = strchr(hstname, '.'))) { + (*ptr) = '\0'; } } diff --git a/orte/util/dash_host/dash_host.c b/orte/util/dash_host/dash_host.c index d38ede035a..c5e3ac3e0a 100644 --- a/orte/util/dash_host/dash_host.c +++ b/orte/util/dash_host/dash_host.c @@ -24,9 +24,6 @@ #include "orte_config.h" #include -#if HAVE_ARPA_INET_H -#include -#endif #include "orte/constants.h" #include "orte/types.h" @@ -34,6 +31,7 @@ #include "orte/util/show_help.h" #include "opal/util/argv.h" #include "opal/util/if.h" +#include "opal/util/net.h" #include "orte/mca/ras/base/base.h" #include "orte/mca/plm/plm_types.h" @@ -211,16 +209,11 @@ int orte_util_add_dash_host_nodes(opal_list_t *nodes, ndname = mini_map[i]; } - // Strip off the FQDN if present - if( !orte_keep_fqdn_hostnames ) { + // Strip off the FQDN if present, ignore IP addresses + if( !orte_keep_fqdn_hostnames && !opal_net_isaddr(ndname) ) { char *ptr; - struct in_addr buf; - /* if the nodename is an IP address, do not mess with it! */ - if (0 == inet_pton(AF_INET, ndname, &buf) && - 0 == inet_pton(AF_INET6, ndname, &buf)) { - if (NULL != (ptr = strchr(ndname, '.'))) { - *ptr = '\0'; - } + if (NULL != (ptr = strchr(ndname, '.'))) { + *ptr = '\0'; } } diff --git a/orte/util/hostfile/hostfile.c b/orte/util/hostfile/hostfile.c index d3a68b3211..286e3acce0 100644 --- a/orte/util/hostfile/hostfile.c +++ b/orte/util/hostfile/hostfile.c @@ -28,9 +28,6 @@ #ifdef HAVE_UNISTD_H #include #endif -#if HAVE_ARPA_INET_H -#include -#endif #include #include #include @@ -41,6 +38,7 @@ #include "opal/mca/mca.h" #include "opal/mca/base/base.h" #include "opal/util/if.h" +#include "opal/util/net.h" #include "opal/mca/installdirs/installdirs.h" #include "orte/util/show_help.h" @@ -168,16 +166,11 @@ static int hostfile_parse_line(int token, opal_list_t* updates, } opal_argv_free (argv); - // Strip off the FQDN if present - if( !orte_keep_fqdn_hostnames ) { + // Strip off the FQDN if present, ignore IP addresses + if( !orte_keep_fqdn_hostnames && !opal_net_isaddr(node_name) ) { char *ptr; - struct in_addr buf; - /* if the nodename is an IP address, do not mess with it! */ - if (0 == inet_pton(AF_INET, node_name, &buf) && - 0 == inet_pton(AF_INET6, node_name, &buf)) { - if (NULL != (ptr = strchr(node_name, '.'))) { - *ptr = '\0'; - } + if (NULL != (ptr = strchr(node_name, '.'))) { + *ptr = '\0'; } } @@ -288,16 +281,11 @@ static int hostfile_parse_line(int token, opal_list_t* updates, } opal_argv_free (argv); - // Strip off the FQDN if present - if( !orte_keep_fqdn_hostnames ) { + // Strip off the FQDN if present, ignore IP addresses + if( !orte_keep_fqdn_hostnames && !opal_net_isaddr(node_name) ) { char *ptr; - struct in_addr buf; - /* if the nodename is an IP address, do not mess with it! */ - if (0 == inet_pton(AF_INET, node_name, &buf) && - 0 == inet_pton(AF_INET6, node_name, &buf)) { - if (NULL != (ptr = strchr(node_name, '.'))) { - *ptr = '\0'; - } + if (NULL != (ptr = strchr(node_name, '.'))) { + *ptr = '\0'; } } diff --git a/orte/util/proc_info.c b/orte/util/proc_info.c index 5a73493515..45ad9c6471 100644 --- a/orte/util/proc_info.c +++ b/orte/util/proc_info.c @@ -13,6 +13,7 @@ * Copyright (c) 2012 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2014-2015 Intel, Inc. All rights reserved + * Copyright (c) 2016 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -32,9 +33,6 @@ #ifdef HAVE_SYS_TYPES_H #include #endif -#if HAVE_ARPA_INET_H -#include -#endif #include #include "opal/mca/base/base.h" @@ -172,16 +170,12 @@ int orte_proc_info(void) /* add this to our list of aliases */ opal_argv_append_nosize(&orte_process_info.aliases, hostname); - if (!orte_keep_fqdn_hostnames) { - /* if the nodename is an IP address, do not mess with it! */ - if (0 == inet_pton(AF_INET, hostname, &buf) && - 0 == inet_pton(AF_INET6, hostname, &buf)) { - /* not an IP address, so remove any domain info */ - if (NULL != (ptr = strchr(hostname, '.'))) { - *ptr = '\0'; - /* add this to our list of aliases */ - opal_argv_append_nosize(&orte_process_info.aliases, hostname); - } + // Strip off the FQDN if present, ignore IP addresses + if( !orte_keep_fqdn_hostnames && !opal_net_isaddr(hostname) ) { + if (NULL != (ptr = strchr(hostname, '.'))) { + *ptr = '\0'; + /* add this to our list of aliases */ + opal_argv_append_nosize(&orte_process_info.aliases, hostname); } } From 112ecc61c5e6b18810dad2f80301c4e209840c73 Mon Sep 17 00:00:00 2001 From: Jeff Squyres Date: Tue, 6 Sep 2016 16:38:15 -0700 Subject: [PATCH 3/3] orte proc_info.c: remove unused variable Signed-off-by: Jeff Squyres (cherry picked from commit open-mpi/ompi@722d5eecf10b993a31ead2025cf83d3d320d66fc) --- orte/util/proc_info.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/orte/util/proc_info.c b/orte/util/proc_info.c index 45ad9c6471..34339a50b0 100644 --- a/orte/util/proc_info.c +++ b/orte/util/proc_info.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2009 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2009-2016 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2014-2015 Intel, Inc. All rights reserved @@ -100,7 +100,6 @@ int orte_proc_info(void) char hostname[OPAL_MAXHOSTNAMELEN]; char **prefixes; bool match; - struct in_addr buf; if (init) { return ORTE_SUCCESS;