From da82bcb9592ed1e1ff6df58eb689ef27aefd581f Mon Sep 17 00:00:00 2001 From: Jeff Squyres Date: Tue, 14 Jun 2016 07:45:06 -0700 Subject: [PATCH] PSM/PSM2: Disable signal handler hijacking by default Per discussion on https://github.com/open-mpi/ompi/pull/1767 (and some subsequent phone calls and off-issue email discussions), the PSM library is hijacking signal handlers by default. Specifically: unless the environment variables `IPATH_NO_BACKTRACE=1` (for PSM / Intel TrueScale) is set, the library constructor for this library will hijack various signal handlers for the purpose of invoking its own error reporting mechanisms. This may be a bit *surprising*, but is not a *problem*, per se. The real problem is that older versions of at least the PSM library do not unregister these signal handlers upon being unloaded from memory. Hence, a segv can actually result in a double segv (i.e., the original segv and then another segv when the now-non-existent signal handler is invoked). This PSM signal hijacking subverts Open MPI's own signal reporting mechanism, which may be a bit surprising for some users (particularly those who do not have Intel TrueScale). As such, we disable it by default so that Open MPI's own error-reporting mechanisms are used. Additionally, there is a typo in the library destructor for the PSM2 library that may cause problems in the unloading of its signal handlers. This problem can be avoided by setting `HFI_NO_BACKTRACE=1` (for PSM2 / Intel OmniPath). This is further compounded by the fact that the PSM / PSM2 libraries can be loaded by the OFI MTL and the usNIC BTL (because they are loaded by libfabric), even when there is no Intel networking hardware present. Having the PSM/PSM2 libraries behave this way when no Intel hardware is present is clearly undesirable (and is likely to be fixed in future releases of the PSM/PSM2 libraries). This commit sets the following two environment variables to disable this behavior from the PSM/PSM2 libraries (if they are not already set): * IPATH_NO_BACKTRACE=1 * HFI_NO_BACKTRACE=1 If the user has set these variables before invoking Open MPI, we will not override their values (i.e., their preferences will be honored). Signed-off-by: Jeff Squyres (cherry picked from commit open-mpi/ompi@5071602c59bdb72fca0e78398eae3184b7ea1433) --- ompi/mpi/java/c/mpi_MPI.c | 6 +++++- opal/runtime/opal.h | 20 ++++++++++++++------ opal/runtime/opal_init.c | 34 +++++++++++++++++++++++++++++++++- 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/ompi/mpi/java/c/mpi_MPI.c b/ompi/mpi/java/c/mpi_MPI.c index 3c2a22a23a..f239e0a663 100644 --- a/ompi/mpi/java/c/mpi_MPI.c +++ b/ompi/mpi/java/c/mpi_MPI.c @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2015-2016 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2015 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -135,6 +135,10 @@ OBJ_CLASS_INSTANCE(ompi_java_buffer_t, */ jint JNI_OnLoad(JavaVM *vm, void *reserved) { + // Ensure that PSM signal hijacking is disabled *before* loading + // the library (see comment in the function for more detail). + opal_init_psm(); + libmpi = dlopen("libmpi." OPAL_DYN_LIB_SUFFIX, RTLD_NOW | RTLD_GLOBAL); #if defined(HAVE_DL_INFO) && defined(HAVE_LIBGEN_H) diff --git a/opal/runtime/opal.h b/opal/runtime/opal.h index ccec2f7fcf..df663a87d9 100644 --- a/opal/runtime/opal.h +++ b/opal/runtime/opal.h @@ -6,18 +6,19 @@ * Copyright (c) 2004-2007 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. - * Copyright (c) 2010-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2010-2016 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. + * Copyright (c) 2014 Intel, Inc. All rights reserved. * $COPYRIGHT$ - * + * * Additional copyrights may follow - * + * * $HEADER$ */ @@ -53,7 +54,7 @@ OPAL_DECLSPEC extern int opal_cache_line_size; OPAL_DECLSPEC int opal_init(int* pargc, char*** pargv); /** - * Finalize the OPAL layer, including the MCA system. + * Finalize the OPAL layer, including the MCA system. * * @retval OPAL_SUCCESS Upon success. * @retval OPAL_ERROR Upon failure. @@ -75,7 +76,14 @@ OPAL_DECLSPEC int opal_finalize(void); OPAL_DECLSPEC int opal_init_util(int* pargc, char*** pargv); /** - * Finalize the OPAL layer, excluding the MCA system. + * Disable PSM/PSM2 signal hijacking. + * + * See comment in the function for more detail. + */ +OPAL_DECLSPEC int opal_init_psm(void); + +/** + * Finalize the OPAL layer, excluding the MCA system. * * @retval OPAL_SUCCESS Upon success. * @retval OPAL_ERROR Upon failure. diff --git a/opal/runtime/opal_init.c b/opal/runtime/opal_init.c index 46a2fa1d08..2c623e71b7 100644 --- a/opal/runtime/opal_init.c +++ b/opal/runtime/opal_init.c @@ -10,7 +10,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2007-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2007-2016 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2007 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2009 Oak Ridge National Labs. All rights reserved. * Copyright (c) 2010-2013 Los Alamos National Security, LLC. @@ -246,6 +246,34 @@ opal_err2str(int errnum, const char **errmsg) } +int opal_init_psm(void) +{ + /* Very early in the init sequence -- before *ANY* MCA components + are opened -- we need to disable some behavior from the PSM and + PSM2 libraries (by default): at least some old versions of + these libraries hijack signal handlers during their library + constructors and then do not un-hijack them when the libraries + are unloaded. + + It is a bit of an abstraction break that we have to put + vendor/transport-specific code in the OPAL core, but we're + out of options, unfortunately. + + NOTE: We only disable this behavior if the corresponding + environment variables are not already set (i.e., if the + user/environment has indicated a preference for this behavior, + we won't override it). */ + if (NULL == getenv("IPATH_NO_BACKTRACE")) { + opal_setenv("IPATH_NO_BACKTRACE", "1", true, &environ); + } + if (NULL == getenv("HFI_NO_BACKTRACE")) { + opal_setenv("HFI_NO_BACKTRACE", "1", true, &environ); + } + + return OPAL_SUCCESS; +} + + int opal_init_util(int* pargc, char*** pargv) { @@ -301,6 +329,10 @@ opal_init_util(int* pargc, char*** pargv) goto return_error; } + // Disable PSM signal hijacking (see comment in function for more + // details) + opal_init_psm(); + /* Setup the parameter system */ if (OPAL_SUCCESS != (ret = mca_base_param_init())) { error = "mca_base_param_init";