Fix issue in abort_print_stack mca variable

This commit allows to control output during abnormal oshmem/ompi application
termination.
Fixed issue in backtrace output. HAVE_BACKTRACE was never set so user was limited
in control of this variable.
Two related mca variables are moved to opal layer. Corresponding aliases are
added for ompi and oshmem.

Author: igor-ivanov

ompi/runtime/ompi_mpi_abort.c

@@ -34,6 +34,7 @@
 #endif
 
 #include "opal/mca/backtrace/backtrace.h"
+#include "opal/runtime/opal_params.h"
 
 #include "ompi/communicator/communicator.h"
 #include "ompi/runtime/mpiruntime.h"
@@ -72,11 +73,11 @@ ompi_mpi_abort(struct ompi_communicator_t* comm,
 
     /* Should we print a stack trace?  Not aggregated because they
        might be different on all processes. */
-    if (ompi_mpi_abort_print_stack) {
+    if (opal_abort_print_stack) {
         char **messages;
         int len, i;
 
-        if (OMPI_SUCCESS == opal_backtrace_buffer(&messages, &len)) {
+        if (OPAL_SUCCESS == opal_backtrace_buffer(&messages, &len)) {
             for (i = 0; i < len; ++i) {
                 fprintf(stderr, "[%s:%d] [%d] func:%s\n", host, (int) pid, 
                         i, messages[i]);
@@ -96,7 +97,7 @@ ompi_mpi_abort(struct ompi_communicator_t* comm,
     if (errcode < 0 ||
         asprintf(&msg, "[%s:%d] aborting with MPI error %s%s", 
                  host, (int) pid, ompi_mpi_errnum_get_string(errcode), 
-                 ompi_mpi_abort_print_stack ? 
+                 opal_abort_print_stack ?
                  " (stack trace available on stderr)" : "") < 0) {
         msg = NULL;
     }
@@ -107,20 +108,20 @@ ompi_mpi_abort(struct ompi_communicator_t* comm,
 
     /* Should we wait for a while before aborting? */
 
-    if (0 != ompi_mpi_abort_delay) {
-        if (ompi_mpi_abort_delay < 0) {
-            fprintf(stderr ,"[%s:%d] Looping forever (MCA parameter mpi_abort_delay is < 0)\n",
+    if (0 != opal_abort_delay) {
+        if (opal_abort_delay < 0) {
+            fprintf(stderr ,"[%s:%d] Looping forever (MCA parameter opal_abort_delay is < 0)\n",
                     host, (int) pid);
             fflush(stderr);
             while (1) { 
                 sleep(5); 
             }
         } else {
             fprintf(stderr, "[%s:%d] Delaying for %d seconds before aborting\n",
-                    host, (int) pid, ompi_mpi_abort_delay);
+                    host, (int) pid, opal_abort_delay);
             do {
                 sleep(1);
-            } while (--ompi_mpi_abort_delay > 0);
+            } while (--opal_abort_delay > 0);
         }
     }
 

ompi/runtime/ompi_mpi_params.c

@@ -38,6 +38,7 @@
 #include "ompi/runtime/params.h"
 #include "ompi/mca/rte/rte.h"
 
+#include "opal/runtime/opal_params.h"
 #include "opal/mca/base/mca_base_param.h"
 #include "opal/util/argv.h"
 #include "opal/util/output.h"
@@ -58,8 +59,6 @@ int ompi_debug_show_mpi_alloc_mem_leaks = 0;
 bool ompi_debug_no_free_handles = false;
 bool ompi_mpi_show_mca_params = false;
 char *ompi_mpi_show_mca_params_file = NULL;
-bool ompi_mpi_abort_print_stack = false;
-int ompi_mpi_abort_delay = 0;
 bool ompi_mpi_keep_fqdn_hostnames = false;
 int ompi_mpi_leave_pinned = -1;
 bool ompi_mpi_leave_pinned_pipeline = false;
@@ -214,33 +213,6 @@ int ompi_mpi_register_params(void)
 
     /* User-level process pinning controls */
 
-    /* MPI_ABORT controls */
-    ompi_mpi_abort_delay = 0;
-    (void) mca_base_var_register("ompi", "mpi", NULL, "abort_delay",
-                                "If nonzero, print out an identifying message when MPI_ABORT is invoked (hostname, PID of the process that called MPI_ABORT) and delay for that many seconds before exiting (a negative delay value means to never abort).  This allows attaching of a debugger before quitting the job.",
-                                 MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                 OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY,
-                                 &ompi_mpi_abort_delay);
-
-    ompi_mpi_abort_print_stack = false;
-    (void) mca_base_var_register("ompi", "mpi", NULL, "abort_print_stack",
-                                 "If nonzero, print out a stack trace when MPI_ABORT is invoked",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0,
-                                /* If we do not have stack trace
-                                   capability, make this a constant
-                                   MCA variable */
-#if OPAL_WANT_PRETTY_PRINT_STACKTRACE && defined(HAVE_BACKTRACE)
-                                 0,
-                                 OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY,
-#else
-                                 MCA_BASE_VAR_FLAG_DEFAULT_ONLY,
-                                 OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_CONSTANT,
-#endif
-                                 &ompi_mpi_abort_print_stack);
-
     ompi_mpi_preconnect_mpi = false;
     value = mca_base_var_register("ompi", "mpi", NULL, "preconnect_mpi",
                                   "Whether to force MPI processes to fully "
@@ -345,6 +317,13 @@ int ompi_mpi_register_params(void)
                                   OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
                                   &ompi_hostname_cutoff);
 
+    (void) mca_base_var_register_synonym(opal_abort_delay_var_index, "ompi", "mpi", NULL,
+                                         "abort_delay",
+                                         MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
+
+    (void) mca_base_var_register_synonym(opal_abort_print_stack_var_index, "ompi", "mpi", NULL,
+                                         "abort_print_stack",
+                                         MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
 
     return OMPI_SUCCESS;
 }

opal/runtime/opal_params.c

@@ -44,6 +44,10 @@
 char *opal_signal_string = NULL;
 char *opal_net_private_ipv4 = NULL;
 char *opal_set_max_sys_limits = NULL;
+bool opal_abort_print_stack = false;
+int opal_abort_print_stack_var_index = -1;
+int opal_abort_delay = 0;
+int opal_abort_delay_var_index = -1;
 
 static bool opal_register_done = false;
 
@@ -153,6 +157,40 @@ int opal_register_params(void)
 	return ret;
     }
 
+    opal_abort_delay = 0;
+    ret = mca_base_var_register("opal", "opal", NULL, "abort_delay",
+                                "If nonzero, print out an identifying message when abort operation is invoked (hostname, PID of the process that called abort) and delay for that many seconds before exiting (a negative delay value means to never abort).  This allows attaching of a debugger before quitting the job.",
+                                 MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                 OPAL_INFO_LVL_5,
+                                 MCA_BASE_VAR_SCOPE_READONLY,
+                                 &opal_abort_delay);
+    if (0 > ret) {
+	return ret;
+    }
+    opal_abort_delay_var_index = ret;
+
+    opal_abort_print_stack = false;
+    ret = mca_base_var_register("opal", "opal", NULL, "abort_print_stack",
+                                 "If nonzero, print out a stack trace when abort is invoked",
+                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0,
+                                /* If we do not have stack trace
+                                   capability, make this a constant
+                                   MCA variable */
+#if OPAL_WANT_PRETTY_PRINT_STACKTRACE
+                                 0,
+                                 OPAL_INFO_LVL_5,
+                                 MCA_BASE_VAR_SCOPE_READONLY,
+#else
+                                 MCA_BASE_VAR_FLAG_DEFAULT_ONLY,
+                                 OPAL_INFO_LVL_5,
+                                 MCA_BASE_VAR_SCOPE_CONSTANT,
+#endif
+                                 &opal_abort_print_stack);
+    if (0 > ret) {
+	return ret;
+    }
+    opal_abort_print_stack_var_index = ret;
+
     /* The ddt engine has a few parameters */
     ret = opal_datatype_register_params();
     if (OPAL_SUCCESS != ret) {

opal/runtime/opal_params.h

@@ -33,4 +33,22 @@ extern char *opal_set_max_sys_limits;
 extern bool opal_progress_debug;
 #endif
 
+/**
+ * Whether an abort operation should print out a stack trace or not.
+ */
+OPAL_DECLSPEC extern bool opal_abort_print_stack;
+OPAL_DECLSPEC extern int opal_abort_print_stack_var_index;
+
+/**
+ * Whether  abort operation  should  print  out an  identifying  message
+ * (e.g., hostname  and PID)  and loop waiting  for a  debugger to
+ * attach.  The value of the integer is how many seconds to wait:
+ *
+ * 0 = do not print the message and do not loop
+ * negative value = print the message and loop forever
+ * positive value = print the message and delay for that many seconds
+ */
+OPAL_DECLSPEC extern int opal_abort_delay;
+OPAL_DECLSPEC extern int opal_abort_delay_var_index;
+
 #endif

oshmem/runtime/oshmem_shmem_abort.c

@@ -24,6 +24,7 @@
 #endif
 
 #include "opal/mca/backtrace/backtrace.h"
+#include "opal/runtime/opal_params.h"
 
 #include "orte/util/proc_info.h"
 #include "orte/runtime/runtime.h"
@@ -71,11 +72,11 @@ int oshmem_shmem_abort(int errcode)
 
     /* Should we print a stack trace?  Not aggregated because they
      might be different on all processes. */
-    if (ompi_mpi_abort_print_stack) {
+    if (opal_abort_print_stack) {
         char **messages;
         int len, i;
 
-        if (OSHMEM_SUCCESS == opal_backtrace_buffer(&messages, &len)) {
+        if (OPAL_SUCCESS == opal_backtrace_buffer(&messages, &len)) {
             for (i = 0; i < len; ++i) {
                 fprintf(stderr,
                         "[%s:%d] [%d] func:%s\n",
@@ -94,6 +95,25 @@ int oshmem_shmem_abort(int errcode)
         }
     }
 
+    /* Should we wait for a while before aborting? */
+
+    if (0 != opal_abort_delay) {
+        if (opal_abort_delay < 0) {
+            fprintf(stderr ,"[%s:%d] Looping forever (MCA parameter opal_abort_delay is < 0)\n",
+                    host, (int) pid);
+            fflush(stderr);
+            while (1) {
+                sleep(5);
+            }
+        } else {
+            fprintf(stderr, "[%s:%d] Delaying for %d seconds before aborting\n",
+                    host, (int) pid, opal_abort_delay);
+            do {
+                sleep(1);
+            } while (--opal_abort_delay > 0);
+        }
+    }
+
     if (!orte_initialized || !oshmem_shmem_initialized) {
         if (orte_show_help_is_available()) {
             /* TODO help message from SHMEM not from MPI is needed*/

oshmem/runtime/oshmem_shmem_params.c

@@ -8,8 +8,12 @@
  * $HEADER$
  */
 
-#include "params.h"
-#include "runtime.h"
+#include "oshmem_config.h"
+
+#include "opal/runtime/opal_params.h"
+
+#include "oshmem/runtime/params.h"
+#include "oshmem/runtime/runtime.h"
 #include "oshmem/constants.h"
 
 
@@ -63,5 +67,13 @@ int oshmem_shmem_register_params(void)
                                  MCA_BASE_VAR_SCOPE_READONLY,
                                  &oshmem_preconnect_all);
 
+    (void) mca_base_var_register_synonym(opal_abort_delay_var_index, "oshmem", "oshmem", NULL,
+                                         "abort_delay",
+                                         MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
+
+    (void) mca_base_var_register_synonym(opal_abort_print_stack_var_index, "oshmem", "oshmem", NULL,
+                                         "abort_print_stack",
+                                         MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
+
     return OSHMEM_SUCCESS;
 }

oshmem/runtime/params.h

@@ -19,10 +19,6 @@ BEGIN_C_DECLS
  * Global variables
  */
 
-/**
- * Whether an MPI_ABORT should print out a stack trace or not.
- */
-OSHMEM_DECLSPEC extern bool ompi_mpi_abort_print_stack;
 
 /** 
  * Whether or not the lock routines are recursive