Enable the PMIx event notification capability and use that for all error notifications, including debugger release. This capability requires use of PMIx 2.0 or above as the features are not available with earlier PMIx releases. When OMPI master is built against an earlier external version, it will fallback to the prior behavior - i.e., debugger will be released via RML and all notifications will go strictly to the default error handler.

Add PMIx 2.0

Remove PMIx 1.1.4

Cleanup copying of component

Add missing file

Touchup a typo in the Makefile.am

Update the pmix ext114 component

Minor cleanups and resync to master

Update to latest PMIx 2.x

Update to the PMIx event notification branch latest changes

Author: Ralph Castain

ompi/errhandler/errhandler.c

@@ -14,7 +14,7 @@
  * Copyright (c) 2009      Sun Microsystems, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2015      Intel, Inc. All rights reserved.
+ * Copyright (c) 2015-2016 Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -42,7 +42,7 @@ opal_pointer_array_t ompi_errhandler_f_to_c_table = {{0}};
 /*
  * default errhandler id
  */
-static int default_errhandler_id = -1;
+static size_t default_errhandler_id = SIZE_MAX;
 
 /*
  * Class information
@@ -163,7 +163,7 @@ int ompi_errhandler_finalize(void)
 
     /* JMS Add stuff here checking for unreleased errorhandlers,
        similar to communicators, info handles, etc. */
-    opal_pmix.deregister_errhandler(default_errhandler_id, NULL, NULL);
+    opal_pmix.deregister_evhandler(default_errhandler_id, NULL, NULL);
 
     /* Remove errhandler F2C table */
 
@@ -222,7 +222,7 @@ ompi_errhandler_t *ompi_errhandler_create(ompi_errhandler_type_t object_type,
 
 /* registration callback */
 void ompi_errhandler_registration_callback(int status,
-                                           int errhandler_ref,
+                                           size_t errhandler_ref,
                                            void *cbdata)
 {
     ompi_errhandler_errtrk_t *errtrk = (ompi_errhandler_errtrk_t*)cbdata;
@@ -236,14 +236,15 @@ void ompi_errhandler_registration_callback(int status,
  * Default errhandler callback
  */
 void ompi_errhandler_callback(int status,
-                              opal_list_t *procs,
-                              opal_list_t *info,
-                              opal_pmix_release_cbfunc_t cbfunc,
+                              const opal_process_name_t *source,
+                              opal_list_t *info, opal_list_t *results,
+                              opal_pmix_notification_complete_fn_t cbfunc,
                               void *cbdata)
 {
-    /* allow the caller to release its data */
+    /* tell the event chain engine to go no further - we
+     * will handle this */
     if (NULL != cbfunc) {
-        cbfunc(cbdata);
+        cbfunc(OMPI_ERR_HANDLERS_COMPLETE, NULL, NULL, NULL, cbdata);
     }
     /* our default action is to abort */
     ompi_mpi_abort(MPI_COMM_WORLD, status);

ompi/errhandler/errhandler.h

@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2008-2012 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2008-2009 Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2015      Intel, Inc. All rights reserved.
+ * Copyright (c) 2015-2016 Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -380,13 +380,13 @@ typedef struct {
 } ompi_errhandler_errtrk_t;
 
 OMPI_DECLSPEC void ompi_errhandler_callback(int status,
-                                            opal_list_t *procs,
-                                            opal_list_t *info,
-                                            opal_pmix_release_cbfunc_t cbfunc,
+                                            const opal_process_name_t *source,
+                                            opal_list_t *info, opal_list_t *results,
+                                            opal_pmix_notification_complete_fn_t cbfunc,
                                             void *cbdata);
 
 OMPI_DECLSPEC void ompi_errhandler_registration_callback(int status,
-                                                         int errhandler_ref,
+                                                         size_t errhandler_ref,
                                                          void *cbdata);
 /**
  * Check to see if an errhandler is intrinsic.

ompi/include/ompi/constants.h

@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2016      Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -63,6 +64,7 @@ enum {
 
     OMPI_ERR_BUFFER                 = OPAL_ERR_BUFFER,
     OMPI_ERR_SILENT                 = OPAL_ERR_SILENT,
+    OMPI_ERR_HANDLERS_COMPLETE      = OPAL_ERR_HANDLERS_COMPLETE,
 
     OMPI_ERR_REQUEST                = OMPI_ERR_BASE - 1,
     OMPI_ERR_RMA_SYNC               = OMPI_ERR_BASE - 2,

ompi/mca/rte/orte/rte_orte_module.c

@@ -95,6 +95,34 @@ void ompi_rte_abort(int error_code, char *fmt, ...)
     exit(-1);
 }
 
+static size_t handler = SIZE_MAX;
+static bool debugger_register_active = true;
+static bool debugger_event_active = true;
+
+static void _release_fn(int status,
+                        const opal_process_name_t *source,
+                        opal_list_t *info, opal_list_t *results,
+                        opal_pmix_notification_complete_fn_t cbfunc,
+                        void *cbdata)
+{
+    /* must let the notifier know we are done */
+    if (NULL != cbfunc) {
+        cbfunc(ORTE_SUCCESS, NULL, NULL, NULL, cbdata);
+    }
+    debugger_event_active = false;
+}
+
+static void _register_fn(int status,
+                         size_t evhandler_ref,
+                         void *cbdata)
+{
+    opal_list_t *codes = (opal_list_t*)cbdata;
+
+    handler = evhandler_ref;
+    OPAL_LIST_RELEASE(codes);
+    debugger_register_active = false;
+}
+
 /*
  * Wait for a debugger if asked.  We support two ways of waiting for
  * attaching debuggers -- see big comment in
@@ -103,7 +131,8 @@ void ompi_rte_abort(int error_code, char *fmt, ...)
 void ompi_rte_wait_for_debugger(void)
 {
     int debugger;
-    orte_rml_recv_cb_t xfer;
+    opal_list_t *codes;
+    opal_value_t *kv;
 
     /* See lengthy comment in orte/tools/orterun/debuggers.c about
        orte_in_parallel_debugger */
@@ -133,23 +162,23 @@ void ompi_rte_wait_for_debugger(void)
 #endif
         }
     } else {
-        /* only the rank=0 proc waits for either a message from the
-         * HNP or for the debugger to attach - everyone else will just
-         * spin in * the grpcomm barrier in ompi_mpi_init until rank=0
-         * joins them.
-         */
-        if (0 != ORTE_PROC_MY_NAME->vpid) {
-            return;
-        }
 
-        /* VPID 0 waits for a message from the HNP */
-        OBJ_CONSTRUCT(&xfer, orte_rml_recv_cb_t);
-        xfer.active = true;
-        orte_rml.recv_buffer_nb(OMPI_NAME_WILDCARD,
-                                ORTE_RML_TAG_DEBUGGER_RELEASE,
-                                ORTE_RML_NON_PERSISTENT,
-                                orte_rml_recv_callback, &xfer);
-        /* let the MPI progress engine run while we wait */
-        OMPI_WAIT_FOR_COMPLETION(xfer.active);
+        /* register an event handler for the ORTE_ERR_DEBUGGER_RELEASE event */
+        codes = OBJ_NEW(opal_list_t);
+        kv = OBJ_NEW(opal_value_t);
+        kv->key = strdup("errorcode");
+        kv->type = OPAL_INT;
+        kv->data.integer = ORTE_ERR_DEBUGGER_RELEASE;
+        opal_list_append(codes, &kv->super);
+
+        opal_pmix.register_evhandler(codes, NULL, _release_fn, _register_fn, codes);
+        /* let the MPI progress engine run while we wait for registration to complete */
+        OMPI_WAIT_FOR_COMPLETION(debugger_register_active);
+
+        /* let the MPI progress engine run while we wait for debugger release */
+        OMPI_WAIT_FOR_COMPLETION(debugger_event_active);
+
+        /* deregister the event handler */
+        opal_pmix.deregister_evhandler(handler, NULL, NULL);
     }
 }

ompi/mpi/c/lookup_name.c

@@ -12,10 +12,10 @@
  *                         All rights reserved.
  * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
  *                         reserved.
- * Copyright (c) 2015      Intel, Inc. All rights reserved.
+ * Copyright (c) 2015-2016 Intel, Inc. All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -94,13 +94,13 @@ int MPI_Lookup_name(const char *service_name, MPI_Info info, char *port_name)
                 rng = OBJ_NEW(opal_value_t);
                 rng->key = strdup(OPAL_PMIX_RANGE);
                 rng->type = OPAL_INT;
-                rng->data.integer = OPAL_PMIX_NAMESPACE;  // share only with procs in same nspace
+                rng->data.integer = OPAL_PMIX_RANGE_NAMESPACE;  // share only with procs in same nspace
                 opal_list_append(&pinfo, &rng->super);
             } else if (0 == strcmp(range, "session")) {
                 rng = OBJ_NEW(opal_value_t);
                 rng->key = strdup(OPAL_PMIX_RANGE);
                 rng->type = OPAL_INT;
-                rng->data.integer = OPAL_PMIX_SESSION; // share only with procs in same session
+                rng->data.integer = OPAL_PMIX_RANGE_SESSION; // share only with procs in same session
                 opal_list_append(&pinfo, &rng->super);
             } else {
                 /* unrecognized scope */

ompi/mpi/c/publish_name.c

@@ -12,10 +12,10 @@
  *                         All rights reserved.
  * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
  *                         reserved.
- * Copyright (c) 2015      Intel, Inc. All rights reserved.
+ * Copyright (c) 2015-2016 Intel, Inc. All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -94,13 +94,13 @@ int MPI_Publish_name(const char *service_name, MPI_Info info,
                 rng = OBJ_NEW(opal_value_t);
                 rng->key = strdup(OPAL_PMIX_RANGE);
                 rng->type = OPAL_INT;
-                rng->data.integer = OPAL_PMIX_NAMESPACE;  // share only with procs in same nspace
+                rng->data.integer = OPAL_PMIX_RANGE_NAMESPACE;  // share only with procs in same nspace
                 opal_list_append(&values, &rng->super);
             } else if (0 == strcmp(range, "session")) {
                 rng = OBJ_NEW(opal_value_t);
                 rng->key = strdup(OPAL_PMIX_RANGE);
                 rng->type = OPAL_INT;
-                rng->data.integer = OPAL_PMIX_SESSION; // share only with procs in same session
+                rng->data.integer = OPAL_PMIX_RANGE_SESSION; // share only with procs in same session
                 opal_list_append(&values, &rng->super);
             } else {
                 /* unrecognized scope */

ompi/mpi/c/unpublish_name.c

@@ -12,10 +12,10 @@
  *                         All rights reserved.
  * Copyright (c) 2012-2013 Los Alamos National Security, LLC.  All rights
  *                         reserved.
- * Copyright (c) 2015      Intel, Inc. All rights reserved.
+ * Copyright (c) 2015-2016 Intel, Inc. All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -96,13 +96,13 @@ int MPI_Unpublish_name(const char *service_name, MPI_Info info,
                 rng = OBJ_NEW(opal_value_t);
                 rng->key = strdup(OPAL_PMIX_RANGE);
                 rng->type = OPAL_INT;
-                rng->data.integer = OPAL_PMIX_NAMESPACE;  // share only with procs in same nspace
+                rng->data.integer = OPAL_PMIX_RANGE_NAMESPACE;  // share only with procs in same nspace
                 opal_list_append(&pinfo, &rng->super);
             } else if (0 == strcmp(range, "session")) {
                 rng = OBJ_NEW(opal_value_t);
                 rng->key = strdup(OPAL_PMIX_RANGE);
                 rng->type = OPAL_INT;
-                rng->data.integer = OPAL_PMIX_SESSION; // share only with procs in same session
+                rng->data.integer = OPAL_PMIX_RANGE_SESSION; // share only with procs in same session
                 opal_list_append(&pinfo, &rng->super);
             } else {
                 /* unrecognized scope */

ompi/runtime/ompi_mpi_init.c

@@ -377,6 +377,8 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
     char *cmd=NULL, *av=NULL;
     ompi_errhandler_errtrk_t errtrk;
     volatile bool active;
+    opal_list_t info;
+    opal_value_t *kv;
     OPAL_TIMING_DECLARE(tm);
     OPAL_TIMING_INIT_EXT(&tm, OPAL_TIMING_GET_TIME_OF_DAY);
 
@@ -519,10 +521,16 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
     /* Register the default errhandler callback  */
     errtrk.status = OPAL_ERROR;
     errtrk.active = true;
-    opal_pmix.register_errhandler(NULL, ompi_errhandler_callback,
-                                  ompi_errhandler_registration_callback,
-                                  (void*)&errtrk);
+    /* we want to go first */
+    OBJ_CONSTRUCT(&info, opal_list_t);
+    kv = OBJ_NEW(opal_value_t);
+    kv->key = strdup(OPAL_PMIX_EVENT_ORDER_PREPEND);
+    opal_list_append(&info, &kv->super);
+    opal_pmix.register_evhandler(NULL, &info, ompi_errhandler_callback,
+                                 ompi_errhandler_registration_callback,
+                                 (void*)&errtrk);
     OMPI_WAIT_FOR_COMPLETION(errtrk.active);
+    OPAL_LIST_DESTRUCT(&info);
     if (OPAL_SUCCESS != errtrk.status) {
         error = "Error handler registration";
         ret = errtrk.status;

opal/Makefile.am

@@ -10,7 +10,7 @@
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
 # Copyright (c) 2009-2015 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2015      Intel, Inc. All rights reserved.
+# Copyright (c) 2015-2016 Intel, Inc. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -76,7 +76,6 @@ nobase_opal_HEADERS = $(headers)
 endif
 
 include class/Makefile.am
-include errhandler/Makefile.am
 include memoryhooks/Makefile.am
 include runtime/Makefile.am
 include threads/Makefile.am