Add pmix120 component, update the error handling functions in the PMIx API.

Update the configure logic for the new pmix120 component

ckpt

Get the pmix120 component to work - still not really registering or handling notifications, but infrastructure now operates

Cleanup some of the symbol scopes, and provide a more comprehensive rename.h file. Will pretty it up later - let's see how this works

Cleanup the rename files to use the pretty macros

Author: Ralph Castain

.gitignore

@@ -303,6 +303,10 @@ opal/mca/installdirs/config/install_dirs.h
 opal/mca/pmix/pmix112/pmix/include/pmix/autogen/config.h
 opal/mca/pmix/pmix112/pmix/include/private/autogen/config.h
 opal/mca/pmix/pmix112/pmix/include/private/autogen/config.h.in
+opal/mca/pmix/pmix120/pmix/include/pmix/autogen/config.h
+opal/mca/pmix/pmix120/pmix/include/private/autogen/config.h
+opal/mca/pmix/pmix120/pmix/include/private/autogen/config.h.in
+
 
 opal/tools/opal-checkpoint/opal-checkpoint
 opal/tools/opal-checkpoint/opal-checkpoint.1

ompi/errhandler/errhandler.c

@@ -14,6 +14,7 @@
  * Copyright (c) 2009      Sun Microsystems, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015      Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -30,13 +31,18 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/errhandler/errhandler_predefined.h"
 #include "opal/class/opal_pointer_array.h"
+#include "opal/mca/pmix/pmix.h"
 
 
 /*
  * Table for Fortran <-> C errhandler handle conversion
  */
 opal_pointer_array_t ompi_errhandler_f_to_c_table = {{0}};
 
+/*
+ * default errhandler id
+ */
+static int default_errhandler_id = -1;
 
 /*
  * Class information
@@ -157,6 +163,7 @@ int ompi_errhandler_finalize(void)
 
     /* JMS Add stuff here checking for unreleased errorhandlers,
        similar to communicators, info handles, etc. */
+    opal_pmix.deregister_errhandler(default_errhandler_id, NULL, NULL);
 
     /* Remove errhandler F2C table */
 
@@ -169,7 +176,7 @@ int ompi_errhandler_finalize(void)
 
 
 ompi_errhandler_t *ompi_errhandler_create(ompi_errhandler_type_t object_type,
-					  ompi_errhandler_generic_handler_fn_t *func,
+					                      ompi_errhandler_generic_handler_fn_t *func,
                                           ompi_errhandler_lang_t lang)
 {
   ompi_errhandler_t *new_errhandler;
@@ -213,20 +220,33 @@ ompi_errhandler_t *ompi_errhandler_create(ompi_errhandler_type_t object_type,
   return new_errhandler;
 }
 
+/* registration callback */
+void ompi_errhandler_registration_callback(int status,
+                                           int errhandler_ref,
+                                           void *cbdata)
+{
+    ompi_errhandler_errtrk_t *errtrk = (ompi_errhandler_errtrk_t*)cbdata;
+
+    default_errhandler_id = errhandler_ref;
+    errtrk->status = status;
+    errtrk->active = false;
+}
+
 /**
- * Default runtime errhandler callback
+ * Default errhandler callback
  */
-int ompi_errhandler_runtime_callback(opal_pointer_array_t *errors) {
-    ompi_rte_error_report_t *err;
-    int errcode = 1;
-
-    if (NULL != errors &&
-        (NULL != (err = (ompi_rte_error_report_t*)opal_pointer_array_get_item(errors, 0)))) {
-        errcode = err->errcode;
+void ompi_errhandler_callback(int status,
+                              opal_list_t *procs,
+                              opal_list_t *info,
+                              opal_pmix_release_cbfunc_t cbfunc,
+                              void *cbdata)
+{
+    /* allow the caller to release its data */
+    if (NULL != cbfunc) {
+        cbfunc(cbdata);
     }
-
-    ompi_mpi_abort(MPI_COMM_WORLD, errcode);
-    return OMPI_SUCCESS;
+    /* our default action is to abort */
+    ompi_mpi_abort(MPI_COMM_WORLD, status);
 }
 
 /**************************************************************************

ompi/errhandler/errhandler.h

@@ -12,6 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2008-2012 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2008-2009 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -30,6 +31,7 @@
 #include "opal/prefetch.h"
 #include "opal/class/opal_object.h"
 #include "opal/class/opal_pointer_array.h"
+#include "opal/mca/pmix/pmix.h"
 
 #include "ompi/mca/rte/rte.h"
 #include "ompi/runtime/mpiruntime.h"
@@ -364,29 +366,28 @@ struct ompi_request_t;
                                             ompi_errhandler_lang_t language);
 
 /**
- * Callback function from runtime layer to alert the MPI layer of an error at
- * the runtime layer.
- *
- * @param errors A pointer array containing structs of type
- *               ompi_rte_error_report_t that consists of at least
- *               {
- *                  ompi_process_name_t proc;
- *                  int errcode;
- *               }
- *               Each RTE is allowed to add additional information
- *               as required
+ * Callback function to alert the MPI layer of an error or notification
+ * from the internal RTE and/or the resource manager.
  *
  * This function is used to alert the MPI layer to a specific fault detected by the
- * runtime layer. This could be a process failure, a lost connection, or the inability
+ * runtime layer or host RM. This could be a process failure, a lost connection, or the inability
  * to send an OOB message. The MPI layer has the option to perform whatever actions it
  * needs to stabilize itself and continue running, abort, etc.
- *
- * Upon completion, the error handler should return OMPI_SUCCESS if the error has
- * been resolved and no further callbacks are to be executed. Return of any other
- * value will cause the RTE to continue executing error callbacks.
  */
-OMPI_DECLSPEC int ompi_errhandler_runtime_callback(opal_pointer_array_t *errors);
-
+typedef struct {
+    volatile bool active;
+    int status;
+} ompi_errhandler_errtrk_t;
+
+OMPI_DECLSPEC void ompi_errhandler_callback(int status,
+                                            opal_list_t *procs,
+                                            opal_list_t *info,
+                                            opal_pmix_release_cbfunc_t cbfunc,
+                                            void *cbdata);
+
+OMPI_DECLSPEC void ompi_errhandler_registration_callback(int status,
+                                                         int errhandler_ref,
+                                                         void *cbdata);
 /**
  * Check to see if an errhandler is intrinsic.
  *

ompi/mca/rte/orte/rte_orte.h

@@ -5,6 +5,7 @@
  * Copyright (c) 2014      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2014      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015      Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -83,12 +84,6 @@ typedef orte_local_rank_t ompi_local_rank_t;
 OMPI_DECLSPEC void __opal_attribute_noreturn__
   ompi_rte_abort(int error_code, char *fmt, ...);
 #define ompi_rte_abort_peers(a, b, c) orte_errmgr.abort_peers(a, b, c)
-#define OMPI_RTE_ERRHANDLER_FIRST ORTE_ERRMGR_CALLBACK_FIRST
-#define OMPI_RTE_ERRHANDLER_LAST ORTE_ERRMGR_CALLBACK_LAST
-#define OMPI_RTE_ERRHANDLER_PREPEND ORTE_ERRMGR_CALLBACK_PREPEND
-#define OMPI_RTE_ERRHANDLER_APPEND ORTE_ERRMGR_CALLBACK_APPEND
-typedef orte_error_t ompi_rte_error_report_t;
-#define ompi_rte_register_errhandler(a, b) orte_errmgr.register_error_callback(a, b)
 #define OMPI_ERROR_LOG ORTE_ERROR_LOG
 
 /* Init and finalize objects and operations */

ompi/mca/rte/rte.h

@@ -3,7 +3,7 @@
  * Copyright (c) 2012-2015 Los Alamos National Security, LLC.  All rights reserved.
  * Copyright (c) 2013      Mellanox Technologies, Inc.
  *                         All rights reserved.
- * Copyright (c) 2014      Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2015 Intel, Inc. All rights reserved.
  *
  * $COPYRIGHT$
  *
@@ -111,8 +111,6 @@
  *     2. int ompi_rte_abort_peers(ompi_process_name_t *procs, size_t nprocs) -
  *        Abort the specified list of peers
  *     3. OMPI_ERROR_LOG(rc) - print error message regarding the given return code
- *     4. ompi_rte_register_errhandler - register a callback function for the RTE
- *        to report asynchronous errors to the caller
  *
  * (e) Init and finalize objects and operations
  *     1. ompi_rte_init - a function to initialize the RTE. The function

ompi/runtime/ompi_mpi_init.c

@@ -378,6 +378,7 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
     size_t nprocs;
     char *error = NULL;
     char *cmd=NULL, *av=NULL;
+    ompi_errhandler_errtrk_t errtrk;
     OPAL_TIMING_DECLARE(tm);
     OPAL_TIMING_INIT_EXT(&tm, OPAL_TIMING_GET_TIME_OF_DAY);
 
@@ -504,11 +505,18 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
         }
     }
 
-    /* Register the default errhandler callback - RTE will ignore if it
-     * doesn't support this capability
-     */
-    ompi_rte_register_errhandler(ompi_errhandler_runtime_callback,
-                                 OMPI_RTE_ERRHANDLER_LAST);
+    /* Register the default errhandler callback  */
+    errtrk.status = OPAL_ERROR;
+    errtrk.active = true;
+    opal_pmix.register_errhandler(NULL, ompi_errhandler_callback,
+                                  ompi_errhandler_registration_callback,
+                                  (void*)&errtrk);
+    OMPI_WAIT_FOR_COMPLETION(errtrk.active);
+    if (OPAL_SUCCESS != errtrk.status) {
+        error = "Error handler registration";
+        ret = errtrk.status;
+        goto error;
+    }
 
     /* Figure out the final MPI thread levels.  If we were not
        compiled for support for MPI threads, then don't allow

opal/mca/pmix/base/base.h

@@ -32,11 +32,17 @@ OPAL_DECLSPEC int opal_pmix_base_select(void);
 
 OPAL_DECLSPEC extern bool opal_pmix_base_allow_delayed_server;
 
-OPAL_DECLSPEC void opal_pmix_base_register_handler(opal_pmix_errhandler_fn_t err);
-OPAL_DECLSPEC void opal_pmix_base_deregister_handler(void);
+OPAL_DECLSPEC void opal_pmix_base_register_handler(opal_list_t *info,
+                                                   opal_pmix_notification_fn_t errhandler,
+                                                   opal_pmix_errhandler_reg_cbfunc_t cbfunc,
+                                                   void *cbdata);
+OPAL_DECLSPEC void opal_pmix_base_deregister_handler(int errhandler,
+                                                     opal_pmix_op_cbfunc_t cbfunc,
+                                                     void *cbdata);
 OPAL_DECLSPEC void opal_pmix_base_errhandler(int status,
                                              opal_list_t *procs,
-                                             opal_list_t *info);
+                                             opal_list_t *info,
+                                             opal_pmix_release_cbfunc_t cbfunc, void *cbdata);
 OPAL_DECLSPEC int opal_pmix_base_exchange(opal_value_t *info,
                                           opal_pmix_pdata_t *pdat,
                                           int timeout);

opal/mca/pmix/base/pmix_base_fns.c

@@ -38,26 +38,40 @@
 
 #define OPAL_PMI_PAD  10
 
-/********     ERRHANDLER SUPPORT     ********/
-static opal_pmix_errhandler_fn_t errhandler = NULL;
-
-void opal_pmix_base_register_handler(opal_pmix_errhandler_fn_t err)
+/********     ERRHANDLER SUPPORT FOR COMPONENTS THAT
+ ********     DO NOT NATIVELY SUPPORT IT
+ ********/
+static opal_pmix_notification_fn_t errhandler = NULL;
+
+void opal_pmix_base_register_handler(opal_list_t *info,
+                                     opal_pmix_notification_fn_t err,
+                                     opal_pmix_errhandler_reg_cbfunc_t cbfunc,
+                                     void *cbdata)
 {
     errhandler = err;
+    if (NULL != cbfunc) {
+        cbfunc(OPAL_SUCCESS, 0, cbdata);
+    }
 }
 
 void opal_pmix_base_errhandler(int status,
                                opal_list_t *procs,
-                               opal_list_t *info)
+                               opal_list_t *info,
+                               opal_pmix_release_cbfunc_t cbfunc, void *cbdata)
 {
     if (NULL != errhandler) {
-        errhandler(status);
+        errhandler(status, procs, info, cbfunc, cbdata);
     }
 }
 
-void opal_pmix_base_deregister_handler(void)
+void opal_pmix_base_deregister_handler(int errid,
+                                       opal_pmix_op_cbfunc_t cbfunc,
+                                       void *cbdata)
 {
     errhandler = NULL;
+    if (NULL != cbfunc) {
+        cbfunc(OPAL_SUCCESS, cbdata);
+    }
 }
 
 struct lookup_caddy_t {

opal/mca/pmix/external/pmix_ext_client.c

@@ -77,7 +77,7 @@ static void myerr(pmix_status_t status,
         iptr = OBJ_NEW(opal_value_t);
         iptr->key = strdup(info[n].key);
         pmix1_value_unload(iptr, &info[n].value);
-        opal_list_append(&plist, &nm->super);
+        opal_list_append(&plist, &iptr->super);
     }
 
     /* call the base errhandler */

opal/mca/pmix/external/pmix_ext_server_south.c

@@ -91,7 +91,7 @@ static void myerr(pmix_status_t status,
         iptr = OBJ_NEW(opal_value_t);
         iptr->key = strdup(info[n].key);
         pmix1_value_unload(iptr, &info[n].value);
-        opal_list_append(&plist, &nm->super);
+        opal_list_append(&plist, &iptr->super);
     }
 
     /* call the base errhandler */