Update the PMIx2 support to include the latest shared memory optimizations

Update ORTE support for dynamic PMIx operations e.g., PMIx_Spawn
Update to track master
Ensure that --disable-pmix-dstore actually disables the dstore. Sync to a few debugger updates

Signed-off-by: Ralph Castain <[email protected]>

Author: Ralph Castain

ompi/dpm/dpm.c

@@ -15,7 +15,7 @@
  * Copyright (c) 2009      Sun Microsystems, Inc.  All rights reserved.
  * Copyright (c) 2011-2015 Los Alamos National Security, LLC.  All rights
  *                         reserved.
- * Copyright (c) 2013-2015 Intel, Inc. All rights reserved
+ * Copyright (c) 2013-2016 Intel, Inc.  All rights reserved.
  * Copyright (c) 2014-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -675,7 +675,7 @@ int ompi_dpm_spawn(int count, const char *array_of_commands[],
 
         /* copy over the name of the executable */
         app->cmd = strdup(array_of_commands[i]);
-        opal_argv_append(&app->argc, &app->argv, app->cmd);
+        opal_argv_append_nosize(&app->argv, app->cmd);
 
         /* record the number of procs to be generated */
         app->maxprocs = array_of_maxprocs[i];
@@ -684,7 +684,7 @@ int ompi_dpm_spawn(int count, const char *array_of_commands[],
         if (MPI_ARGVS_NULL != array_of_argv &&
             MPI_ARGV_NULL != array_of_argv[i]) {
             for (j=0; NULL != array_of_argv[i][j]; j++) {
-                opal_argv_append(&app->argc, &app->argv, array_of_argv[i][j]);
+                opal_argv_append_nosize(&app->argv, array_of_argv[i][j]);
             }
         }
 

opal/mca/pmix/base/pmix_base_frame.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2015 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015-2016 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
@@ -41,12 +41,12 @@ static int opal_pmix_base_frame_register(mca_base_register_flag_t flags)
 {
     opal_pmix_base_async_modex = false;
     (void) mca_base_var_register("opal", "pmix", "base", "async_modex", "Use asynchronous modex mode",
-				 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9,
-				 MCA_BASE_VAR_SCOPE_READONLY, &opal_pmix_base_async_modex);
+                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                 MCA_BASE_VAR_SCOPE_READONLY, &opal_pmix_base_async_modex);
     opal_pmix_collect_all_data = true;
     (void) mca_base_var_register("opal", "pmix", "base", "collect_data", "Collect all data during modex",
-				 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9,
-				 MCA_BASE_VAR_SCOPE_READONLY, &opal_pmix_collect_all_data);
+                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                 MCA_BASE_VAR_SCOPE_READONLY, &opal_pmix_collect_all_data);
     return OPAL_SUCCESS;
 }
 
@@ -74,10 +74,10 @@ static int opal_pmix_base_frame_open(mca_base_open_flag_t flags)
 }
 
 MCA_BASE_FRAMEWORK_DECLARE(opal, pmix, "OPAL PMI Client Framework",
-			   opal_pmix_base_frame_register,
-			   opal_pmix_base_frame_open,
-			   opal_pmix_base_frame_close,
-			   mca_pmix_base_static_components, 0);
+                           opal_pmix_base_frame_register,
+                           opal_pmix_base_frame_open,
+                           opal_pmix_base_frame_close,
+                           mca_pmix_base_static_components, 0);
 
 /****  PMIX FRAMEWORK OBJECTS  ****/
 static void lkcon(opal_pmix_pdata_t *p)
@@ -91,8 +91,8 @@ static void lkdes(opal_pmix_pdata_t *p)
     OBJ_DESTRUCT(&p->value);
 }
 OBJ_CLASS_INSTANCE(opal_pmix_pdata_t,
-		   opal_list_item_t,
-		   lkcon, lkdes);
+                   opal_list_item_t,
+                   lkcon, lkdes);
 
 static void mdcon(opal_pmix_modex_data_t *p)
 {
@@ -104,38 +104,41 @@ static void mdcon(opal_pmix_modex_data_t *p)
 static void mddes(opal_pmix_modex_data_t *p)
 {
     if (NULL != p->blob) {
-	free(p->blob);
+        free(p->blob);
     }
 }
 OBJ_CLASS_INSTANCE(opal_pmix_modex_data_t,
-		   opal_list_item_t,
-		   mdcon, mddes);
+                   opal_list_item_t,
+                   mdcon, mddes);
 
 static void apcon(opal_pmix_app_t *p)
 {
     p->cmd = NULL;
-    p->argc = 0;
     p->argv = NULL;
     p->env = NULL;
+    p->cwd = NULL;
     p->maxprocs = 0;
     OBJ_CONSTRUCT(&p->info, opal_list_t);
 }
 static void apdes(opal_pmix_app_t *p)
 {
     if (NULL != p->cmd) {
-	free(p->cmd);
+        free(p->cmd);
     }
     if (NULL != p->argv) {
-	opal_argv_free(p->argv);
+        opal_argv_free(p->argv);
     }
     if (NULL != p->env) {
-	opal_argv_free(p->env);
+        opal_argv_free(p->env);
+    }
+    if (NULL != p->cwd) {
+        free(p->cwd);
     }
     OPAL_LIST_DESTRUCT(&p->info);
 }
 OBJ_CLASS_INSTANCE(opal_pmix_app_t,
-		   opal_list_item_t,
-		   apcon, apdes);
+                   opal_list_item_t,
+                   apcon, apdes);
 
 static void qcon(opal_pmix_query_t *p)
 {

opal/mca/pmix/pmix2x/configure.m4

@@ -41,7 +41,7 @@ AC_DEFUN([MCA_opal_pmix_pmix2x_CONFIG],[
                   [AC_HELP_STRING([--enable-pmix-dstore],
                                   [Enable PMIx shared memory data store (default: disabled)])])
     AC_MSG_CHECKING([if PMIx shared memory data store is enabled])
-    if test "$enable_pmix2_dstore" != "no"; then
+    if test "$enable_pmix_dstore" != "no"; then
         AC_MSG_RESULT([yes])
         opal_pmix_pmix2x_sm_flag=--enable-dstore
     else

opal/mca/pmix/pmix2x/pmix/VERSION

@@ -30,7 +30,7 @@ greek=
 # command, or with the date (if "git describe" fails) in the form of
 # "date<date>".
 
-repo_rev=git22b754e
+repo_rev=git9089b99
 
 # If tarball_version is not empty, it is used as the version string in
 # the tarball filename, regardless of all other versions listed in
@@ -44,7 +44,7 @@ tarball_version=
 
 # The date when this release was created
 
-date="Dec 06, 2016"
+date="Dec 13, 2016"
 
 # The shared library version of each of PMIx's public libraries.
 # These versions are maintained in accordance with the "Library

opal/mca/pmix/pmix2x/pmix/config/pmix.m4

@@ -328,11 +328,11 @@ AC_DEFUN([PMIX_SETUP_CORE],[
                       netdb.h ucred.h])
 
     AC_CHECK_HEADERS([sys/mount.h], [], [],
-                    [AC_INCLUDES_DEFAULT
-                     #if HAVE_SYS_PARAM_H
-                     #include <sys/param.h>
-                     #endif
-                     ])
+                     [AC_INCLUDES_DEFAULT
+                      #if HAVE_SYS_PARAM_H
+                      #include <sys/param.h>
+                      #endif
+                      ])
 
     AC_CHECK_HEADERS([sys/sysctl.h], [], [],
                      [AC_INCLUDES_DEFAULT
@@ -358,6 +358,7 @@ AC_DEFUN([PMIX_SETUP_CORE],[
                       #endif
                       ])
 
+
     # Note that sometimes we have <stdbool.h>, but it doesn't work (e.g.,
     # have both Portland and GNU installed; using pgcc will find GNU's
     # <stdbool.h>, which all it does -- by standard -- is define "bool" to

opal/mca/pmix/pmix2x/pmix/examples/dynamic.c

@@ -95,7 +95,6 @@ int main(int argc, char **argv)
             exit(1);
         }
         app->maxprocs = 2;
-        app->argc = 1;
         app->argv = (char**)malloc(2 * sizeof(char*));
         if (0 > asprintf(&app->argv[0], "%s/client", dir)) {
             exit(1);

opal/mca/pmix/pmix2x/pmix/include/pmix_common.h

@@ -155,6 +155,7 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_TDIR_RMCLEAN                   "pmix.tdir.rmclean"     // (bool)  Resource Manager will clean session directories
 
 /* information about relative ranks as assigned by the RM */
+#define PMIX_NSPACE                         "pmix.nspace"           // (char*) nspace of a job
 #define PMIX_JOBID                          "pmix.jobid"            // (char*) jobid assigned by scheduler
 #define PMIX_APPNUM                         "pmix.appnum"           // (uint32_t) app number within the job
 #define PMIX_RANK                           "pmix.rank"             // (pmix_rank_t) process rank within the job
@@ -275,16 +276,20 @@ typedef uint32_t pmix_rank_t;
                                                                      //     returns (pmix_data_array_t) an array of pmix_proc_info_t for
                                                                      //     procs in job on same node
 #define PMIX_QUERY_AUTHORIZATIONS           "pmix.qry.auths"         // return operations tool is authorized to perform"
+#define PMIX_QUERY_SPAWN_SUPPORT            "pmix.qry.spawn"         // return a comma-delimited list of supported spawn attributes
+#define PMIX_QUERY_DEBUG_SUPPORT            "pmix.qry.debug"         // return a comma-delimited list of supported debug attributes
 
 /* log attributes */
 #define PMIX_LOG_STDERR                     "pmix.log.stderr"        // (bool) log data to stderr
 #define PMIX_LOG_STDOUT                     "pmix.log.stdout"        // (bool) log data to stdout
 #define PMIX_LOG_SYSLOG                     "pmix.log.syslog"        // (bool) log data to syslog - defaults to ERROR priority unless
                                                                      //        modified by directive
 /* debugger attributes */
-#define PMIX_SPAWN_UNDER_DEBUGGER           "pmix.dbg.pause"         // (bool) job is being spawned under debugger - instruct it to pause on start
-#define PMIX_JOB_BEING_DEBUGGED             "pmix.dbg.job"           // (char*) nspace of the job to be debugged - the RM/PMIx server are
-                                                                     //     to provide the job-level info of that job to each debugger daemon
+#define PMIX_DEBUG_STOP_ON_EXEC             "pmix.dbg.exec"          // (bool) job is being spawned under debugger - instruct it to pause on start
+#define PMIX_DEBUG_STOP_IN_INIT             "pmix.dbg.init"          // (bool) instruct job to stop during PMIx init
+#define PMIX_DEBUG_WAIT_FOR_NOTIFY          "pmix.dbg.notify"        // (bool) block at desired point until receiving debugger release notification
+#define PMIX_DEBUG_JOB                      "pmix.dbg.job"           // (char*) nspace of the job to be debugged - the RM/PMIx server are
+#define PMIX_DEBUG_WAITING_FOR_NOTIFY       "pmix.dbg.waiting"       // (bool) job to be debugged is waiting for a release
 
 /****    PROCESS STATE DEFINITIONS    ****/
 typedef uint8_t pmix_proc_state_t;
@@ -581,18 +586,17 @@ typedef struct pmix_proc_info {
 
 
 /****    PMIX VALUE STRUCT    ****/
+typedef struct pmix_info_t pmix_info_t;
+
 typedef struct pmix_data_array {
     pmix_data_type_t type;
     size_t size;
     void *array;
 } pmix_data_array_t;
 
-/**** DEPRECATED ****/
-struct pmix_info;
-
 typedef struct pmix_info_array {
     size_t size;
-    struct pmix_info *array;
+    pmix_info_t *array;
 } pmix_info_array_t;
 /********************/
 
@@ -754,23 +758,29 @@ typedef struct pmix_value {
         }                                               \
     } while (0)
 
-/* expose two functions that are resolved in the
+/* expose some functions that are resolved in the
  * PMIx library, but part of a header that
  * includes internal functions - we don't
  * want to expose the entire header here
  */
 void pmix_value_load(pmix_value_t *v, void *data, pmix_data_type_t type);
 pmix_status_t pmix_value_xfer(pmix_value_t *kv, pmix_value_t *src);
+pmix_status_t pmix_argv_append_nosize(char ***argv, const char *arg);
+pmix_status_t pmix_setenv(const char *name, const char *value,
+                              bool overwrite, char ***env);
 
-
+#define PMIX_ARGV_APPEND(a, b) \
+    pmix_argv_append_nosize(&(a), (b))
+#define PMIX_SETENV(a, b, c) \
+    pmix_setenv((a), (b), true, (c))
 
 
 /****    PMIX INFO STRUCT    ****/
-typedef struct pmix_info {
+struct pmix_info_t {
     char key[PMIX_MAX_KEYLEN+1];    // ensure room for the NULL terminator
     pmix_info_directives_t flags;   // bit-mask of flags
     pmix_value_t value;
-} pmix_info_t;
+};
 
 /* utility macros for working with pmix_info_t structs */
 #define PMIX_INFO_CREATE(m, n)                                  \
@@ -874,9 +884,9 @@ typedef struct pmix_pdata {
 /****    PMIX APP STRUCT    ****/
 typedef struct pmix_app {
     char *cmd;
-    int argc;
     char **argv;
     char **env;
+    char *cwd;
     int maxprocs;
     pmix_info_t *info;
     size_t ninfo;
@@ -916,6 +926,9 @@ typedef struct pmix_app {
             }                                                   \
             free((m)->env);                                     \
         }                                                       \
+        if (NULL != (m)->cwd) {                                 \
+            free((m)->cwd);                                     \
+        }                                                       \
         if (NULL != (m)->info) {                                \
             for (_ii=0; _ii < (m)->ninfo; _ii++) {              \
                 PMIX_INFO_DESTRUCT(&(m)->info[_ii]);            \

opal/mca/pmix/pmix2x/pmix/src/buffer_ops/copy.c

@@ -557,13 +557,15 @@ PMIX_EXPORT pmix_status_t pmix_value_xfer(pmix_value_t *p, pmix_value_t *src)
                     if (NULL != sa[n].cmd) {
                         pa[n].cmd = strdup(sa[n].cmd);
                     }
-                    pa[n].argc = sa[n].argc;
                     if (NULL != sa[n].argv) {
                         pa[n].argv = pmix_argv_copy(sa[n].argv);
                     }
                     if (NULL != sa[n].env) {
                         pa[n].env = pmix_argv_copy(sa[n].env);
                     }
+                    if (NULL != sa[n].cwd) {
+                        pa[n].cwd = strdup(sa[n].cwd);
+                    }
                     pa[n].maxprocs = sa[n].maxprocs;
                     if (0 < sa[n].ninfo && NULL != sa[n].info) {
                         PMIX_INFO_CREATE(pa[n].info, sa[n].ninfo);
@@ -844,9 +846,11 @@ pmix_status_t pmix_bfrop_copy_app(pmix_app_t **dest, pmix_app_t *src,
 
     *dest = (pmix_app_t*)malloc(sizeof(pmix_app_t));
     (*dest)->cmd = strdup(src->cmd);
-    (*dest)->argc = src->argc;
     (*dest)->argv = pmix_argv_copy(src->argv);
     (*dest)->env = pmix_argv_copy(src->env);
+    if (NULL != src->cwd) {
+        (*dest)->cwd = strdup(src->cwd);
+    }
     (*dest)->maxprocs = src->maxprocs;
     (*dest)->ninfo = src->ninfo;
     (*dest)->info = (pmix_info_t*)malloc(src->ninfo * sizeof(pmix_info_t));
@@ -1174,13 +1178,15 @@ pmix_status_t pmix_bfrop_copy_darray(pmix_data_array_t **dest,
                 if (NULL != sa[n].cmd) {
                     pa[n].cmd = strdup(sa[n].cmd);
                 }
-                pa[n].argc = sa[n].argc;
                 if (NULL != sa[n].argv) {
                     pa[n].argv = pmix_argv_copy(sa[n].argv);
                 }
                 if (NULL != sa[n].env) {
                     pa[n].env = pmix_argv_copy(sa[n].env);
                 }
+                if (NULL != sa[n].cwd) {
+                    pa[n].cwd = strdup(sa[n].cwd);
+                }
                 pa[n].maxprocs = sa[n].maxprocs;
                 if (0 < sa[n].ninfo && NULL != sa[n].info) {
                     PMIX_INFO_CREATE(pa[n].info, sa[n].ninfo);

opal/mca/pmix/pmix2x/pmix/src/buffer_ops/pack.c

@@ -753,10 +753,11 @@ pmix_status_t pmix_bfrop_pack_app(pmix_buffer_t *buffer, const void *src,
             return ret;
         }
         /* argv */
-        if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_int(buffer, &app[i].argc, 1, PMIX_INT))) {
+        nvals = pmix_argv_count(app[i].argv);
+        if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_int(buffer, &nvals, 1, PMIX_INT32))) {
             return ret;
         }
-        for (j=0; j < app[i].argc; j++) {
+        for (j=0; j < nvals; j++) {
             if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_string(buffer, &app[i].argv[j], 1, PMIX_STRING))) {
                 return ret;
             }
@@ -771,6 +772,10 @@ pmix_status_t pmix_bfrop_pack_app(pmix_buffer_t *buffer, const void *src,
                 return ret;
             }
         }
+        /* cwd */
+        if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_string(buffer, &app[i].cwd, 1, PMIX_STRING))) {
+            return ret;
+        }
         /* maxprocs */
         if (PMIX_SUCCESS != (ret = pmix_bfrop_pack_int(buffer, &app[i].maxprocs, 1, PMIX_INT))) {
             return ret;

opal/mca/pmix/pmix2x/pmix/src/buffer_ops/unpack.c

@@ -939,11 +939,11 @@ pmix_status_t pmix_bfrop_unpack_app(pmix_buffer_t *buffer, void *dest,
         }
         /* unpack argc */
         m=1;
-        if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_int(buffer, &ptr[i].argc, &m, PMIX_INT))) {
+        if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_int(buffer, &nval, &m, PMIX_INT32))) {
             return ret;
         }
         /* unpack argv */
-        for (k=0; k < ptr[i].argc; k++) {
+        for (k=0; k < nval; k++) {
             m=1;
             tmp = NULL;
             if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_string(buffer, &tmp, &m, PMIX_STRING))) {
@@ -972,6 +972,11 @@ pmix_status_t pmix_bfrop_unpack_app(pmix_buffer_t *buffer, void *dest,
             pmix_argv_append_nosize(&ptr[i].env, tmp);
             free(tmp);
         }
+        /* unpack cwd */
+        m=1;
+        if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_string(buffer, &ptr[i].cwd, &m, PMIX_STRING))) {
+            return ret;
+        }
         /* unpack maxprocs */
         m=1;
         if (PMIX_SUCCESS != (ret = pmix_bfrop_unpack_int(buffer, &ptr[i].maxprocs, &m, PMIX_INT))) {