Merge pull request #4746 from karasevb/pmix21_update

Sync to PMIx v2.1.0

Author: bwbarrett

opal/mca/pmix/pmix2x/pmix/NEWS

@@ -1,4 +1,4 @@
-Copyright (c) 2015-2017 Intel, Inc. All rights reserved.
+Copyright (c) 2015-2018 Intel, Inc. All rights reserved.
 Copyright (c) 2017      IBM Corporation.  All rights reserved.
 $COPYRIGHT$
 
@@ -21,7 +21,7 @@ example, a bug might be fixed in the master, and then moved to the
 current release as well as the "stable" bug fix release branch.
 
 
-2.1.0 -- 26 Oct 2017
+2.1.0 -- 1 Feb 2018
 ----------------------
 **** NOTE: This release contains the first implementation of cross-version
 **** support. Servers using v2.1.0 are capable of supporting clients using
@@ -32,6 +32,30 @@ current release as well as the "stable" bug fix release branch.
 - Enable support for remote tool connections (PR #540, #542)
 - Cleanup libevent configure logi to support default install paths (PR #541)
 - Debounce "unreachable" notifications for tools when they disconnect (PR #544)
+- Enable the regex generator to support node names that include multiple
+  sets of numbers
+
+
+2.0.3 -- 1 Feb 2018
+----------------------
+- Fix event notification so all sides of multi-library get notified
+  of other library's existence
+- Update syslog protection to support Mac High Sierra OS
+- Remove usock component - unable to support v1.x clients due
+  to datatype differences
+- Cleanup security handshake
+- Cleanup separation of PMI-1/2 libraries and PMIx symbols
+- Protect against overly-large messages
+- Update data buffer APIs to support cross-version operations
+- Protect receive callbacks from NULL and/or empty buffers as this
+  can occur when the peer on a connection disappears.
+- Fix tool connection search so it properly descends into the directory
+  tree while searching for the server's contact file.
+- Fix store_local so it doesn't reject a new nspace as that can happen
+  when working with tools
+- Ensure we always complete PMIx_Finalize - don't return if something
+  goes wrong in the middle of the procedure
+- Fix several tool connection issues
 
 
 2.0.2 -- 19 Oct 2017
@@ -115,6 +139,14 @@ current release as well as the "stable" bug fix release branch.
   and to themselves
 
 
+1.2.5 -- 1 Feb 2018
+----------------------
+- Fix cross-version issue when v1.2 client interacts with v2.1 server (PR #564)
+- Update client connection for cross-version support (PR #591)
+- Fix write memory barrier ASM for PowerPC (PR #606)
+- Add protection from overly-large messages
+
+
 1.2.4 -- 13 Oct. 2017
 ----------------------
 - Silence some unnecessary warning messages (PR #487)

opal/mca/pmix/pmix2x/pmix/VERSION

@@ -30,7 +30,7 @@ greek=
 # command, or with the date (if "git describe" fails) in the form of
 # "date<date>".
 
-repo_rev=git1617e76
+repo_rev=git79b2db3
 
 # If tarball_version is not empty, it is used as the version string in
 # the tarball filename, regardless of all other versions listed in
@@ -44,7 +44,7 @@ tarball_version=
 
 # The date when this release was created
 
-date="Oct 31, 2017"
+date="Feb 01, 2018"
 
 # The shared library version of each of PMIx's public libraries.
 # These versions are maintained in accordance with the "Library
@@ -75,6 +75,6 @@ date="Oct 31, 2017"
 # Version numbers are described in the Libtool current:revision:age
 # format.
 
-libpmix_so_version=3:3:1
+libpmix_so_version=3:10:1
 libpmi_so_version=1:0:0
 libpmi2_so_version=1:0:0

opal/mca/pmix/pmix2x/pmix/etc/pmix-mca-params.conf

@@ -10,7 +10,7 @@
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
 # Copyright (c) 2006-2017 Cisco Systems, Inc.  All rights reserved
-# Copyright (c) 2017      Intel, Inc. All rights reserved.
+# Copyright (c) 2017-2018 Intel, Inc. All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -53,7 +53,7 @@
 # directory.  For example:
 
 # Change component loading path
-#   component_path = /usr/local/lib/pmix:~/my_pmix_components
+#   mca_base_component_path = /usr/local/lib/pmix:~/my_pmix_components
 
 # See "pinfo --param all all --level 9" for a full listing of PMIx
 # MCA parameters available and their default values.

opal/mca/pmix/pmix2x/pmix/examples/client.c

@@ -13,7 +13,7 @@
  *                         All rights reserved.
  * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
@@ -97,11 +97,11 @@ int main(int argc, char **argv)
     int rc;
     pmix_value_t value;
     pmix_value_t *val = &value;
-    char *tmp;
+    char *tmp, *ptr, *p;
     pmix_proc_t proc;
-    uint32_t nprocs, n;
+    uint32_t nprocs, m, n, local_cnt, *localpeers;
     pmix_info_t *info;
-    bool flag;
+    bool flag, local;
     volatile int active;
     pmix_status_t dbg = PMIX_ERR_DEBUGGER_RELEASE;
 
@@ -196,7 +196,7 @@ int main(int argc, char **argv)
     value.type = PMIX_UINT64;
     value.data.uint64 = 1234;
     if (PMIX_SUCCESS != (rc = PMIx_Put(PMIX_LOCAL, tmp, &value))) {
-        fprintf(stderr, "Client ns %s rank %d: PMIx_Put internal failed: %d\n", myproc.nspace, myproc.rank, rc);
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Put local failed: %d\n", myproc.nspace, myproc.rank, rc);
         goto done;
     }
     free(tmp);
@@ -207,7 +207,7 @@ int main(int argc, char **argv)
     value.type = PMIX_STRING;
     value.data.string = "1234";
     if (PMIX_SUCCESS != (rc = PMIx_Put(PMIX_REMOTE, tmp, &value))) {
-        fprintf(stderr, "Client ns %s rank %d: PMIx_Put internal failed: %d\n", myproc.nspace, myproc.rank, rc);
+        fprintf(stderr, "Client ns %s rank %d: PMIx_Put remote failed: %d\n", myproc.nspace, myproc.rank, rc);
         goto done;
     }
     free(tmp);
@@ -230,53 +230,96 @@ int main(int argc, char **argv)
     }
     PMIX_INFO_FREE(info, 1);
 
+    /* get the number of local peers */
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_LOCAL_SIZE, NULL, 0, &val))) {
+         fprintf(stderr, "Client ns %s rank %d: PMIx_Get PMIX_LOCAL_SIZE failed: %d", myproc.nspace, myproc.rank, rc);
+         goto done;
+     }
+     local_cnt = val->data.uint32;
+     PMIX_VALUE_RELEASE(val);
+
+     /* create an array for the peers */
+     localpeers = (uint32_t*)malloc(local_cnt * sizeof(int));
+
+    /* get the list of local peers */
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_LOCAL_PEERS, NULL, 0, &val))) {
+         fprintf(stderr, "Client ns %s rank %d: PMIx_Get PMIX_LOCAL_PEERS failed: %d", myproc.nspace, myproc.rank, rc);
+         goto done;
+     }
+     ptr = strdup(val->data.string);
+     PMIX_VALUE_RELEASE(val);
+
+     /* populate the peers array */
+     p = strtok(ptr, ",");
+     localpeers[0] = strtoul(p, NULL, 10);
+     for (n=1; n < local_cnt; n++) {
+        p = strtok(NULL, ",");
+        localpeers[n] = strtoul(p, NULL, 10);
+     }
+     free(ptr);
+
     /* check the returned data */
     for (n=0; n < nprocs; n++) {
-        if (0 > asprintf(&tmp, "%s-%d-local", myproc.nspace, myproc.rank)) {
-            exit(1);
+        if (n == myproc.rank) {
+            continue;
         }
-        if (PMIX_SUCCESS != (rc = PMIx_Get(&myproc, tmp, NULL, 0, &val))) {
-            fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s failed: %d\n", myproc.nspace, myproc.rank, tmp, rc);
-            goto done;
-        }
-        if (PMIX_UINT64 != val->type) {
-            fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong type: %d\n", myproc.nspace, myproc.rank, tmp, val->type);
-            PMIX_VALUE_RELEASE(val);
-            free(tmp);
-            goto done;
+        proc.rank = n;
+        local = false;
+        for (m=0; m < local_cnt; m++) {
+            if (localpeers[m] == proc.rank) {
+                local = true;
+                break;
+            }
         }
-        if (1234 != val->data.uint64) {
-            fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong value: %d\n", myproc.nspace, myproc.rank, tmp, (int)val->data.uint64);
+        if (local) {
+            if (0 > asprintf(&tmp, "%s-%d-local", proc.nspace, proc.rank)) {
+                exit(1);
+            }
+            if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, tmp, NULL, 0, &val))) {
+                fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s failed: %d\n", myproc.nspace, myproc.rank, tmp, rc);
+                goto done;
+            }
+            if (PMIX_UINT64 != val->type) {
+                fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong type: %d\n", myproc.nspace, myproc.rank, tmp, val->type);
+                PMIX_VALUE_RELEASE(val);
+                free(tmp);
+                goto done;
+            }
+            if (1234 != val->data.uint64) {
+                fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong value: %d\n", myproc.nspace, myproc.rank, tmp, (int)val->data.uint64);
+                PMIX_VALUE_RELEASE(val);
+                free(tmp);
+                goto done;
+            }
+            fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned correct\n", myproc.nspace, myproc.rank, tmp);
             PMIX_VALUE_RELEASE(val);
             free(tmp);
-            goto done;
-        }
-        fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned correct\n", myproc.nspace, myproc.rank, tmp);
-        PMIX_VALUE_RELEASE(val);
-        free(tmp);
-        if (0 > asprintf(&tmp, "%s-%d-remote", myproc.nspace, myproc.rank)) {
-            exit(1);
-        }
-        if (PMIX_SUCCESS != (rc = PMIx_Get(&myproc, tmp, NULL, 0, &val))) {
-            fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s failed: %d\n", myproc.nspace, myproc.rank, tmp, rc);
-            goto done;
-        }
-        if (PMIX_STRING != val->type) {
-            fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong type: %d\n", myproc.nspace, myproc.rank, tmp, val->type);
-            PMIX_VALUE_RELEASE(val);
-            free(tmp);
-            goto done;
-        }
-        if (0 != strcmp(val->data.string, "1234")) {
-            fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong value: %s\n", myproc.nspace, myproc.rank, tmp, val->data.string);
+        } else {
+            if (0 > asprintf(&tmp, "%s-%d-remote", proc.nspace, proc.rank)) {
+                exit(1);
+            }
+            if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, tmp, NULL, 0, &val))) {
+                fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s failed: %d\n", myproc.nspace, myproc.rank, tmp, rc);
+                goto done;
+            }
+            if (PMIX_STRING != val->type) {
+                fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong type: %d\n", myproc.nspace, myproc.rank, tmp, val->type);
+                PMIX_VALUE_RELEASE(val);
+                free(tmp);
+                goto done;
+            }
+            if (0 != strcmp(val->data.string, "1234")) {
+                fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong value: %s\n", myproc.nspace, myproc.rank, tmp, val->data.string);
+                PMIX_VALUE_RELEASE(val);
+                free(tmp);
+                goto done;
+            }
+            fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned correct\n", myproc.nspace, myproc.rank, tmp);
             PMIX_VALUE_RELEASE(val);
             free(tmp);
-            goto done;
         }
-        fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned correct\n", myproc.nspace, myproc.rank, tmp);
-        PMIX_VALUE_RELEASE(val);
-        free(tmp);
     }
+    free(localpeers);
 
  done:
     /* finalize us */

opal/mca/pmix/pmix2x/pmix/include/pmi.h

@@ -48,11 +48,8 @@
 #ifndef PMI_H
 #define PMI_H
 
-#ifdef PMIX_HAVE_VISIBILITY
-#define PMIX_EXPORT __attribute__((__visibility__("default")))
-#else
-#define PMIX_EXPORT
-#endif
+/* Structure and constant definitions */
+#include <pmix_common.h>
 
 /* prototypes for the PMI interface in MPICH2 */
 

opal/mca/pmix/pmix2x/pmix/include/pmi2.h

@@ -7,11 +7,8 @@
 #ifndef PMI2_H_INCLUDED
 #define PMI2_H_INCLUDED
 
-#ifdef PMIX_HAVE_VISIBILITY
-#define PMIX_EXPORT __attribute__((__visibility__("default")))
-#else
-#define PMIX_EXPORT
-#endif
+/* Structure and constant definitions */
+#include <pmix_common.h>
 
 #define PMI2_MAX_KEYLEN 64
 #define PMI2_MAX_VALLEN 1024

opal/mca/pmix/pmix2x/pmix/include/pmix_common.h.in

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2013-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2013-2018 Intel, Inc.  All rights reserved.
  * Copyright (c) 2016-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016      IBM Corporation.  All rights reserved.
@@ -1675,7 +1675,6 @@ PMIX_EXPORT const char* PMIx_Get_version(void);
 PMIX_EXPORT pmix_status_t PMIx_Store_internal(const pmix_proc_t *proc,
                                               const char *key, pmix_value_t *val);
 
-
 /**
  * Top-level interface function to pack one or more values into a
  * buffer.
@@ -1694,6 +1693,17 @@ PMIX_EXPORT pmix_status_t PMIx_Store_internal(const pmix_proc_t *proc,
  * will return an error code (generated upon unpacking) -
  * the error cannot be detected during packing.
  *
+ * The identity of the intended recipient of the packed buffer (i.e., the
+ * process that will be unpacking it) is used solely to resolve any data type
+ * differences between PMIx versions. The recipient must, therefore, be
+ * known to the user prior to calling the pack function so that the
+ * PMIx library is aware of the version the recipient is using.
+ *
+ * @param *target Pointer to a pmix_proc_t structure containing the
+ * nspace/rank of the process that will be unpacking the final buffer.
+ * A NULL value may be used to indicate that the target is based on
+ * the same PMIx version as the caller.
+ *
  * @param *buffer A pointer to the buffer into which the value is to
  * be packed.
  *
@@ -1727,7 +1737,8 @@ PMIX_EXPORT pmix_status_t PMIx_Store_internal(const pmix_proc_t *proc,
  * status_code = PMIx_Data_pack(buffer, &src, 1, PMIX_INT32);
  * @endcode
  */
-PMIX_EXPORT pmix_status_t PMIx_Data_pack(pmix_data_buffer_t *buffer,
+PMIX_EXPORT pmix_status_t PMIx_Data_pack(const pmix_proc_t *target,
+                                         pmix_data_buffer_t *buffer,
                                          void *src, int32_t num_vals,
                                          pmix_data_type_t type);
 
@@ -1774,6 +1785,17 @@ PMIX_EXPORT pmix_status_t PMIx_Data_pack(pmix_data_buffer_t *buffer,
  * will return an error code generated upon unpacking - these errors
  * cannot be detected during packing.
  *
+ * The identity of the source of the packed buffer (i.e., the
+ * process that packed it) is used solely to resolve any data type
+ * differences between PMIx versions. The source must, therefore, be
+ * known to the user prior to calling the unpack function so that the
+ * PMIx library is aware of the version the source used.
+ *
+ * @param *source Pointer to a pmix_proc_t structure containing the
+ * nspace/rank of the process that packed the provided buffer.
+ * A NULL value may be used to indicate that the source is based on
+ * the same PMIx version as the caller.
+ *
  * @param *buffer A pointer to the buffer from which the value will be
  * extracted.
  *
@@ -1823,7 +1845,8 @@ PMIX_EXPORT pmix_status_t PMIx_Data_pack(pmix_data_buffer_t *buffer,
  *
  * @endcode
  */
-PMIX_EXPORT pmix_status_t PMIx_Data_unpack(pmix_data_buffer_t *buffer, void *dest,
+PMIX_EXPORT pmix_status_t PMIx_Data_unpack(const pmix_proc_t *source,
+                                           pmix_data_buffer_t *buffer, void *dest,
                                            int32_t *max_num_values,
                                            pmix_data_type_t type);
 

opal/mca/pmix/pmix2x/pmix/include/pmix_rename.h.in

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016      Intel, Inc. All rights reserved
+ * Copyright (c) 2016-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$