pmix/external: correctly use PMIx_server_register_nspace()

PMIx_server_register_nspace() is an asynchronous operation, so
the pmix glue wait for it completes before returning.

Signed-off-by: Gilles Gouaillardet <[email protected]>

(back-ported from commit dccb189)

Author: ggouaillardet

opal/mca/pmix/external/pmix_ext.c

@@ -1,7 +1,7 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2014-2015 Intel, Inc.  All rights reserved.
- * Copyright (c) 2014-2015 Research Organization for Information Science
+ * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014      Mellanox Technologies, Inc.
  *                         All rights reserved.
@@ -490,6 +490,7 @@ static void opcon(pmix_ext_opcaddy_t *p)
     p->ninfo = 0;
     p->apps = NULL;
     p->sz = 0;
+    p->active = false;
     p->opcbfunc = NULL;
     p->mdxcbfunc = NULL;
     p->valcbfunc = NULL;

opal/mca/pmix/external/pmix_ext.h

@@ -2,6 +2,8 @@
  * Copyright (c) 2014-2015 Intel, Inc.  All rights reserved.
  * Copyright (c) 2014-2015 Mellanox Technologies, Inc.
  *                         All rights reserved.
+ * Copyright (c) 2017      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -61,6 +63,7 @@ typedef struct {
     size_t ninfo;
     pmix_app_t *apps;
     size_t sz;
+    volatile bool active;
     opal_pmix_op_cbfunc_t opcbfunc;
     opal_pmix_modex_cbfunc_t mdxcbfunc;
     opal_pmix_value_cbfunc_t valcbfunc;

opal/mca/pmix/external/pmix_ext_server_south.c

@@ -1,7 +1,7 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2014-2015 Intel, Inc.  All rights reserved.
- * Copyright (c) 2014-2016 Research Organization for Information Science
+ * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014      Mellanox Technologies, Inc.
  *                         All rights reserved.
@@ -54,6 +54,14 @@ static void release_cbfunc(void *cbdata)
     pmix_ext_opalcaddy_t *cd = (pmix_ext_opalcaddy_t*)cbdata;
     OBJ_RELEASE(cd);
 }
+
+#define PMIX_WAIT_FOR_COMPLETION(a)             \
+    do {                                        \
+        while ((a)) {                           \
+            usleep(10);                         \
+        }                                       \
+    } while (0);
+
 static void myerr(pmix_status_t status,
                   pmix_proc_t procs[], size_t nprocs,
                   pmix_info_t info[], size_t ninfo)
@@ -178,7 +186,11 @@ static void opcbfunc(pmix_status_t status, void *cbdata)
     if (NULL != op->opcbfunc) {
         op->opcbfunc(pmix_ext_convert_rc(status), op->cbdata);
     }
-    OBJ_RELEASE(op);
+    if (op->active) {
+        op->active = false;
+    } else {
+        OBJ_RELEASE(op);
+    }
 }
 
 int pmix_ext_server_register_nspace(opal_jobid_t jobid,
@@ -192,7 +204,7 @@ int pmix_ext_server_register_nspace(opal_jobid_t jobid,
     size_t sz, szmap, m, n;
     char nspace[PMIX_MAX_NSLEN];
     pmix_status_t rc;
-    pmix_ext_opcaddy_t *op;
+    pmix_ext_opcaddy_t op;
     opal_list_t *pmapinfo;
     opal_pmix_ext_jobid_trkr_t *job;
 
@@ -238,15 +250,15 @@ int pmix_ext_server_register_nspace(opal_jobid_t jobid,
     }
 
     /* setup the caddy */
-    op = OBJ_NEW(pmix_ext_opcaddy_t);
-    op->info = pinfo;
-    op->sz = sz;
-    op->opcbfunc = cbfunc;
-    op->cbdata = cbdata;
+    OBJ_CONSTRUCT(&op, pmix_ext_opcaddy_t);
+    op.info = pinfo;
+    op.sz = sz;
+    op.opcbfunc = cbfunc;
+    op.cbdata = cbdata;
     rc = PMIx_server_register_nspace(nspace, nlocalprocs, pinfo, sz,
-                                     opcbfunc, op);
-    if (PMIX_SUCCESS != rc) {
-        OBJ_RELEASE(op);
+                                     opcbfunc, &op);
+    if (PMIX_SUCCESS == rc) {
+        PMIX_WAIT_FOR_COMPLETION(op.active);
     }
     return pmix_ext_convert_rc(rc);
 }