minor revisions, renaming, indents, macros, adding flush to ctx_destroy

Author: Tomislav Janjusic

opal/mca/common/ucx/common_ucx.c

@@ -155,14 +155,7 @@ void opal_common_ucx_mca_proc_added(void)
 
 OPAL_DECLSPEC int opal_common_ucx_mca_pmix_fence_nb(int *fenced)
 {
-    int ret = OPAL_SUCCESS;
-
-    if (OPAL_SUCCESS != (ret = opal_pmix.fence_nb(NULL, 0,
-                    opal_common_ucx_mca_fence_complete_cb, (void*)fenced))){
-        return ret;
-    }
-
-    return ret;
+    return opal_pmix.fence_nb(NULL, 0, opal_common_ucx_mca_fence_complete_cb, (void *) fenced);
 }
 
 OPAL_DECLSPEC int opal_common_ucx_mca_pmix_fence(ucp_worker_h worker)
@@ -194,7 +187,7 @@ static void opal_common_ucx_wait_all_requests(void **reqs, int count, ucp_worker
     }
 }
 
-OPAL_DECLSPEC int opal_common_ucx_del_procs_nb(opal_common_ucx_del_proc_t *procs, size_t count,
+OPAL_DECLSPEC int opal_common_ucx_del_procs_nofence(opal_common_ucx_del_proc_t *procs, size_t count,
                                             size_t my_rank, size_t max_disconnect, ucp_worker_h worker) {
     size_t num_reqs;
     size_t max_reqs;
@@ -249,12 +242,8 @@ OPAL_DECLSPEC int opal_common_ucx_del_procs_nb(opal_common_ucx_del_proc_t *procs
 OPAL_DECLSPEC int opal_common_ucx_del_procs(opal_common_ucx_del_proc_t *procs, size_t count,
                                             size_t my_rank, size_t max_disconnect, ucp_worker_h worker)
 {
-    int ret = OPAL_SUCCESS;
-    opal_common_ucx_del_procs_nb(procs, count, my_rank, max_disconnect, worker);
-    if (OPAL_SUCCESS != (ret = opal_common_ucx_mca_pmix_fence(worker))) {
-        return ret;
-    }
+    opal_common_ucx_del_procs_nofence(procs, count, my_rank, max_disconnect, worker);
 
-    return OPAL_SUCCESS;
+    return opal_common_ucx_mca_pmix_fence(worker);
 }
 

oshmem/mca/spml/ucx/spml_ucx.c

@@ -515,15 +515,14 @@ static inline void _ctx_add(mca_spml_ucx_ctx_array_t *array, mca_spml_ucx_ctx_t
 
     if (array->ctxs_count < array->ctxs_num) {
         array->ctxs[array->ctxs_count] = ctx;
-    }
-    else {
-        array->ctxs = realloc(array->ctxs, (array->ctxs_num + 8) * sizeof(mca_spml_ucx_ctx_t *));
+    } else {
+        array->ctxs = realloc(array->ctxs, (array->ctxs_num + MCA_SPML_UCX_CTXS_ARRAY_INC) * sizeof(mca_spml_ucx_ctx_t *));
         opal_atomic_wmb ();
-        for (i = array->ctxs_num; i < array->ctxs_num + 8; i++) {
+        for (i = array->ctxs_num; i < array->ctxs_num + MCA_SPML_UCX_CTXS_ARRAY_INC; i++) {
             array->ctxs[i] = NULL;
         }
         array->ctxs[array->ctxs_num] = ctx;
-        array->ctxs_num += 8;
+        array->ctxs_num += MCA_SPML_UCX_CTXS_ARRAY_INC;
     }
 
     opal_atomic_wmb ();
@@ -637,13 +636,16 @@ int mca_spml_ucx_ctx_create(long options, shmem_ctx_t *ctx)
 
 void mca_spml_ucx_ctx_destroy(shmem_ctx_t ctx)
 {
+    mca_spml_ucx_ctx_t *ucx_ctx = (mca_spml_ucx_ctx_t *)ctx;
     MCA_SPML_CALL(quiet(ctx));
 
     SHMEM_MUTEX_LOCK(mca_spml_ucx.internal_mutex);
     _ctx_remove(&mca_spml_ucx.active_array, (mca_spml_ucx_ctx_t *)ctx);
     _ctx_add(&mca_spml_ucx.idle_array, (mca_spml_ucx_ctx_t *)ctx);
     SHMEM_MUTEX_UNLOCK(mca_spml_ucx.internal_mutex);
 
+    ucp_worker_flush(ucx_ctx->ucp_worker);
+
     if (!mca_spml_ucx.active_array.ctxs_count) {
         opal_progress_unregister(spml_ucx_ctx_progress);
     }

oshmem/mca/spml/ucx/spml_ucx.h

@@ -193,6 +193,9 @@ static inline int ucx_status_to_oshmem_nb(ucs_status_t status)
 #endif
 }
 
+#define MCA_SPML_UCX_CTXS_ARRAY_SIZE 64
+#define MCA_SPML_UCX_CTXS_ARRAY_INC  64
+
 END_C_DECLS
 
 #endif

oshmem/mca/spml/ucx/spml_ucx_component.c

@@ -178,11 +178,11 @@ static int spml_ucx_init(void)
     }
 
     mca_spml_ucx.active_array.ctxs_count = mca_spml_ucx.idle_array.ctxs_count = 0;
-    mca_spml_ucx.active_array.ctxs_num = mca_spml_ucx.idle_array.ctxs_num = 8;
+    mca_spml_ucx.active_array.ctxs_num = mca_spml_ucx.idle_array.ctxs_num = MCA_SPML_UCX_CTXS_ARRAY_SIZE;
     mca_spml_ucx.active_array.ctxs = calloc(mca_spml_ucx.active_array.ctxs_num,
                                             sizeof(mca_spml_ucx_ctx_t *));
     mca_spml_ucx.idle_array.ctxs = calloc(mca_spml_ucx.idle_array.ctxs_num,
-                                            sizeof(mca_spml_ucx_ctx_t *));
+                                          sizeof(mca_spml_ucx_ctx_t *));
 
     SHMEM_MUTEX_INIT(mca_spml_ucx.internal_mutex);
 
@@ -251,7 +251,7 @@ static void _ctx_cleanup(mca_spml_ucx_ctx_t *ctx)
         ctx->ucp_peers[i].ucp_conn = NULL;
     }
 
-    opal_common_ucx_del_procs_nb(del_procs, nprocs, oshmem_my_proc_id(),
+    opal_common_ucx_del_procs_nofence(del_procs, nprocs, oshmem_my_proc_id(),
                                  mca_spml_ucx.num_disconnect,
                                  ctx->ucp_worker);
     free(del_procs);