Merge pull request #5248 from hoopoepg/topic/pml-ucx-hang-on-finalize-v3.1

PML/UCX: fixed hang on MPI_Finalize - v3.1.x

Author: bwbarrett

ompi/mca/pml/ucx/pml_ucx.c

@@ -394,6 +394,7 @@ int mca_pml_ucx_del_procs(struct ompi_proc_t **procs, size_t nprocs)
     void *dreq, **dreqs;
     ucp_ep_h ep;
     size_t i;
+    ucs_status_t ret;
 
     max_reqs = ompi_pml_ucx.num_disconnect;
     if (max_reqs > nprocs) {
@@ -435,6 +436,14 @@ int mca_pml_ucx_del_procs(struct ompi_proc_t **procs, size_t nprocs)
 
     mca_pml_ucx_waitall(dreqs, &num_reqs);
     free(dreqs);
+    /* flush worker to allow all pending operations to complete.
+     * ignore error (we can do nothing here), just try to
+     * finalize gracefully */
+    ret = ucp_worker_flush(ompi_pml_ucx.ucp_worker);
+    if (UCS_OK != ret) {
+        PML_UCX_ERROR("ucp_worker_flush failed: %s",
+                      ucs_status_string(ret));
+    }
 
     opal_pmix.fence(NULL, 0);