Skip to content

Commit 3c8b927

Browse files
devrealdevreal
committed
Revert "osc/ucx: implement rput and rget using ucp_worker_flush_nb"
This reverts commit eee891f.
1 parent 9f95367 commit 3c8b927

File tree

3 files changed

+23
-85
lines changed

3 files changed

+23
-85
lines changed

‎ompi/mca/osc/ucx/osc_ucx_comm.c‎

Lines changed: 22 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1071,28 +1071,21 @@ int ompi_osc_ucx_rput(const void *origin_addr, int origin_count,
10711071
return ret;
10721072
}
10731073

1074-
mca_osc_ucx_component.num_incomplete_req_ops++;
1075-
ret = opal_common_ucx_wpmem_flush_ep_nb(mem, target, req_completion, ucx_req);
1074+
ret = opal_common_ucx_wpmem_fence(mem);
1075+
if (ret != OMPI_SUCCESS) {
1076+
OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_fence failed: %d", ret);
1077+
return OMPI_ERROR;
1078+
}
10761079

1080+
mca_osc_ucx_component.num_incomplete_req_ops++;
1081+
/* TODO: investigate whether ucp_worker_flush_nb is a better choice here */
1082+
ret = opal_common_ucx_wpmem_fetch_nb(module->state_mem, UCP_ATOMIC_FETCH_OP_FADD,
1083+
0, target, &(module->req_result),
1084+
sizeof(uint64_t), remote_addr & (~0x7),
1085+
req_completion, ucx_req);
10771086
if (ret != OMPI_SUCCESS) {
10781087
OMPI_OSC_UCX_REQUEST_RETURN(ucx_req);
10791088
return ret;
1080-
1081-
/* fallback to using an atomic op to acquire a request handle */
1082-
ret = opal_common_ucx_wpmem_fence(mem);
1083-
if (ret != OMPI_SUCCESS) {
1084-
OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_fence failed: %d", ret);
1085-
return OMPI_ERROR;
1086-
}
1087-
1088-
ret = opal_common_ucx_wpmem_fetch_nb(mem, UCP_ATOMIC_FETCH_OP_FADD,
1089-
0, target, &(module->req_result),
1090-
sizeof(uint64_t), remote_addr & (~0x7),
1091-
req_completion, ucx_req);
1092-
if (ret != OMPI_SUCCESS) {
1093-
OMPI_OSC_UCX_REQUEST_RETURN(ucx_req);
1094-
return ret;
1095-
}
10961089
}
10971090

10981091
*request = &ucx_req->super;
@@ -1127,28 +1120,21 @@ int ompi_osc_ucx_rget(void *origin_addr, int origin_count,
11271120
return ret;
11281121
}
11291122

1130-
mca_osc_ucx_component.num_incomplete_req_ops++;
1131-
ret = opal_common_ucx_wpmem_flush_ep_nb(mem, target, req_completion, ucx_req);
1123+
ret = opal_common_ucx_wpmem_fence(mem);
1124+
if (ret != OMPI_SUCCESS) {
1125+
OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_fence failed: %d", ret);
1126+
return OMPI_ERROR;
1127+
}
11321128

1129+
mca_osc_ucx_component.num_incomplete_req_ops++;
1130+
/* TODO: investigate whether ucp_worker_flush_nb is a better choice here */
1131+
ret = opal_common_ucx_wpmem_fetch_nb(module->state_mem, UCP_ATOMIC_FETCH_OP_FADD,
1132+
0, target, &(module->req_result),
1133+
sizeof(uint64_t), remote_addr & (~0x7),
1134+
req_completion, ucx_req);
11331135
if (ret != OMPI_SUCCESS) {
11341136
OMPI_OSC_UCX_REQUEST_RETURN(ucx_req);
11351137
return ret;
1136-
1137-
/* fallback to using an atomic op to acquire a request handle */
1138-
ret = opal_common_ucx_wpmem_fence(mem);
1139-
if (ret != OMPI_SUCCESS) {
1140-
OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_fence failed: %d", ret);
1141-
return OMPI_ERROR;
1142-
}
1143-
1144-
ret = opal_common_ucx_wpmem_fetch_nb(mem, UCP_ATOMIC_FETCH_OP_FADD,
1145-
0, target, &(module->req_result),
1146-
sizeof(uint64_t), remote_addr & (~0x7),
1147-
req_completion, ucx_req);
1148-
if (ret != OMPI_SUCCESS) {
1149-
OMPI_OSC_UCX_REQUEST_RETURN(ucx_req);
1150-
return ret;
1151-
}
11521138
}
11531139

11541140
*request = &ucx_req->super;

‎opal/mca/common/ucx/common_ucx_wpool.c‎

Lines changed: 1 addition & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -761,7 +761,7 @@ OPAL_DECLSPEC int opal_common_ucx_winfo_flush(opal_common_ucx_winfo_t *winfo, in
761761
((opal_common_ucx_request_t *) req)->winfo = winfo;
762762
}
763763

764-
if (OPAL_COMMON_UCX_FLUSH_B == type) {
764+
if (OPAL_COMMON_UCX_FLUSH_B) {
765765
rc = opal_common_ucx_wait_request_mt(req, "ucp_ep_flush_nb");
766766
} else {
767767
*req_ptr = req;
@@ -818,57 +818,13 @@ OPAL_DECLSPEC int opal_common_ucx_ctx_flush(opal_common_ucx_ctx_t *ctx,
818818
if (rc != OPAL_SUCCESS) {
819819
MCA_COMMON_UCX_ERROR("opal_common_ucx_flush failed: %d", rc);
820820
rc = OPAL_ERROR;
821-
break;
822821
}
823822
}
824823
opal_mutex_unlock(&ctx->mutex);
825824

826825
return rc;
827826
}
828827

829-
830-
OPAL_DECLSPEC int opal_common_ucx_wpmem_flush_ep_nb(opal_common_ucx_wpmem_t *mem,
831-
int target,
832-
opal_common_ucx_user_req_handler_t user_req_cb,
833-
void *user_req_ptr)
834-
{
835-
#if HAVE_DECL_UCP_EP_FLUSH_NB
836-
int rc = OPAL_SUCCESS;
837-
ucp_ep_h ep = NULL;
838-
ucp_rkey_h rkey = NULL;
839-
opal_common_ucx_winfo_t *winfo = NULL;
840-
841-
if (NULL == mem) {
842-
return OPAL_SUCCESS;
843-
}
844-
845-
rc = opal_common_ucx_tlocal_fetch(mem, target, &ep, &rkey, &winfo);
846-
if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
847-
MCA_COMMON_UCX_ERROR("tlocal_fetch failed: %d", rc);
848-
return rc;
849-
}
850-
851-
opal_mutex_lock(&winfo->mutex);
852-
opal_common_ucx_request_t *req;
853-
req = ucp_worker_flush_nb(winfo->worker, 0, opal_common_ucx_req_completion);
854-
if (UCS_PTR_IS_PTR(req)) {
855-
req->ext_req = user_req_ptr;
856-
req->ext_cb = user_req_cb;
857-
req->winfo = winfo;
858-
} else {
859-
if (user_req_cb != NULL) {
860-
(*user_req_cb)(user_req_ptr);
861-
}
862-
}
863-
opal_mutex_unlock(&winfo->mutex);
864-
return rc;
865-
#else
866-
return OPAL_ERR_NOT_SUPPORTED;
867-
#endif // HAVE_DECL_UCP_EP_FLUSH_NB
868-
869-
}
870-
871-
872828
OPAL_DECLSPEC int opal_common_ucx_wpmem_fence(opal_common_ucx_wpmem_t *mem)
873829
{
874830
ucs_status_t status = UCS_OK;

‎opal/mca/common/ucx/common_ucx_wpool.h‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -247,10 +247,6 @@ OPAL_DECLSPEC void opal_common_ucx_wpmem_free(opal_common_ucx_wpmem_t *mem);
247247

248248
OPAL_DECLSPEC int opal_common_ucx_ctx_flush(opal_common_ucx_ctx_t *ctx,
249249
opal_common_ucx_flush_scope_t scope, int target);
250-
OPAL_DECLSPEC int opal_common_ucx_wpmem_flush_ep_nb(opal_common_ucx_wpmem_t *mem,
251-
int target,
252-
opal_common_ucx_user_req_handler_t user_req_cb,
253-
void *user_req_ptr);
254250
OPAL_DECLSPEC int opal_common_ucx_wpmem_fence(opal_common_ucx_wpmem_t *mem);
255251

256252
OPAL_DECLSPEC int opal_common_ucx_winfo_flush(opal_common_ucx_winfo_t *winfo, int target,

0 commit comments

Comments
 (0)