Skip to content

Commit e439bda

Browse files
hppritchahppritcha
authored
Merge pull request #2420 from alex-mikheev/topic/oshmem_mkey_cache_v2.x
Topic/oshmem mkey cache v2.x
2 parents f8065c9 + b34ed20 commit e439bda

24 files changed

+633
-976
lines changed

‎oshmem/mca/atomic/mxm/atomic_mxm.h‎

Lines changed: 68 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
/* This component does uses SPML:IKRIT */
2121
#include "oshmem/mca/spml/ikrit/spml_ikrit.h"
22+
#include "oshmem/runtime/runtime.h"
2223

2324

2425
BEGIN_C_DECLS
@@ -60,16 +61,76 @@ struct mca_atomic_mxm_module_t {
6061
typedef struct mca_atomic_mxm_module_t mca_atomic_mxm_module_t;
6162
OBJ_CLASS_DECLARATION(mca_atomic_mxm_module_t);
6263

63-
END_C_DECLS
6464

65-
#if MXM_API >= MXM_VERSION(2,0)
66-
static inline mxm_mem_key_t *to_mxm_mkey(sshmem_mkey_t *mkey) {
65+
static inline uint8_t mca_atomic_mxm_order(size_t nlong)
66+
{
67+
if (OPAL_LIKELY(8 == nlong)) {
68+
return 3;
69+
}
70+
71+
if (OPAL_LIKELY(4 == nlong)) {
72+
return 2;
73+
}
74+
75+
if (2 == nlong) {
76+
return 1;
77+
}
6778

68-
if (0 == mkey->len) {
69-
return &mxm_empty_mem_key;
79+
if (1 == nlong) {
80+
return 0;
7081
}
71-
return (mxm_mem_key_t *)mkey->u.data;
82+
83+
ATOMIC_ERROR("Type size must be 1/2/4 or 8 bytes.");
84+
oshmem_shmem_abort(-1);
85+
return OSHMEM_ERR_BAD_PARAM;
7286
}
73-
#endif
87+
88+
static inline void mca_atomic_mxm_req_init(mxm_send_req_t *sreq, int pe, void *target, size_t nlong)
89+
{
90+
uint8_t nlong_order;
91+
void *remote_addr;
92+
mxm_mem_key_t *mkey;
93+
94+
nlong_order = mca_atomic_mxm_order(nlong);
95+
96+
mkey = mca_spml_ikrit_get_mkey(pe, target, MXM_PTL_RDMA, &remote_addr);
97+
98+
/* mxm request init */
99+
sreq->base.state = MXM_REQ_NEW;
100+
sreq->base.mq = mca_atomic_mxm_spml_self->mxm_mq;
101+
sreq->base.conn = mca_atomic_mxm_spml_self->mxm_peers[pe].mxm_hw_rdma_conn;
102+
sreq->base.completed_cb = NULL;
103+
sreq->base.data_type = MXM_REQ_DATA_BUFFER;
104+
105+
sreq->base.data.buffer.memh = MXM_INVALID_MEM_HANDLE;
106+
sreq->base.data.buffer.length = nlong;
107+
108+
sreq->op.atomic.remote_vaddr = (uintptr_t) remote_addr;
109+
sreq->op.atomic.remote_mkey = mkey;
110+
sreq->op.atomic.order = nlong_order;
111+
112+
sreq->flags = 0;
113+
}
114+
115+
static inline void mca_atomic_mxm_post(mxm_send_req_t *sreq)
116+
{
117+
mxm_error_t mxm_err;
118+
119+
mxm_err = mxm_req_send(sreq);
120+
if (OPAL_UNLIKELY(MXM_OK != mxm_err)) {
121+
ATOMIC_ERROR("mxm_req_send failed, mxm_error = %d",
122+
mxm_err);
123+
oshmem_shmem_abort(-1);
124+
}
125+
126+
mxm_req_wait(&sreq->base);
127+
if (OPAL_UNLIKELY(MXM_OK != sreq->base.error)) {
128+
ATOMIC_ERROR("mxm_req_wait got non MXM_OK error: %d",
129+
sreq->base.error);
130+
oshmem_shmem_abort(-1);
131+
}
132+
}
133+
134+
END_C_DECLS
74135

75136
#endif /* MCA_ATOMIC_MXM_H */

‎oshmem/mca/atomic/mxm/atomic_mxm_cswap.c‎

Lines changed: 2 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -31,96 +31,20 @@ int mca_atomic_mxm_cswap(void *target,
3131
size_t nlong,
3232
int pe)
3333
{
34-
unsigned my_pe;
35-
uint8_t nlong_order;
36-
void *remote_addr;
37-
int ptl_id;
3834
mxm_send_req_t sreq;
39-
mxm_error_t mxm_err;
40-
sshmem_mkey_t *r_mkey;
4135

42-
my_pe = oshmem_my_proc_id();
43-
ptl_id = -1;
44-
mxm_err = MXM_OK;
36+
mca_atomic_mxm_req_init(&sreq, pe, target, nlong);
4537

46-
switch (nlong) {
47-
case 1:
48-
nlong_order = 0;
49-
break;
50-
case 2:
51-
nlong_order = 1;
52-
break;
53-
case 4:
54-
nlong_order = 2;
55-
break;
56-
case 8:
57-
nlong_order = 3;
58-
break;
59-
default:
60-
ATOMIC_ERROR("[#%d] Type size must be 1/2/4 or 8 bytes.", my_pe);
61-
oshmem_shmem_abort(-1);
62-
return OSHMEM_ERR_BAD_PARAM;
63-
}
64-
65-
ptl_id = OSHMEM_PROC_DATA(oshmem_proc_group_all(pe))->transport_ids[0];
66-
if (MXM_PTL_SHM == ptl_id) {
67-
ptl_id = MXM_PTL_RDMA;
68-
}
69-
r_mkey = mca_memheap_base_get_cached_mkey(pe, target, ptl_id, &remote_addr);
70-
if (!r_mkey) {
71-
ATOMIC_ERROR("[#%d] %p is not address of symmetric variable",
72-
my_pe, target);
73-
oshmem_shmem_abort(-1);
74-
return OSHMEM_ERR_BAD_PARAM;
75-
}
76-
77-
/* mxm request init */
78-
sreq.base.state = MXM_REQ_NEW;
79-
sreq.base.mq = mca_atomic_mxm_spml_self->mxm_mq;
80-
sreq.base.conn = mca_atomic_mxm_spml_self->mxm_peers[pe]->mxm_hw_rdma_conn;
81-
sreq.base.completed_cb = NULL;
82-
sreq.base.data_type = MXM_REQ_DATA_BUFFER;
83-
84-
/* set data */
8538
sreq.base.data.buffer.ptr = (void *) value;
86-
sreq.base.data.buffer.length = nlong;
87-
sreq.base.data.buffer.memh = MXM_INVALID_MEM_HANDLE;
88-
89-
sreq.op.atomic.remote_vaddr = (uintptr_t) remote_addr;
90-
#if MXM_API < MXM_VERSION(2,0)
91-
sreq.base.flags = 0;
92-
sreq.op.atomic.remote_memh = MXM_INVALID_MEM_HANDLE;
93-
#else
94-
sreq.flags = 0;
95-
sreq.op.atomic.remote_mkey = to_mxm_mkey(r_mkey);
96-
#endif
97-
sreq.op.atomic.order = nlong_order;
98-
9939
if (NULL == cond) {
10040
sreq.opcode = MXM_REQ_OP_ATOMIC_SWAP;
10141
} else {
102-
#if MXM_API < MXM_VERSION(2,0)
103-
memcpy(&sreq.op.atomic.value8, cond, nlong);
104-
#else
10542
memcpy(&sreq.op.atomic.value, cond, nlong);
106-
#endif
10743
sreq.opcode = MXM_REQ_OP_ATOMIC_CSWAP;
10844
}
10945

110-
if (MXM_OK != (mxm_err = mxm_req_send(&sreq))) {
111-
ATOMIC_ERROR("[#%d] mxm_req_send failed, mxm_error = %d",
112-
my_pe, mxm_err);
113-
oshmem_shmem_abort(-1);
114-
return OSHMEM_ERROR;
115-
}
46+
mca_atomic_mxm_post(&sreq);
11647

117-
mxm_req_wait(&sreq.base);
118-
if (MXM_OK != sreq.base.error) {
119-
ATOMIC_ERROR("[#%d] mxm_req_wait got non MXM_OK error: %d",
120-
my_pe, sreq.base.error);
121-
oshmem_shmem_abort(-1);
122-
return OSHMEM_ERROR;
123-
}
12448
memcpy(prev, value, nlong);
12549

12650
return OSHMEM_SUCCESS;

‎oshmem/mca/atomic/mxm/atomic_mxm_fadd.c‎

Lines changed: 3 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -32,106 +32,20 @@ int mca_atomic_mxm_fadd(void *target,
3232
int pe,
3333
struct oshmem_op_t *op)
3434
{
35-
unsigned my_pe;
36-
uint8_t nlong_order;
37-
void *remote_addr;
38-
int ptl_id;
3935
mxm_send_req_t sreq;
40-
mxm_error_t mxm_err;
41-
sshmem_mkey_t *r_mkey;
4236
static char dummy_buf[8];
4337

44-
my_pe = oshmem_my_proc_id();
45-
ptl_id = -1;
46-
mxm_err = MXM_OK;
38+
mca_atomic_mxm_req_init(&sreq, pe, target, nlong);
4739

48-
switch (nlong) {
49-
case 1:
50-
nlong_order = 0;
51-
break;
52-
case 2:
53-
nlong_order = 1;
54-
break;
55-
case 4:
56-
nlong_order = 2;
57-
break;
58-
case 8:
59-
nlong_order = 3;
60-
break;
61-
default:
62-
ATOMIC_ERROR("[#%d] Type size must be 1/2/4 or 8 bytes.", my_pe);
63-
oshmem_shmem_abort(-1);
64-
return OSHMEM_ERR_BAD_PARAM;
65-
}
66-
67-
ptl_id = OSHMEM_PROC_DATA(oshmem_proc_group_all(pe))->transport_ids[0];
68-
if (MXM_PTL_SHM == ptl_id) {
69-
ptl_id = MXM_PTL_RDMA;
70-
}
71-
r_mkey = mca_memheap_base_get_cached_mkey(pe, target, ptl_id, &remote_addr);
72-
if (!r_mkey) {
73-
ATOMIC_ERROR("[#%d] %p is not address of symmetric variable",
74-
my_pe, target);
75-
oshmem_shmem_abort(-1);
76-
return OSHMEM_ERR_BAD_PARAM;
77-
}
78-
79-
/* mxm request init */
80-
sreq.base.state = MXM_REQ_NEW;
81-
sreq.base.mq = mca_atomic_mxm_spml_self->mxm_mq;
82-
sreq.base.conn = mca_atomic_mxm_spml_self->mxm_peers[pe]->mxm_hw_rdma_conn;
83-
sreq.base.completed_cb = NULL;
84-
sreq.base.data_type = MXM_REQ_DATA_BUFFER;
85-
86-
sreq.op.atomic.remote_vaddr = (uintptr_t) remote_addr;
87-
#if MXM_API < MXM_VERSION(2,0)
88-
sreq.op.atomic.remote_memh = MXM_INVALID_MEM_HANDLE;
89-
memcpy(&sreq.op.atomic.value8, value, nlong);
90-
#else
91-
sreq.op.atomic.remote_mkey = to_mxm_mkey(r_mkey);
9240
memcpy(&sreq.op.atomic.value, value, nlong);
93-
#endif
94-
sreq.op.atomic.order = nlong_order;
95-
96-
/* Do we need atomic 'add' or atomic 'fetch and add'? */
41+
sreq.opcode = MXM_REQ_OP_ATOMIC_FADD;
9742
if (NULL == prev) {
9843
sreq.base.data.buffer.ptr = dummy_buf;
99-
sreq.base.data.buffer.length = nlong;
100-
sreq.base.data.buffer.memh = MXM_INVALID_MEM_HANDLE;
101-
#if MXM_API < MXM_VERSION(2,0)
102-
sreq.base.flags = MXM_REQ_FLAG_SEND_SYNC;
103-
sreq.opcode = MXM_REQ_OP_ATOMIC_ADD;
104-
#else
105-
sreq.flags = 0;
106-
sreq.opcode = MXM_REQ_OP_ATOMIC_FADD;
107-
#endif
10844
} else {
10945
sreq.base.data.buffer.ptr = prev;
110-
sreq.base.data.buffer.length = nlong;
111-
sreq.base.data.buffer.memh = MXM_INVALID_MEM_HANDLE;
112-
#if MXM_API < MXM_VERSION(2,0)
113-
sreq.base.flags = 0;
114-
#else
115-
sreq.flags = 0;
116-
#endif
117-
118-
sreq.opcode = MXM_REQ_OP_ATOMIC_FADD;
11946
}
12047

121-
if (MXM_OK != (mxm_err = mxm_req_send(&sreq))) {
122-
ATOMIC_ERROR("[#%d] mxm_req_send failed, mxm_error = %d",
123-
my_pe, mxm_err);
124-
oshmem_shmem_abort(-1);
125-
return OSHMEM_ERROR;
126-
}
127-
128-
mxm_req_wait(&sreq.base);
129-
if (MXM_OK != sreq.base.error) {
130-
ATOMIC_ERROR("[#%d] mxm_req_wait got non MXM_OK error: %d",
131-
my_pe, sreq.base.error);
132-
oshmem_shmem_abort(-1);
133-
return OSHMEM_ERROR;
134-
}
48+
mca_atomic_mxm_post(&sreq);
13549

13650
return OSHMEM_SUCCESS;
13751
}

0 commit comments

Comments
 (0)