Skip to content
This repository was archived by the owner on Sep 30, 2022. It is now read-only.

Commit 7fd34e2

Browse files
ggouaillardetggouaillardet
committed
oshmem: get rid of oshmem_proc_t and use ompi_proc_t instead
store oshmem related per proc data in an oshmem_proc_data_t struct, that is stored in the padding section of an ompi_proc_t this data can be accessed via the OSHMEM_PROC_DATA(proc) macro Fixes open-mpi/ompi#2023 (back-ported from commit open-mpi/ompi@0a25420)
1 parent edb35d5 commit 7fd34e2

File tree

17 files changed

+110
-98
lines changed

17 files changed

+110
-98
lines changed

‎ompi/proc/proc.h‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,13 +65,16 @@ BEGIN_C_DECLS
6565
* a tag increases the memory consumed by Open MPI, so should only be done
6666
* if unavoidable.
6767
*/
68+
69+
#define OMPI_PROC_PADDING_SIZE 16
70+
6871
struct ompi_proc_t {
6972
opal_proc_t super;
7073

7174
/* endpoint data */
7275
void *proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MAX];
7376

74-
char padding[16]; /* for future extensions (OSHMEM uses this area also)*/
77+
char padding[OMPI_PROC_PADDING_SIZE]; /* for future extensions (OSHMEM uses this area also)*/
7578
};
7679
typedef struct ompi_proc_t ompi_proc_t;
7780
OBJ_CLASS_DECLARATION(ompi_proc_t);

‎oshmem/include/oshmem/types.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
/*
22
* Copyright (c) 2013 Mellanox Technologies, Inc.
33
* All rights reserved.
4+
* Copyright (c) 2016 Research Organization for Information Science
5+
* and Technology (RIST). All rights reserved.
46
* $COPYRIGHT$
57
*
68
* Additional copyrights may follow
@@ -16,7 +18,7 @@
1618
* dependencies.
1719
*/
1820

19-
struct oshmem_proc_t;
21+
struct ompi_proc_t;
2022
struct oshmem_group_t;
2123
struct oshmem_op_t;
2224

‎oshmem/mca/atomic/mxm/atomic_mxm_cswap.c‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
/*
22
* Copyright (c) 2013 Mellanox Technologies, Inc.
33
* All rights reserved.
4+
* Copyright (c) 2016 Research Organization for Information Science
5+
* and Technology (RIST). All rights reserved.
46
* $COPYRIGHT$
57
*
68
* Additional copyrights may follow
@@ -72,7 +74,7 @@ int mca_atomic_mxm_cswap(void *target,
7274
return OSHMEM_ERR_BAD_PARAM;
7375
}
7476

75-
ptl_id = oshmem_proc_group_all(pe)->transport_ids[0];
77+
ptl_id = OSHMEM_PROC_DATA(oshmem_proc_group_all(pe))->transport_ids[0];
7678
if (MXM_PTL_SHM == ptl_id) {
7779
ptl_id = MXM_PTL_RDMA;
7880
}

‎oshmem/mca/atomic/mxm/atomic_mxm_fadd.c‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
/*
22
* Copyright (c) 2013 Mellanox Technologies, Inc.
33
* All rights reserved.
4+
* Copyright (c) 2016 Research Organization for Information Science
5+
* and Technology (RIST). All rights reserved.
46
* $COPYRIGHT$
57
*
68
* Additional copyrights may follow
@@ -74,7 +76,7 @@ int mca_atomic_mxm_fadd(void *target,
7476
return OSHMEM_ERR_BAD_PARAM;
7577
}
7678

77-
ptl_id = oshmem_proc_group_all(pe)->transport_ids[0];
79+
ptl_id = OSHMEM_PROC_DATA(oshmem_proc_group_all(pe))->transport_ids[0];
7880
if (MXM_PTL_SHM == ptl_id) {
7981
ptl_id = MXM_PTL_RDMA;
8082
}

‎oshmem/mca/memheap/base/memheap_base_mkey.c‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
/*
33
* Copyright (c) 2013-2015 Mellanox Technologies, Inc.
44
* All rights reserved.
5-
* Copyright (c) 2015 Research Organization for Information Science
5+
* Copyright (c) 2015-2016 Research Organization for Information Science
66
* and Technology (RIST). All rights reserved.
77
* Copyright (c) 2015 Intel, Inc. All rights reserved.
88
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
@@ -93,7 +93,7 @@ int mca_memheap_seg_cmp(const void *k, const void *v)
9393
*/
9494
static int pack_local_mkeys(opal_buffer_t *msg, int pe, int seg, int all_trs)
9595
{
96-
oshmem_proc_t *proc;
96+
ompi_proc_t *proc;
9797
int i, n, tr_id;
9898
sshmem_mkey_t *mkey;
9999

@@ -111,7 +111,7 @@ static int pack_local_mkeys(opal_buffer_t *msg, int pe, int seg, int all_trs)
111111
MEMHEAP_VERBOSE(5, "found %d transports to %d", n, pe);
112112
for (i = 0; i < n; i++) {
113113
if (!all_trs) {
114-
tr_id = proc->transport_ids[i];
114+
tr_id = OSHMEM_PROC_DATA(proc)->transport_ids[i];
115115
}
116116
else {
117117
tr_id = i;
@@ -171,7 +171,7 @@ static void unpack_remote_mkeys(opal_buffer_t *msg, int remote_pe)
171171
int32_t n;
172172
int32_t tr_id;
173173
int i;
174-
oshmem_proc_t *proc;
174+
ompi_proc_t *proc;
175175

176176
proc = oshmem_proc_group_find(oshmem_group_all, remote_pe);
177177
cnt = 1;

‎oshmem/mca/scoll/fca/scoll_fca_module.c‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
/**
22
* Copyright (c) 2013 Mellanox Technologies, Inc.
33
* All rights reserved.
4-
* Copyright (c) 2014 Research Organization for Information Science
4+
* Copyright (c) 2014-2016 Research Organization for Information Science
55
* and Technology (RIST). All rights reserved.
66
* $COPYRIGHT$
77
*
@@ -104,7 +104,7 @@ static int have_remote_peers(struct oshmem_group_t *group,
104104
size_t size,
105105
int *local_peers)
106106
{
107-
struct oshmem_proc_t *proc;
107+
struct ompi_proc_t *proc;
108108
size_t i;
109109
int ret;
110110

@@ -128,7 +128,7 @@ static int have_remote_peers(struct oshmem_group_t *group,
128128
static int _get_local_ranks(mca_scoll_fca_module_t *fca_module)
129129
{
130130
struct oshmem_group_t *comm = fca_module->comm;
131-
oshmem_proc_t* proc;
131+
ompi_proc_t* proc;
132132
int i, rank;
133133

134134
/* Count the local ranks */

‎oshmem/mca/spml/base/base.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
* Copyright (c) 2013 Mellanox Technologies, Inc.
33
* All rights reserved.
44
* Copyright (c) 2015 Cisco Systems, Inc. All rights reserved.
5+
* Copyright (c) 2016 Research Organization for Information Science
6+
* and Technology (RIST). All rights reserved.
57
* $COPYRIGHT$
68
*
79
* Additional copyrights may follow
@@ -57,7 +59,7 @@ OSHMEM_DECLSPEC int mca_spml_base_spml_selected(const char *name);
5759
* Verify that all new procs are using the currently selected component
5860
*/
5961
OSHMEM_DECLSPEC int mca_spml_base_spml_check_selected(const char *my_spml,
60-
oshmem_proc_t **procs,
62+
ompi_proc_t **procs,
6163
size_t nprocs);
6264

6365
OSHMEM_DECLSPEC int mca_spml_base_wait(void* addr,

‎oshmem/mca/spml/base/spml_base_request.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
* All rights reserved.
55
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
66
* reserved.
7-
* Copyright (c) 2015 Research Organization for Information Science
7+
* Copyright (c) 2015-2016 Research Organization for Information Science
88
* and Technology (RIST). All rights reserved.
99
* $COPYRIGHT$
1010
*
@@ -72,7 +72,7 @@ struct mca_spml_base_request_t {
7272
void *req_addr; /**< pointer to application buffer */
7373
size_t req_count; /**< count of user datatype elements *//* TODO: Need to remove since we are going to remove datatype*/
7474
int32_t req_peer; /**< peer process - rank of process executing the parallel program */
75-
oshmem_proc_t* req_proc; /**< peer process */
75+
ompi_proc_t* req_proc; /**< peer process */
7676
uint64_t req_sequence; /**< sequence number for shmem one sided ordering */
7777
};
7878
typedef struct mca_spml_base_request_t mca_spml_base_request_t;

‎oshmem/mca/spml/ikrit/spml_ikrit.c‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
/*
33
* Copyright (c) 2013-2015 Mellanox Technologies, Inc.
44
* All rights reserved.
5-
* Copyright (c) 2014 Research Organization for Information Science
5+
* Copyright (c) 2014-2016 Research Organization for Information Science
66
* and Technology (RIST). All rights reserved.
77
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
88
* reserved.
@@ -331,31 +331,31 @@ int mca_spml_ikrit_enable(bool enable)
331331

332332
static int create_ptl_idx(int dst_pe)
333333
{
334-
oshmem_proc_t *proc;
334+
ompi_proc_t *proc;
335335

336336
proc = oshmem_proc_group_find(oshmem_group_all, dst_pe);
337337

338-
proc->transport_ids = (char *) malloc(MXM_PTL_LAST * sizeof(char));
339-
if (!proc->transport_ids)
338+
OSHMEM_PROC_DATA(proc)->transport_ids = (char *) malloc(MXM_PTL_LAST * sizeof(char));
339+
if (NULL == OSHMEM_PROC_DATA(proc)->transport_ids)
340340
return OSHMEM_ERROR;
341341

342-
proc->num_transports = 1;
342+
OSHMEM_PROC_DATA(proc)->num_transports = 1;
343343
#if MXM_API < MXM_VERSION(2,0)
344344
if (oshmem_my_proc_id() == dst_pe)
345-
proc->transport_ids[0] = MXM_PTL_SELF;
345+
OSHMEM_PROC_DATA(proc)->transport_ids[0] = MXM_PTL_SELF;
346346
else
347347
#endif
348-
proc->transport_ids[0] = MXM_PTL_RDMA;
348+
OSHMEM_PROC_DATA(proc)->transport_ids[0] = MXM_PTL_RDMA;
349349
return OSHMEM_SUCCESS;
350350
}
351351

352352
static void destroy_ptl_idx(int dst_pe)
353353
{
354-
oshmem_proc_t *proc;
354+
ompi_proc_t *proc;
355355

356356
proc = oshmem_proc_group_find(oshmem_group_all, dst_pe);
357-
if (proc->transport_ids)
358-
free(proc->transport_ids);
357+
if (NULL != OSHMEM_PROC_DATA(proc)->transport_ids)
358+
free(OSHMEM_PROC_DATA(proc)->transport_ids);
359359
}
360360

361361
static void mxm_peer_construct(mxm_peer_t *p)
@@ -375,7 +375,7 @@ OBJ_CLASS_INSTANCE( mxm_peer_t,
375375
mxm_peer_construct,
376376
mxm_peer_destruct);
377377

378-
int mca_spml_ikrit_del_procs(oshmem_proc_t** procs, size_t nprocs)
378+
int mca_spml_ikrit_del_procs(ompi_proc_t** procs, size_t nprocs)
379379
{
380380
size_t i, n;
381381
int my_rank = oshmem_my_proc_id();
@@ -406,7 +406,7 @@ int mca_spml_ikrit_del_procs(oshmem_proc_t** procs, size_t nprocs)
406406
return OSHMEM_SUCCESS;
407407
}
408408

409-
int mca_spml_ikrit_add_procs(oshmem_proc_t** procs, size_t nprocs)
409+
int mca_spml_ikrit_add_procs(ompi_proc_t** procs, size_t nprocs)
410410
{
411411
spml_ikrit_mxm_ep_conn_info_t *ep_info = NULL;
412412
spml_ikrit_mxm_ep_conn_info_t *ep_hw_rdma_info = NULL;
@@ -420,7 +420,7 @@ int mca_spml_ikrit_add_procs(oshmem_proc_t** procs, size_t nprocs)
420420
mxm_error_t err;
421421
size_t i, n;
422422
int rc = OSHMEM_ERROR;
423-
oshmem_proc_t *proc_self;
423+
ompi_proc_t *proc_self;
424424
int my_rank = oshmem_my_proc_id();
425425

426426
OBJ_CONSTRUCT(&mca_spml_ikrit.active_peers, opal_list_t);
@@ -588,9 +588,9 @@ int mca_spml_ikrit_add_procs(oshmem_proc_t** procs, size_t nprocs)
588588
continue;
589589

590590
/* use zcopy for put/get via sysv shared memory */
591-
procs[i]->transport_ids[0] = MXM_PTL_SHM;
592-
procs[i]->transport_ids[1] = MXM_PTL_RDMA;
593-
procs[i]->num_transports = 2;
591+
OSHMEM_PROC_DATA(procs[i])->transport_ids[0] = MXM_PTL_SHM;
592+
OSHMEM_PROC_DATA(procs[i])->transport_ids[1] = MXM_PTL_RDMA;
593+
OSHMEM_PROC_DATA(procs[i])->num_transports = 2;
594594
}
595595

596596
SPML_VERBOSE(50, "*** ADDED PROCS ***");
@@ -742,7 +742,7 @@ int mca_spml_ikrit_deregister(sshmem_mkey_t *mkeys)
742742

743743
static inline int get_ptl_id(int dst)
744744
{
745-
oshmem_proc_t *proc;
745+
ompi_proc_t *proc;
746746

747747
/* get endpoint and btl */
748748
proc = oshmem_proc_group_all(dst);
@@ -751,7 +751,7 @@ static inline int get_ptl_id(int dst)
751751
oshmem_shmem_abort(-1);
752752
return -1;
753753
}
754-
return proc->transport_ids[0];
754+
return OSHMEM_PROC_DATA(proc)->transport_ids[0];
755755
}
756756

757757
int mca_spml_ikrit_oob_get_mkeys(int pe, uint32_t seg, sshmem_mkey_t *mkeys)

‎oshmem/mca/spml/ikrit/spml_ikrit.h‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
* All rights reserved.
55
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
66
* reserved.
7+
* Copyright (c) 2016 Research Organization for Information Science
8+
* and Technology (RIST). All rights reserved.
79
* $COPYRIGHT$
810
*
911
* Additional copyrights may follow
@@ -161,8 +163,8 @@ extern int mca_spml_ikrit_oob_get_mkeys(int pe,
161163
uint32_t seg,
162164
sshmem_mkey_t *mkeys);
163165

164-
extern int mca_spml_ikrit_add_procs(oshmem_proc_t** procs, size_t nprocs);
165-
extern int mca_spml_ikrit_del_procs(oshmem_proc_t** procs, size_t nprocs);
166+
extern int mca_spml_ikrit_add_procs(ompi_proc_t** procs, size_t nprocs);
167+
extern int mca_spml_ikrit_del_procs(ompi_proc_t** procs, size_t nprocs);
166168
extern int mca_spml_ikrit_fence(void);
167169
extern int spml_ikrit_progress(void);
168170

0 commit comments

Comments
 (0)