oshmem: get rid of oshmem_proc_to

store oshmem related per proc data in a oshmem_proc_data_t, that is
written in the padding section of a ompi_proc_t

this data can be accessed via the OSHMEM_PROC_DATA(proc) macro

Fixes #2023

Author: ggouaillardet

ompi/proc/proc.h

@@ -65,13 +65,16 @@ BEGIN_C_DECLS
  * a tag increases the memory consumed by Open MPI, so should only be done
  * if unavoidable.
  */
+
+#define OMPI_PROC_PADDING_SIZE 16
+
 struct ompi_proc_t {
     opal_proc_t                     super;
 
     /* endpoint data */
     void *proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MAX];
 
-    char padding[16];         /* for future extensions (OSHMEM uses this area also)*/
+    char padding[OMPI_PROC_PADDING_SIZE]; /* for future extensions (OSHMEM uses this area also)*/
 };
 typedef struct ompi_proc_t ompi_proc_t;
 OBJ_CLASS_DECLARATION(ompi_proc_t);

oshmem/include/oshmem/types.h

@@ -1,6 +1,8 @@
 /*
  * Copyright (c) 2013      Mellanox Technologies, Inc.
  *                         All rights reserved.
+ * Copyright (c) 2016      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -16,7 +18,7 @@
  * dependencies.
  */
 
-   struct oshmem_proc_t;
+   struct ompi_proc_t;
    struct oshmem_group_t;
    struct oshmem_op_t;
 

oshmem/mca/atomic/mxm/atomic_mxm_cswap.c

@@ -1,6 +1,8 @@
 /*
  * Copyright (c) 2013      Mellanox Technologies, Inc.
  *                         All rights reserved.
+ * Copyright (c) 2016      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -60,7 +62,7 @@ int mca_atomic_mxm_cswap(void *target,
         return OSHMEM_ERR_BAD_PARAM;
     }
 
-    ptl_id = oshmem_proc_group_all(pe)->transport_ids[0];
+    ptl_id = OSHMEM_PROC_DATA(oshmem_proc_group_all(pe))->transport_ids[0];
     if (MXM_PTL_SHM == ptl_id) {
         ptl_id = MXM_PTL_RDMA;
     }

oshmem/mca/atomic/mxm/atomic_mxm_fadd.c

@@ -1,6 +1,8 @@
 /*
  * Copyright (c) 2013      Mellanox Technologies, Inc.
  *                         All rights reserved.
+ * Copyright (c) 2016      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -62,7 +64,7 @@ int mca_atomic_mxm_fadd(void *target,
         return OSHMEM_ERR_BAD_PARAM;
     }
 
-    ptl_id = oshmem_proc_group_all(pe)->transport_ids[0];
+    ptl_id = OSHMEM_PROC_DATA(oshmem_proc_group_all(pe))->transport_ids[0];
     if (MXM_PTL_SHM == ptl_id) {
         ptl_id = MXM_PTL_RDMA;
     }

oshmem/mca/memheap/base/memheap_base_mkey.c

@@ -2,7 +2,7 @@
 /*
  * Copyright (c) 2013-2015 Mellanox Technologies, Inc.
  *                         All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
+ * Copyright (c) 2015-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2015      Intel, Inc. All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
@@ -93,7 +93,7 @@ int mca_memheap_seg_cmp(const void *k, const void *v)
  */
 static int pack_local_mkeys(opal_buffer_t *msg, int pe, int seg, int all_trs)
 {
-    oshmem_proc_t *proc;
+    ompi_proc_t *proc;
     int i, n, tr_id;
     sshmem_mkey_t *mkey;
 
@@ -111,7 +111,7 @@ static int pack_local_mkeys(opal_buffer_t *msg, int pe, int seg, int all_trs)
     MEMHEAP_VERBOSE(5, "found %d transports to %d", n, pe);
     for (i = 0; i < n; i++) {
         if (!all_trs) {
-            tr_id = proc->transport_ids[i];
+            tr_id = OSHMEM_PROC_DATA(proc)->transport_ids[i];
         }
         else {
             tr_id = i;
@@ -171,7 +171,7 @@ static void unpack_remote_mkeys(opal_buffer_t *msg, int remote_pe)
     int32_t n;
     int32_t tr_id;
     int i;
-    oshmem_proc_t *proc;
+    ompi_proc_t *proc;
 
     proc = oshmem_proc_group_find(oshmem_group_all, remote_pe);
     cnt = 1;

oshmem/mca/scoll/fca/scoll_fca_module.c

@@ -1,7 +1,7 @@
 /**
  * Copyright (c) 2013      Mellanox Technologies, Inc.
  *                         All rights reserved.
- * Copyright (c) 2014      Research Organization for Information Science
+ * Copyright (c) 2014-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
@@ -106,7 +106,7 @@ static int have_remote_peers(struct oshmem_group_t *group,
                              size_t size,
                              int *local_peers)
 {
-    struct oshmem_proc_t *proc;
+    struct ompi_proc_t *proc;
     size_t i;
     int ret;
 
@@ -130,7 +130,7 @@ static int have_remote_peers(struct oshmem_group_t *group,
 static int _get_local_ranks(mca_scoll_fca_module_t *fca_module)
 {
     struct oshmem_group_t *comm = fca_module->comm;
-    oshmem_proc_t* proc;
+    ompi_proc_t* proc;
     int i, rank;
 
     /* Count the local ranks */

oshmem/mca/spml/base/base.h

@@ -2,6 +2,8 @@
  * Copyright (c) 2013      Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2016      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -57,7 +59,7 @@ OSHMEM_DECLSPEC int mca_spml_base_spml_selected(const char *name);
  * Verify that all new procs are using the currently selected component
  */
 OSHMEM_DECLSPEC int mca_spml_base_spml_check_selected(const char *my_spml,
-                                                      oshmem_proc_t **procs,
+                                                      ompi_proc_t **procs,
                                                       size_t nprocs);
 
 OSHMEM_DECLSPEC int mca_spml_base_wait(void* addr,

oshmem/mca/spml/base/spml_base_request.h

@@ -4,7 +4,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2015      Research Organization for Information Science
+ * Copyright (c) 2015-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
@@ -72,7 +72,7 @@ struct mca_spml_base_request_t {
     void *req_addr; /**< pointer to application buffer */
     size_t req_count; /**< count of user datatype elements *//* TODO: Need to remove since we are going to remove datatype*/
     int32_t req_peer; /**< peer process - rank of process executing the parallel program */
-    oshmem_proc_t* req_proc; /**< peer process */
+    ompi_proc_t* req_proc; /**< peer process */
     uint64_t req_sequence; /**< sequence number for shmem one sided ordering */
 };
 typedef struct mca_spml_base_request_t mca_spml_base_request_t;

oshmem/mca/spml/ikrit/spml_ikrit.c

@@ -2,7 +2,7 @@
 /*
  * Copyright (c) 2013-2015 Mellanox Technologies, Inc.
  *                         All rights reserved.
- * Copyright (c) 2014      Research Organization for Information Science
+ * Copyright (c) 2014-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
@@ -332,31 +332,31 @@ int mca_spml_ikrit_enable(bool enable)
 
 static int create_ptl_idx(int dst_pe)
 {
-    oshmem_proc_t *proc;
+    ompi_proc_t *proc;
 
     proc = oshmem_proc_group_find(oshmem_group_all, dst_pe);
 
-    proc->transport_ids = (char *) malloc(MXM_PTL_LAST * sizeof(char));
-    if (!proc->transport_ids)
+    OSHMEM_PROC_DATA(proc)->transport_ids = (char *) malloc(MXM_PTL_LAST * sizeof(char));
+    if (NULL == OSHMEM_PROC_DATA(proc)->transport_ids)
         return OSHMEM_ERROR;
 
-    proc->num_transports = 1;
+    OSHMEM_PROC_DATA(proc)->num_transports = 1;
 #if MXM_API < MXM_VERSION(2,0)
     if (oshmem_my_proc_id() == dst_pe)
-        proc->transport_ids[0] = MXM_PTL_SELF;
+        OSHMEM_PROC_DATA(proc)->transport_ids[0] = MXM_PTL_SELF;
     else
 #endif
-        proc->transport_ids[0] = MXM_PTL_RDMA;
+        OSHMEM_PROC_DATA(proc)->transport_ids[0] = MXM_PTL_RDMA;
     return OSHMEM_SUCCESS;
 }
 
 static void destroy_ptl_idx(int dst_pe)
 {
-    oshmem_proc_t *proc;
+    ompi_proc_t *proc;
 
     proc = oshmem_proc_group_find(oshmem_group_all, dst_pe);
-    if (proc->transport_ids)
-        free(proc->transport_ids);
+    if (NULL != OSHMEM_PROC_DATA(proc)->transport_ids)
+        free(OSHMEM_PROC_DATA(proc)->transport_ids);
 }
 
 static void mxm_peer_construct(mxm_peer_t *p)
@@ -376,7 +376,7 @@ OBJ_CLASS_INSTANCE( mxm_peer_t,
                    mxm_peer_construct,
                    mxm_peer_destruct);
 
-int mca_spml_ikrit_del_procs(oshmem_proc_t** procs, size_t nprocs)
+int mca_spml_ikrit_del_procs(ompi_proc_t** procs, size_t nprocs)
 {
     size_t i, n;
     int my_rank = oshmem_my_proc_id();
@@ -407,7 +407,7 @@ int mca_spml_ikrit_del_procs(oshmem_proc_t** procs, size_t nprocs)
     return OSHMEM_SUCCESS;
 }
 
-int mca_spml_ikrit_add_procs(oshmem_proc_t** procs, size_t nprocs)
+int mca_spml_ikrit_add_procs(ompi_proc_t** procs, size_t nprocs)
 {
     spml_ikrit_mxm_ep_conn_info_t *ep_info = NULL;
     spml_ikrit_mxm_ep_conn_info_t *ep_hw_rdma_info = NULL;
@@ -421,7 +421,7 @@ int mca_spml_ikrit_add_procs(oshmem_proc_t** procs, size_t nprocs)
     mxm_error_t err;
     size_t i, n;
     int rc = OSHMEM_ERROR;
-    oshmem_proc_t *proc_self;
+    ompi_proc_t *proc_self;
     int my_rank = oshmem_my_proc_id();
 
     OBJ_CONSTRUCT(&mca_spml_ikrit.active_peers, opal_list_t);
@@ -589,9 +589,9 @@ int mca_spml_ikrit_add_procs(oshmem_proc_t** procs, size_t nprocs)
             continue;
 
         /* use zcopy for put/get via sysv shared memory */
-        procs[i]->transport_ids[0] = MXM_PTL_SHM;
-        procs[i]->transport_ids[1] = MXM_PTL_RDMA;
-        procs[i]->num_transports = 2;
+        OSHMEM_PROC_DATA(procs[i])->transport_ids[0] = MXM_PTL_SHM;
+        OSHMEM_PROC_DATA(procs[i])->transport_ids[1] = MXM_PTL_RDMA;
+        OSHMEM_PROC_DATA(procs[i])->num_transports = 2;
     }
 
     SPML_VERBOSE(50, "*** ADDED PROCS ***");
@@ -743,7 +743,7 @@ int mca_spml_ikrit_deregister(sshmem_mkey_t *mkeys)
 
 static inline int get_ptl_id(int dst)
 {
-    oshmem_proc_t *proc;
+    ompi_proc_t *proc;
 
     /* get endpoint and btl */
     proc = oshmem_proc_group_all(dst);
@@ -752,7 +752,7 @@ static inline int get_ptl_id(int dst)
         oshmem_shmem_abort(-1);
         return -1;
     }
-    return proc->transport_ids[0];
+    return OSHMEM_PROC_DATA(proc)->transport_ids[0];
 }
 
 int mca_spml_ikrit_oob_get_mkeys(int pe, uint32_t seg, sshmem_mkey_t *mkeys)

oshmem/mca/spml/ikrit/spml_ikrit.h

@@ -4,6 +4,8 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2016      Research Organization for Information Science
+ *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -166,8 +168,8 @@ extern int mca_spml_ikrit_oob_get_mkeys(int pe,
                                         uint32_t seg,
                                         sshmem_mkey_t *mkeys);
 
-extern int mca_spml_ikrit_add_procs(oshmem_proc_t** procs, size_t nprocs);
-extern int mca_spml_ikrit_del_procs(oshmem_proc_t** procs, size_t nprocs);
+extern int mca_spml_ikrit_add_procs(ompi_proc_t** procs, size_t nprocs);
+extern int mca_spml_ikrit_del_procs(ompi_proc_t** procs, size_t nprocs);
 extern int mca_spml_ikrit_fence(void);
 extern int spml_ikrit_progress(void);