mpid/shm: remove MPIDU_shm_seg_t and refactor

MPIDU_shm_seg_t was used by mpidu_shm_alloc.c, mpidu_init_shm.c, and
mpidu_init_shm_alloc.c. However, the usages are all slightly different
and some fields are only used in one but not the other. It is simpler to
locally define it or, in the case of mpidu_init_shm.c, just use static
globals.

Author: hzhou

src/mpid/common/shm/mpidu_init_shm.c

@@ -8,7 +8,6 @@
 #include "mpl_shm.h"
 #include "mpidimpl.h"
 #include "mpir_pmi.h"
-#include "mpidu_shm_seg.h"
 
 static int init_shm_initialized;
 
@@ -58,7 +57,10 @@ typedef struct Init_shm_barrier {
     MPL_atomic_int_t wait;
 } Init_shm_barrier_t;
 
-static MPIDU_shm_seg_t memory;
+static size_t init_shm_len;
+static MPL_shm_hnd_t init_shm_hnd;
+static char *init_shm_addr;
+
 static Init_shm_barrier_t *barrier;
 static void *baseaddr;
 
@@ -70,7 +72,7 @@ static int Init_shm_barrier_init(int is_root)
 
     MPIR_FUNC_ENTER;
 
-    barrier = (Init_shm_barrier_t *) memory.base_addr;
+    barrier = (Init_shm_barrier_t *) init_shm_addr;
     if (is_root) {
         MPL_atomic_store_int(&barrier->val, 0);
         MPL_atomic_store_int(&barrier->wait, 0);
@@ -136,20 +138,20 @@ int MPIDU_Init_shm_init(void)
         char *serialized_hnd = NULL;
         int serialized_hnd_size = 0;
 
-        mpl_err = MPL_shm_hnd_init(&(memory.hnd));
+        mpl_err = MPL_shm_hnd_init(&init_shm_hnd);
         MPIR_ERR_CHKANDJUMP(mpl_err, mpi_errno, MPI_ERR_OTHER, "**alloc_shar_mem");
 
-        memory.segment_len = segment_len;
+        init_shm_len = segment_len;
 
         if (MPIDU_Init_shm_local_rank == 0) {
             /* root prepare shm segment */
-            mpl_err = MPL_shm_seg_create_and_attach(memory.hnd, memory.segment_len,
-                                                    (void **) &(memory.base_addr), 0);
+            mpl_err = MPL_shm_seg_create_and_attach(init_shm_hnd, init_shm_len,
+                                                    (void **) &(init_shm_addr), 0);
             MPIR_ERR_CHKANDJUMP(mpl_err, mpi_errno, MPI_ERR_OTHER, "**alloc_shar_mem");
 
             MPIR_Assert(MPIR_Process.node_local_map[0] == MPIR_Process.rank);
 
-            mpl_err = MPL_shm_hnd_get_serialized_by_ref(memory.hnd, &serialized_hnd);
+            mpl_err = MPL_shm_hnd_get_serialized_by_ref(init_shm_hnd, &serialized_hnd);
             MPIR_ERR_CHKANDJUMP(mpl_err, mpi_errno, MPI_ERR_OTHER, "**alloc_shar_mem");
             serialized_hnd_size = strlen(serialized_hnd) + 1;
             MPIR_Assert(serialized_hnd_size < MPIR_pmi_max_val_size());
@@ -169,11 +171,10 @@ int MPIDU_Init_shm_init(void)
 
         if (MPIDU_Init_shm_local_rank > 0) {
             /* non-root attach shm segment */
-            mpl_err = MPL_shm_hnd_deserialize(memory.hnd, serialized_hnd, strlen(serialized_hnd));
+            mpl_err = MPL_shm_hnd_deserialize(init_shm_hnd, serialized_hnd, strlen(serialized_hnd));
             MPIR_ERR_CHKANDJUMP(mpl_err, mpi_errno, MPI_ERR_OTHER, "**alloc_shar_mem");
 
-            mpl_err = MPL_shm_seg_attach(memory.hnd, memory.segment_len,
-                                         (void **) &memory.base_addr, 0);
+            mpl_err = MPL_shm_seg_attach(init_shm_hnd, init_shm_len, (void **) &init_shm_addr, 0);
             MPIR_ERR_CHKANDJUMP(mpl_err, mpi_errno, MPI_ERR_OTHER, "**attach_shar_mem");
 
             mpi_errno = Init_shm_barrier_init(FALSE);
@@ -184,13 +185,12 @@ int MPIDU_Init_shm_init(void)
         MPIR_ERR_CHECK(mpi_errno);
 
         if (MPIDU_Init_shm_local_rank == 0) {
-            /* memory->hnd no longer needed */
-            mpl_err = MPL_shm_seg_remove(memory.hnd);
+            /* init_shm_hnd no longer needed */
+            mpl_err = MPL_shm_seg_remove(init_shm_hnd);
             MPIR_ERR_CHKANDJUMP(mpl_err, mpi_errno, MPI_ERR_OTHER, "**remove_shar_mem");
         }
 
-        baseaddr = memory.base_addr + MPIDU_SHM_CACHE_LINE_LEN;
-        memory.symmetrical = 0;
+        baseaddr = init_shm_addr + MPIDU_SHM_CACHE_LINE_LEN;
 
         mpi_errno = Init_shm_barrier();
     }
@@ -215,10 +215,10 @@ int MPIDU_Init_shm_finalize(void)
         goto fn_exit;
     }
 
-    mpl_err = MPL_shm_seg_detach(memory.hnd, (void **) &(memory.base_addr), memory.segment_len);
+    mpl_err = MPL_shm_seg_detach(init_shm_hnd, (void **) &(init_shm_addr), init_shm_len);
     MPIR_ERR_CHKANDJUMP(mpl_err, mpi_errno, MPI_ERR_OTHER, "**detach_shar_mem");
 
-    MPL_shm_hnd_finalize(&(memory.hnd));
+    MPL_shm_hnd_finalize(&(init_shm_hnd));
 
     init_shm_initialized = 0;
 

src/mpid/common/shm/mpidu_init_shm_alloc.c

@@ -6,7 +6,6 @@
 #include <mpidimpl.h>
 #include "mpl_shm.h"
 #include "mpidu_init_shm.h"
-#include "mpidu_shm_seg.h"
 
 #include <stdlib.h>
 #ifdef HAVE_UNISTD_H
@@ -22,16 +21,24 @@
 extern int MPIDU_Init_shm_local_size;
 extern int MPIDU_Init_shm_local_rank;
 
+struct memory_seg {
+    size_t segment_len;
+    MPL_shm_hnd_t hnd;
+    char *base_addr;
+    bool symmetrical;
+    bool is_shm;
+};
+
 typedef struct memory_list {
     void *ptr;
-    MPIDU_shm_seg_t *memory;
+    struct memory_seg *memory;
     struct memory_list *next;
 } memory_list_t;
 
 static memory_list_t *memory_head = NULL;
 static memory_list_t *memory_tail = NULL;
 
-static int check_alloc(MPIDU_shm_seg_t * memory);
+static int check_alloc(struct memory_seg *memory);
 
 /* MPIDU_Init_shm_alloc(len, ptr_p)
 
@@ -42,7 +49,7 @@ int MPIDU_Init_shm_alloc(size_t len, void **ptr)
     int mpi_errno = MPI_SUCCESS, mpl_err = 0;
     void *current_addr;
     size_t segment_len = len;
-    MPIDU_shm_seg_t *memory = NULL;
+    struct memory_seg *memory = NULL;
     memory_list_t *memory_node = NULL;
     MPIR_CHKPMEM_DECL();
 
@@ -101,7 +108,8 @@ int MPIDU_Init_shm_alloc(size_t len, void **ptr)
             MPIR_ERR_CHKANDJUMP(mpl_err, mpi_errno, MPI_ERR_OTHER, "**remove_shar_mem");
         }
         current_addr = memory->base_addr;
-        memory->symmetrical = 0;
+        memory->symmetrical = false;
+        memory->is_shm = true;
 
         mpi_errno = check_alloc(memory);
         MPIR_ERR_CHECK(mpi_errno);
@@ -134,7 +142,7 @@ int MPIDU_Init_shm_alloc(size_t len, void **ptr)
 int MPIDU_Init_shm_free(void *ptr)
 {
     int mpi_errno = MPI_SUCCESS, mpl_err = 0;
-    MPIDU_shm_seg_t *memory = NULL;
+    struct memory_seg *memory = NULL;
     memory_list_t *el = NULL;
 
     MPIR_FUNC_ENTER;
@@ -193,7 +201,7 @@ int MPIDU_Init_shm_is_symm(void *ptr)
 /* check_alloc() checks to see whether the shared memory segment is
    allocated at the same virtual memory address at each process.
 */
-static int check_alloc(MPIDU_shm_seg_t * memory)
+static int check_alloc(struct memory_seg *memory)
 {
     int mpi_errno = MPI_SUCCESS;
     int is_sym;
@@ -226,9 +234,9 @@ static int check_alloc(MPIDU_shm_seg_t * memory)
     }
 
     if (is_sym) {
-        memory->symmetrical = 1;
+        memory->symmetrical = true;
     } else {
-        memory->symmetrical = 0;
+        memory->symmetrical = false;
     }
 
     MPIR_FUNC_EXIT;

src/mpid/common/shm/mpidu_shm_alloc.c

@@ -6,7 +6,6 @@
 #include <mpidimpl.h>
 #include "mpl_shm.h"
 #include "mpidu_shm.h"
-#include "mpidu_shm_seg.h"
 
 #include <stdlib.h>
 #ifdef HAVE_UNISTD_H
@@ -58,6 +57,16 @@ enum {
     SYMSHM_OTHER_FAIL           /* other failure reported by MPL shm */
 };
 
+typedef struct MPIDU_shm_seg {
+    size_t segment_len;
+    /* Handle to shm seg */
+    MPL_shm_hnd_t hnd;
+    /* Pointers */
+    char *base_addr;
+    /* Misc */
+    int symmetrical;
+} MPIDU_shm_seg_t;
+
 /* Linked list internally used to keep track
  * of allocate shared memory segments */
 typedef struct seg_list {