timings: Update/extend OSHMEM timings

Signed-off-by: Artem Polyakov <[email protected]>

Author: artpol84

oshmem/mca/memheap/base/base.h

@@ -62,7 +62,7 @@ typedef struct mca_memheap_map {
 extern mca_memheap_map_t mca_memheap_base_map;
 extern mca_memheap_base_config_t mca_memheap_base_config;
 
-int mca_memheap_base_alloc_init(mca_memheap_map_t *, size_t, long);
+int mca_memheap_base_alloc_init(mca_memheap_map_t *, size_t, long, char *);
 void mca_memheap_base_alloc_exit(mca_memheap_map_t *);
 int mca_memheap_base_static_init(mca_memheap_map_t *);
 void mca_memheap_base_static_exit(mca_memheap_map_t *);

oshmem/mca/memheap/base/memheap_base_alloc.c

@@ -17,13 +17,17 @@
 #include "oshmem/mca/sshmem/base/base.h"
 #include "oshmem/mca/memheap/memheap.h"
 #include "oshmem/mca/memheap/base/base.h"
+#include "ompi/util/timings.h"
 
 
-int mca_memheap_base_alloc_init(mca_memheap_map_t *map, size_t size, long hint)
+int mca_memheap_base_alloc_init(mca_memheap_map_t *map, size_t size, long hint,
+                                char *timing_prefix)
 {
     int ret = OSHMEM_SUCCESS;
     char * seg_filename = NULL;
 
+    OPAL_TIMING_ENV_INIT_PREFIX(timing_prefix, timing);
+
     assert(map);
     if (hint == 0) {
         assert(HEAP_SEG_INDEX == map->n_segments);
@@ -33,8 +37,13 @@ int mca_memheap_base_alloc_init(mca_memheap_map_t *map, size_t size, long hint)
 
     map_segment_t *s = &map->mem_segs[map->n_segments];
     seg_filename = oshmem_get_unique_file_name(oshmem_my_proc_id());
+
+    OPAL_TIMING_ENV_NEXT(timing, "oshmem_get_unique_file_name()");
+
     ret = mca_sshmem_segment_create(s, seg_filename, size, hint);
 
+    OPAL_TIMING_ENV_NEXT(timing, "mca_sshmem_segment_create()");
+
     if (OSHMEM_SUCCESS == ret) {
         map->n_segments++;
         MEMHEAP_VERBOSE(1,
@@ -43,6 +52,7 @@ int mca_memheap_base_alloc_init(mca_memheap_map_t *map, size_t size, long hint)
     }
 
     free(seg_filename);
+    OPAL_TIMING_ENV_NEXT(timing, "DONE");
 
     return ret;
 }

oshmem/mca/memheap/base/memheap_base_mkey.c

@@ -27,6 +27,7 @@
 #include "oshmem/mca/memheap/memheap.h"
 #include "oshmem/mca/memheap/base/base.h"
 #include "oshmem/mca/spml/spml.h"
+#include "opal/util/timings.h"
 
 /* Turn ON/OFF debug output from build (default 0) */
 #ifndef MEMHEAP_BASE_DEBUG
@@ -529,14 +530,16 @@ void mca_memheap_modex_recv_all(void)
     int rc = OSHMEM_SUCCESS;
     size_t buffer_size;
 
+    OPAL_TIMING_ENV_INIT(recv_all);
+
     if (!mca_memheap_base_key_exchange) {
         oshmem_shmem_barrier();
         return;
     }
-
+    OPAL_TIMING_ENV_NEXT(recv_all, "barrier");
     nprocs = oshmem_num_procs();
     my_pe = oshmem_my_proc_id();
-
+    OPAL_TIMING_ENV_NEXT(recv_all, "proc position");
     /* buffer allocation for num_transports
      * message sizes and offsets */
 
@@ -560,6 +563,7 @@ void mca_memheap_modex_recv_all(void)
         rc = OSHMEM_ERR_OUT_OF_RESOURCE;
         goto exit_fatal;
     }
+    OPAL_TIMING_ENV_NEXT(recv_all, "alloc bufs");
 
     /* serialize our own mkeys */
     msg = OBJ_NEW(opal_buffer_t);
@@ -582,6 +586,9 @@ void mca_memheap_modex_recv_all(void)
     opal_dss.unload(msg, &send_buffer, &size);
     MEMHEAP_VERBOSE(1, "local keys packed into %d bytes, %d segments", size, memheap_map->n_segments);
 
+    OPAL_TIMING_ENV_NEXT(recv_all, "serialize data");
+
+
     /* we need to send num_transports and message sizes separately
      * since message sizes depend on types of btl used */
 
@@ -591,12 +598,17 @@ void mca_memheap_modex_recv_all(void)
         goto exit_fatal;
     }
 
+    OPAL_TIMING_ENV_NEXT(recv_all, "allgather: transport cnt");
+
+
     rc = oshmem_shmem_allgather(&size, rcv_size, sizeof(int));
     if (MPI_SUCCESS != rc) {
         MEMHEAP_ERROR("allgather failed");
         goto exit_fatal;
     }
 
+    OPAL_TIMING_ENV_NEXT(recv_all, "allgather: size info");
+
     /* calculating offsets (displacements) for allgatherv */
 
     rcv_offsets[0] = 0;
@@ -613,13 +625,17 @@ void mca_memheap_modex_recv_all(void)
         goto exit_fatal;
     }
 
+    OPAL_TIMING_ENV_NEXT(recv_all, "alloc data buf");
+
     rc = oshmem_shmem_allgatherv(send_buffer, rcv_buffer, size, rcv_size, rcv_offsets);
     if (MPI_SUCCESS != rc) {
         free (rcv_buffer);
         MEMHEAP_ERROR("allgatherv failed");
         goto exit_fatal;
     }
 
+    OPAL_TIMING_ENV_NEXT(recv_all, "Perform mkey exchange");
+
     opal_dss.load(msg, rcv_buffer, buffer_size);
 
     /* deserialize mkeys */
@@ -651,6 +667,8 @@ void mca_memheap_modex_recv_all(void)
         }
     }
 
+    OPAL_TIMING_ENV_NEXT(recv_all, "Unpack data");
+
     OPAL_THREAD_UNLOCK(&memheap_oob.lck);
 
 exit_fatal:
@@ -670,6 +688,7 @@ void mca_memheap_modex_recv_all(void)
         OBJ_RELEASE(msg);
     }
 
+    OPAL_TIMING_ENV_NEXT(recv_all, "Cleanup");
     /* This function requires abort in any error case */
     if (OSHMEM_SUCCESS != rc) {
         oshmem_shmem_abort(rc);

oshmem/mca/memheap/base/memheap_base_select.c

@@ -23,7 +23,7 @@
 #include "oshmem/mca/memheap/base/base.h"
 #include "oshmem/include/shmemx.h"
 #include "oshmem/mca/sshmem/base/base.h"
-
+#include "ompi/util/timings.h"
 
 mca_memheap_base_config_t mca_memheap_base_config = {
     .device_nic_mem_seg_size = 0
@@ -56,6 +56,8 @@ int mca_memheap_base_select()
     mca_memheap_base_component_t *best_component = NULL;
     mca_memheap_base_module_t *best_module = NULL;
 
+    OPAL_TIMING_ENV_INIT(timing);
+
     if( OPAL_SUCCESS != mca_base_select("memheap", oshmem_memheap_base_framework.framework_output,
                                         &oshmem_memheap_base_framework.framework_components,
                                         (mca_base_module_t **) &best_module,
@@ -64,11 +66,15 @@ int mca_memheap_base_select()
         return OSHMEM_ERROR;
     }
 
+    OPAL_TIMING_ENV_NEXT(timing, "env");
+
     context = _memheap_create();
     if (NULL == context) {
         return OSHMEM_ERROR;
     }
 
+    OPAL_TIMING_ENV_NEXT(timing, "_memheap_create()");
+
     if (OSHMEM_SUCCESS != best_component->memheap_init(context)) {
         opal_show_help("help-oshmem-memheap.txt",
                        "find-available:none-found",
@@ -77,6 +83,8 @@ int mca_memheap_base_select()
         return OSHMEM_ERROR;
     }
 
+    OPAL_TIMING_ENV_NEXT(timing, "best_component->memheap_init()");
+
     /* Calculate memheap size in case it was not set during component initialization */
     best_module->memheap_size = context->user_size;
     setenv(SHMEM_HEAP_TYPE,
@@ -89,6 +97,7 @@ int mca_memheap_base_select()
                     best_component->memheap_version.mca_type_name, 
                     best_component->memheap_version.mca_component_name);
 
+    OPAL_TIMING_ENV_NEXT(timing, "DONE");
     return OSHMEM_SUCCESS;
 }
 
@@ -103,43 +112,60 @@ static memheap_context_t* _memheap_create(void)
     static memheap_context_t context;
     size_t user_size, size;
 
+    OPAL_TIMING_ENV_INIT(timing);
+
     user_size = _memheap_size();
     if (user_size < MEMHEAP_BASE_MIN_SIZE) {
         MEMHEAP_ERROR("Requested memheap size is less than minimal meamheap size (%llu < %llu)",
                       (unsigned long long)user_size, MEMHEAP_BASE_MIN_SIZE);
         return NULL ;
     }
+
+    OPAL_TIMING_ENV_NEXT(timing, "_memheap_size()");
+
     /* Inititialize symmetric area */
     if (OSHMEM_SUCCESS == rc) {
         rc = mca_memheap_base_alloc_init(&mca_memheap_base_map,
-                                         user_size + MEMHEAP_BASE_PRIVATE_SIZE, 0);
+                                         user_size + MEMHEAP_BASE_PRIVATE_SIZE, 0,
+                                         "regular_mem");
     }
 
+    OPAL_TIMING_ENV_NEXT(timing, "mca_memheap_base_alloc_init()");
+
     /* Initialize atomic symmetric area */
     size = mca_memheap_base_config.device_nic_mem_seg_size;
     if ((OSHMEM_SUCCESS == rc) && (size > 0)) {
         rc = mca_memheap_base_alloc_init(&mca_memheap_base_map, size,
-                                         SHMEM_HINT_DEVICE_NIC_MEM);
+                                         SHMEM_HINT_DEVICE_NIC_MEM,
+                                         "device_mem");
         if (rc == OSHMEM_ERR_NOT_IMPLEMENTED) {
             /* do not treat NOT_IMPLEMENTED as error */
             rc = OSHMEM_SUCCESS;
         }
     }
 
+    OPAL_TIMING_ENV_NEXT(timing, "mca_memheap_base_alloc_init(DEVICE_MEM)");
+
+
     /* Inititialize static/global variables area */
     if (OSHMEM_SUCCESS == rc) {
         rc = mca_memheap_base_static_init(&mca_memheap_base_map);
     }
 
+    OPAL_TIMING_ENV_NEXT(timing, "mca_memheap_base_static_init()");
+
     /* Memory Registration */
     if (OSHMEM_SUCCESS == rc) {
         rc = mca_memheap_base_reg(&mca_memheap_base_map);
     }
 
+    OPAL_TIMING_ENV_NEXT(timing, "mca_memheap_base_reg()");
+
     /* Init OOB channel */
     if (OSHMEM_SUCCESS == rc) {
         rc = memheap_oob_init(&mca_memheap_base_map);
     }
+    OPAL_TIMING_ENV_NEXT(timing, "memheap_oob_init()");
 
     if (OSHMEM_SUCCESS == rc) {
         context.user_size = user_size;
@@ -151,6 +177,7 @@ static memheap_context_t* _memheap_create(void)
                 (void*) ((unsigned char*) mca_memheap_base_map.mem_segs[HEAP_SEG_INDEX].super.va_base
                         + context.user_size);
     }
+    OPAL_TIMING_ENV_NEXT(timing, "DONE");
 
     return ((OSHMEM_SUCCESS == rc) ? &context : NULL );
 }