Add initial MPI_T-like profiling support in Mercury

src/CMakeLists.txt

@@ -185,6 +185,8 @@ set(MERCURY_SRCS
   ${CMAKE_CURRENT_SOURCE_DIR}/mercury_core_header.c
   ${CMAKE_CURRENT_SOURCE_DIR}/mercury_header.c
   ${CMAKE_CURRENT_SOURCE_DIR}/mercury_proc.c
+  ${CMAKE_CURRENT_SOURCE_DIR}/mercury_prof_interface.c
+  ${CMAKE_CURRENT_SOURCE_DIR}/mercury_prof_pvar_impl.c
   ${CMAKE_CURRENT_SOURCE_DIR}/proc_extra/mercury_string_object.c
 )
 set(MERCURY_HL_SRCS
@@ -240,6 +242,8 @@ set(MERCURY_HEADERS
   ${CMAKE_CURRENT_BINARY_DIR}/mercury_config.h
   ${CMAKE_CURRENT_SOURCE_DIR}/mercury.h
   ${CMAKE_CURRENT_SOURCE_DIR}/mercury_bulk.h
+  ${CMAKE_CURRENT_SOURCE_DIR}/mercury_prof_interface.h
+  ${CMAKE_CURRENT_SOURCE_DIR}/mercury_prof_types.h
   ${CMAKE_CURRENT_SOURCE_DIR}/mercury_core.h
   ${CMAKE_CURRENT_SOURCE_DIR}/mercury_core_header.h
   ${CMAKE_CURRENT_SOURCE_DIR}/mercury_core_types.h

src/mercury.c

@@ -22,6 +22,10 @@
 #include <string.h>
 #include <assert.h>
 
+/* PVAR profiling support */
+#include "mercury_prof_pvar_impl.h"
+HG_PROF_PVAR_UINT_COUNTER_DECL_EXTERN(hg_pvar_hg_forward_count);
+
 /****************/
 /* Local Macros */
 /****************/
@@ -959,6 +963,7 @@ HG_Init_opt(const char *na_info_string, hg_bool_t na_listen,
     const struct hg_init_info *hg_init_info)
 {
     struct hg_private_class *hg_class = NULL;
+    int ret;
 #ifdef HG_HAS_VERBOSE_ERROR
     const char *log_level = NULL;
 
@@ -987,6 +992,11 @@ HG_Init_opt(const char *na_info_string, hg_bool_t na_listen,
     HG_Core_set_more_data_callback(hg_class->hg_class.core_class,
         hg_more_data_cb, hg_more_data_free_cb);
 
+
+    /* EXPERIMENTAL: Initialize PVAR profiling data structures */
+    ret = hg_prof_pvar_init();
+    assert(ret == HG_SUCCESS);
+
     return (hg_class_t *) hg_class;
 
 error:
@@ -1918,6 +1928,10 @@ HG_Forward(hg_handle_t handle, hg_cb_t callback, void *arg, void *in_struct)
     HG_CHECK_HG_ERROR(done, ret, "Could not forward call (%s)",
         HG_Error_to_string(ret));
 
+    /* PVAR profiling support: Increment the value of the PVAR with the name "hg_pvar_hg_forward_count" */
+    fprintf(stderr, "[MERCURY_PROF_INTERFACE] Incrementing the value of PVAR with name hg_pvar_hg_forward_count\n");
+    HG_PROF_PVAR_COUNTER_INC(hg_pvar_hg_forward_count, 1);
+
 done:
     return ret;
 }

src/mercury_prof_interface.c

@@ -0,0 +1,198 @@
+/*
+ * Copyright (C) 2013-2020 Argonne National Laboratory, Department of Energy,
+ *                    UChicago Argonne, LLC and The HDF Group.
+ * All rights reserved.
+ *
+ * The full copyright notice, including terms governing use, modification,
+ * and redistribution, is contained in the COPYING file that can be
+ * found at the root of the source code distribution tree.
+ */
+
+#include "mercury_bulk.h"
+#include "mercury_core.h"
+#include "mercury_private.h"
+#include "mercury_error.h"
+
+#include "mercury_atomic.h"
+#include "mercury_prof_interface.h"
+#include "mercury_prof_pvar_impl.h"
+
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+/* Variable denoting whether or not the profiling interface has been initialized */
+static int hg_prof_is_initialized = 0;
+
+/* Client-side PVAR handle that contains all information about the PVAR. Data structure is opaque to the client. 
+ * Handle exists to make PVAR reads quicker */
+struct hg_prof_pvar_handle {
+   hg_prof_class_t pvar_class;
+   hg_prof_datatype_t pvar_datatype;
+   hg_prof_bind_t pvar_bind;
+   int continuous;
+   void * addr;
+   int is_started;
+   int count;
+   char name[128];
+   char description[128];
+};
+
+/* Client-side PVAR session. PVARs are associated with a session. As of now, we only support one default session */
+struct hg_prof_pvar_session {
+   hg_prof_pvar_handle_t * pvar_handle_array;
+   int num_pvars;
+   int reference_counter;
+};
+
+struct hg_prof_pvar_session default_session;
+
+/* Setter and getter functions for profiling interface initialization */
+static void hg_prof_set_is_initialized(int val)
+{
+  hg_prof_is_initialized = val;
+}
+
+static int hg_prof_get_is_initialized() {
+   return hg_prof_is_initialized;
+}
+
+/* Client-side API to initialize the PVAR profiling interface */
+hg_return_t HG_Prof_init() {
+
+  default_session.reference_counter = 0;
+  default_session.num_pvars = NUM_PVARS;
+  default_session.pvar_handle_array = (hg_prof_pvar_handle_t *)malloc(sizeof(hg_prof_pvar_handle_t)*NUM_PVARS);
+  hg_prof_set_is_initialized(1);
+
+  return HG_SUCCESS;
+}
+
+/* Client-side API to finalize the PVAR profiling interface */
+hg_return_t HG_Prof_finalize() {
+
+  if(hg_prof_get_is_initialized()) {
+    hg_prof_set_is_initialized(0);
+  }
+
+  fprintf(stderr, "[MERCURY_PROF_INTERFACE] Successfully shutting down profiling interface\n");
+  return HG_SUCCESS;
+}
+
+/* Client-side API to retrieve the number of PVARs currently exported */
+int HG_Prof_pvar_get_num() {
+  if(hg_prof_get_is_initialized()) {
+    return NUM_PVARS;
+  } else {
+    return 0;
+  }
+}
+
+/* Gather information about every PVAR exported. This API is necessary in order for the client to discover the types, bindings, etc.
+ * The client can then allocate the necessary data structures using this information */
+hg_return_t HG_Prof_pvar_get_info(int pvar_index, char *name, int *name_len, hg_prof_class_t *var_class, hg_prof_datatype_t *datatype, char *desc, int *desc_len, hg_prof_bind_t *bind, int *continuous) {
+
+  if(!hg_prof_get_is_initialized())
+    return HG_NA_ERROR;
+
+  assert(pvar_index < NUM_PVARS);
+
+  unsigned int key = pvar_index;
+  hg_prof_pvar_data_t * val;
+
+  /* Lookup the internal PVAR hash table to gather information about this PVAR */
+  val = hg_hash_table_lookup(pvar_table, (hg_hash_table_key_t)(&key));
+  strcpy(name, (*val).name);
+  *name_len = strlen(name);
+  strcpy(desc, (*val).description);
+  *desc_len = strlen(desc);
+  *var_class = (*val).pvar_class;
+  *datatype = (*val).pvar_datatype;
+  *bind = (*val).pvar_bind;
+  *continuous = (*val).continuous;
+
+  return HG_SUCCESS;
+}
+
+/* Create a session. In this case, return a reference to the default session that is currently supported */
+hg_return_t HG_Prof_pvar_session_create(hg_prof_pvar_session_t *session) {
+  if(!hg_prof_get_is_initialized())
+    return HG_NA_ERROR;
+
+  default_session.reference_counter += 1;
+
+  /* Only support one tool at the moment */
+  assert(default_session.reference_counter == 1);
+
+  *session = &default_session;
+
+  return HG_SUCCESS;
+}
+
+/* Allocate a handle for a PVAR at a given index.
+ * This handle will later be used by the client to query the value for the PVAR
+ * This handle is an opaque object */
+hg_return_t HG_Prof_pvar_handle_alloc(hg_prof_pvar_session_t session, int pvar_index, void *obj_handle, hg_prof_pvar_handle_t *handle, int *count) {
+
+  if(!hg_prof_get_is_initialized())
+    return HG_NA_ERROR;
+
+  /* Only supporting a default session and null bind type at the moment */
+  assert(session == &default_session);
+  assert(obj_handle == NULL);
+
+  struct hg_prof_pvar_session s = *session;
+  unsigned int key = pvar_index;
+  hg_prof_pvar_data_t * val;
+
+  s.pvar_handle_array[pvar_index] = (hg_prof_pvar_handle_t)malloc(sizeof(struct hg_prof_pvar_handle));
+  val = hg_hash_table_lookup(pvar_table, (hg_hash_table_key_t)(&key)); 
+
+  /* Copy out information from the internal PVAR hash table */
+  (*s.pvar_handle_array[pvar_index]).pvar_class = (*val).pvar_class;
+  (*s.pvar_handle_array[pvar_index]).pvar_datatype = (*val).pvar_datatype;
+  (*s.pvar_handle_array[pvar_index]).pvar_bind = (*val).pvar_bind;
+  (*s.pvar_handle_array[pvar_index]).continuous = (*val).continuous;
+  (*s.pvar_handle_array[pvar_index]).is_started = 0;
+  (*s.pvar_handle_array[pvar_index]).addr = (*val).addr;
+  if((*val).continuous)
+    (*s.pvar_handle_array[pvar_index]).is_started = 1;
+  strcpy((*s.pvar_handle_array[pvar_index]).name, (*val).name);
+  strcpy((*s.pvar_handle_array[pvar_index]).description, (*val).description);
+  *count = (*val).count;
+
+  /* Return handle */
+  *handle = s.pvar_handle_array[pvar_index];
+
+  fprintf(stderr, "[MERCURY_PROF_INTERFACE] Successfully allocated handle for PVAR: %s\n", (*s.pvar_handle_array[pvar_index]).name);
+
+  return HG_SUCCESS;
+}
+
+/* Start the PVAR is it is not continuous and has not been started yet */
+hg_return_t HG_Prof_pvar_start(hg_prof_pvar_session_t session, hg_prof_pvar_handle_t handle) {
+  if(!hg_prof_get_is_initialized())
+    return HG_NA_ERROR;
+  if (!(*handle).continuous && !((*handle).is_started))
+     (*handle).is_started = 1;
+  return HG_SUCCESS;
+}
+
+/* Read the value of the PVAR when the client supplies the handle.
+ * Note that the handle is necessary as the input (instead of the pvar_index) because there may be multiple PVAR sessions in flight */
+hg_return_t HG_Prof_pvar_read(hg_prof_pvar_session_t session, hg_prof_pvar_handle_t handle, void *buf) {
+  if(!hg_prof_get_is_initialized())
+    return HG_NA_ERROR;
+
+
+  /* Assert first that handle belongs to the session provided. NOT DOING THIS HERE FOR NOW */
+
+  struct hg_prof_pvar_handle h = (*handle);
+  switch(h.pvar_datatype) {
+    case HG_UINT:
+      /*for(int i = 0; i < h.count; h++)*/ /* Need to support PVAR arrays, just a placeholder that assumes PVAR count is 1 */
+      *((unsigned int *)buf) = *((unsigned int *)h.addr);
+      break;
+  }
+  return HG_SUCCESS;
+}

src/mercury_prof_interface.h

@@ -0,0 +1,37 @@
+/*
+ * Copyright (C) 2013-2020 Argonne National Laboratory, Department of Energy,
+ *                    UChicago Argonne, LLC and The HDF Group.
+ * All rights reserved.
+ *
+ * The full copyright notice, including terms governing use, modification,
+ * and redistribution, is contained in the COPYING file that can be
+ * found at the root of the source code distribution tree.
+ */
+
+#ifndef MERCURY_PROF_INTERFACE_H
+#define MERCURY_PROF_INTERFACE_H
+
+#include "mercury_prof_types.h"
+
+/*Initialize and finalize routines */
+HG_PUBLIC hg_return_t HG_Prof_init();
+HG_PUBLIC hg_return_t HG_Prof_finalize();
+
+/*Create a session */
+HG_PUBLIC hg_return_t HG_Prof_pvar_session_create(hg_prof_pvar_session_t *session);
+
+/* Gather information about PVARs */
+HG_PUBLIC int HG_Prof_pvar_get_num();
+HG_PUBLIC hg_return_t HG_Prof_pvar_get_info(int pvar_index, char *name, int *name_len, 
+			hg_prof_class_t *var_class, hg_prof_datatype_t *datatype, 
+			char *desc, int *desc_len, hg_prof_bind_t *bind, int *continuous);
+
+/* Allocate handles */
+HG_PUBLIC hg_return_t HG_Prof_pvar_handle_alloc(hg_prof_pvar_session_t session, 
+			int pvar_index, void *obj_handle, hg_prof_pvar_handle_t *handle, int *count);
+
+/* Start and read PVARs */
+HG_PUBLIC hg_return_t HG_Prof_pvar_start(hg_prof_pvar_session_t session, hg_prof_pvar_handle_t handle);
+HG_PUBLIC hg_return_t HG_Prof_pvar_read(hg_prof_pvar_session_t session, hg_prof_pvar_handle_t handle, void *buf);
+
+#endif /* MERCURY_PROF_INTERFACE_H */

src/mercury_prof_pvar_impl.c

@@ -0,0 +1,73 @@
+/*
+ * Copyright (C) 2013-2020 Argonne National Laboratory, Department of Energy,
+ *                    UChicago Argonne, LLC and The HDF Group.
+ * All rights reserved.
+ *
+ * The full copyright notice, including terms governing use, modification,
+ * and redistribution, is contained in the COPYING file that can be
+ * found at the root of the source code distribution tree.
+ */
+
+#include "mercury_bulk.h"
+#include "mercury_core.h"
+#include "mercury_private.h"
+#include "mercury_error.h"
+
+#include "mercury_atomic.h"
+#include "mercury_prof_interface.h"
+#include "mercury_prof_pvar_impl.h"
+
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+/* Internal routines for the pvar_hash_table data structure */
+static HG_INLINE int
+hg_prof_uint_equal(void *vlocation1, void *vlocation2)
+{
+    return *((unsigned int *) vlocation1) == *((unsigned int *) vlocation2);
+}
+
+/*---------------------------------------------------------------------------*/
+static HG_INLINE unsigned int
+hg_prof_uint_hash(void *vlocation)
+{
+    return *((unsigned int *) vlocation);
+}
+
+hg_hash_table_t *pvar_table;
+
+/* Declarate a PVAR that counts the number of times the HG_Forward call has been invoked */
+HG_PROF_PVAR_UINT_COUNTER_DECL(hg_pvar_hg_forward_count);
+
+/* Store the details of the PVAR in an internal hash table */
+void HG_PROF_PVAR_REGISTER_impl(hg_prof_class_t varclass, hg_prof_datatype_t dtype, const char* name, void *addr, int count,
+    hg_prof_bind_t bind, int continuous, const char * desc) {
+
+    unsigned int * key = NULL;
+    key = (unsigned int *)malloc(sizeof(unsigned int));
+    *key = hg_hash_table_num_entries(pvar_table);
+    hg_prof_pvar_data_t * pvar_info = NULL;
+    pvar_info = (hg_prof_pvar_data_t *)malloc(sizeof(hg_prof_pvar_data_t));
+    (*pvar_info).pvar_class = varclass;
+    (*pvar_info).pvar_datatype = dtype;
+    (*pvar_info).pvar_bind = bind;
+    (*pvar_info).count = count;
+    (*pvar_info).addr = addr;
+    strcpy((*pvar_info).name, name);
+    strcpy((*pvar_info).description, desc);
+    (*pvar_info).continuous = continuous;
+    hg_hash_table_insert(pvar_table, (hg_hash_table_key_t)key, (hg_hash_table_value_t)(pvar_info));
+}
+
+/* Internal routine that gets invoked during mercury's own initialization routine.
+ * General routine for initializing the PVAR data structures and registering any PVARs that are not bound to a specific module. */
+hg_return_t hg_prof_pvar_init() {
+
+    /*Initialize internal PVAR data structures*/
+    pvar_table = hg_hash_table_new(hg_prof_uint_hash, hg_prof_uint_equal);
+    /* Register available PVARs */
+    HG_PROF_PVAR_UINT_COUNTER_REGISTER(HG_UINT, HG_PROF_BIND_NO_OBJECT, hg_pvar_hg_forward_count, "Number of times HG_Forward has been invoked");
+
+return HG_SUCCESS;
+}