ch4/ipc: Add debug logging for sender ipc handle cache

Add logging for handle cache insertion and removal as well as hits and
misses when searching. This could be useful for debugging handle cache
issues in the field.

Author: raffenet

src/mpid/ch4/shm/ipc/gpu/gpu_post.c

@@ -206,9 +206,11 @@ static int ipc_track_cache_search(MPL_gavl_tree_t gavl_tree, const void *addr, u
     void *obj = MPL_gavl_tree_search(gavl_tree, addr, len);
 
     if (obj) {
+        MPL_DBG_MSG_P(MPIDI_CH4_DBG_IPC, VERBOSE, "cached gpu ipc handle HIT for %p", addr);
         *handle_out = *((MPL_gpu_ipc_mem_handle_t *) obj);
         *found = true;
     } else {
+        MPL_DBG_MSG_P(MPIDI_CH4_DBG_IPC, VERBOSE, "cached gpu ipc handle MISS for %p", addr);
         *found = false;
     }
 
@@ -220,6 +222,8 @@ static int ipc_track_cache_insert(MPL_gavl_tree_t gavl_tree, const void *addr, u
 {
     int mpi_errno = MPI_SUCCESS;
 
+    MPL_DBG_MSG_P(MPIDI_CH4_DBG_IPC, VERBOSE, "caching NEW gpu ipc handle for %p", addr);
+
     MPL_gpu_ipc_mem_handle_t *cache_obj = MPL_malloc(sizeof(handle), MPL_MEM_OTHER);
     MPIR_ERR_CHKANDJUMP(!cache_obj, mpi_errno, MPI_ERR_OTHER, "**nomem");
 
@@ -241,6 +245,8 @@ static int ipc_track_cache_remove(const void *addr, uintptr_t len, int local_dev
     int mpi_errno = MPI_SUCCESS;
     int mpl_err;
 
+    MPL_DBG_MSG_P(MPIDI_CH4_DBG_IPC, VERBOSE, "removing STALE gpu ipc handle for %p", addr);
+
     for (int i = 0; i < MPIR_Process.local_size; ++i) {
         MPL_gavl_tree_t track_tree = MPIDI_GPUI_global.ipc_handle_track_trees[i][local_dev_id];
         mpl_err = MPL_gavl_tree_delete_range(track_tree, addr, len);

src/mpid/ch4/src/ch4_globals.c

@@ -163,4 +163,5 @@ MPL_dbg_class MPIDI_CH4_DBG_GENERAL;
 MPL_dbg_class MPIDI_CH4_DBG_MAP;
 MPL_dbg_class MPIDI_CH4_DBG_COMM;
 MPL_dbg_class MPIDI_CH4_DBG_MEMORY;
+MPL_dbg_class MPIDI_CH4_DBG_IPC;
 #endif

src/mpid/ch4/src/ch4_init.c

@@ -445,6 +445,7 @@ int MPID_Init(int requested, int *provided)
     MPIDI_CH4_DBG_MAP = MPL_dbg_class_alloc("CH4_MAP", "ch4_map");
     MPIDI_CH4_DBG_COMM = MPL_dbg_class_alloc("CH4_COMM", "ch4_comm");
     MPIDI_CH4_DBG_MEMORY = MPL_dbg_class_alloc("CH4_MEMORY", "ch4_memory");
+    MPIDI_CH4_DBG_IPC = MPL_dbg_class_alloc("CH4_IPC", "ch4_ipc");
 #endif
 
 #ifdef HAVE_SIGNAL

src/mpid/ch4/src/ch4_types.h

@@ -300,6 +300,7 @@ extern MPL_dbg_class MPIDI_CH4_DBG_GENERAL;
 extern MPL_dbg_class MPIDI_CH4_DBG_MAP;
 extern MPL_dbg_class MPIDI_CH4_DBG_COMM;
 extern MPL_dbg_class MPIDI_CH4_DBG_MEMORY;
+extern MPL_dbg_class MPIDI_CH4_DBG_IPC;
 #endif
 
 #endif /* CH4_TYPES_H_INCLUDED */