vader/xpmem: fix finalize

opal/class/opal_object.h

@@ -11,8 +11,8 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2007-2014 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2014      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2014-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
  *                         reserved.
  * $COPYRIGHT$
@@ -284,10 +284,26 @@ static inline opal_object_t *opal_obj_new_debug(opal_class_t* type, const char*
         assert(NULL != ((opal_object_t *) (object))->obj_class);        \
         assert(OPAL_OBJ_MAGIC_ID == ((opal_object_t *) (object))->obj_magic_id); \
         opal_obj_update((opal_object_t *) (object), 1);                 \
-        assert(((opal_object_t *) (object))->obj_reference_count >= 0); \
+        assert(((opal_object_t *) (object))->obj_reference_count > 1); \
     } while (0)
 #else
-#define OBJ_RETAIN(object)  opal_obj_update((opal_object_t *) (object), 1);
+#define OBJ_RETAIN(object)  opal_obj_update((opal_object_t *) (object), 1)
+#endif
+
+/**
+ * Check an object is still valid
+ *
+ * @param object        Pointer to the object
+ */
+#if OPAL_ENABLE_DEBUG
+#define OBJ_CHECK(object)                                              \
+    do {                                                                \
+        assert(NULL != ((opal_object_t *) (object))->obj_class);        \
+        assert(OPAL_OBJ_MAGIC_ID == ((opal_object_t *) (object))->obj_magic_id); \
+        assert(((opal_object_t *) (object))->obj_reference_count > 0); \
+    } while (0)
+#else
+#define OBJ_CHECK(object)
 #endif
 
 /**

opal/mca/btl/vader/btl_vader_endpoint.h

@@ -88,6 +88,7 @@ typedef struct mca_btl_base_endpoint_t {
 
     opal_mutex_t pending_frags_lock; /**< protect pending_frags */
     opal_list_t pending_frags; /**< fragments pending fast box space */
+    opal_list_t regs; /**< registration objects to be freed */
     bool waiting;           /**< endpoint is on the component wait list */
 } mca_btl_base_endpoint_t;
 

opal/mca/btl/vader/btl_vader_module.c

@@ -550,13 +550,15 @@ static void mca_btl_vader_endpoint_constructor (mca_btl_vader_endpoint_t *ep)
     OBJ_CONSTRUCT(&ep->pending_frags_lock, opal_mutex_t);
     ep->fifo = NULL;
     ep->fbox_out.fbox = NULL;
+    OBJ_CONSTRUCT(&ep->regs, opal_list_t);
 }
 
 #if OPAL_BTL_VADER_HAVE_XPMEM
 #endif
 
 static void mca_btl_vader_endpoint_destructor (mca_btl_vader_endpoint_t *ep)
 {
+    OBJ_DESTRUCT(&ep->regs);
     OBJ_DESTRUCT(&ep->pending_frags);
     OBJ_DESTRUCT(&ep->pending_frags_lock);
 

opal/mca/btl/vader/btl_vader_xpmem.c

@@ -5,6 +5,8 @@
  * Copyright (c) 2014      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2019      Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -154,22 +156,30 @@ mca_rcache_base_registration_t *vader_get_registation (struct mca_btl_base_endpo
 static int mca_btl_vader_endpoint_xpmem_rcache_cleanup (mca_rcache_base_registration_t *reg, void *ctx)
 {
     mca_btl_vader_endpoint_t *ep = (mca_btl_vader_endpoint_t *) ctx;
+    OBJ_CHECK(reg);
     if ((intptr_t) reg->alloc_base == ep->peer_smp_rank) {
         /* otherwise dereg will fail on assert */
-        reg->ref_count = 0;
-        OBJ_RELEASE(reg);
+        if (0 == OPAL_THREAD_FETCH_ADD32(&reg->ref_count, -1)) {
+            opal_list_append(&ep->regs, &reg->super.super);
+        }
     }
 
     return OPAL_SUCCESS;
 }
 
 void mca_btl_vader_xpmem_cleanup_endpoint (struct mca_btl_base_endpoint_t *ep)
 {
+    mca_rcache_base_registration_t *reg, *next;
     /* clean out the registration cache */
     (void) mca_rcache_base_vma_iterate (mca_btl_vader_component.vma_module,
                                         NULL, (size_t) -1, true,
                                         mca_btl_vader_endpoint_xpmem_rcache_cleanup,
                                         (void *) ep);
+    OPAL_LIST_FOREACH_SAFE(reg, next, &ep->regs, mca_rcache_base_registration_t) {
+        mca_rcache_base_vma_delete(mca_btl_vader_component.vma_module, reg);
+        opal_list_remove_item(&ep->regs, &reg->super.super);
+        OBJ_RELEASE(reg);
+    }
     if (ep->segment_base) {
         xpmem_release (ep->segment_data.xpmem.apid);
         ep->segment_data.xpmem.apid = 0;