mpool/rgpusm: update for rcache threading fixes

This commit brings the rgpusm mpool in line with the changes made to
the rcache to make it thread safe. There is no master equivalent of
this commit.

Signed-off-by: Nathan Hjelm <[email protected]>

Author: hjelmn

opal/mca/mpool/rgpusm/mpool_rgpusm_module.c

@@ -15,7 +15,7 @@
  * Copyright (c) 2007      Mellanox Technologies. All rights reserved.
  * Copyright (c) 2010      IBM Corporation.  All rights reserved.
  * Copyright (c) 2012-2015 NVIDIA Corporation.  All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2015-2016 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  *
  * $COPYRIGHT$
@@ -113,11 +113,11 @@ static inline bool mca_mpool_rgpusm_deregister_lru (mca_mpool_base_module_t *mpo
     mpool->rcache->rcache_delete(mpool->rcache, old_reg);
 
     /* Drop the rcache lock while we deregister the memory */
-    OPAL_THREAD_UNLOCK(&mpool->rcache->lock);
+    opal_mutex_unlock (&mpool->rcache->lock);
     assert(old_reg->ref_count == 0);
     rc = mpool_rgpusm->resources.deregister_mem(mpool_rgpusm->resources.reg_data,
                                                 old_reg);
-    OPAL_THREAD_LOCK(&mpool->rcache->lock);
+    opal_mutex_lock (&mpool->rcache->lock);
 
     /* This introduces a potential leak of registrations if
        the deregistration fails to occur as we no longer have
@@ -242,7 +242,7 @@ int mca_mpool_rgpusm_register (mca_mpool_base_module_t *mpool, void *addr,
     }
 
     /* Check to see if memory is registered and stored in the cache. */
-    OPAL_THREAD_LOCK(&mpool->rcache->lock);
+    opal_mutex_lock (&mpool->rcache->lock);
     mpool->rcache->rcache_find(mpool->rcache, addr, size, reg);
 
     /* If *reg is not NULL, we have a registration.  Let us see if the
@@ -306,7 +306,7 @@ int mca_mpool_rgpusm_register (mca_mpool_base_module_t *mpool, void *addr,
                                   (opal_list_item_t*)(*reg));
         }
         (*reg)->ref_count++;
-        OPAL_THREAD_UNLOCK(&mpool->rcache->lock);
+        opal_mutex_unlock (&mpool->rcache->lock);
         opal_output(-1, "reg->ref_count=%d", (int)(*reg)->ref_count);
         opal_output_verbose(80, mca_mpool_rgpusm_component.output,
                            "RGPUSM: Found entry in cache addr=%p, size=%d", addr, (int)size);
@@ -322,7 +322,7 @@ int mca_mpool_rgpusm_register (mca_mpool_base_module_t *mpool, void *addr,
 
     item = opal_free_list_get (&mpool_rgpusm->reg_list);
     if(NULL == item) {
-        OPAL_THREAD_UNLOCK(&mpool->rcache->lock);
+        opal_mutex_unlock (&mpool->rcache->lock);
         return OPAL_ERR_OUT_OF_RESOURCE;
     }
     rgpusm_reg = (mca_mpool_common_cuda_reg_t*)item;
@@ -399,7 +399,7 @@ int mca_mpool_rgpusm_register (mca_mpool_base_module_t *mpool, void *addr,
     }
 
     if(rc != OPAL_SUCCESS) {
-        OPAL_THREAD_UNLOCK(&mpool->rcache->lock);
+        opal_mutex_unlock (&mpool->rcache->lock);
         opal_free_list_return (&mpool_rgpusm->reg_list, item);
         return rc;
     }
@@ -439,7 +439,7 @@ int mca_mpool_rgpusm_register (mca_mpool_base_module_t *mpool, void *addr,
     }
 
     if(rc != OPAL_SUCCESS) {
-        OPAL_THREAD_UNLOCK(&mpool->rcache->lock);
+        opal_mutex_unlock (&mpool->rcache->lock);
         opal_free_list_return (&mpool_rgpusm->reg_list, item);
         /* We cannot recover from this.  We can be here if the size of
          * the cache is smaller than the amount of memory we are
@@ -454,10 +454,8 @@ int mca_mpool_rgpusm_register (mca_mpool_base_module_t *mpool, void *addr,
 
     rgpusm_reg->base.ref_count++;
     *reg = (mca_mpool_base_registration_t *)rgpusm_reg;
-    OPAL_THREAD_UNLOCK(&mpool->rcache->lock);
+    opal_mutex_unlock (&mpool->rcache->lock);
 
-    /* Cleanup any vmas that we have deferred deletion on */
-    mpool->rcache->rcache_clean(mpool->rcache);
     return OPAL_SUCCESS;
 }
 
@@ -483,7 +481,7 @@ int mca_mpool_rgpusm_find(struct mca_mpool_base_module_t *mpool, void *addr,
     base = addr;
     bound = base + size - 1; /* To keep cache hits working correctly */
 
-    OPAL_THREAD_LOCK(&mpool->rcache->lock);
+    opal_mutex_lock (&mpool->rcache->lock);
     opal_output(-1, "Looking for addr=%p, size=%d", addr, (int)size);
     rc = mpool->rcache->rcache_find(mpool->rcache, addr, size, reg);
     if(*reg != NULL && mca_mpool_rgpusm_component.leave_pinned) {
@@ -495,12 +493,12 @@ int mca_mpool_rgpusm_find(struct mca_mpool_base_module_t *mpool, void *addr,
     } else {
         mpool_rgpusm->stat_cache_notfound++;
     }
-    OPAL_THREAD_UNLOCK(&mpool->rcache->lock);
+    opal_mutex_unlock (&mpool->rcache->lock);
 
     return rc;
 }
 
-static inline bool registration_is_cachebale(mca_mpool_base_registration_t *reg)
+static inline bool registration_is_cacheable(mca_mpool_base_registration_t *reg)
 {
      return !(reg->flags &
              (MCA_MPOOL_FLAGS_CACHE_BYPASS |
@@ -514,14 +512,14 @@ int mca_mpool_rgpusm_deregister(struct mca_mpool_base_module_t *mpool,
     int rc = OPAL_SUCCESS;
     assert(reg->ref_count > 0);
 
-    OPAL_THREAD_LOCK(&mpool->rcache->lock);
+    opal_mutex_lock (&mpool->rcache->lock);
     reg->ref_count--;
     opal_output(-1, "Deregister: reg->ref_count=%d", (int)reg->ref_count);
     if(reg->ref_count > 0) {
-        OPAL_THREAD_UNLOCK(&mpool->rcache->lock);
+        opal_mutex_unlock (&mpool->rcache->lock);
         return OPAL_SUCCESS;
     }
-    if(mca_mpool_rgpusm_component.leave_pinned && registration_is_cachebale(reg))
+    if(mca_mpool_rgpusm_component.leave_pinned && registration_is_cacheable(reg))
     {
         /* if leave_pinned is set don't deregister memory, but put it
          * on LRU list for future use */
@@ -535,7 +533,7 @@ int mca_mpool_rgpusm_deregister(struct mca_mpool_base_module_t *mpool,
             mpool->rcache->rcache_delete(mpool->rcache, reg);
 
         /* Drop the rcache lock before deregistring the memory */
-        OPAL_THREAD_UNLOCK(&mpool->rcache->lock);
+        opal_mutex_unlock (&mpool->rcache->lock);
 
         {
              mca_mpool_rgpusm_module_t *mpool_rgpusm = (mca_mpool_rgpusm_module_t *)mpool;
@@ -545,17 +543,14 @@ int mca_mpool_rgpusm_deregister(struct mca_mpool_base_module_t *mpool,
                                                          reg);
          }
 
-        OPAL_THREAD_LOCK(&mpool->rcache->lock);
+        opal_mutex_lock (&mpool->rcache->lock);
 
         if(OPAL_SUCCESS == rc) {
             opal_free_list_return (&mpool_rgpusm->reg_list,
                                    (opal_free_list_item_t*)reg);
         }
     }
-    OPAL_THREAD_UNLOCK(&mpool->rcache->lock);
-
-    /* Cleanup any vmas that we have deferred deletion on */
-    mpool->rcache->rcache_clean(mpool->rcache);
+    opal_mutex_unlock (&mpool->rcache->lock);
 
     return rc;
 }
@@ -572,7 +567,7 @@ int mca_mpool_rgpusm_deregister_no_lock(struct mca_mpool_base_module_t *mpool,
     if(reg->ref_count > 0) {
         return OPAL_SUCCESS;
     }
-    if(mca_mpool_rgpusm_component.leave_pinned && registration_is_cachebale(reg))
+    if(mca_mpool_rgpusm_component.leave_pinned && registration_is_cacheable(reg))
     {
         /* if leave_pinned is set don't deregister memory, but put it
          * on LRU list for future use */
@@ -599,15 +594,28 @@ int mca_mpool_rgpusm_deregister_no_lock(struct mca_mpool_base_module_t *mpool,
     return rc;
 }
 
-#define RGPUSM_MPOOL_NREGS 100
+static int iterate_dereg_finalize (mca_mpool_base_registration_t *rgpusm_reg, void *ctx)
+{
+    mca_mpool_rgpusm_module_t *mpool_rgpusm = (mca_mpool_rgpusm_module_t *) ctx;
+
+    if ((mca_mpool_base_module_t *) mpool_rgpusm != rgpusm_reg->mpool) {
+        return 0;
+    }
+
+    if (registration_is_cacheable (rgpusm_reg)) {
+        opal_list_remove_item (&mpool_rgpusm->lru_list, (opal_list_item_t *) rgpusm_reg);
+    }
+
+    /* set the reference count to 0 otherwise dereg will fail on assert */
+    rgpusm_reg->ref_count = 0;
+    (void) mpool_rgpusm->resources.deregister_mem (mpool_rgpusm->resources.reg_data, rgpusm_reg);
+
+    return 0;
+}
 
 void mca_mpool_rgpusm_finalize(struct mca_mpool_base_module_t *mpool)
 {
     mca_mpool_rgpusm_module_t *mpool_rgpusm = (mca_mpool_rgpusm_module_t*)mpool;
-    mca_mpool_base_registration_t *reg;
-    mca_mpool_base_registration_t *regs[RGPUSM_MPOOL_NREGS];
-    int reg_cnt, i;
-    int rc;
 
     /* Statistic */
     if(true == mca_mpool_rgpusm_component.print_stats) {
@@ -619,49 +627,11 @@ void mca_mpool_rgpusm_finalize(struct mca_mpool_base_module_t *mpool)
                 mpool_rgpusm->stat_evicted);
     }
 
-    OPAL_THREAD_LOCK(&mpool->rcache->lock);
-    do {
-        reg_cnt = mpool->rcache->rcache_find_all(mpool->rcache, 0, (size_t)-1,
-                regs, RGPUSM_MPOOL_NREGS);
-        opal_output(-1, "Registration size at finalize = %d", reg_cnt);
-
-        for(i = 0; i < reg_cnt; i++) {
-            reg = regs[i];
 
-            if(reg->ref_count) {
-                reg->ref_count = 0; /* otherway dereg will fail on assert */
-            } else if (mca_mpool_rgpusm_component.leave_pinned) {
-                opal_list_remove_item(&mpool_rgpusm->lru_list,
-                        (opal_list_item_t*)reg);
-            }
-
-            /* Remove from rcache first */
-            mpool->rcache->rcache_delete(mpool->rcache, reg);
-
-            /* Drop lock before deregistering memory */
-            OPAL_THREAD_UNLOCK(&mpool->rcache->lock);
-            assert(reg->ref_count == 0);
-            rc = mpool_rgpusm->resources.deregister_mem(mpool_rgpusm->resources.reg_data,
-                                                   reg);
-            OPAL_THREAD_LOCK(&mpool->rcache->lock);
-
-            if(rc != OPAL_SUCCESS) {
-                /* Potentially lose track of registrations
-                   do we have to put it back? */
-                continue;
-            }
-
-            opal_free_list_return (&mpool_rgpusm->reg_list,
-                                   (opal_free_list_item_t *) reg);
-        }
-    } while(reg_cnt == RGPUSM_MPOOL_NREGS);
+    (void) mpool->rcache->rcache_iterate (mpool->rcache, NULL, (size_t) -1,
+                                          iterate_dereg_finalize, (void *) mpool);
 
     OBJ_DESTRUCT(&mpool_rgpusm->lru_list);
     OBJ_DESTRUCT(&mpool_rgpusm->reg_list);
-    OPAL_THREAD_UNLOCK(&mpool->rcache->lock);
-
-    /* Cleanup any vmas that we have deferred deletion on */
-    mpool->rcache->rcache_clean(mpool->rcache);
-
 }