rcache/base: update VMA tree to use opal_interval_tree_t

This commit replaces the current VMA tree implementation with one that
uses the new opal_interval_tree_t class. Since the VMA tree lock is no
longer used this commit also updates rcache/grdma and btl/vader to
take better care when searching for existing registrations.

Signed-off-by: Nathan Hjelm <[email protected]>

Author: hjelmn

opal/mca/btl/vader/btl_vader_xpmem.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2014      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
@@ -33,7 +33,6 @@ int mca_btl_vader_xpmem_init (void)
 }
 
 struct vader_check_reg_ctx_t {
-    mca_rcache_base_vma_module_t *vma_module;
     mca_btl_base_endpoint_t *ep;
     mca_rcache_base_registration_t **reg;
     uintptr_t base;
@@ -58,13 +57,24 @@ static int vader_check_reg (mca_rcache_base_registration_t *reg, void *ctx)
         return 1;
     }
 
-    /* remove this pointer from the rcache and decrement its reference count
-       (so it is detached later) */
-    mca_rcache_base_vma_delete (vader_ctx->vma_module, reg);
-
     return 2;
 }
 
+void vader_return_registration (mca_rcache_base_registration_t *reg, struct mca_btl_base_endpoint_t *ep)
+{
+    mca_rcache_base_vma_module_t *vma_module =  mca_btl_vader_component.vma_module;
+    int32_t ref_count;
+
+    ref_count = opal_atomic_add_fetch_32 (&reg->ref_count, -1);
+    if (OPAL_UNLIKELY(0 == ref_count && !(reg->flags & MCA_RCACHE_FLAGS_PERSIST))) {
+        mca_rcache_base_vma_delete (vma_module, reg);
+
+        opal_memchecker_base_mem_noaccess (reg->rcache_context, (uintptr_t)(reg->bound - reg->base));
+        (void)xpmem_detach (reg->rcache_context);
+        OBJ_RELEASE (reg);
+    }
+}
+
 /* look up the remote pointer in the peer rcache and attach if
  * necessary */
 mca_rcache_base_registration_t *vader_get_registation (struct mca_btl_base_endpoint_t *ep, void *rem_ptr,
@@ -73,7 +83,7 @@ mca_rcache_base_registration_t *vader_get_registation (struct mca_btl_base_endpo
     mca_rcache_base_vma_module_t *vma_module = mca_btl_vader_component.vma_module;
     uint64_t attach_align = 1 << mca_btl_vader_component.log_attach_align;
     mca_rcache_base_registration_t *reg = NULL;
-    vader_check_reg_ctx_t check_ctx = {.ep = ep, .reg = &reg, .vma_module = vma_module};
+    vader_check_reg_ctx_t check_ctx = {.ep = ep, .reg = &reg};
     xpmem_addr_t xpmem_addr;
     uintptr_t base, bound;
     int rc;
@@ -88,16 +98,17 @@ mca_rcache_base_registration_t *vader_get_registation (struct mca_btl_base_endpo
     check_ctx.bound = bound;
 
     /* several segments may match the base pointer */
-    rc = mca_rcache_base_vma_iterate (vma_module, (void *) base, bound - base, vader_check_reg, &check_ctx);
+    rc = mca_rcache_base_vma_iterate (vma_module, (void *) base, bound - base, true, vader_check_reg, &check_ctx);
     if (2 == rc) {
+        /* remove this pointer from the rcache and decrement its reference count
+           (so it is detached later) */
+        mca_rcache_base_vma_delete (vma_module, reg);
+
         /* start the new segment from the lower of the two bases */
         base = (uintptr_t) reg->base < base ? (uintptr_t) reg->base : base;
 
-        if (OPAL_LIKELY(0 == opal_atomic_add_fetch_32 (&reg->ref_count, -1))) {
-            /* this pointer is not in use */
-            (void) xpmem_detach (reg->rcache_context);
-            OBJ_RELEASE(reg);
-        }
+        /* remove the last reference to this registration */
+        vader_return_registration (reg, ep);
 
         reg = NULL;
     }
@@ -127,7 +138,9 @@ mca_rcache_base_registration_t *vader_get_registation (struct mca_btl_base_endpo
 
             opal_memchecker_base_mem_defined (reg->rcache_context, bound - base);
 
-            mca_rcache_base_vma_insert (vma_module, reg, 0);
+            if (!(flags & MCA_RCACHE_FLAGS_PERSIST)) {
+                mca_rcache_base_vma_insert (vma_module, reg, 0);
+            }
         }
     }
 
@@ -138,30 +151,13 @@ mca_rcache_base_registration_t *vader_get_registation (struct mca_btl_base_endpo
     return reg;
 }
 
-void vader_return_registration (mca_rcache_base_registration_t *reg, struct mca_btl_base_endpoint_t *ep)
-{
-    mca_rcache_base_vma_module_t *vma_module =  mca_btl_vader_component.vma_module;
-    int32_t ref_count;
-
-    ref_count = opal_atomic_add_fetch_32 (&reg->ref_count, -1);
-    if (OPAL_UNLIKELY(0 == ref_count && !(reg->flags & MCA_RCACHE_FLAGS_PERSIST))) {
-        /* protect rcache access */
-        mca_rcache_base_vma_delete (vma_module, reg);
-
-        opal_memchecker_base_mem_noaccess (reg->rcache_context, (uintptr_t)(reg->bound - reg->base));
-        (void)xpmem_detach (reg->rcache_context);
-        OBJ_RELEASE (reg);
-    }
-}
-
 static int mca_btl_vader_endpoint_xpmem_rcache_cleanup (mca_rcache_base_registration_t *reg, void *ctx)
 {
     mca_rcache_base_vma_module_t *vma_module = mca_btl_vader_component.vma_module;
     mca_btl_vader_endpoint_t *ep = (mca_btl_vader_endpoint_t *) ctx;
     if ((intptr_t) reg->alloc_base == ep->peer_smp_rank) {
         /* otherwise dereg will fail on assert */
         reg->ref_count = 0;
-        (void) mca_rcache_base_vma_delete (vma_module, reg);
         OBJ_RELEASE(reg);
     }
 
@@ -172,7 +168,7 @@ void mca_btl_vader_xpmem_cleanup_endpoint (struct mca_btl_base_endpoint_t *ep)
 {
     /* clean out the registration cache */
     (void) mca_rcache_base_vma_iterate (mca_btl_vader_component.vma_module,
-                                        NULL, (size_t) -1,
+                                        NULL, (size_t) -1, true,
                                         mca_btl_vader_endpoint_xpmem_rcache_cleanup,
                                         (void *) ep);
     if (ep->segment_base) {

opal/mca/rcache/base/rcache_base_frame.c

@@ -10,7 +10,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2012-2016 Los Alamos National Security, LLC.
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC.
  *                         All rights reserved
  * Copyright (c) 2015-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -73,11 +73,6 @@ OBJ_CLASS_INSTANCE(mca_rcache_base_registration_t, opal_free_list_item_t,
  * Global variables
  */
 opal_list_t mca_rcache_base_modules = {{0}};
-opal_free_list_t mca_rcache_base_vma_tree_items = {{{0}}};
-bool mca_rcache_base_vma_tree_items_inited = false;
-unsigned int mca_rcache_base_vma_tree_items_min = TREE_ITEMS_MIN;
-int mca_rcache_base_vma_tree_items_max = TREE_ITEMS_MAX;
-unsigned int mca_rcache_base_vma_tree_items_inc = TREE_ITEMS_INC;
 
 OBJ_CLASS_INSTANCE(mca_rcache_base_selected_module_t, opal_list_item_t, NULL, NULL);
 
@@ -114,9 +109,6 @@ static int mca_rcache_base_close(void)
         (void) mca_base_framework_close (&opal_memory_base_framework);
     }
 
-    OBJ_DESTRUCT(&mca_rcache_base_vma_tree_items);
-    mca_rcache_base_vma_tree_items_inited = false;
-
     /* All done */
     /* Close all remaining available components */
     return mca_base_framework_components_close(&opal_rcache_base_framework, NULL);
@@ -133,37 +125,12 @@ static int mca_rcache_base_open(mca_base_open_flag_t flags)
 
     OBJ_CONSTRUCT(&mca_rcache_base_modules, opal_list_t);
 
-    /* the free list is only initialized when a VMA tree is created */
-    OBJ_CONSTRUCT(&mca_rcache_base_vma_tree_items, opal_free_list_t);
-
      /* Open up all available components */
     return mca_base_framework_components_open(&opal_rcache_base_framework, flags);
 }
 
 static int mca_rcache_base_register_mca_variables (mca_base_register_flag_t flags)
 {
-
-    mca_rcache_base_vma_tree_items_min = TREE_ITEMS_MIN;
-    (void) mca_base_framework_var_register (&opal_rcache_base_framework, "vma_tree_items_min",
-                                            "Minimum number of VMA tree items to allocate (default: "
-                                            STRINGIFY(TREE_ITEMS_MIN) ")", MCA_BASE_VAR_TYPE_UNSIGNED_INT,
-                                            NULL, MCA_BASE_VAR_BIND_NO_OBJECT, 0, OPAL_INFO_LVL_6,
-                                            MCA_BASE_VAR_SCOPE_READONLY, &mca_rcache_base_vma_tree_items_min);
-
-    mca_rcache_base_vma_tree_items_max = TREE_ITEMS_MAX;
-    (void) mca_base_framework_var_register (&opal_rcache_base_framework, "vma_tree_items_max",
-                                            "Maximum number of VMA tree items to allocate (default: "
-                                            STRINGIFY(TREE_ITEMS_MAX) ", -1: unlimited)", MCA_BASE_VAR_TYPE_INT,
-                                            NULL, MCA_BASE_VAR_BIND_NO_OBJECT, 0, OPAL_INFO_LVL_6,
-                                            MCA_BASE_VAR_SCOPE_READONLY, &mca_rcache_base_vma_tree_items_max);
-
-    mca_rcache_base_vma_tree_items_inc = TREE_ITEMS_INC;
-    (void) mca_base_framework_var_register (&opal_rcache_base_framework, "vma_tree_items_inc",
-                                            "Number of VMA tree items to allocate at a time (default: "
-                                            STRINGIFY(TREE_ITEMS_INC) ")", MCA_BASE_VAR_TYPE_UNSIGNED_INT,
-                                            NULL, MCA_BASE_VAR_BIND_NO_OBJECT, 0, OPAL_INFO_LVL_6,
-                                            MCA_BASE_VAR_SCOPE_READONLY, &mca_rcache_base_vma_tree_items_inc);
-
     return OPAL_SUCCESS;
 }
 

opal/mca/rcache/base/rcache_base_vma.c

@@ -14,7 +14,7 @@
  * Copyright (c) 2009-2013 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2009      IBM Corporation.  All rights reserved.
  * Copyright (c) 2013      NVIDIA Corporation.  All rights reserved.
- * Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
  *                         reserved.
  *
  * $COPYRIGHT$
@@ -53,14 +53,6 @@ OBJ_CLASS_INSTANCE(mca_rcache_base_vma_module_t, opal_object_t,
 
 mca_rcache_base_vma_module_t *mca_rcache_base_vma_module_alloc (void)
 {
-    if (!mca_rcache_base_vma_tree_items_inited) {
-        opal_free_list_init (&mca_rcache_base_vma_tree_items, sizeof (mca_rcache_base_vma_item_t),
-                             8, OBJ_CLASS(mca_rcache_base_vma_item_t), 0, 8,
-                             mca_rcache_base_vma_tree_items_min, mca_rcache_base_vma_tree_items_max,
-                             mca_rcache_base_vma_tree_items_inc, NULL, 0, NULL, NULL, NULL);
-        mca_rcache_base_vma_tree_items_inited = true;
-    }
-
     return OBJ_NEW(mca_rcache_base_vma_module_t);
 }
 
@@ -154,15 +146,20 @@ int mca_rcache_base_vma_delete (mca_rcache_base_vma_module_t *vma_module,
 }
 
 int mca_rcache_base_vma_iterate (mca_rcache_base_vma_module_t *vma_module,
-                                 unsigned char *base, size_t size,
+                                 unsigned char *base, size_t size, bool partial_ok,
                                  int (*callback_fn) (struct mca_rcache_base_registration_t *, void *),
                                  void *ctx)
 {
-    return mca_rcache_base_vma_tree_iterate (vma_module, base, size, callback_fn, ctx);
+    return mca_rcache_base_vma_tree_iterate (vma_module, base, size, partial_ok, callback_fn, ctx);
 }
 
 void mca_rcache_base_vma_dump_range (mca_rcache_base_vma_module_t *vma_module,
                                      unsigned char *base, size_t size, char *msg)
 {
     mca_rcache_base_vma_tree_dump_range (vma_module, base, size, msg);
 }
+
+size_t mca_rcache_base_vma_size (mca_rcache_base_vma_module_t *vma_module)
+{
+    return mca_rcache_base_vma_tree_size (vma_module);
+}

opal/mca/rcache/base/rcache_base_vma.h

@@ -13,7 +13,7 @@
  *
  * Copyright (c) 2006      Voltaire. All rights reserved.
  * Copyright (c) 2009      IBM Corporation.  All rights reserved.
- * Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
  *                         reserved.
  *
  * $COPYRIGHT$
@@ -33,7 +33,7 @@
 
 #include "opal_config.h"
 #include "opal/class/opal_list.h"
-#include "opal/class/opal_rb_tree.h"
+#include "opal/class/opal_interval_tree.h"
 #include "opal/class/opal_lifo.h"
 
 BEGIN_C_DECLS
@@ -42,7 +42,7 @@ struct mca_rcache_base_registration_t;
 
 struct mca_rcache_base_vma_module_t {
     opal_object_t super;
-    opal_rb_tree_t rb_tree;
+    opal_interval_tree_t tree;
     opal_list_t vma_list;
     opal_lifo_t vma_gc_lifo;
     size_t reg_cur_cache_size;
@@ -77,6 +77,7 @@ void mca_rcache_base_vma_dump_range (mca_rcache_base_vma_module_t *vma_module,
  * @param[in] vma_module  vma tree
  * @param[in] base        base address of region
  * @param[in] size        size of region
+ * @param[in] partial_ok  partial overlap of range is ok
  * @param[in] callback_fn function to call for each matching registration handle
  * @param[in] ctx         callback context
  *
@@ -87,10 +88,12 @@ void mca_rcache_base_vma_dump_range (mca_rcache_base_vma_module_t *vma_module,
  * other than OPAL_SUCCESS.
  */
 int mca_rcache_base_vma_iterate (mca_rcache_base_vma_module_t *vma_module,
-                                 unsigned char *base, size_t size,
+                                 unsigned char *base, size_t size, bool partial_ok,
                                  int (*callback_fn) (struct mca_rcache_base_registration_t *, void *),
                                  void *ctx);
 
+size_t mca_rcache_base_vma_size (mca_rcache_base_vma_module_t *vma_module);
+
 END_C_DECLS
 
 #endif /* MCA_RCACHE_BASE_VMA_H */