btl/scif: update for mpool/rcache rewrite

This commit brings the scif btl up to date with changes made on master
to rework the mpool and rcache frameworks.

Signed-off-by: Nathan Hjelm <[email protected]>

Author: hjelmn

opal/mca/btl/scif/btl_scif.h

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2013-2016 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2014      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -16,16 +16,15 @@
 
 #include "opal_config.h"
 
-#include "opal/mca/mpool/mpool.h"
-#include "opal/mca/mpool/base/base.h"
-#include "opal/mca/mpool/grdma/mpool_grdma.h"
 #include "opal/util/output.h"
 #include "opal_stdint.h"
 #include "opal/util/proc.h"
 
 #include "opal/mca/btl/btl.h"
 #include "opal/mca/btl/base/base.h"
 #include "opal/mca/btl/base/btl_base_error.h"
+#include "opal/mca/rcache/rcache.h"
+#include "opal/mca/rcache/base/base.h"
 
 #include <scif.h>
 #include <errno.h>
@@ -93,6 +92,8 @@ typedef struct mca_btl_scif_module_t {
 
     volatile bool exiting;
     bool listening;
+
+    mca_rcache_base_module_t *rcache;
 } mca_btl_scif_module_t;
 
 typedef struct mca_btl_scif_component_t {
@@ -235,7 +236,7 @@ struct mca_btl_scif_registration_handle_t {
 typedef struct mca_btl_scif_registration_handle_t mca_btl_scif_registration_handle_t;
 
 typedef struct mca_btl_scif_reg_t {
-    mca_mpool_base_registration_t base;
+    mca_rcache_base_registration_t base;
     /** per-endpoint btl handles for this registration */
     mca_btl_scif_registration_handle_t *handles;
 } mca_btl_scif_reg_t;

opal/mca/btl/scif/btl_scif_add_procs.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2013-2016 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2014      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -17,8 +17,7 @@
 #include "btl_scif.h"
 #include "btl_scif_frag.h"
 
-static int
-mca_btl_scif_setup_mpools (mca_btl_scif_module_t *scif_module);
+static int mca_btl_scif_setup_rcache (mca_btl_scif_module_t *scif_module);
 static void *mca_btl_scif_connect_accept (void *arg);
 
 int mca_btl_scif_add_procs(struct mca_btl_base_module_t* btl,
@@ -48,16 +47,16 @@ int mca_btl_scif_add_procs(struct mca_btl_base_module_t* btl,
         procs_on_board++;
     }
 
-    /* allocate space for the detected peers and setup the mpool */
+    /* allocate space for the detected peers and setup the rcache */
     if (NULL == scif_module->endpoints) {
         scif_module->endpoints = calloc (procs_on_board, sizeof (mca_btl_base_endpoint_t));
         if (OPAL_UNLIKELY(NULL == scif_module->endpoints)) {
             return OPAL_ERR_OUT_OF_RESOURCE;
         }
 
-        rc = mca_btl_scif_setup_mpools (scif_module);
+        rc = mca_btl_scif_setup_rcache (scif_module);
         if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-            BTL_ERROR(("btl/scif error setting up mpools/free lists"));
+            BTL_ERROR(("btl/scif error setting up rcache or free lists"));
             return rc;
         }
     }
@@ -178,15 +177,15 @@ static int scif_dereg_mem (void *reg_data, mca_mpool_base_registration_t *reg)
 }
 
 static int scif_reg_mem (void *reg_data, void *base, size_t size,
-                         mca_mpool_base_registration_t *reg)
+                         mca_rcache_base_registration_t *reg)
 {
     mca_btl_scif_reg_t *scif_reg = (mca_btl_scif_reg_t *)reg;
     int rc = OPAL_SUCCESS;
     unsigned int i;
 
     scif_reg->handles = calloc (mca_btl_scif_module.endpoint_count, sizeof (scif_reg->handles[0]));
 
-    /* intialize all scif offsets to -1 and initialize the pointer back to the mpool registration */
+    /* intialize all scif offsets to -1 and initialize the pointer back to the rcache registration */
     for (i = 0 ; i < mca_btl_scif_module.endpoint_count ; ++i) {
         scif_reg->handles[i].btl_handle.scif_offset = -1;
         scif_reg->handles[i].btl_handle.scif_base = (intptr_t) base;
@@ -211,22 +210,20 @@ static int scif_reg_mem (void *reg_data, void *base, size_t size,
     return rc;
 }
 
-static int
-mca_btl_scif_setup_mpools (mca_btl_scif_module_t *scif_module)
+static int mca_btl_scif_setup_rcache (mca_btl_scif_module_t *scif_module)
 {
-    struct mca_mpool_base_resources_t mpool_resources;
+    mca_rcache_base_resources_t rcache_resources;
     int rc;
 
-    /* initialize the grdma mpool */
-    mpool_resources.pool_name      = "scif";
-    mpool_resources.reg_data       = (void *) scif_module;
-    mpool_resources.sizeof_reg     = sizeof (mca_btl_scif_reg_t);
-    mpool_resources.register_mem   = scif_reg_mem;
-    mpool_resources.deregister_mem = scif_dereg_mem;
-    scif_module->super.btl_mpool =
-        mca_mpool_base_module_create("grdma", scif_module, &mpool_resources);
-    if (NULL == scif_module->super.btl_mpool) {
-        BTL_ERROR(("error creating grdma mpool"));
+    /* initialize the grdma rcache */
+    rcache_resources.cache_name     = "scif";
+    rcache_resources.reg_data       = (void *) scif_module;
+    rcache_resources.sizeof_reg     = sizeof (mca_btl_scif_reg_t);
+    rcache_resources.register_mem   = scif_reg_mem;
+    rcache_resources.deregister_mem = scif_dereg_mem;
+    scif_module->rcache = mca_rcache_base_module_create ("grdma", scif_module, &rcache_resources);
+    if (NULL == scif_module->rcache) {
+        BTL_ERROR(("error creating grdma rcache"));
         return OPAL_ERROR;
     }
 

opal/mca/btl/scif/btl_scif_frag.h

@@ -60,8 +60,8 @@ static inline int mca_btl_scif_frag_alloc (mca_btl_base_endpoint_t *ep,
 static inline int mca_btl_scif_frag_return (mca_btl_scif_base_frag_t *frag)
 {
     if (frag->registration) {
-        frag->endpoint->btl->super.btl_mpool->mpool_deregister(frag->endpoint->btl->super.btl_mpool,
-                                                               &frag->registration->base);
+        frag->endpoint->btl->rcache->rcache_deregister (frag->endpoint->btl->rcache,
+                                                        &frag->registration->base);
         frag->registration = NULL;
     }
 

opal/mca/btl/scif/btl_scif_module.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2013-2016 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2014      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -123,6 +123,11 @@ mca_btl_scif_module_finalize (struct mca_btl_base_module_t *btl)
         scif_module->endpoints = NULL;
     }
 
+    if (NULL != scif_module->rcache) {
+        mca_rcache_base_module_destroy (scif_module->rcache);
+        scif_module->rcache = NULL;
+    }
+
     /* close the listening endpoint */
     if (mca_btl_scif_module.listening && -1 != mca_btl_scif_module.scif_fd) {
         /* wake up the scif thread */
@@ -180,6 +185,7 @@ static mca_btl_base_registration_handle_t *mca_btl_scif_register_mem (struct mca
                                                                       mca_btl_base_endpoint_t *endpoint,
                                                                       void *base, size_t size, uint32_t flags)
 {
+    mca_btl_scif_module_t *scif_module = &mca_btl_scif_module;
     mca_btl_scif_reg_t *scif_reg;
     int access_flags = flags & MCA_BTL_REG_FLAG_ACCESS_ANY;
     int rc;
@@ -200,8 +206,8 @@ static mca_btl_base_registration_handle_t *mca_btl_scif_register_mem (struct mca
         }
     }
 
-    rc = btl->btl_mpool->mpool_register(btl->btl_mpool, base, size, 0, access_flags,
-                                        (mca_mpool_base_registration_t **) &scif_reg);
+    rc = scif_module->rcache->rcache_register (scif_module->rcache, base, size, 0, access_flags,
+                                               (mca_rcache_base_registration_t **) &scif_reg);
     if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
         return NULL;
     }
@@ -210,7 +216,7 @@ static mca_btl_base_registration_handle_t *mca_btl_scif_register_mem (struct mca
     if ((off_t) -1 == scif_reg->handles[endpoint->id].btl_handle.scif_offset) {
         size_t seg_size = (size_t)((uintptr_t) scif_reg->base.bound - (uintptr_t) scif_reg->base.base) + 1;
 
-        /* NTH: until we determine a way to pass permissions to the mpool just make all segments
+        /* NTH: until we determine a way to pass permissions to the rcache just make all segments
          * read/write */
         scif_reg->handles[endpoint->id].btl_handle.scif_offset =
             scif_register (endpoint->scif_epd, scif_reg->base.base, seg_size, 0, SCIF_PROT_READ |
@@ -225,9 +231,10 @@ static mca_btl_base_registration_handle_t *mca_btl_scif_register_mem (struct mca
 static int mca_btl_scif_deregister_mem (struct mca_btl_base_module_t *btl, mca_btl_base_registration_handle_t *handle)
 {
     mca_btl_scif_registration_handle_t *scif_handle = (mca_btl_scif_registration_handle_t *) handle;
+    mca_btl_scif_module_t *scif_module = &mca_btl_scif_module;
     mca_btl_scif_reg_t *scif_reg = scif_handle->reg;
 
-    btl->btl_mpool->mpool_deregister (btl->btl_mpool, &scif_reg->base);
+    scif_module->rcache->rcache_deregister (scif_module->rcache, &scif_reg->base);
 
     return OPAL_SUCCESS;
 }