Merge pull request #1495 from hjelmn/new_hooks

Add new patcher memory hooks

Author: hjelmn

config/opal_configure_options.m4

@@ -283,7 +283,8 @@ else
     OPAL_ENABLE_DLOPEN_SUPPORT=1
     AC_MSG_RESULT([yes])
 fi
-
+AC_DEFINE_UNQUOTED(OPAL_ENABLE_DLOPEN_SUPPORT, $OPAL_ENABLE_DLOPEN_SUPPORT,
+    [Whether we want to enable dlopen support])
 
 #
 # Heterogeneous support

contrib/platform/lanl/cray_xe6/debug-common

@@ -1,4 +1,3 @@
-enable_dlopen=no
 enable_mem_profile=no
 enable_binaries=yes
 enable_heterogeneous=no
@@ -33,8 +32,5 @@ enable_mca_direct=pml-ob1
 # enable development headers
 with_devel_headers=yes
 
-# enable ptmalloc (enables lazy deregistration)
-with_memory_manager=linux
-
 # disable valgrind
 with_valgrind=no

contrib/platform/lanl/cray_xe6/optimized-common

@@ -1,5 +1,3 @@
-enable_dlopen=no
-
 enable_mem_profile=no
 enable_binaries=yes
 
@@ -40,8 +38,5 @@ enable_mca_direct=pml-ob1
 # enable development headers
 with_devel_headers=yes
 
-# enable ptmalloc (enables lazy deregistration)
-with_memory_manager=linux
-
 # disable valgrind
 with_valgrind=no

contrib/platform/lanl/toss/toss-common

@@ -1,7 +1,6 @@
 # (c) 2013 Los Alamos National Security, LLC. All rights reserved.
 # Open MPI common configuration for TOSS/TOSS2 v1.7.x/1.8.x
 
-enable_dlopen=no
 enable_binaries=yes
 enable_heterogeneous=no
 enable_shared=yes

ompi/runtime/ompi_mpi_init.c

@@ -98,15 +98,13 @@
 #endif
 #include "ompi/runtime/ompi_cr.h"
 
-#if defined(MEMORY_LINUX_PTMALLOC2) && MEMORY_LINUX_PTMALLOC2
-#include "opal/mca/memory/linux/memory_linux.h"
+#include "opal/mca/memory/base/base.h"
 /* So this sucks, but with OPAL in its own library that is brought in
    implicity from libmpi, there are times when the malloc initialize
    hook in the memory component doesn't work.  So we have to do it
    from here, since any MPI code is going to call MPI_Init... */
 OPAL_DECLSPEC void (*__malloc_initialize_hook) (void) =
-    opal_memory_linux_malloc_init_hook;
-#endif /* defined(MEMORY_LINUX_PTMALLOC2) && MEMORY_LINUX_PTMALLOC2 */
+    opal_memory_base_malloc_init_hook;
 
 /* This is required for the boundaries of the hash tables used to store
  * the F90 types returned by the MPI_Type_create_f90_XXX functions.

opal/common_sym_whitelist.txt

@@ -4,3 +4,4 @@ opal_show_help_yyleng
 opal_show_help_yytext
 opal_util_keyval_yyleng
 opal_util_keyval_yytext
+__curbrk

opal/mca/btl/openib/btl_openib_component.c

@@ -2562,39 +2562,6 @@ btl_openib_component_init(int *num_btl_modules,
         goto no_btls;
     }
 
-    /* If we are using ptmalloc2 and there are no posix threads
-       available, this will cause memory corruption.  Refuse to run.
-       Right now, ptmalloc2 is the only memory manager that we have on
-       OS's that support OpenFabrics that provide both FREE and MUNMAP
-       support, so the following test is [currently] good enough... */
-    value = opal_mem_hooks_support_level();
-
-    /* If we have a memory manager available, and
-       opal_leave_pinned==-1, then unless the user explicitly set
-       opal_leave_pinned_pipeline==0, then set opal_leave_pinned to 1.
-
-       We have a memory manager if we have both FREE and MUNMAP
-       support */
-    if ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) ==
-        ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) & value)) {
-        if (0 == opal_leave_pinned_pipeline &&
-            -1 == opal_leave_pinned) {
-            opal_leave_pinned = 1;
-        }
-    } else {
-        opal_leave_pinned = 0;
-        opal_leave_pinned_pipeline = 0;
-    }
-
-#if OPAL_CUDA_SUPPORT
-    if (mca_btl_openib_component.cuda_want_gdr && (0 == opal_leave_pinned)) {
-        opal_show_help("help-mpi-btl-openib.txt",
-                       "CUDA_gdr_and_nopinned", true,
-                       opal_process_info.nodename);
-        goto no_btls;
-    }
-#endif /* OPAL_CUDA_SUPPORT */
-
     index = mca_base_var_find("ompi", "btl", "openib", "max_inline_data");
     if (index >= 0) {
         if (OPAL_SUCCESS == mca_base_var_get_value(index, NULL, &source, NULL)) {
@@ -2931,6 +2898,22 @@ btl_openib_component_init(int *num_btl_modules,
         mca_btl_openib_component.if_exclude_list = NULL;
     }
 
+    /* If we are using ptmalloc2 and there are no posix threads
+       available, this will cause memory corruption.  Refuse to run.
+       Right now, ptmalloc2 is the only memory manager that we have on
+       OS's that support OpenFabrics that provide both FREE and MUNMAP
+       support, so the following test is [currently] good enough... */
+    value = opal_mem_hooks_support_level();
+
+#if OPAL_CUDA_SUPPORT
+   if (mca_btl_openib_component.cuda_want_gdr && (0 == opal_leave_pinned)) {
+        opal_show_help("help-mpi-btl-openib.txt",
+                       "CUDA_gdr_and_nopinned", true,
+                       opal_process_info.nodename);
+        goto no_btls;
+    }
+#endif /* OPAL_CUDA_SUPPORT */
+
     mca_btl_openib_component.memory_registration_verbose = opal_output_open(NULL);
     opal_output_set_verbosity (mca_btl_openib_component.memory_registration_verbose,
                                mca_btl_openib_component.memory_registration_verbose_level);

opal/mca/btl/ugni/btl_ugni_component.c

@@ -331,26 +331,6 @@ btl_ugni_component_close(void)
     return OPAL_SUCCESS;
 }
 
-static void mca_btl_ugni_autoset_leave_pinned (void) {
-    if (MCA_BTL_UGNI_RCACHE_UDREG != mca_btl_ugni_component.rcache_type) {
-        int value = opal_mem_hooks_support_level();
-        if ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) ==
-            ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) & value)) {
-            /* Set leave pinned to 1 if leave pinned pipeline is not set */
-            if (-1 == opal_leave_pinned) {
-                opal_leave_pinned = !opal_leave_pinned_pipeline;
-            }
-        } else {
-            opal_leave_pinned = 0;
-            opal_leave_pinned_pipeline = 0;
-        }
-    } else if (-1 == opal_leave_pinned) {
-        /* if udreg is in use we can set leave pinned without checking for the
-         * memory hooks. */
-        opal_leave_pinned = !opal_leave_pinned_pipeline;
-    }
-}
-
 static mca_btl_base_module_t **
 mca_btl_ugni_component_init (int *num_btl_modules,
                              bool enable_progress_threads,
@@ -409,8 +389,6 @@ mca_btl_ugni_component_init (int *num_btl_modules,
         }
     }
 
-    mca_btl_ugni_autoset_leave_pinned ();
-
     mca_btl_ugni_module.super.btl_rdma_pipeline_send_length = mca_btl_ugni_module.super.btl_eager_limit;
 
     for (i = 0 ; i < mca_btl_ugni_component.ugni_num_btls ; ++i) {

opal/mca/memory/base/base.h

@@ -32,5 +32,7 @@ BEGIN_C_DECLS
  */
 OPAL_DECLSPEC extern mca_base_framework_t opal_memory_base_framework;
 
+OPAL_DECLSPEC void opal_memory_base_malloc_init_hook (void);
+
 END_C_DECLS
 #endif /* OPAL_BASE_MEMORY_H */

opal/mca/memory/base/memory_base_open.c

@@ -13,6 +13,8 @@
  * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -43,20 +45,22 @@ static int empty_process(void)
     return OPAL_SUCCESS;
 }
 
+static int empty_query (int *priority)
+{
+    *priority = 0;
+    return OPAL_SUCCESS;
+}
 
 /*
  * Local variables
  */
 static opal_memory_base_component_2_0_0_t empty_component = {
-    /* Don't care about the version info */
-    { 0, },
-    /* Don't care about the data */
-    { 0, },
     /* Empty / safe functions to call if no memory componet is selected */
-    empty_process,
-    opal_memory_base_component_register_empty,
-    opal_memory_base_component_deregister_empty,
-    opal_memory_base_component_set_alignment_empty,
+    .memoryc_query = empty_query,
+    .memoryc_process = empty_process,
+    .memoryc_register = opal_memory_base_component_register_empty,
+    .memoryc_deregister = opal_memory_base_component_deregister_empty,
+    .memoryc_set_alignment = opal_memory_base_component_set_alignment_empty,
 };
 
 
@@ -66,30 +70,49 @@ static opal_memory_base_component_2_0_0_t empty_component = {
 opal_memory_base_component_2_0_0_t *opal_memory = &empty_component;
 
 
+void opal_memory_base_malloc_init_hook (void)
+{
+    if (opal_memory->memoryc_init_hook) {
+        opal_memory->memoryc_init_hook ();
+    }
+}
 
 /*
  * Function for finding and opening either all MCA components, or the one
  * that was specifically requested via a MCA parameter.
  */
 static int opal_memory_base_open(mca_base_open_flag_t flags)
 {
+    mca_base_component_list_item_t *item, *next;
+    opal_memory_base_component_2_0_0_t *tmp;
+    int priority, highest_priority = 0;
     int ret;
 
-    /* Open up all available components */
+    /* can only be zero or one */
+    OPAL_LIST_FOREACH(item, &opal_memory_base_framework.framework_components, mca_base_component_list_item_t) {
+        tmp = (opal_memory_base_component_2_0_0_t *) item->cli_component;
+        ret = tmp->memoryc_query (&priority);
+        if (OPAL_SUCCESS != ret || priority < highest_priority) {
+            continue;
+        }
+
+        highest_priority = priority;
+        opal_memory = tmp;
+    }
+
+    OPAL_LIST_FOREACH_SAFE(item, next, &opal_memory_base_framework.framework_components, mca_base_component_list_item_t) {
+        if ((void *) opal_memory != (void *) item->cli_component) {
+            mca_base_component_unload (item->cli_component, opal_memory_base_framework.framework_output);
+            opal_list_remove_item (&opal_memory_base_framework.framework_components, &item->super);
+        }
+    }
+
+    /* open remaining component */
     ret = mca_base_framework_components_open (&opal_memory_base_framework, flags);
     if (ret != OPAL_SUCCESS) {
         return ret;
     }
 
-    /* can only be zero or one */
-    if (opal_list_get_size(&opal_memory_base_framework.framework_components) == 1) {
-        mca_base_component_list_item_t *item;
-        item = (mca_base_component_list_item_t*)
-            opal_list_get_first(&opal_memory_base_framework.framework_components);
-        opal_memory = (opal_memory_base_component_2_0_0_t*)
-            item->cli_component;
-    }
-
     /* All done */
     return OPAL_SUCCESS;
 }