Merge pull request #1079 from hjelmn/v2.x_mem_hooks

v2.x mem hook update

Author: hppritcha

AUTHORS

@@ -43,6 +43,7 @@ [email protected]                 Graham Fagg                 UTK
 [email protected]               Gilles Gouaillardet         RIST
 [email protected]                Ginger Young                LANL
 [email protected]               Gleb Natapov                Voltaire
+[email protected]             Geoffrey Paulsen            IBM
 [email protected]               Galen Shipman               LANL
 [email protected]                Greg Watson                 LANL
 [email protected]             Thomas Herault              INRIA

config/opal_configure_options.m4

@@ -302,7 +302,8 @@ else
     OPAL_ENABLE_DLOPEN_SUPPORT=1
     AC_MSG_RESULT([yes])
 fi
-
+AC_DEFINE_UNQUOTED(OPAL_ENABLE_DLOPEN_SUPPORT, $OPAL_ENABLE_DLOPEN_SUPPORT,
+    [Whether we want to enable dlopen support])
 
 #
 # Heterogeneous support

contrib/platform/lanl/cray_xe6/debug-common

@@ -1,4 +1,3 @@
-enable_dlopen=no
 enable_mem_profile=no
 enable_binaries=yes
 enable_heterogeneous=no
@@ -33,8 +32,5 @@ enable_mca_direct=pml-ob1
 # enable development headers
 with_devel_headers=yes
 
-# enable ptmalloc (enables lazy deregistration)
-with_memory_manager=linux
-
 # disable valgrind
 with_valgrind=no

contrib/platform/lanl/cray_xe6/optimized-common

@@ -1,5 +1,3 @@
-enable_dlopen=no
-
 enable_mem_profile=no
 enable_binaries=yes
 
@@ -40,8 +38,5 @@ enable_mca_direct=pml-ob1
 # enable development headers
 with_devel_headers=yes
 
-# enable ptmalloc (enables lazy deregistration)
-with_memory_manager=linux
-
 # disable valgrind
 with_valgrind=no

contrib/platform/lanl/toss/toss-common

@@ -1,7 +1,6 @@
 # (c) 2013 Los Alamos National Security, LLC. All rights reserved.
 # Open MPI common configuration for TOSS/TOSS2 v1.7.x/1.8.x
 
-enable_dlopen=no
 enable_binaries=yes
 enable_heterogeneous=no
 enable_shared=yes

ompi/runtime/ompi_mpi_init.c

@@ -93,15 +93,13 @@
 #include "ompi/dpm/dpm.h"
 #include "ompi/mpiext/mpiext.h"
 
-#if defined(MEMORY_LINUX_PTMALLOC2) && MEMORY_LINUX_PTMALLOC2
-#include "opal/mca/memory/linux/memory_linux.h"
+#include "opal/mca/memory/base/base.h"
 /* So this sucks, but with OPAL in its own library that is brought in
    implicity from libmpi, there are times when the malloc initialize
    hook in the memory component doesn't work.  So we have to do it
    from here, since any MPI code is going to call MPI_Init... */
 OPAL_DECLSPEC void (*__malloc_initialize_hook) (void) =
-    opal_memory_linux_malloc_init_hook;
-#endif /* defined(MEMORY_LINUX_PTMALLOC2) && MEMORY_LINUX_PTMALLOC2 */
+    opal_memory_base_malloc_init_hook;
 
 /* This is required for the boundaries of the hash tables used to store
  * the F90 types returned by the MPI_Type_create_f90_XXX functions.

opal/mca/memory/base/base.h

@@ -32,5 +32,7 @@ BEGIN_C_DECLS
  */
 OPAL_DECLSPEC extern mca_base_framework_t opal_memory_base_framework;
 
+OPAL_DECLSPEC void opal_memory_base_malloc_init_hook (void);
+
 END_C_DECLS
 #endif /* OPAL_BASE_MEMORY_H */

opal/mca/memory/base/memory_base_open.c

@@ -13,6 +13,8 @@
  * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -43,20 +45,22 @@ static int empty_process(void)
     return OPAL_SUCCESS;
 }
 
+static int empty_query (int *priority)
+{
+    *priority = 0;
+    return OPAL_SUCCESS;
+}
 
 /*
  * Local variables
  */
 static opal_memory_base_component_2_0_0_t empty_component = {
-    /* Don't care about the version info */
-    { 0, },
-    /* Don't care about the data */
-    { 0, },
     /* Empty / safe functions to call if no memory componet is selected */
-    empty_process,
-    opal_memory_base_component_register_empty,
-    opal_memory_base_component_deregister_empty,
-    opal_memory_base_component_set_alignment_empty,
+    .memoryc_query = empty_query,
+    .memoryc_process = empty_process,
+    .memoryc_register = opal_memory_base_component_register_empty,
+    .memoryc_deregister = opal_memory_base_component_deregister_empty,
+    .memoryc_set_alignment = opal_memory_base_component_set_alignment_empty,
 };
 
 
@@ -65,31 +69,76 @@ static opal_memory_base_component_2_0_0_t empty_component = {
  */
 opal_memory_base_component_2_0_0_t *opal_memory = &empty_component;
 
+#if MEMORY_LINUX_PTMALLOC2
+/*
+ * Note that this is a minor abstraction violation (that has actually
+ * existed for quite a long time -- it used to be up in
+ * ompi_mpi_init(); yoinks!): we're including a component's header
+ * file here.  This is an unfortunate necessity: the linux/ptmallox
+ * system hooks in pre-main, and has to be initialized before any
+ * component or module has even been created.  Sad panda.
+ */
+#include "opal/mca/memory/linux/memory_linux.h"
+#endif
 
+void opal_memory_base_malloc_init_hook (void)
+{
+#if MEMORY_LINUX_PTMALLOC2
+    /* See above comment about linux/ptmalloc2 about why this
+       abstraction violation is here. */
+    opal_memory->memoryc_init_hook = opal_memory_linux_malloc_init_hook;
+#endif
+
+    if (opal_memory->memoryc_init_hook) {
+        opal_memory->memoryc_init_hook ();
+    }
+}
 
 /*
  * Function for finding and opening either all MCA components, or the one
  * that was specifically requested via a MCA parameter.
  */
 static int opal_memory_base_open(mca_base_open_flag_t flags)
 {
+    mca_base_component_list_item_t *item, *next;
+    opal_memory_base_component_2_0_0_t *tmp;
+    int priority, highest_priority = 0;
     int ret;
 
-    /* Open up all available components */
+    /* can only be zero or one */
+    OPAL_LIST_FOREACH(item, &opal_memory_base_framework.framework_components, mca_base_component_list_item_t) {
+        tmp = (opal_memory_base_component_2_0_0_t *) item->cli_component;
+
+        ret = tmp->memoryc_query (&priority);
+#if MEMORY_LINUX_PTMALLOC2
+        /* See above comment about linux/ptmalloc2 about why this
+           abstraction violation is here. */
+        if (0 == strcmp (tmp->mca_component_name, "linux")) {
+            /* if ptmalloc is enabled always use it */
+            priority = 1000000;
+        }
+#endif
+        if (OPAL_SUCCESS != ret || priority < highest_priority) {
+            continue;
+        }
+
+        highest_priority = priority;
+        opal_memory = tmp;
+    }
+
+    OPAL_LIST_FOREACH_SAFE(item, next, &opal_memory_base_framework.framework_components, mca_base_component_list_item_t) {
+        if ((void *) opal_memory != (void *) item->cli_component) {
+            mca_base_component_unload (item->cli_component, opal_memory_base_framework.framework_output);
+            opal_list_remove_item (&opal_memory_base_framework.framework_components, &item->super);
+        }
+    }
+
+    /* open remaining component */
     ret = mca_base_framework_components_open (&opal_memory_base_framework, flags);
     if (ret != OPAL_SUCCESS) {
         return ret;
     }
 
-    /* can only be zero or one */
-    if (opal_list_get_size(&opal_memory_base_framework.framework_components) == 1) {
-        mca_base_component_list_item_t *item;
-        item = (mca_base_component_list_item_t*)
-            opal_list_get_first(&opal_memory_base_framework.framework_components);
-        opal_memory = (opal_memory_base_component_2_0_0_t*)
-            item->cli_component;
-    }
-
     /* All done */
     return OPAL_SUCCESS;
 }

opal/mca/memory/linux/configure.m4

@@ -52,17 +52,14 @@ AC_DEFUN([MCA_opal_memory_linux_CONFIG],[
         ;;
     esac
 
+    # Must specifically request this component
     AS_IF([test "$with_memory_manager" = "linux"],
           [memory_linux_ptmalloc2_happy=yes
            memory_linux_ummu_happy=yes
            memory_linux_requested=1],
           [memory_linux_requested=0
-           AS_IF([test -z "$with_memory_manager" || test "$with_memory_manager" = "yes"],
-                 [memory_linux_ptmalloc2_happy=yes
-                  memory_linux_ummu_happy=yes],
-                 [memory_linux_ptmalloc2_happy=no
-                  memory_linux_ummu_happy=no])])
-
+           memory_linux_ptmalloc2_happy=no
+          memory_linux_ummu_happy=no])
 
     ######################################################################
     # if memory hook available

opal/mca/memory/linux/hooks.c

@@ -744,6 +744,10 @@ void opal_memory_linux_malloc_init_hook(void)
     check_result_t r1, lp, lpp;
     bool want_rcache = false, found_driver = false;
 
+    if (!opal_memory_linux_opened) {
+        return;
+    }
+
     /* First, check for a FAKEROOT environment.  If we're in a
        fakeroot, then access() (and likely others) have been replaced
        and are not safe to call here in this pre-main environment.  So

opal/mca/memory/linux/memory_linux.h

@@ -83,6 +83,8 @@ OPAL_DECLSPEC void opal_memory_linux_malloc_init_hook(void);
 OPAL_DECLSPEC void opal_memory_linux_malloc_set_alignment(int use_memalign, size_t memalign_threshold);
 #endif /* MEMORY_LINUX_MALLOC_ALIGN_ENABLED */
 
+extern bool opal_memory_linux_opened;
+
 END_C_DECLS
 
 #endif

opal/mca/memory/linux/memory_linux_component.c

@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2009-2014 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2013-2016 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -60,6 +60,7 @@
 static int linux_open(void);
 static int linux_close(void);
 static int linux_register(void);
+static int linux_query(int *);
 
 #if MEMORY_LINUX_UMMUNOTIFY
 static bool ummunotify_opened = false;
@@ -69,6 +70,9 @@ static bool ptmalloc2_opened = false;
 #endif
 
 bool opal_memory_linux_disable = false;
+static int mca_memory_linux_priority;
+
+bool opal_memory_linux_opened = false;
 
 opal_memory_linux_component_t mca_memory_linux_component = {
     /* First, the opal_memory_base_component_2_0_0_t */
@@ -96,6 +100,8 @@ opal_memory_linux_component_t mca_memory_linux_component = {
         /* Memory framework functions.  These function pointer values
            are replaced by memory_linux_ummunotify.c at run time if we
            end up using ummunotify support. */
+        .memoryc_init_hook = opal_memory_linux_malloc_init_hook,
+        .memoryc_query = linux_query,
         .memoryc_register = opal_memory_base_component_register_empty,
         .memoryc_deregister = opal_memory_base_component_deregister_empty,
 #if MEMORY_LINUX_MALLOC_ALIGN_ENABLED
@@ -243,11 +249,25 @@ static int linux_register(void)
     if (0 > ret) {
         return ret;
     }
+
+    mca_memory_linux_priority = 50;
+    ret = mca_base_component_var_register (&mca_memory_linux_component.super.memoryc_version,
+                                           "priority", "Priority of the linux memory hook component",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_5,
+                                           MCA_BASE_VAR_SCOPE_CONSTANT, &mca_memory_linux_priority);
+    if (0 > ret) {
+        return ret;
+    }
 #endif /* MEMORY_LINUX_MALLOC_ALIGN_ENABLED */
 
     return (0 > ret) ? ret : OPAL_SUCCESS;
 }
 
+static int linux_query (int *priority)
+{
+    *priority = mca_memory_linux_priority;
+    return OPAL_SUCCESS;
+}
 
 static int linux_open(void)
 {
@@ -318,6 +338,7 @@ static int linux_open(void)
         __malloc_hook = _opal_memory_linux_malloc_align_hook;
     }
 #endif /* MEMORY_LINUX_MALLOC_ALIGN_ENABLED */
+    opal_memory_linux_opened = true;
 
     return OPAL_SUCCESS;
 }

opal/mca/memory/malloc_solaris/memory_malloc_solaris_component.c

@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2007-2011 Oracle and/or its affiliates.  All rights reserved.
  * Copyright (c) 2009-2011 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2015-2016 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -46,6 +46,7 @@ int  __munmap(caddr_t addr, size_t len);
 #endif
 
 static int opal_memory_malloc_open(void);
+static int opal_memory_malloc_query(int *);
 
 const opal_memory_base_component_2_0_0_t mca_memory_malloc_solaris_component = {
     /* First, the mca_component_t struct containing meta information
@@ -68,6 +69,7 @@ const opal_memory_base_component_2_0_0_t mca_memory_malloc_solaris_component = {
 
     /* This component doesn't need these functions, but need to
        provide safe/empty register/deregister functions to call */
+    .memoryc_query = opal_memory_malloc_query,
     .memoryc_register = opal_memory_base_component_register_empty,
     .memoryc_deregister = opal_memory_base_component_deregister_empty,
     .memoryc_set_alignment = opal_memory_base_component_set_alignment_empty,
@@ -93,6 +95,11 @@ opal_memory_malloc_open(void)
     return OPAL_SUCCESS;
 }
 
+static int opal_memory_malloc_query (int *priority)
+{
+    *priority = 79;
+    return OPAL_SUCCESS;
+}
 
 /*
  * Three ways to call munmap.  Prefered is to call __munmap, which