diff --git a/ompi/include/ompi/constants.h b/ompi/include/ompi/constants.h index ed9e463853b..7683e027bbb 100644 --- a/ompi/include/ompi/constants.h +++ b/ompi/include/ompi/constants.h @@ -10,6 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2016 Intel, Inc. All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -75,6 +76,7 @@ enum { OMPI_ERR_WIN = OMPI_ERR_BASE - 7, OMPI_ERR_RMA_FLAVOR = OMPI_ERR_BASE - 8, }; +/* it'd be nice to have a new OMPI_ERR_DISABLED error code. */ #define OMPI_ERR_MAX (OMPI_ERR_BASE - 100) diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt.h b/ompi/mca/osc/pt2pt/osc_pt2pt.h index bbb35f55629..ea66edcb6bd 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt.h +++ b/ompi/mca/osc/pt2pt/osc_pt2pt.h @@ -15,6 +15,7 @@ * Copyright (c) 2015-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 FUJITSU LIMITED. All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -88,6 +89,21 @@ struct ompi_osc_pt2pt_component_t { /** Is the progress function enabled? */ bool progress_enable; + + /** opal_output_verbose stream id */ + int output; + + /** MCA parameter: priority of this component */ + int priority; + + /** MCA parameter: Verbose level of this component */ + int verbose; + + /** MCA parameters: Enable osc_pt2pt component */ + bool enable; + + /** Allow osc_pt2pt to run when user passes in MPI_THREAD_MULTIPLE */ + bool allow_thread_multiple; }; typedef struct ompi_osc_pt2pt_component_t ompi_osc_pt2pt_component_t; diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt_component.c b/ompi/mca/osc/pt2pt/osc_pt2pt_component.c index e41a8306b7d..139d369020c 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt_component.c +++ b/ompi/mca/osc/pt2pt/osc_pt2pt_component.c @@ -16,6 +16,7 @@ * Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved. * Copyright (c) 2015-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -34,6 +35,7 @@ #include "ompi/mca/osc/base/osc_base_obj_convert.h" + static int component_register(void); static int component_init(bool enable_progress_threads, bool enable_mpi_threads); static int component_finalize(void); @@ -124,6 +126,45 @@ static bool check_config_value_bool(char *key, ompi_info_t *info, bool result) static int component_register (void) { + mca_osc_pt2pt_component.enable = true; + (void) mca_base_component_var_register(&mca_osc_pt2pt_component.super.osc_version, + "enable", + "Enable/Disable osc_pt2pt (default: true)", + MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, + OPAL_INFO_LVL_1, + MCA_BASE_VAR_SCOPE_READONLY, + &mca_osc_pt2pt_component.enable); + + mca_osc_pt2pt_component.allow_thread_multiple = true; + (void) mca_base_component_var_register(&mca_osc_pt2pt_component.super.osc_version, + "allow_thread_multiple", + "Allow osc_pt2pt to run when user passes in MPI_THREAD_MULTIPLE (default: true)", + MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, + OPAL_INFO_LVL_1, + MCA_BASE_VAR_SCOPE_READONLY, + &mca_osc_pt2pt_component.allow_thread_multiple); + + mca_osc_pt2pt_component.verbose = ompi_osc_base_framework.framework_verbose; + (void) mca_base_component_var_register(&mca_osc_pt2pt_component.super.osc_version, + "verbose", + "Verbosity of osc_pt2pt", + MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + OPAL_INFO_LVL_9, + MCA_BASE_VAR_SCOPE_READONLY, + &mca_osc_pt2pt_component.verbose); + + mca_osc_pt2pt_component.output = opal_output_open(NULL); + opal_output_set_verbosity(mca_osc_pt2pt_component.output, mca_osc_pt2pt_component.verbose); + + mca_osc_pt2pt_component.priority = 10; + (void) mca_base_component_var_register(&mca_osc_pt2pt_component.super.osc_version, + "priority", + "[0-99] Priority of osc_pt2pt", + MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + OPAL_INFO_LVL_9, + MCA_BASE_VAR_SCOPE_READONLY, + &mca_osc_pt2pt_component.priority); + ompi_osc_pt2pt_no_locks = false; (void) mca_base_component_var_register(&mca_osc_pt2pt_component.super.osc_version, "no_locks", @@ -208,6 +249,18 @@ component_init(bool enable_progress_threads, { int ret; + if (!mca_osc_pt2pt_component.enable) { + opal_output_verbose(2,mca_osc_pt2pt_component.output, "osc_pt2pt component_init DISABLED.\n"); + return -1; + } + + if (!mca_osc_pt2pt_component.allow_thread_multiple && enable_mpi_threads) { + opal_output_verbose(1, mca_osc_pt2pt_component.output, + "osc_pt2pt Disabled for MPI_THREAD_INIT (MCA: osc_pt2pt_allow_thread_multiple == 0)\n"); + return -1; + } + + OBJ_CONSTRUCT(&mca_osc_pt2pt_component.lock, opal_mutex_t); OBJ_CONSTRUCT(&mca_osc_pt2pt_component.pending_operations, opal_list_t); OBJ_CONSTRUCT(&mca_osc_pt2pt_component.pending_operations_lock, opal_mutex_t); @@ -247,6 +300,10 @@ component_init(bool enable_progress_threads, return ret; } + opal_output_verbose(3,mca_osc_pt2pt_component.output, + "osc_pt2pt component_init succeeded. Priority: %d\n", + mca_osc_pt2pt_component.priority); + return ret; } @@ -256,6 +313,10 @@ component_finalize(void) { size_t num_modules; + if (!mca_osc_pt2pt_component.enable) { + OPAL_OUTPUT_VERBOSE((2,mca_osc_pt2pt_component.output, "%s:%d: osc_pt2pt component_finalize, on DISABLED comp.\n", __FILE__, __LINE__)); + } + if (mca_osc_pt2pt_component.progress_enable) { opal_progress_unregister (component_progress); } @@ -276,6 +337,12 @@ component_finalize(void) OBJ_DESTRUCT(&mca_osc_pt2pt_component.pending_receives); OBJ_DESTRUCT(&mca_osc_pt2pt_component.pending_receives_lock); + if (mca_osc_pt2pt_component.enable) { + opal_output_verbose(2,mca_osc_pt2pt_component.output, "%s:%d: osc_pt2pt component_finalize.\n", __FILE__, __LINE__); + } + opal_output_close(mca_osc_pt2pt_component.output); + mca_osc_pt2pt_component.output = 0; + return OMPI_SUCCESS; } @@ -285,9 +352,10 @@ component_query(struct ompi_win_t *win, void **base, size_t size, int disp_unit, struct ompi_communicator_t *comm, struct ompi_info_t *info, int flavor) { + if (!mca_osc_pt2pt_component.enable) return -1; if (MPI_WIN_FLAVOR_SHARED == flavor) return -1; - return 10; + return mca_osc_pt2pt_component.priority; } @@ -304,6 +372,8 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit component */ if (MPI_WIN_FLAVOR_SHARED == flavor) return OMPI_ERR_NOT_SUPPORTED; + if (!mca_osc_pt2pt_component.enable) return OMPI_ERR_NOT_SUPPORTED; /* OMPI_ERR_DISABLED */ + /* create module structure with all fields initialized to zero */ module = (ompi_osc_pt2pt_module_t*) calloc(1, sizeof(ompi_osc_pt2pt_module_t)); diff --git a/ompi/mca/osc/rdma/osc_rdma.h b/ompi/mca/osc/rdma/osc_rdma.h index eb22898ca59..562df1f874f 100644 --- a/ompi/mca/osc/rdma/osc_rdma.h +++ b/ompi/mca/osc/rdma/osc_rdma.h @@ -13,6 +13,7 @@ * Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved. * Copyright (c) 2016 Intel, Inc. All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -94,10 +95,19 @@ struct ompi_osc_rdma_component_t { bool acc_use_amo; /** Priority of the osc/rdma component */ - unsigned int priority; + int priority; /** aggregation free list */ opal_free_list_t aggregate; + + /** MCA parameter: Verbose level of this component */ + int verbose; + + /** MCA parameters: Enable osc_pt2pt component */ + bool enable; + + /** Allow osc_rdma to run when user passes in MPI_THREAD_MULTIPLE */ + bool allow_thread_multiple; }; typedef struct ompi_osc_rdma_component_t ompi_osc_rdma_component_t; diff --git a/ompi/mca/osc/rdma/osc_rdma_component.c b/ompi/mca/osc/rdma/osc_rdma_component.c index c951a767610..bb4baeda466 100644 --- a/ompi/mca/osc/rdma/osc_rdma_component.c +++ b/ompi/mca/osc/rdma/osc_rdma_component.c @@ -16,6 +16,7 @@ * Copyright (c) 2012-2015 Sandia National Laboratories. All rights reserved. * Copyright (c) 2015 NVIDIA Corporation. All rights reserved. * Copyright (c) 2015 Intel, Inc. All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -214,6 +215,20 @@ static int ompi_osc_rdma_component_register (void) MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_GROUP, &mca_osc_rdma_component.priority); + mca_osc_rdma_component.enable = true; + (void) mca_base_component_var_register(&mca_osc_rdma_component.super.osc_version, + "enable", "Enable osc_rdma component (default: true)", + MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_1, + MCA_BASE_VAR_SCOPE_READONLY, &mca_osc_rdma_component.enable); + + mca_osc_rdma_component.allow_thread_multiple = true; + (void) mca_base_component_var_register(&mca_osc_rdma_component.super.osc_version, + "allow_thread_multiple", "Allow osc_rdma component when running with " + "MPI_THREAD_MULTIPLE. (default: true)", + MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_1, + MCA_BASE_VAR_SCOPE_READONLY, &mca_osc_rdma_component.allow_thread_multiple); + + ompi_osc_rdma_btl_names = "openib,ugni"; (void) mca_base_component_var_register (&mca_osc_rdma_component.super.osc_version, "btls", "Comma-delimited list of BTL component names to allow without verifying " @@ -247,6 +262,17 @@ static int ompi_osc_rdma_component_init (bool enable_progress_threads, { int ret; + if (!mca_osc_rdma_component.enable) { + opal_output_verbose(2, ompi_osc_base_framework.framework_output, "osc_rdma component_init DISABLED.\n"); + return -1; + } + + if (!mca_osc_rdma_component.allow_thread_multiple && ompi_mpi_thread_multiple) { + opal_output_verbose(1, ompi_osc_base_framework.framework_output, + "osc_rdma Disabled for MPI_THREAD_INIT (MCA: osc_rdma_allow_thread_multiple == 0)\n"); + return -1; + } + OBJ_CONSTRUCT(&mca_osc_rdma_component.lock, opal_mutex_t); OBJ_CONSTRUCT(&mca_osc_rdma_component.request_gc, opal_list_t); OBJ_CONSTRUCT(&mca_osc_rdma_component.buffer_gc, opal_list_t); @@ -296,6 +322,7 @@ static int ompi_osc_rdma_component_init (bool enable_progress_threads, mca_osc_rdma_component.aggregation_limit = 0; } + return ret; } @@ -325,7 +352,7 @@ static int ompi_osc_rdma_component_query (struct ompi_win_t *win, void **base, s struct ompi_communicator_t *comm, struct ompi_info_t *info, int flavor) { - + if (!mca_osc_rdma_component.enable) return -1; if (MPI_WIN_FLAVOR_SHARED == flavor) { return -1; } @@ -1023,6 +1050,8 @@ static int ompi_osc_rdma_component_select (struct ompi_win_t *win, void **base, int ret; char *name; + if (!mca_osc_rdma_component.enable) { return OMPI_ERR_NOT_SUPPORTED; } /* OMPI_ERR_DISABLED */ + /* the osc/sm component is the exclusive provider for support for shared * memory windows */ if (MPI_WIN_FLAVOR_SHARED == flavor) {