Skip to content

Topic/osc controls #2626

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

Topic/osc controls #2626

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
917c049
Adding mca parameters to osc_rdma: enable and allow_thread_multiple
gpaulsen Dec 20, 2016
cfef4f4
Adding mca parameters to osc_pt2pt:
gpaulsen Dec 20, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions ompi/include/ompi/constants.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2016 Intel, Inc. All rights reserved.
* Copyright (c) 2016 IBM Corporation. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand Down Expand Up @@ -75,6 +76,7 @@ enum {
OMPI_ERR_WIN = OMPI_ERR_BASE - 7,
OMPI_ERR_RMA_FLAVOR = OMPI_ERR_BASE - 8,
};
/* it'd be nice to have a new OMPI_ERR_DISABLED error code. */

#define OMPI_ERR_MAX (OMPI_ERR_BASE - 100)

Expand Down
16 changes: 16 additions & 0 deletions ompi/mca/osc/pt2pt/osc_pt2pt.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
* Copyright (c) 2015-2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2016 FUJITSU LIMITED. All rights reserved.
* Copyright (c) 2016 IBM Corporation. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand Down Expand Up @@ -88,6 +89,21 @@ struct ompi_osc_pt2pt_component_t {

/** Is the progress function enabled? */
bool progress_enable;

/** opal_output_verbose stream id */
int output;

/** MCA parameter: priority of this component */
int priority;

/** MCA parameter: Verbose level of this component */
int verbose;

/** MCA parameters: Enable osc_pt2pt component */
bool enable;

/** Allow osc_pt2pt to run when user passes in MPI_THREAD_MULTIPLE */
bool allow_thread_multiple;
};
typedef struct ompi_osc_pt2pt_component_t ompi_osc_pt2pt_component_t;

Expand Down
72 changes: 71 additions & 1 deletion ompi/mca/osc/pt2pt/osc_pt2pt_component.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
* Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved.
* Copyright (c) 2015-2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2016 IBM Corporation. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand All @@ -34,6 +35,7 @@

#include "ompi/mca/osc/base/osc_base_obj_convert.h"


static int component_register(void);
static int component_init(bool enable_progress_threads, bool enable_mpi_threads);
static int component_finalize(void);
Expand Down Expand Up @@ -124,6 +126,45 @@ static bool check_config_value_bool(char *key, ompi_info_t *info, bool result)

static int component_register (void)
{
mca_osc_pt2pt_component.enable = true;
(void) mca_base_component_var_register(&mca_osc_pt2pt_component.super.osc_version,
"enable",
"Enable/Disable osc_pt2pt (default: true)",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_1,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_osc_pt2pt_component.enable);

mca_osc_pt2pt_component.allow_thread_multiple = true;
(void) mca_base_component_var_register(&mca_osc_pt2pt_component.super.osc_version,
"allow_thread_multiple",
"Allow osc_pt2pt to run when user passes in MPI_THREAD_MULTIPLE (default: true)",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_1,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_osc_pt2pt_component.allow_thread_multiple);

mca_osc_pt2pt_component.verbose = ompi_osc_base_framework.framework_verbose;
(void) mca_base_component_var_register(&mca_osc_pt2pt_component.super.osc_version,
"verbose",
"Verbosity of osc_pt2pt",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_osc_pt2pt_component.verbose);

mca_osc_pt2pt_component.output = opal_output_open(NULL);
opal_output_set_verbosity(mca_osc_pt2pt_component.output, mca_osc_pt2pt_component.verbose);

mca_osc_pt2pt_component.priority = 10;
(void) mca_base_component_var_register(&mca_osc_pt2pt_component.super.osc_version,
"priority",
"[0-99] Priority of osc_pt2pt",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_osc_pt2pt_component.priority);

ompi_osc_pt2pt_no_locks = false;
(void) mca_base_component_var_register(&mca_osc_pt2pt_component.super.osc_version,
"no_locks",
Expand Down Expand Up @@ -208,6 +249,18 @@ component_init(bool enable_progress_threads,
{
int ret;

if (!mca_osc_pt2pt_component.enable) {
opal_output_verbose(2,mca_osc_pt2pt_component.output, "osc_pt2pt component_init DISABLED.\n");
return -1;
}

if (!mca_osc_pt2pt_component.allow_thread_multiple && enable_mpi_threads) {
opal_output_verbose(1, mca_osc_pt2pt_component.output,
"osc_pt2pt Disabled for MPI_THREAD_INIT (MCA: osc_pt2pt_allow_thread_multiple == 0)\n");
return -1;
}


OBJ_CONSTRUCT(&mca_osc_pt2pt_component.lock, opal_mutex_t);
OBJ_CONSTRUCT(&mca_osc_pt2pt_component.pending_operations, opal_list_t);
OBJ_CONSTRUCT(&mca_osc_pt2pt_component.pending_operations_lock, opal_mutex_t);
Expand Down Expand Up @@ -247,6 +300,10 @@ component_init(bool enable_progress_threads,
return ret;
}

opal_output_verbose(3,mca_osc_pt2pt_component.output,
"osc_pt2pt component_init succeeded. Priority: %d\n",
mca_osc_pt2pt_component.priority);

return ret;
}

Expand All @@ -256,6 +313,10 @@ component_finalize(void)
{
size_t num_modules;

if (!mca_osc_pt2pt_component.enable) {
OPAL_OUTPUT_VERBOSE((2,mca_osc_pt2pt_component.output, "%s:%d: osc_pt2pt component_finalize, on DISABLED comp.\n", __FILE__, __LINE__));
}

if (mca_osc_pt2pt_component.progress_enable) {
opal_progress_unregister (component_progress);
}
Expand All @@ -276,6 +337,12 @@ component_finalize(void)
OBJ_DESTRUCT(&mca_osc_pt2pt_component.pending_receives);
OBJ_DESTRUCT(&mca_osc_pt2pt_component.pending_receives_lock);

if (mca_osc_pt2pt_component.enable) {
opal_output_verbose(2,mca_osc_pt2pt_component.output, "%s:%d: osc_pt2pt component_finalize.\n", __FILE__, __LINE__);
}
opal_output_close(mca_osc_pt2pt_component.output);
mca_osc_pt2pt_component.output = 0;

return OMPI_SUCCESS;
}

Expand All @@ -285,9 +352,10 @@ component_query(struct ompi_win_t *win, void **base, size_t size, int disp_unit,
struct ompi_communicator_t *comm, struct ompi_info_t *info,
int flavor)
{
if (!mca_osc_pt2pt_component.enable) return -1;
if (MPI_WIN_FLAVOR_SHARED == flavor) return -1;

return 10;
return mca_osc_pt2pt_component.priority;
}


Expand All @@ -304,6 +372,8 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
component */
if (MPI_WIN_FLAVOR_SHARED == flavor) return OMPI_ERR_NOT_SUPPORTED;

if (!mca_osc_pt2pt_component.enable) return OMPI_ERR_NOT_SUPPORTED; /* OMPI_ERR_DISABLED */

/* create module structure with all fields initialized to zero */
module = (ompi_osc_pt2pt_module_t*)
calloc(1, sizeof(ompi_osc_pt2pt_module_t));
Expand Down
12 changes: 11 additions & 1 deletion ompi/mca/osc/rdma/osc_rdma.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
* Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved.
* Copyright (c) 2016 Intel, Inc. All rights reserved.
* Copyright (c) 2016 IBM Corporation. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand Down Expand Up @@ -94,10 +95,19 @@ struct ompi_osc_rdma_component_t {
bool acc_use_amo;

/** Priority of the osc/rdma component */
unsigned int priority;
int priority;

/** aggregation free list */
opal_free_list_t aggregate;

/** MCA parameter: Verbose level of this component */
int verbose;

/** MCA parameters: Enable osc_pt2pt component */
bool enable;

/** Allow osc_rdma to run when user passes in MPI_THREAD_MULTIPLE */
bool allow_thread_multiple;
};
typedef struct ompi_osc_rdma_component_t ompi_osc_rdma_component_t;

Expand Down
31 changes: 30 additions & 1 deletion ompi/mca/osc/rdma/osc_rdma_component.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
* Copyright (c) 2012-2015 Sandia National Laboratories. All rights reserved.
* Copyright (c) 2015 NVIDIA Corporation. All rights reserved.
* Copyright (c) 2015 Intel, Inc. All rights reserved.
* Copyright (c) 2016 IBM Corporation. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand Down Expand Up @@ -214,6 +215,20 @@ static int ompi_osc_rdma_component_register (void)
MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0, OPAL_INFO_LVL_3,
MCA_BASE_VAR_SCOPE_GROUP, &mca_osc_rdma_component.priority);

mca_osc_rdma_component.enable = true;
(void) mca_base_component_var_register(&mca_osc_rdma_component.super.osc_version,
"enable", "Enable osc_rdma component (default: true)",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_1,
MCA_BASE_VAR_SCOPE_READONLY, &mca_osc_rdma_component.enable);

mca_osc_rdma_component.allow_thread_multiple = true;
(void) mca_base_component_var_register(&mca_osc_rdma_component.super.osc_version,
"allow_thread_multiple", "Allow osc_rdma component when running with "
"MPI_THREAD_MULTIPLE. (default: true)",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_1,
MCA_BASE_VAR_SCOPE_READONLY, &mca_osc_rdma_component.allow_thread_multiple);


ompi_osc_rdma_btl_names = "openib,ugni";
(void) mca_base_component_var_register (&mca_osc_rdma_component.super.osc_version, "btls",
"Comma-delimited list of BTL component names to allow without verifying "
Expand Down Expand Up @@ -247,6 +262,17 @@ static int ompi_osc_rdma_component_init (bool enable_progress_threads,
{
int ret;

if (!mca_osc_rdma_component.enable) {
opal_output_verbose(2, ompi_osc_base_framework.framework_output, "osc_rdma component_init DISABLED.\n");
return -1;
}

if (!mca_osc_rdma_component.allow_thread_multiple && ompi_mpi_thread_multiple) {
opal_output_verbose(1, ompi_osc_base_framework.framework_output,
"osc_rdma Disabled for MPI_THREAD_INIT (MCA: osc_rdma_allow_thread_multiple == 0)\n");
return -1;
}

OBJ_CONSTRUCT(&mca_osc_rdma_component.lock, opal_mutex_t);
OBJ_CONSTRUCT(&mca_osc_rdma_component.request_gc, opal_list_t);
OBJ_CONSTRUCT(&mca_osc_rdma_component.buffer_gc, opal_list_t);
Expand Down Expand Up @@ -296,6 +322,7 @@ static int ompi_osc_rdma_component_init (bool enable_progress_threads,
mca_osc_rdma_component.aggregation_limit = 0;
}


return ret;
}

Expand Down Expand Up @@ -325,7 +352,7 @@ static int ompi_osc_rdma_component_query (struct ompi_win_t *win, void **base, s
struct ompi_communicator_t *comm, struct ompi_info_t *info,
int flavor)
{

if (!mca_osc_rdma_component.enable) return -1;
if (MPI_WIN_FLAVOR_SHARED == flavor) {
return -1;
}
Expand Down Expand Up @@ -1023,6 +1050,8 @@ static int ompi_osc_rdma_component_select (struct ompi_win_t *win, void **base,
int ret;
char *name;

if (!mca_osc_rdma_component.enable) { return OMPI_ERR_NOT_SUPPORTED; } /* OMPI_ERR_DISABLED */

/* the osc/sm component is the exclusive provider for support for shared
* memory windows */
if (MPI_WIN_FLAVOR_SHARED == flavor) {
Expand Down