Skip to content

ofi - add MCA parameters to not use FI_HMEM #11981

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 12, 2023
Merged

ofi - add MCA parameters to not use FI_HMEM #11981

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 24 additions & 6 deletions ompi/mca/mtl/ofi/mtl_ofi_component.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,15 @@ ompi_mtl_ofi_component_register(void)
MCA_BASE_VAR_SCOPE_READONLY,
&ompi_mtl_ofi.num_ofi_contexts);

ompi_mtl_ofi.disable_hmem = false;
mca_base_component_var_register(&mca_mtl_ofi_component.super.mtl_version,
"disable_hmem",
"Disable HMEM usage",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_3,
MCA_BASE_VAR_SCOPE_READONLY,
&ompi_mtl_ofi.disable_hmem);

return opal_common_ofi_mca_register(&mca_mtl_ofi_component.super.mtl_version);
}

Expand Down Expand Up @@ -626,8 +635,10 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,

/* Request device transfer capabilities */
#if defined(FI_HMEM)
hints->caps |= FI_HMEM;
hints->domain_attr->mr_mode |= FI_MR_HMEM | FI_MR_ALLOCATED;
if (false == ompi_mtl_ofi.disable_hmem) {
hints->caps |= FI_HMEM;
hints->domain_attr->mr_mode |= FI_MR_HMEM | FI_MR_ALLOCATED;
}
#endif

no_hmem:
Expand Down Expand Up @@ -791,10 +802,17 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,

*accelerator_support = false;
#if defined(FI_HMEM)
if (!(prov->caps & FI_HMEM)) {
opal_output_verbose(50, opal_common_ofi.output,
"%s:%d: Libfabric provider does not support device buffers. Continuing with device to host copies.\n",
__FILE__, __LINE__);
if (!(prov->caps & FI_HMEM) || (true == ompi_mtl_ofi.disable_hmem)) {
if (!(prov->caps & FI_HMEM) && (false == ompi_mtl_ofi.disable_hmem)) {
opal_output_verbose(50, opal_common_ofi.output,
"%s:%d: Libfabric provider does not support device buffers. Continuing with device to host copies.\n",
__FILE__, __LINE__);
}
if (true == ompi_mtl_ofi.disable_hmem) {
opal_output_verbose(50, opal_common_ofi.output,
"%s:%d: Support for device buffers disabled by MCA parameter. Continuing with device to host copies.\n",
__FILE__, __LINE__);
}
} else {
*accelerator_support = true;
ompi_mtl_ofi.hmem_needs_reg = true;
Expand Down
5 changes: 3 additions & 2 deletions ompi/mca/mtl/ofi/mtl_ofi_types.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
* Copyright (c) 2013-2018 Intel, Inc. All rights reserved
*
* Copyright (c) 2015 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2022 Triad National Security, LLC. All rights
* reserved.
* Copyright (c) 2022-2023 Triad National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand Down Expand Up @@ -59,6 +59,7 @@ typedef struct mca_mtl_ofi_module_t {
int enable_sep; /* MCA to enable/disable SEP feature */
int thread_grouping; /* MCA for thread grouping feature */
int num_ofi_contexts; /* MCA for number of contexts to use */
bool disable_hmem; /* MCA to enable/disable request for FI_HMEM support from provider */

/** Endpoint name length */
size_t epnamelen;
Expand Down
2 changes: 2 additions & 0 deletions opal/mca/btl/ofi/btl_ofi.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,8 @@ struct mca_btl_ofi_component_t {
size_t max_inject_size;
bool disable_inject;

bool disable_hmem;

/** All BTL OFI modules (1 per tl) */
mca_btl_ofi_module_t *modules[MCA_BTL_OFI_MAX_MODULES];
};
Expand Down
16 changes: 14 additions & 2 deletions opal/mca/btl/ofi/btl_ofi_component.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,16 @@ static int mca_btl_ofi_component_register(void)
MCA_BASE_VAR_SCOPE_READONLY,
&mca_btl_ofi_component.disable_inject);

mca_btl_ofi_component.disable_hmem = false;
mca_base_component_var_register(&mca_btl_ofi_component.super.btl_version,
"disable_hmem",
"Disable HMEM usage",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_5,
MCA_BASE_VAR_SCOPE_READONLY,
&mca_btl_ofi_component.disable_hmem);


/* for now we want this component to lose to the MTL. */
module->super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_HIGH - 50;

Expand Down Expand Up @@ -345,8 +355,10 @@ static mca_btl_base_module_t **mca_btl_ofi_component_init(int *num_btl_modules,

#if defined(FI_HMEM)
/* Request device transfer capabilities, separate from required_caps */
hints.caps |= FI_HMEM;
hints.domain_attr->mr_mode |= FI_MR_HMEM;
if (false == mca_btl_ofi_component.disable_hmem) {
hints.caps |= FI_HMEM;
hints.domain_attr->mr_mode |= FI_MR_HMEM;
}
no_hmem:
#endif

Expand Down