Skip to content

v3.1.x: btl/vader: move backing files into /dev/shm on Linux #5421

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Aug 6, 2018
Merged

v3.1.x: btl/vader: move backing files into /dev/shm on Linux #5421

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7ae4c29
btl/vader: move backing files into /dev/shm on Linux
hjelmn Dec 13, 2017
203d44c
btl/vader: make the backing file job specific
ggouaillardet Jan 30, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion opal/mca/btl/vader/btl_vader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
* All rights reserved.
* Copyright (c) 2006-2007 Voltaire. All rights reserved.
* Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2010-2015 Los Alamos National Security, LLC. All rights
* Copyright (c) 2010-2017 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2015 Mellanox Technologies. All rights reserved.
*
Expand Down Expand Up @@ -136,6 +136,8 @@ struct mca_btl_vader_component_t {
opal_list_t pending_endpoints; /**< list of endpoints with pending fragments */
opal_list_t pending_fragments; /**< fragments pending remote completion */

char *backing_directory; /**< directory to place shared memory backing files */

/* knem stuff */
#if OPAL_BTL_VADER_HAVE_KNEM
unsigned int knem_dma_min; /**< minimum size to enable DMA for knem transfers (0 disables) */
Expand Down
23 changes: 18 additions & 5 deletions opal/mca/btl/vader/btl_vader_component.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,11 @@
* All rights reserved.
* Copyright (c) 2006-2007 Voltaire. All rights reserved.
* Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2010-2015 Los Alamos National Security, LLC.
* Copyright (c) 2010-2017 Los Alamos National Security, LLC.
* All rights reserved.
* Copyright (c) 2011 NVIDIA Corporation. All rights reserved.
* Copyright (c) 2014-2015 Intel, Inc. All rights reserved.
* Copyright (c) 2014-2016 Research Organization for Information Science
* Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
* Copyright (c) 2014-2018 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
Expand Down Expand Up @@ -211,6 +211,19 @@ static int mca_btl_vader_component_register (void)
OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_GROUP, &mca_btl_vader_component.single_copy_mechanism);
OBJ_RELEASE(new_enum);

if (0 == access ("/dev/shm", W_OK)) {
mca_btl_vader_component.backing_directory = "/dev/shm";
} else {
mca_btl_vader_component.backing_directory = opal_process_info.job_session_dir;
}
(void) mca_base_component_var_register (&mca_btl_vader_component.super.btl_version, "backing_directory",
"Directory to place backing files for shared memory communication. "
"This directory should be on a local filesystem such as /tmp or "
"/dev/shm (default: (linux) /dev/shm, (others) session directory)",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3,
MCA_BASE_VAR_SCOPE_READONLY, &mca_btl_vader_component.backing_directory);


#if OPAL_BTL_VADER_HAVE_KNEM
/* Currently disabling DMA mode by default; it's not clear that this is useful in all applications and architectures. */
mca_btl_vader_component.knem_dma_min = 0;
Expand Down Expand Up @@ -491,8 +504,8 @@ static mca_btl_base_module_t **mca_btl_vader_component_init (int *num_btls,
if (MCA_BTL_VADER_XPMEM != mca_btl_vader_component.single_copy_mechanism) {
char *sm_file;

rc = asprintf(&sm_file, "%s" OPAL_PATH_SEP "vader_segment.%s.%d", opal_process_info.proc_session_dir,
opal_process_info.nodename, MCA_BTL_VADER_LOCAL_RANK);
rc = asprintf(&sm_file, "%s" OPAL_PATH_SEP "vader_segment.%s.%x.%d", mca_btl_vader_component.backing_directory,
opal_process_info.nodename, OPAL_PROC_MY_NAME.jobid, MCA_BTL_VADER_LOCAL_RANK);
if (0 > rc) {
free (btls);
return NULL;
Expand Down