Skip to content

Commit ec5e7c0

Browse files
committed
Merge branch '5.0.x' into job-local-tweak
2 parents 0c02ded + 2e8e993 commit ec5e7c0

File tree

9 files changed

+389
-36
lines changed

9 files changed

+389
-36
lines changed

easybuild/framework/easyblock.py

Lines changed: 25 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@
7878
from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES
7979
from easybuild.tools.config import EASYBUILD_SOURCES_URL, EBPYTHONPREFIXES # noqa
8080
from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES
81-
from easybuild.tools.config import PYTHONPATH, SEARCH_PATH_BIN_DIRS, SEARCH_PATH_LIB_DIRS
81+
from easybuild.tools.config import MOD_SEARCH_PATH_HEADERS, PYTHONPATH, SEARCH_PATH_BIN_DIRS, SEARCH_PATH_LIB_DIRS
8282
from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath
8383
from easybuild.tools.config import install_path, log_path, package_path, source_paths
8484
from easybuild.tools.environment import restore_env, sanitize_env
@@ -100,7 +100,7 @@
100100
from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for
101101
from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
102102
from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX
103-
from easybuild.tools.modules import Lmod, ModEnvVarType, ModuleLoadEnvironment
103+
from easybuild.tools.modules import Lmod, ModEnvVarType, ModuleLoadEnvironment, MODULE_LOAD_ENV_HEADERS
104104
from easybuild.tools.modules import curr_module_paths, invalidate_module_caches_for, get_software_root
105105
from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name
106106
from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ALL, PROGRESS_BAR_EASYCONFIG, PROGRESS_BAR_EXTENSIONS
@@ -133,8 +133,9 @@ class LibSymlink(Enum):
133133
- LIB_TO_LIB64: 'lib' is a symlink to 'lib64'
134134
- LIB64_TO_LIB: 'lib64' is a symlink to 'lib'
135135
- NEITHER: neither 'lib' is a symlink to 'lib64', nor 'lib64' is a symlink to 'lib'
136+
- BOTH_TO_DIR: 'lib' and 'lib64' are symlinks to some other directory
136137
- """
137-
LIB_TO_LIB64, LIB64_TO_LIB, NEITHER = range(3)
138+
LIB_TO_LIB64, LIB64_TO_LIB, NEITHER, BOTH_TO_DIR = range(4)
138139

139140

140141
class EasyBlock(object):
@@ -220,7 +221,19 @@ def __init__(self, ec, logfile=None):
220221
self.modules_header = read_file(modules_header_path)
221222

222223
# environment variables on module load
223-
self.module_load_environment = ModuleLoadEnvironment()
224+
mod_load_aliases = {}
225+
# apply --module-search-path-headers: easyconfig parameter has precedence
226+
mod_load_cpp_headers = self.cfg['module_search_path_headers'] or build_option('module_search_path_headers')
227+
228+
try:
229+
mod_load_aliases[MODULE_LOAD_ENV_HEADERS] = MOD_SEARCH_PATH_HEADERS[mod_load_cpp_headers]
230+
except KeyError as err:
231+
raise EasyBuildError(
232+
f"Unknown value selected for option module-search-path-headers: {mod_load_cpp_headers}. "
233+
f"Choose one of: {', '.join(MOD_SEARCH_PATH_HEADERS)}"
234+
) from err
235+
236+
self.module_load_environment = ModuleLoadEnvironment(aliases=mod_load_aliases)
224237

225238
# determine install subdirectory, based on module name
226239
self.install_subdir = None
@@ -1668,12 +1681,12 @@ def make_module_req(self):
16681681

16691682
if self.make_module_req_guess.__qualname__ != "EasyBlock.make_module_req_guess":
16701683
# Deprecated make_module_req_guess method used in child Easyblock
1671-
# Update environment with custom make_module_req_guess
1684+
# adjust environment with custom make_module_req_guess
16721685
self.log.deprecated(
16731686
"make_module_req_guess() is deprecated, use EasyBlock.module_load_environment instead.",
16741687
"6.0",
16751688
)
1676-
self.module_load_environment.update(self.make_module_req_guess())
1689+
self.module_load_environment.replace(self.make_module_req_guess())
16771690

16781691
# Expand and inject path-like environment variables into module file
16791692
env_var_requirements = {
@@ -1760,7 +1773,9 @@ def check_install_lib_symlink(self):
17601773

17611774
self._install_lib_symlink = LibSymlink.NEITHER
17621775
if os.path.exists(lib_dir) and os.path.exists(lib64_dir):
1763-
if os.path.islink(lib_dir) and os.path.samefile(lib_dir, lib64_dir):
1776+
if os.path.islink(lib_dir) and os.path.islink(lib64_dir):
1777+
self._install_lib_symlink = LibSymlink.BOTH_TO_DIR
1778+
elif os.path.islink(lib_dir) and os.path.samefile(lib_dir, lib64_dir):
17641779
self._install_lib_symlink = LibSymlink.LIB_TO_LIB64
17651780
elif os.path.islink(lib64_dir) and os.path.samefile(lib_dir, lib64_dir):
17661781
self._install_lib_symlink = LibSymlink.LIB64_TO_LIB
@@ -3325,8 +3340,8 @@ def sanity_check_rpath(self, rpath_dirs=None, check_readelf_rpath=True):
33253340
# For example, libcuda.so.1 should never be RPATH-ed by design,
33263341
# see https://github.com/easybuilders/easybuild-framework/issues/4095
33273342
filter_rpath_sanity_libs = build_option('filter_rpath_sanity_libs')
3328-
msg = "Ignoring the following libraries if they are not found by RPATH sanity check: {filter_rpath_sanity_libs}"
3329-
self.log.info(msg)
3343+
self.log.info("Ignoring the following libraries if they are not found by RPATH sanity check: %s",
3344+
filter_rpath_sanity_libs)
33303345

33313346
if rpath_dirs is None:
33323347
rpath_dirs = self.cfg['bin_lib_subdirs'] or self.bin_lib_subdirs()
@@ -4468,7 +4483,7 @@ def build_and_install_one(ecdict, init_env):
44684483
try:
44694484
app_class = get_easyblock_class(easyblock, name=name)
44704485
app = app_class(ecdict['ec'])
4471-
_log.info("Obtained application instance of for %s (easyblock: %s)" % (name, easyblock))
4486+
_log.info("Obtained application instance for %s (easyblock: %s)" % (name, easyblock))
44724487
except EasyBuildError as err:
44734488
print_error("Failed to get application instance for %s (easyblock: %s): %s" % (name, easyblock, err.msg),
44744489
silent=silent)

easybuild/framework/easyconfig/default.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -208,10 +208,11 @@
208208
'moduleloadnoconflict': [False, "Don't check for conflicts, unload other versions instead ", MODULES],
209209
'module_depends_on': [None, 'Use depends_on (Lmod 7.6.1+) for dependencies in generated module '
210210
'(implies recursive unloading of modules) [DEPRECATED]', MODULES],
211+
'module_search_path_headers': [None, "Environment variable set by modules on load "
212+
"with search paths to header files (if None, use $CPATH)", MODULES],
211213
'recursive_module_unload': [None, "Recursive unload of all dependencies when unloading module "
212-
"(True/False to hard enable/disable; None implies honoring "
213-
"the --recursive-module-unload EasyBuild configuration setting",
214-
MODULES],
214+
"(True/False to hard enable/disable; None implies honoring the "
215+
"--recursive-module-unload EasyBuild configuration setting", MODULES],
215216

216217
# MODULES documentation easyconfig parameters
217218
# (docurls is part of MANDATORY)

easybuild/tools/config.py

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,15 @@
182182
EBPYTHONPREFIXES = 'EBPYTHONPREFIXES'
183183
PYTHON_SEARCH_PATH_TYPES = [PYTHONPATH, EBPYTHONPREFIXES]
184184

185+
# options to handle header search paths in environment of modules
186+
MOD_SEARCH_PATH_HEADERS_CPATH = 'cpath'
187+
MOD_SEARCH_PATH_HEADERS_INCLUDE_PATHS = 'include_paths'
188+
MOD_SEARCH_PATH_HEADERS = {
189+
MOD_SEARCH_PATH_HEADERS_CPATH: ['CPATH'],
190+
MOD_SEARCH_PATH_HEADERS_INCLUDE_PATHS: ['C_INCLUDE_PATH', 'CPLUS_INCLUDE_PATH', 'OBJC_INCLUDE_PATH'],
191+
}
192+
DEFAULT_MOD_SEARCH_PATH_HEADERS = MOD_SEARCH_PATH_HEADERS_CPATH
193+
185194

186195
class Singleton(ABCMeta):
187196
"""Serves as metaclass for classes that should implement the Singleton pattern.
@@ -304,6 +313,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
304313
'ignore_locks',
305314
'ignore_test_failure',
306315
'install_latest_eb_release',
316+
'keep_debug_symbols',
307317
'logtostdout',
308318
'minimal_toolchains',
309319
'module_only',
@@ -338,7 +348,6 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
338348
'cleanup_tmpdir',
339349
'extended_dry_run_ignore_errors',
340350
'fixed_installdir_naming_scheme',
341-
'keep_debug_symbols',
342351
'lib_lib64_symlink',
343352
'lib64_fallback_sanity_check',
344353
'lib64_lib_symlink',
@@ -389,6 +398,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
389398
DEFAULT_MINIMAL_BUILD_ENV: [
390399
'minimal_build_env',
391400
],
401+
DEFAULT_MOD_SEARCH_PATH_HEADERS: [
402+
'module_search_path_headers',
403+
],
392404
DEFAULT_PKG_RELEASE: [
393405
'package_release',
394406
],

easybuild/tools/modules.py

Lines changed: 109 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@
5757
from easybuild.tools.utilities import get_subclasses, nub
5858

5959

60+
MODULE_LOAD_ENV_HEADERS = 'HEADERS'
61+
6062
# software root/version environment variable name prefixes
6163
ROOT_ENV_VAR_NAME_PREFIX = "EBROOT"
6264
VERSION_ENV_VAR_NAME_PREFIX = "EBVERSION"
@@ -241,18 +243,38 @@ def is_path(self):
241243

242244

243245
class ModuleLoadEnvironment:
244-
"""Changes to environment variables that should be made when environment module is loaded"""
246+
"""
247+
Changes to environment variables that should be made when environment module is loaded.
248+
- Environment variables are defined as ModuleEnvironmentVariables instances
249+
with attribute name equal to environment variable name.
250+
- Aliases are arbitrary names that serve to apply changes to lists of
251+
environment variables
252+
- Only environment variables attributes are public. Other attributes like
253+
aliases are private.
254+
"""
245255

246-
def __init__(self):
256+
def __init__(self, aliases=None):
247257
"""
248258
Initialize default environment definition
249259
Paths are relative to root of installation directory
260+
261+
:aliases: dict defining environment variables aliases
250262
"""
263+
self._aliases = {}
264+
if aliases is not None:
265+
try:
266+
for alias_name, alias_vars in aliases.items():
267+
self.update_alias(alias_name, alias_vars)
268+
except AttributeError as err:
269+
raise EasyBuildError(
270+
"Wrong format for aliases defitions passed to ModuleLoadEnvironment. "
271+
f"Expected a dictionary but got: {type(aliases)}."
272+
) from err
273+
251274
self.ACLOCAL_PATH = [os.path.join('share', 'aclocal')]
252275
self.CLASSPATH = ['*.jar']
253276
self.CMAKE_LIBRARY_PATH = ['lib64'] # only needed for installations with standalone lib64
254277
self.CMAKE_PREFIX_PATH = ['']
255-
self.CPATH = SEARCH_PATH_HEADER_DIRS
256278
self.GI_TYPELIB_PATH = [os.path.join(x, 'girepository-*') for x in SEARCH_PATH_LIB_DIRS]
257279
self.LD_LIBRARY_PATH = SEARCH_PATH_LIB_DIRS
258280
self.LIBRARY_PATH = SEARCH_PATH_LIB_DIRS
@@ -261,11 +283,29 @@ def __init__(self):
261283
self.PKG_CONFIG_PATH = [os.path.join(x, 'pkgconfig') for x in SEARCH_PATH_LIB_DIRS + ['share']]
262284
self.XDG_DATA_DIRS = ['share']
263285

286+
# environment variables with known aliases
287+
# e.g. search paths to C/C++ headers
288+
for envar_name in self._aliases.get(MODULE_LOAD_ENV_HEADERS, []):
289+
setattr(self, envar_name, SEARCH_PATH_HEADER_DIRS)
290+
264291
def __setattr__(self, name, value):
265292
"""
266293
Specific restrictions for ModuleLoadEnvironment attributes:
294+
- public attributes are instances of ModuleEnvironmentVariable with uppercase names
295+
- private attributes are allowed with any name
296+
"""
297+
if name.startswith('_'):
298+
# do not control protected/private attributes
299+
return super().__setattr__(name, value)
300+
301+
return self.__set_module_environment_variable(name, value)
302+
303+
def __set_module_environment_variable(self, name, value):
304+
"""
305+
Specific restrictions for ModuleEnvironmentVariable attributes:
267306
- attribute names are uppercase
268-
- attributes are instances of ModuleEnvironmentVariable
307+
- dictionaries are unpacked into arguments of ModuleEnvironmentVariable
308+
- controls variables with special types (e.g. PATH, LD_LIBRARY_PATH)
269309
"""
270310
if name != name.upper():
271311
raise EasyBuildError(f"Names of ModuleLoadEnvironment attributes must be uppercase, got '{name}'")
@@ -284,17 +324,24 @@ def __setattr__(self, name, value):
284324

285325
return super().__setattr__(name, ModuleEnvironmentVariable(contents, **kwargs))
286326

327+
@property
328+
def vars(self):
329+
"""Return list of public ModuleEnvironmentVariable"""
330+
331+
return [envar for envar in self.__dict__ if not str(envar).startswith('_')]
332+
287333
def __iter__(self):
288334
"""Make the class iterable"""
289-
yield from self.__dict__
335+
yield from self.vars
290336

291337
def items(self):
292338
"""
293339
Return key-value pairs for each attribute that is a ModuleEnvironmentVariable
294340
- key = attribute name
295341
- value = its "contents" attribute
296342
"""
297-
return self.__dict__.items()
343+
for attr in self.vars:
344+
yield attr, getattr(self, attr)
298345

299346
def update(self, new_env):
300347
"""Update contents of environment from given dictionary"""
@@ -304,6 +351,20 @@ def update(self, new_env):
304351
except AttributeError as err:
305352
raise EasyBuildError("Cannot update ModuleLoadEnvironment from a non-dict variable") from err
306353

354+
def replace(self, new_env):
355+
"""Replace contents of environment with given dictionary"""
356+
for var in self.vars:
357+
self.remove(var)
358+
self.update(new_env)
359+
360+
def remove(self, var_name):
361+
"""
362+
Remove ModuleEnvironmentVariable attribute from instance
363+
Silently goes through if attribute is already missing
364+
"""
365+
if var_name in self.vars:
366+
delattr(self, var_name)
367+
307368
@property
308369
def as_dict(self):
309370
"""
@@ -319,6 +380,48 @@ def environ(self):
319380
"""
320381
return {envar_name: str(envar_contents) for envar_name, envar_contents in self.items()}
321382

383+
def alias(self, alias):
384+
"""
385+
Return iterator to search path variables for given alias
386+
"""
387+
try:
388+
yield from [getattr(self, envar) for envar in self._aliases[alias]]
389+
except KeyError as err:
390+
raise EasyBuildError(f"Unknown search path alias: {alias}") from err
391+
except AttributeError as err:
392+
raise EasyBuildError(f"Missing environment variable in '{alias} alias") from err
393+
394+
def alias_vars(self, alias):
395+
"""
396+
Return list of environment variable names aliased by given alias
397+
"""
398+
try:
399+
return self._aliases[alias]
400+
except KeyError as err:
401+
raise EasyBuildError(f"Unknown search path alias: {alias}") from err
402+
403+
def update_alias(self, alias, value):
404+
"""
405+
Update existing or non-existing alias with given search paths variables
406+
"""
407+
if isinstance(value, str):
408+
value = [value]
409+
410+
try:
411+
self._aliases[alias] = [str(envar) for envar in value]
412+
except TypeError as err:
413+
raise TypeError("ModuleLoadEnvironment aliases must be a list of strings") from err
414+
415+
def set_alias_vars(self, alias, value):
416+
"""
417+
Set value of search paths variables for given alias
418+
"""
419+
try:
420+
for envar_name in self._aliases[alias]:
421+
setattr(self, envar_name, value)
422+
except KeyError as err:
423+
raise EasyBuildError(f"Unknown search path alias: {alias}") from err
424+
322425

323426
class ModulesTool(object):
324427
"""An abstract interface to a tool that deals with modules."""
@@ -1013,7 +1116,6 @@ def run_module(self, *args, **kwargs):
10131116
# stdout will contain python code (to change environment etc)
10141117
# stderr will contain text (just like the normal module command)
10151118
stdout, stderr = res.output, res.stderr
1016-
self.log.debug("Output of module command '%s': stdout: %s; stderr: %s", cmd, stdout, stderr)
10171119

10181120
# also catch and check exit code
10191121
if kwargs.get('check_exit_code', True) and res.exit_code != EasyBuildExit.SUCCESS:

easybuild/tools/options.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@
6969
from easybuild.tools.config import DEFAULT_JOB_EB_CMD, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS
7070
from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL
7171
from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL
72+
from easybuild.tools.config import DEFAULT_MOD_SEARCH_PATH_HEADERS, MOD_SEARCH_PATH_HEADERS
7273
from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_EXTRA_SOURCE_URLS
7374
from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT
7475
from easybuild.tools.config import DEFAULT_PR_TARGET_ACCOUNT, DEFAULT_FILTER_RPATH_SANITY_LIBS
@@ -465,7 +466,7 @@ def override_options(self):
465466
'insecure-download': ("Don't check the server certificate against the available certificate authorities.",
466467
None, 'store_true', False),
467468
'install-latest-eb-release': ("Install latest known version of easybuild", None, 'store_true', False),
468-
'keep-debug-symbols': ("Sets default value of debug toolchain option", None, 'store_true', True),
469+
'keep-debug-symbols': ("Sets default value of debug toolchain option", None, 'store_true', False),
469470
'lib-lib64-symlink': ("Automatically create symlinks for lib/ pointing to lib64/ if the former is missing",
470471
None, 'store_true', True),
471472
'lib64-fallback-sanity-check': ("Fallback in sanity check to lib64/ equivalent for missing libraries",
@@ -615,6 +616,9 @@ def config_options(self):
615616
'module-extensions': ("Include 'extensions' statement in generated module file (Lua syntax only)",
616617
None, 'store_true', True),
617618
'module-naming-scheme': ("Module naming scheme to use", None, 'store', DEFAULT_MNS),
619+
'module-search-path-headers': ("Environment variable set by modules on load with search paths "
620+
"to header files", 'choice', 'store', DEFAULT_MOD_SEARCH_PATH_HEADERS,
621+
sorted(MOD_SEARCH_PATH_HEADERS.keys())),
618622
'module-syntax': ("Syntax to be used for module files", 'choice', 'store', DEFAULT_MODULE_SYNTAX,
619623
sorted(avail_module_generators().keys())),
620624
'moduleclasses': (("Extend supported module classes "

0 commit comments

Comments
 (0)