Skip to content

Commit 8ef6d6e

Browse files
authored
Merge pull request #4655 from lexming/cpath-mod-opt
add `module-search-path-headers` configuration option to control how modules set search paths to header files
2 parents 3f1b6aa + bebf495 commit 8ef6d6e

File tree

7 files changed

+290
-14
lines changed

7 files changed

+290
-14
lines changed

easybuild/framework/easyblock.py

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@
7878
from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES
7979
from easybuild.tools.config import EASYBUILD_SOURCES_URL, EBPYTHONPREFIXES # noqa
8080
from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES
81-
from easybuild.tools.config import PYTHONPATH, SEARCH_PATH_BIN_DIRS, SEARCH_PATH_LIB_DIRS
81+
from easybuild.tools.config import MOD_SEARCH_PATH_HEADERS, PYTHONPATH, SEARCH_PATH_BIN_DIRS, SEARCH_PATH_LIB_DIRS
8282
from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath
8383
from easybuild.tools.config import install_path, log_path, package_path, source_paths
8484
from easybuild.tools.environment import restore_env, sanitize_env
@@ -100,7 +100,7 @@
100100
from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for
101101
from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
102102
from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX
103-
from easybuild.tools.modules import Lmod, ModEnvVarType, ModuleLoadEnvironment
103+
from easybuild.tools.modules import Lmod, ModEnvVarType, ModuleLoadEnvironment, MODULE_LOAD_ENV_HEADERS
104104
from easybuild.tools.modules import curr_module_paths, invalidate_module_caches_for, get_software_root
105105
from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name
106106
from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ALL, PROGRESS_BAR_EASYCONFIG, PROGRESS_BAR_EXTENSIONS
@@ -221,7 +221,19 @@ def __init__(self, ec, logfile=None):
221221
self.modules_header = read_file(modules_header_path)
222222

223223
# environment variables on module load
224-
self.module_load_environment = ModuleLoadEnvironment()
224+
mod_load_aliases = {}
225+
# apply --module-search-path-headers: easyconfig parameter has precedence
226+
mod_load_cpp_headers = self.cfg['module_search_path_headers'] or build_option('module_search_path_headers')
227+
228+
try:
229+
mod_load_aliases[MODULE_LOAD_ENV_HEADERS] = MOD_SEARCH_PATH_HEADERS[mod_load_cpp_headers]
230+
except KeyError as err:
231+
raise EasyBuildError(
232+
f"Unknown value selected for option module-search-path-headers: {mod_load_cpp_headers}. "
233+
f"Choose one of: {', '.join(MOD_SEARCH_PATH_HEADERS)}"
234+
) from err
235+
236+
self.module_load_environment = ModuleLoadEnvironment(aliases=mod_load_aliases)
225237

226238
# determine install subdirectory, based on module name
227239
self.install_subdir = None

easybuild/framework/easyconfig/default.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -208,10 +208,11 @@
208208
'moduleloadnoconflict': [False, "Don't check for conflicts, unload other versions instead ", MODULES],
209209
'module_depends_on': [None, 'Use depends_on (Lmod 7.6.1+) for dependencies in generated module '
210210
'(implies recursive unloading of modules) [DEPRECATED]', MODULES],
211+
'module_search_path_headers': [None, "Environment variable set by modules on load "
212+
"with search paths to header files (if None, use $CPATH)", MODULES],
211213
'recursive_module_unload': [None, "Recursive unload of all dependencies when unloading module "
212-
"(True/False to hard enable/disable; None implies honoring "
213-
"the --recursive-module-unload EasyBuild configuration setting",
214-
MODULES],
214+
"(True/False to hard enable/disable; None implies honoring the "
215+
"--recursive-module-unload EasyBuild configuration setting", MODULES],
215216

216217
# MODULES documentation easyconfig parameters
217218
# (docurls is part of MANDATORY)

easybuild/tools/config.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,15 @@
182182
EBPYTHONPREFIXES = 'EBPYTHONPREFIXES'
183183
PYTHON_SEARCH_PATH_TYPES = [PYTHONPATH, EBPYTHONPREFIXES]
184184

185+
# options to handle header search paths in environment of modules
186+
MOD_SEARCH_PATH_HEADERS_CPATH = 'cpath'
187+
MOD_SEARCH_PATH_HEADERS_INCLUDE_PATHS = 'include_paths'
188+
MOD_SEARCH_PATH_HEADERS = {
189+
MOD_SEARCH_PATH_HEADERS_CPATH: ['CPATH'],
190+
MOD_SEARCH_PATH_HEADERS_INCLUDE_PATHS: ['C_INCLUDE_PATH', 'CPLUS_INCLUDE_PATH', 'OBJC_INCLUDE_PATH'],
191+
}
192+
DEFAULT_MOD_SEARCH_PATH_HEADERS = MOD_SEARCH_PATH_HEADERS_CPATH
193+
185194

186195
class Singleton(ABCMeta):
187196
"""Serves as metaclass for classes that should implement the Singleton pattern.
@@ -389,6 +398,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
389398
DEFAULT_MINIMAL_BUILD_ENV: [
390399
'minimal_build_env',
391400
],
401+
DEFAULT_MOD_SEARCH_PATH_HEADERS: [
402+
'module_search_path_headers',
403+
],
392404
DEFAULT_PKG_RELEASE: [
393405
'package_release',
394406
],

easybuild/tools/modules.py

Lines changed: 103 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@
5757
from easybuild.tools.utilities import get_subclasses, nub
5858

5959

60+
MODULE_LOAD_ENV_HEADERS = 'HEADERS'
61+
6062
# software root/version environment variable name prefixes
6163
ROOT_ENV_VAR_NAME_PREFIX = "EBROOT"
6264
VERSION_ENV_VAR_NAME_PREFIX = "EBVERSION"
@@ -241,18 +243,38 @@ def is_path(self):
241243

242244

243245
class ModuleLoadEnvironment:
244-
"""Changes to environment variables that should be made when environment module is loaded"""
246+
"""
247+
Changes to environment variables that should be made when environment module is loaded.
248+
- Environment variables are defined as ModuleEnvironmentVariables instances
249+
with attribute name equal to environment variable name.
250+
- Aliases are arbitrary names that serve to apply changes to lists of
251+
environment variables
252+
- Only environment variables attributes are public. Other attributes like
253+
aliases are private.
254+
"""
245255

246-
def __init__(self):
256+
def __init__(self, aliases=None):
247257
"""
248258
Initialize default environment definition
249259
Paths are relative to root of installation directory
260+
261+
:aliases: dict defining environment variables aliases
250262
"""
263+
self._aliases = {}
264+
if aliases is not None:
265+
try:
266+
for alias_name, alias_vars in aliases.items():
267+
self.update_alias(alias_name, alias_vars)
268+
except AttributeError as err:
269+
raise EasyBuildError(
270+
"Wrong format for aliases defitions passed to ModuleLoadEnvironment. "
271+
f"Expected a dictionary but got: {type(aliases)}."
272+
) from err
273+
251274
self.ACLOCAL_PATH = [os.path.join('share', 'aclocal')]
252275
self.CLASSPATH = ['*.jar']
253276
self.CMAKE_LIBRARY_PATH = ['lib64'] # only needed for installations with standalone lib64
254277
self.CMAKE_PREFIX_PATH = ['']
255-
self.CPATH = SEARCH_PATH_HEADER_DIRS
256278
self.GI_TYPELIB_PATH = [os.path.join(x, 'girepository-*') for x in SEARCH_PATH_LIB_DIRS]
257279
self.LD_LIBRARY_PATH = SEARCH_PATH_LIB_DIRS
258280
self.LIBRARY_PATH = SEARCH_PATH_LIB_DIRS
@@ -261,11 +283,29 @@ def __init__(self):
261283
self.PKG_CONFIG_PATH = [os.path.join(x, 'pkgconfig') for x in SEARCH_PATH_LIB_DIRS + ['share']]
262284
self.XDG_DATA_DIRS = ['share']
263285

286+
# environment variables with known aliases
287+
# e.g. search paths to C/C++ headers
288+
for envar_name in self._aliases.get(MODULE_LOAD_ENV_HEADERS, []):
289+
setattr(self, envar_name, SEARCH_PATH_HEADER_DIRS)
290+
264291
def __setattr__(self, name, value):
265292
"""
266293
Specific restrictions for ModuleLoadEnvironment attributes:
294+
- public attributes are instances of ModuleEnvironmentVariable with uppercase names
295+
- private attributes are allowed with any name
296+
"""
297+
if name.startswith('_'):
298+
# do not control protected/private attributes
299+
return super().__setattr__(name, value)
300+
301+
return self.__set_module_environment_variable(name, value)
302+
303+
def __set_module_environment_variable(self, name, value):
304+
"""
305+
Specific restrictions for ModuleEnvironmentVariable attributes:
267306
- attribute names are uppercase
268-
- attributes are instances of ModuleEnvironmentVariable
307+
- dictionaries are unpacked into arguments of ModuleEnvironmentVariable
308+
- controls variables with special types (e.g. PATH, LD_LIBRARY_PATH)
269309
"""
270310
if name != name.upper():
271311
raise EasyBuildError(f"Names of ModuleLoadEnvironment attributes must be uppercase, got '{name}'")
@@ -284,17 +324,24 @@ def __setattr__(self, name, value):
284324

285325
return super().__setattr__(name, ModuleEnvironmentVariable(contents, **kwargs))
286326

327+
@property
328+
def vars(self):
329+
"""Return list of public ModuleEnvironmentVariable"""
330+
331+
return [envar for envar in self.__dict__ if not str(envar).startswith('_')]
332+
287333
def __iter__(self):
288334
"""Make the class iterable"""
289-
yield from self.__dict__
335+
yield from self.vars
290336

291337
def items(self):
292338
"""
293339
Return key-value pairs for each attribute that is a ModuleEnvironmentVariable
294340
- key = attribute name
295341
- value = its "contents" attribute
296342
"""
297-
return self.__dict__.items()
343+
for attr in self.vars:
344+
yield attr, getattr(self, attr)
298345

299346
def update(self, new_env):
300347
"""Update contents of environment from given dictionary"""
@@ -304,6 +351,14 @@ def update(self, new_env):
304351
except AttributeError as err:
305352
raise EasyBuildError("Cannot update ModuleLoadEnvironment from a non-dict variable") from err
306353

354+
def remove(self, var_name):
355+
"""
356+
Remove ModuleEnvironmentVariable attribute from instance
357+
Silently goes through if attribute is already missing
358+
"""
359+
if var_name in self.vars:
360+
delattr(self, var_name)
361+
307362
@property
308363
def as_dict(self):
309364
"""
@@ -319,6 +374,48 @@ def environ(self):
319374
"""
320375
return {envar_name: str(envar_contents) for envar_name, envar_contents in self.items()}
321376

377+
def alias(self, alias):
378+
"""
379+
Return iterator to search path variables for given alias
380+
"""
381+
try:
382+
yield from [getattr(self, envar) for envar in self._aliases[alias]]
383+
except KeyError as err:
384+
raise EasyBuildError(f"Unknown search path alias: {alias}") from err
385+
except AttributeError as err:
386+
raise EasyBuildError(f"Missing environment variable in '{alias} alias") from err
387+
388+
def alias_vars(self, alias):
389+
"""
390+
Return list of environment variable names aliased by given alias
391+
"""
392+
try:
393+
return self._aliases[alias]
394+
except KeyError as err:
395+
raise EasyBuildError(f"Unknown search path alias: {alias}") from err
396+
397+
def update_alias(self, alias, value):
398+
"""
399+
Update existing or non-existing alias with given search paths variables
400+
"""
401+
if isinstance(value, str):
402+
value = [value]
403+
404+
try:
405+
self._aliases[alias] = [str(envar) for envar in value]
406+
except TypeError as err:
407+
raise TypeError("ModuleLoadEnvironment aliases must be a list of strings") from err
408+
409+
def set_alias_vars(self, alias, value):
410+
"""
411+
Set value of search paths variables for given alias
412+
"""
413+
try:
414+
for envar_name in self._aliases[alias]:
415+
setattr(self, envar_name, value)
416+
except KeyError as err:
417+
raise EasyBuildError(f"Unknown search path alias: {alias}") from err
418+
322419

323420
class ModulesTool(object):
324421
"""An abstract interface to a tool that deals with modules."""

easybuild/tools/options.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@
6969
from easybuild.tools.config import DEFAULT_JOB_EB_CMD, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS
7070
from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL
7171
from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL
72+
from easybuild.tools.config import DEFAULT_MOD_SEARCH_PATH_HEADERS, MOD_SEARCH_PATH_HEADERS
7273
from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_EXTRA_SOURCE_URLS
7374
from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT
7475
from easybuild.tools.config import DEFAULT_PR_TARGET_ACCOUNT, DEFAULT_FILTER_RPATH_SANITY_LIBS
@@ -615,6 +616,9 @@ def config_options(self):
615616
'module-extensions': ("Include 'extensions' statement in generated module file (Lua syntax only)",
616617
None, 'store_true', True),
617618
'module-naming-scheme': ("Module naming scheme to use", None, 'store', DEFAULT_MNS),
619+
'module-search-path-headers': ("Environment variable set by modules on load with search paths "
620+
"to header files", 'choice', 'store', DEFAULT_MOD_SEARCH_PATH_HEADERS,
621+
sorted(MOD_SEARCH_PATH_HEADERS.keys())),
618622
'module-syntax': ("Syntax to be used for module files", 'choice', 'store', DEFAULT_MODULE_SYNTAX,
619623
sorted(avail_module_generators().keys())),
620624
'moduleclasses': (("Extend supported module classes "

test/framework/easyblock.py

Lines changed: 94 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -532,7 +532,7 @@ def test_make_module_req(self):
532532
for env_var in default_mod_load_vars:
533533
delattr(eb.module_load_environment, env_var)
534534

535-
self.assertEqual(len(vars(eb.module_load_environment)), 0)
535+
self.assertEqual(len(eb.module_load_environment.vars), 0)
536536

537537
# check for behavior when a string value is used as value of module_load_environment
538538
eb.module_load_environment.PATH = 'bin'
@@ -668,6 +668,99 @@ def test_make_module_req(self):
668668
eb.close_log()
669669
os.remove(eb.logfile)
670670

671+
def test_module_search_path_headers(self):
672+
"""Test functionality of module-search-path-headers option"""
673+
sp_headers_mode = {
674+
"cpath": ["CPATH"],
675+
"include_paths": ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"],
676+
}
677+
678+
self.contents = '\n'.join([
679+
'easyblock = "ConfigureMake"',
680+
'name = "pi"',
681+
'version = "3.14"',
682+
'homepage = "http://example.com"',
683+
'description = "test easyconfig"',
684+
'toolchain = SYSTEM',
685+
])
686+
self.writeEC()
687+
688+
for build_opt, sp_headers in sp_headers_mode.items():
689+
update_build_option('module_search_path_headers', build_opt)
690+
eb = EasyBlock(EasyConfig(self.eb_file))
691+
eb.installdir = config.install_path()
692+
try:
693+
os.makedirs(os.path.join(eb.installdir, 'include'))
694+
write_file(os.path.join(eb.installdir, 'include', 'header.h'), 'dummy header file')
695+
except FileExistsError:
696+
pass
697+
698+
with eb.module_generator.start_module_creation():
699+
guess = eb.make_module_req()
700+
701+
if not sp_headers:
702+
# none option adds nothing to module file
703+
if get_module_syntax() == 'Tcl':
704+
tcl_ref_pattern = r"^prepend-path\s+CPATH\s+\$root/include$"
705+
self.assertFalse(re.search(tcl_ref_pattern, guess, re.M))
706+
elif get_module_syntax() == 'Lua':
707+
lua_ref_pattern = r'^prepend_path\("CPATH", pathJoin\(root, "include"\)\)$'
708+
self.assertFalse(re.search(lua_ref_pattern, guess, re.M))
709+
else:
710+
for env_var in sp_headers:
711+
if get_module_syntax() == 'Tcl':
712+
tcl_ref_pattern = rf"^prepend-path\s+{env_var}\s+\$root/include$"
713+
self.assertTrue(re.search(tcl_ref_pattern, guess, re.M))
714+
elif get_module_syntax() == 'Lua':
715+
lua_ref_pattern = rf'^prepend_path\("{env_var}", pathJoin\(root, "include"\)\)$'
716+
self.assertTrue(re.search(lua_ref_pattern, guess, re.M))
717+
718+
# test with easyconfig parameter
719+
for ec_param, sp_headers in sp_headers_mode.items():
720+
self.contents += f'\nmodule_search_path_headers = "{ec_param}"'
721+
self.writeEC()
722+
eb = EasyBlock(EasyConfig(self.eb_file))
723+
eb.installdir = config.install_path()
724+
try:
725+
os.makedirs(os.path.join(eb.installdir, 'include'))
726+
write_file(os.path.join(eb.installdir, 'include', 'header.h'), 'dummy header file')
727+
except FileExistsError:
728+
pass
729+
730+
for build_opt in sp_headers_mode:
731+
update_build_option('module_search_path_headers', build_opt)
732+
with eb.module_generator.start_module_creation():
733+
guess = eb.make_module_req()
734+
if not sp_headers:
735+
# none option adds nothing to module file
736+
if get_module_syntax() == 'Tcl':
737+
tcl_ref_pattern = r"^prepend-path\s+CPATH\s+\$root/include$"
738+
self.assertFalse(re.search(tcl_ref_pattern, guess, re.M))
739+
elif get_module_syntax() == 'Lua':
740+
lua_ref_pattern = r'^prepend_path\("CPATH", pathJoin\(root, "include"\)\)$'
741+
self.assertFalse(re.search(lua_ref_pattern, guess, re.M))
742+
else:
743+
for env_var in sp_headers:
744+
if get_module_syntax() == 'Tcl':
745+
tcl_ref_pattern = rf"^prepend-path\s+{env_var}\s+\$root/include$"
746+
self.assertTrue(re.search(tcl_ref_pattern, guess, re.M))
747+
elif get_module_syntax() == 'Lua':
748+
lua_ref_pattern = rf'^prepend_path\("{env_var}", pathJoin\(root, "include"\)\)$'
749+
self.assertTrue(re.search(lua_ref_pattern, guess, re.M))
750+
751+
# test wrong easyconfig parameter
752+
self.contents += '\nmodule_search_path_headers = "WRONG_OPT"'
753+
self.writeEC()
754+
ec = EasyConfig(self.eb_file)
755+
756+
error_pattern = "Unknown value selected for option module-search-path-headers"
757+
with eb.module_generator.start_module_creation():
758+
self.assertErrorRegex(EasyBuildError, error_pattern, EasyBlock, ec)
759+
760+
# cleanup
761+
eb.close_log()
762+
os.remove(eb.logfile)
763+
671764
def test_make_module_extra(self):
672765
"""Test for make_module_extra."""
673766
init_config(build_options={'silent': True})

0 commit comments

Comments
 (0)