Skip to content

Commit fe1439e

Browse files
Merge pull request #3114 from easybuilders/4.1.x
release EasyBuild v4.1.0
2 parents 4b89db2 + 2e5d9f0 commit fe1439e

40 files changed

+2456
-408
lines changed

.github/workflows/unit_tests.yml

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,12 @@ jobs:
66
runs-on: ubuntu-18.04
77
strategy:
88
matrix:
9-
python: [2.7, 3.5, 3.6, 3.7]
10-
modules_tool: [Lmod-6.6.3, Lmod-7.8.22, Lmod-8.1.14, modules-tcl-1.147, modules-3.2.10, modules-4.1.4]
9+
python: [2.7, 3.5, 3.6, 3.7, 3.8]
10+
modules_tool: [Lmod-7.8.22, Lmod-8.2.3, modules-tcl-1.147, modules-3.2.10, modules-4.1.4]
1111
module_syntax: [Lua, Tcl]
1212
# exclude some configuration for non-Lmod modules tool:
1313
# - don't test with Lua module syntax (only supported in Lmod)
14-
# - don't test with Python 3.5 and 3.7 (only with 2.7 and 3.6), to limit test configurations
14+
# - exclude Python 3.x versions other than 3.6, to limit test configurations
1515
exclude:
1616
- modules_tool: modules-tcl-1.147
1717
module_syntax: Lua
@@ -23,14 +23,20 @@ jobs:
2323
python: 3.5
2424
- modules_tool: modules-tcl-1.147
2525
python: 3.7
26+
- modules_tool: modules-tcl-1.147
27+
python: 3.8
2628
- modules_tool: modules-3.2.10
2729
python: 3.5
2830
- modules_tool: modules-3.2.10
2931
python: 3.7
32+
- modules_tool: modules-3.2.10
33+
python: 3.8
3034
- modules_tool: modules-4.1.4
3135
python: 3.5
3236
- modules_tool: modules-4.1.4
3337
python: 3.7
38+
- modules_tool: modules-4.1.4
39+
python: 3.8
3440
fail-fast: false
3541
steps:
3642
- uses: actions/checkout@v1

.travis.yml

Lines changed: 8 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,10 @@
11
language: python
2-
python: 2.6
3-
dist: trusty
2+
python: 2.7
3+
dist: xenial
44
env:
55
matrix:
6-
# purposely specifying slowest builds first, to gain time overall
7-
- LMOD_VERSION=6.5.1
8-
- LMOD_VERSION=6.5.1 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
96
- LMOD_VERSION=7.8.22
107
- LMOD_VERSION=7.8.22 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
11-
- LMOD_VERSION=8.0.9
12-
- LMOD_VERSION=8.0.9 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
138
- ENV_MOD_VERSION=3.2.10 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesC TEST_EASYBUILD_MODULE_SYNTAX=Tcl
149
- ENV_MOD_TCL_VERSION=1.147 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl TEST_EASYBUILD_MODULE_SYNTAX=Tcl
1510
- ENV_MOD_VERSION=4.1.4 TEST_EASYBUILD_MODULE_SYNTAX=Tcl TEST_EASYBUILD_MODULES_TOOL=EnvironmentModules # Tmod 4.1.4 is used in RHEL8
@@ -19,44 +14,15 @@ matrix:
1914
# mark build as finished as soon as job has failed
2015
fast_finish: true
2116
include:
22-
# also test default configuration with Python 2.7
17+
# also test default configuration with Python 2.6
18+
- python: 2.6
19+
env: LMOD_VERSION=7.8.22 TEST_EASYBUILD_SILENCE_DEPRECATION_WARNINGS=Python26
20+
dist: trusty
21+
# single configuration to test with Lmod 6 and Python 2.7
2322
- python: 2.7
24-
env: LMOD_VERSION=6.6.3
25-
dist: xenial
23+
env: LMOD_VERSION=6.5.1 TEST_EASYBUILD_SILENCE_DEPRECATION_WARNINGS=Lmod6
2624
# also test with Python 3.6
2725
- python: 3.6
28-
env: LMOD_VERSION=6.6.3
29-
dist: xenial
30-
- python: 3.6
31-
env: LMOD_VERSION=6.5.1 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
32-
dist: xenial
33-
- python: 3.6
34-
env: LMOD_VERSION=7.8.22
35-
dist: xenial
36-
- python: 3.6
37-
env: LMOD_VERSION=7.8.22 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
38-
dist: xenial
39-
- python: 3.6
40-
env: LMOD_VERSION=8.0.9
41-
dist: xenial
42-
- python: 3.6
43-
env: LMOD_VERSION=8.0.9 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
44-
dist: xenial
45-
- python: 3.6
46-
env: ENV_MOD_VERSION=3.2.10 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesC TEST_EASYBUILD_MODULE_SYNTAX=Tcl
47-
dist: xenial
48-
- python: 3.6
49-
env: ENV_MOD_TCL_VERSION=1.147 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl TEST_EASYBUILD_MODULE_SYNTAX=Tcl
50-
dist: xenial
51-
- python: 3.6
52-
env: ENV_MOD_VERSION=4.1.4 TEST_EASYBUILD_MODULE_SYNTAX=Tcl TEST_EASYBUILD_MODULES_TOOL=EnvironmentModules # Tmod 4.1.4 is used in RHEL8
53-
dist: xenial
54-
# also test most common configuration with Python 3.5 and 3.7
55-
- python: 3.5
56-
env: LMOD_VERSION=7.8.22
57-
dist: xenial
58-
- python: 3.7
59-
dist: xenial
6026
env: LMOD_VERSION=7.8.22
6127
addons:
6228
apt:

RELEASE_NOTES

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,49 @@ For more detailed information, please see the git log.
33

44
These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
55

6+
v4.1.0 (December 4th 2019)
7+
--------------------------
8+
9+
update/bugfix release
10+
11+
- various enhancements, including:
12+
- performance improvements:
13+
- skip validation when copying EasyConfig object for extension (#3071)
14+
- correctly specify that ActiveMNS & co are singleton classes when using Python 3 (#3073)
15+
- don't call out to prohibitively expensive getRootLoggerName in getLogger, just use 'root' instead (#3074)
16+
- fix inconsistent module path usage that leads to repeated reloading in HMNS (#3099)
17+
- add support for specifying different dependency version based on processor architecture (#3047)
18+
- support use of glob patterns for paths to files with external modules metadata (#3075)
19+
- take into account that external modules may not be visible directly (due to module hierarchy) (#3083)
20+
- add support for including 'extensions' statement in Lua modules with Lmod 8.2.8+ (#3085, #3107, #3110)
21+
- add support for --sync-pr-with-develop (#3087)
22+
- add support for --new-branch-github, --new-pr-from-branch, --sync-branch-with-develop, --update-branch-github (#3103)
23+
- fix typo in docstring for new_branch_github (#3106)
24+
- various bug fixes, including:
25+
- correctly handle iccifortcuda toolchain with standalone iccifort in det_toolchain_compilers (#3055)
26+
- init git repo with git.repo.clone() instead of copy_dir() (#3062)
27+
- fix regular expression so depends-on statements are recognized correctly in Tcl module files (#3065)
28+
- update GitPython to latest version that supports Python 2.6 in requirements.txt to fix broken test_new_update_pr (#3066)
29+
- imply --disable-pre-create-installdir with --inject-checksums (#3069)
30+
- handle patches in extensions more like normal patches (#3067)
31+
- take into account that platform.linux_distribution and platform.dist was removed in Python 3.8 (#3078)
32+
- always include mandatory easyconfig parameters in dumped easyconfig (#3081)
33+
- hide backup module file when using Lmod 6.x (fixes #9302) (#3089)
34+
- add better error message when mandatory key is missing from a dictionary easyconfig parameter (#3092)
35+
- also create symlinks for default modules in class module folders (#3093)
36+
- fix semantics of --set-default-module: only set default for specified easyconfigs, not for the ones that are installed as dependencies via --robot (#3094)
37+
- fix various issues in extracting comments from original easyconfig file and including them again in dumped easyconfig (#3095)
38+
- don't use %(version)s template in exts_default_options in dumped easyconfig (#3096)
39+
- fix generated module statements in case only a single version is listed in multi_deps (#3097)
40+
- fix broken test_show_system_info on macOS due to 'Python' binary (#3105)
41+
- take into account that dependency version could be a dict rather than a string value in template_constant_dict (#3111)
42+
- other changes:
43+
- deprecate running EasyBuild with Python 2.6 via new check_python_version() function (#3076)
44+
- deprecate support for using Lmod 6.x (#3077)
45+
- trim set of test configurations in Travis CI (#3086)
46+
- flesh out env_vars_external_module from Toolchain._simulated_load_dependency_module (#3088)
47+
48+
649
v4.0.1 (October 15th 2019)
750
--------------------------
851

easybuild/base/fancylogger.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -452,7 +452,9 @@ def getLogger(name=None, fname=False, clsname=False, fancyrecord=None):
452452
nameparts = []
453453

454454
if not is_fancyroot():
455-
nameparts.append(getRootLoggerName())
455+
# deliberately not calling getRootLoggerName function to determine actual root logger name,
456+
# because it is prohibitively expensive in some texts (even when using 'python -O')
457+
nameparts.append('root')
456458

457459
if fancyrecord is None:
458460
# Altough we could set it as default value in the function definition

easybuild/framework/easyblock.py

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@
8686
from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for
8787
from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
8888
from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX
89-
from easybuild.tools.modules import curr_module_paths, invalidate_module_caches_for, get_software_root
89+
from easybuild.tools.modules import Lmod, curr_module_paths, invalidate_module_caches_for, get_software_root
9090
from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name
9191
from easybuild.tools.package.utilities import package
9292
from easybuild.tools.py2vs3 import extract_method_name, string_type
@@ -175,6 +175,7 @@ def __init__(self, ec):
175175
self.module_generator = module_generator(self, fake=True)
176176
self.mod_filepath = self.module_generator.get_module_filepath()
177177
self.mod_file_backup = None
178+
self.set_default_module = self.cfg.set_default_module
178179

179180
# modules footer/header
180181
self.modules_footer = None
@@ -453,8 +454,8 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None):
453454
raise EasyBuildError('No file found for patch %s', patch_spec)
454455

455456
if extension:
456-
self.log.info("Fetched extension patches: %s" % patches)
457-
return [patch['path'] for patch in patches]
457+
self.log.info("Fetched extension patches: %s", patches)
458+
return patches
458459
else:
459460
self.log.info("Added patches: %s" % self.patches)
460461

@@ -544,6 +545,7 @@ def fetch_extension_sources(self, skip_checksums=False):
544545

545546
if not skip_checksums:
546547
for patch in ext_patches:
548+
patch = patch['path']
547549
# report both MD5 and SHA256 checksums,
548550
# since both are valid default checksum types
549551
for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256):
@@ -553,6 +555,7 @@ def fetch_extension_sources(self, skip_checksums=False):
553555
# verify checksum (if provided)
554556
self.log.debug('Verifying checksums for extension patches...')
555557
for idx, patch in enumerate(ext_patches):
558+
patch = patch['path']
556559
checksum = self.get_checksum_for(checksums[1:], filename=patch, index=idx)
557560
if verify_checksum(patch, checksum):
558561
self.log.info('Checksum for extension patch %s verified', patch)
@@ -1672,6 +1675,11 @@ def check_readiness_step(self):
16721675
# which is better than hiding them (since --show-hidden still reveals them)
16731676
hidden = isinstance(self.module_generator, ModuleGeneratorTcl)
16741677

1678+
# with old Lmod versions, the backup module should also be hidden when using Lua syntax;
1679+
# see https://github.com/easybuilders/easybuild-easyconfigs/issues/9302
1680+
if isinstance(self.module_generator, ModuleGeneratorLua) and isinstance(self.modules_tool, Lmod):
1681+
hidden = LooseVersion(self.modules_tool.version) < LooseVersion('7.0.0')
1682+
16751683
self.mod_file_backup = back_up_file(self.mod_filepath, hidden=hidden, strip_fn=strip_fn)
16761684
print_msg("backup of existing module file stored at %s" % self.mod_file_backup, log=self.log)
16771685

@@ -2596,8 +2604,12 @@ def _set_module_as_default(self, fake=False):
25962604
if self.dry_run:
25972605
dry_run_msg("Marked %s v%s as default version" % (self.name, version))
25982606
else:
2599-
mod_folderpath = os.path.dirname(self.module_generator.get_module_filepath(fake=fake))
2600-
self.module_generator.set_as_default(mod_folderpath, version)
2607+
mod_dir_path = os.path.dirname(self.module_generator.get_module_filepath(fake=fake))
2608+
if fake:
2609+
mod_symlink_paths = []
2610+
else:
2611+
mod_symlink_paths = ActiveMNS().det_module_symlink_paths(self.cfg)
2612+
self.module_generator.set_as_default(mod_dir_path, version, mod_symlink_paths=mod_symlink_paths)
26012613

26022614
def cleanup_step(self):
26032615
"""
@@ -2708,7 +2720,7 @@ def make_module_step(self, fake=False):
27082720

27092721
# always set default for temporary module file,
27102722
# to avoid that it gets overruled by an existing module file that is set as default
2711-
if fake or build_option('set_default_module'):
2723+
if fake or self.set_default_module:
27122724
self._set_module_as_default(fake=fake)
27132725

27142726
return modpath
@@ -3484,8 +3496,8 @@ def make_checksum_lines(checksums, indent_level):
34843496
print_msg(" * %s: %s" % (src_fn, checksum), log=_log)
34853497
ext_checksums.append((src_fn, checksum))
34863498
for ext_patch in ext.get('patches', []):
3487-
patch_fn = os.path.basename(ext_patch)
3488-
checksum = compute_checksum(ext_patch, checksum_type)
3499+
patch_fn = os.path.basename(ext_patch['path'])
3500+
checksum = compute_checksum(ext_patch['path'], checksum_type)
34893501
print_msg(" * %s: %s" % (patch_fn, checksum), log=_log)
34903502
ext_checksums.append((patch_fn, checksum))
34913503

easybuild/framework/easyconfig/easyconfig.py

Lines changed: 24 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,8 @@
6767
from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version
6868
from easybuild.tools.module_naming_scheme.utilities import det_hidden_modname, is_valid_module_name
6969
from easybuild.tools.modules import modules_tool
70-
from easybuild.tools.py2vs3 import OrderedDict, string_type
71-
from easybuild.tools.systemtools import check_os_dependency
70+
from easybuild.tools.py2vs3 import OrderedDict, create_base_metaclass, string_type
71+
from easybuild.tools.systemtools import check_os_dependency, pick_dep_version
7272
from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME, is_system_toolchain
7373
from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES, TOOLCHAIN_CAPABILITY_CUDA
7474
from easybuild.tools.toolchain.utilities import get_toolchain, search_toolchain
@@ -410,7 +410,7 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi
410410
if rawtxt is None:
411411
self.path = path
412412
self.rawtxt = read_file(path)
413-
self.log.debug("Raw contents from supplied easyconfig file %s: %s" % (path, self.rawtxt))
413+
self.log.debug("Raw contents from supplied easyconfig file %s: %s", path, self.rawtxt)
414414
else:
415415
self.rawtxt = rawtxt
416416
self.log.debug("Supplied raw easyconfig contents: %s" % self.rawtxt)
@@ -497,6 +497,8 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi
497497
self.short_mod_name = mns.det_short_module_name(self)
498498
self.mod_subdir = mns.det_module_subdir(self)
499499

500+
self.set_default_module = False
501+
500502
self.software_license = None
501503

502504
def filename(self):
@@ -530,12 +532,15 @@ def extend_params(self, extra, overwrite=True):
530532
self.mandatory.append(key)
531533
self.log.debug("Updated list of mandatory easyconfig parameters: %s", self.mandatory)
532534

533-
def copy(self):
535+
def copy(self, validate=None):
534536
"""
535537
Return a copy of this EasyConfig instance.
536538
"""
539+
if validate is None:
540+
validate = self.validation
541+
537542
# create a new EasyConfig instance
538-
ec = EasyConfig(self.path, validate=self.validation, hidden=self.hidden, rawtxt=self.rawtxt)
543+
ec = EasyConfig(self.path, validate=validate, hidden=self.hidden, rawtxt=self.rawtxt)
539544
# take a copy of the actual config dictionary (which already contains the extra options)
540545
ec._config = copy.deepcopy(self._config)
541546
# since rawtxt is defined, self.path may not get inherited, make sure it does
@@ -601,7 +606,7 @@ def parse(self):
601606
type(self.build_specs))
602607
self.log.debug("Obtained specs dict %s" % arg_specs)
603608

604-
self.log.info("Parsing easyconfig file %s with rawcontent: %s" % (self.path, self.rawtxt))
609+
self.log.info("Parsing easyconfig file %s with rawcontent: %s", self.path, self.rawtxt)
605610
self.parser.set_specifications(arg_specs)
606611
local_vars = self.parser.get_config_dict()
607612
self.log.debug("Parsed easyconfig as a dictionary: %s" % local_vars)
@@ -1269,6 +1274,7 @@ def _parse_dependency(self, dep, hidden=False, build_only=False):
12691274
# provides information on what this module represents (software name/version, install prefix, ...)
12701275
'external_module_metadata': {},
12711276
}
1277+
12721278
if isinstance(dep, dict):
12731279
dependency.update(dep)
12741280

@@ -1305,6 +1311,9 @@ def _parse_dependency(self, dep, hidden=False, build_only=False):
13051311
else:
13061312
raise EasyBuildError("Dependency %s of unsupported type: %s", dep, type(dep))
13071313

1314+
# Find the version to use on this system
1315+
dependency['version'] = pick_dep_version(dependency['version'])
1316+
13081317
if dependency['external_module']:
13091318
# check whether the external module is hidden
13101319
if dependency['full_mod_name'].split('/')[-1].startswith('.'):
@@ -1482,6 +1491,10 @@ def __getitem__(self, key):
14821491

14831492
return value
14841493

1494+
def is_mandatory_param(self, key):
1495+
"""Check whether specified easyconfig parameter is mandatory."""
1496+
return key in self.mandatory
1497+
14851498
def get_ref(self, key):
14861499
"""
14871500
Obtain reference to original/untemplated value of specified easyconfig parameter
@@ -2209,10 +2222,12 @@ def fix_deprecated_easyconfigs(paths):
22092222
print_msg("\nAll done! Fixed %d easyconfigs (out of %d found).\n", fixed_cnt, cnt, prefix=False)
22102223

22112224

2212-
class ActiveMNS(object):
2213-
"""Wrapper class for active module naming scheme."""
2225+
# singleton metaclass: only one instance is created
2226+
BaseActiveMNS = create_base_metaclass('BaseActiveMNS', Singleton, object)
22142227

2215-
__metaclass__ = Singleton
2228+
2229+
class ActiveMNS(BaseActiveMNS):
2230+
"""Wrapper class for active module naming scheme."""
22162231

22172232
def __init__(self, *args, **kwargs):
22182233
"""Initialize logger."""

easybuild/framework/easyconfig/format/format.py

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,8 @@
5252
DEPENDENCY_PARAMETERS = ['builddependencies', 'dependencies', 'hiddendependencies']
5353

5454
# values for these keys will not be templated in dump()
55-
EXCLUDED_KEYS_REPLACE_TEMPLATES = ['description', 'easyblock', 'exts_list', 'homepage', 'name', 'toolchain',
56-
'version', 'multi_deps'] + DEPENDENCY_PARAMETERS
55+
EXCLUDED_KEYS_REPLACE_TEMPLATES = ['description', 'easyblock', 'exts_default_options', 'exts_list',
56+
'homepage', 'multi_deps', 'name', 'toolchain', 'version'] + DEPENDENCY_PARAMETERS
5757

5858
# ordered groups of keys to obtain a nice looking easyconfig file
5959
GROUPED_PARAMS = [
@@ -69,7 +69,10 @@
6969
['preinstallopts', 'installopts'],
7070
['parallel', 'maxparallel'],
7171
]
72-
LAST_PARAMS = ['sanity_check_paths', 'moduleclass']
72+
LAST_PARAMS = ['exts_default_options', 'exts_list',
73+
'sanity_check_paths', 'sanity_check_commands',
74+
'modextrapaths', 'modextravars',
75+
'moduleclass']
7376

7477
SANITY_CHECK_PATHS_DIRS = 'dirs'
7578
SANITY_CHECK_PATHS_FILES = 'files'

0 commit comments

Comments
 (0)