Skip to content

Commit 74a32f8

Browse files
Merge pull request #3640 from easybuilders/4.3.x
release EasyBuild v4.3.4
2 parents f4a9479 + cdb4bf4 commit 74a32f8

39 files changed

+1283
-240
lines changed

.github/workflows/eb_command.yml

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
2+
name: Tests for the 'eb' command
3+
on: [push, pull_request]
4+
jobs:
5+
test-eb:
6+
runs-on: ubuntu-18.04
7+
strategy:
8+
matrix:
9+
python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9]
10+
fail-fast: false
11+
steps:
12+
- uses: actions/checkout@v2
13+
14+
- name: set up Python
15+
uses: actions/setup-python@v2
16+
with:
17+
python-version: ${{matrix.python}}
18+
architecture: x64
19+
20+
- name: install OS & Python packages
21+
run: |
22+
# check Python version
23+
python -V
24+
# update to latest pip, check version
25+
pip install --upgrade pip
26+
pip --version
27+
# install packages required for modules tool
28+
sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
29+
# fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
30+
# needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
31+
if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
32+
sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
33+
fi
34+
35+
- name: install modules tool
36+
run: |
37+
# avoid downloading modules tool sources into easybuild-framework dir
38+
cd $HOME
39+
export INSTALL_DEP=$GITHUB_WORKSPACE/easybuild/scripts/install_eb_dep.sh
40+
# install Lmod
41+
source $INSTALL_DEP Lmod-8.4.26 $HOME
42+
# changes in environment are not passed to other steps, so need to create files...
43+
echo $MOD_INIT > mod_init
44+
echo $PATH > path
45+
if [ ! -z $MODULESHOME ]; then echo $MODULESHOME > moduleshome; fi
46+
47+
- name: install EasyBuild framework
48+
run: |
49+
# install from source distribution tarball, to test release as published on PyPI
50+
python setup.py sdist
51+
ls dist
52+
export PREFIX=/tmp/$USER/$GITHUB_SHA
53+
pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
54+
55+
- name: run tests for 'eb' command
56+
env:
57+
EB_VERBOSE: 1
58+
run: |
59+
# run tests *outside* of checked out easybuild-framework directory,
60+
# to ensure we're testing installed version (see previous step)
61+
cd $HOME
62+
# initialize environment for modules tool
63+
if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi
64+
source $(cat $HOME/mod_init); type module
65+
# make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that);
66+
# also pick up changes to $PATH set by sourcing $MOD_INIT
67+
export PREFIX=/tmp/$USER/$GITHUB_SHA
68+
export PATH=$PREFIX/bin:$(cat $HOME/path)
69+
export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH
70+
# run --version, capture (verbose) output
71+
eb --version | tee eb_version.out 2>&1
72+
# determine active Python version
73+
pymajver=$(python -c 'import sys; print(sys.version_info[0])')
74+
pymajminver=$(python -c 'import sys; print(".".join(str(x) for x in sys.version_info[:2]))')
75+
# check patterns in verbose output
76+
for pattern in "^>> Considering .python.\.\.\." "^>> .python. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> 'python' is able to import 'easybuild.main', so retaining it" "^>> Selected Python command: python \(.*/bin/python\)" "^This is EasyBuild 4\.[0-9.]\+"; do
77+
echo "Looking for pattern \"${pattern}\" in eb_version.out..."
78+
grep "$pattern" eb_version.out
79+
done
80+
# also check when specifying Python command via $EB_PYTHON
81+
for eb_python in "python${pymajver}" "python${pymajminver}"; do
82+
export EB_PYTHON="${eb_python}"
83+
eb --version | tee eb_version.out 2>&1
84+
for pattern in "^>> Considering .${eb_python}.\.\.\." "^>> .${eb_python}. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> '${eb_python}' is able to import 'easybuild.main', so retaining it" "^>> Selected Python command: ${eb_python} \(.*/bin/${eb_python}\)" "^This is EasyBuild 4\.[0-9.]\+"; do
85+
echo "Looking for pattern \"${pattern}\" in eb_version.out..."
86+
grep "$pattern" eb_version.out
87+
done
88+
done

.github/workflows/unit_tests.yml

Lines changed: 52 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -2,61 +2,70 @@
22
name: EasyBuild framework unit tests
33
on: [push, pull_request]
44
jobs:
5+
setup:
6+
runs-on: ubuntu-latest
7+
outputs:
8+
lmod7: Lmod-7.8.22
9+
lmod8: Lmod-8.4.27
10+
modulesTcl: modules-tcl-1.147
11+
modules3: modules-3.2.10
12+
modules4: modules-4.1.4
13+
steps:
14+
- run: "true"
515
build:
16+
needs: setup
617
runs-on: ubuntu-18.04
718
strategy:
819
matrix:
9-
python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9]
10-
modules_tool: [Lmod-7.8.22, Lmod-8.2.9, modules-tcl-1.147, modules-3.2.10, modules-4.1.4]
20+
python: [2.7, 3.6]
21+
modules_tool:
22+
# use variables defined by 'setup' job above, see also
23+
# https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context
24+
- ${{needs.setup.outputs.lmod7}}
25+
- ${{needs.setup.outputs.lmod8}}
26+
- ${{needs.setup.outputs.modulesTcl}}
27+
- ${{needs.setup.outputs.modules3}}
28+
- ${{needs.setup.outputs.modules4}}
1129
module_syntax: [Lua, Tcl]
1230
lc_all: [""]
13-
# exclude some configuration for non-Lmod modules tool:
14-
# - don't test with Lua module syntax (only supported in Lmod)
15-
# - exclude Python 3.x versions other than 3.6, to limit test configurations
31+
# don't test with Lua module syntax (only supported in Lmod)
1632
exclude:
17-
- modules_tool: modules-tcl-1.147
33+
- modules_tool: ${{needs.setup.outputs.modulesTcl}}
1834
module_syntax: Lua
19-
- modules_tool: modules-3.2.10
35+
- modules_tool: ${{needs.setup.outputs.modules3}}
2036
module_syntax: Lua
21-
- modules_tool: modules-4.1.4
37+
- modules_tool: ${{needs.setup.outputs.modules4}}
2238
module_syntax: Lua
23-
- modules_tool: modules-tcl-1.147
24-
python: 3.5
25-
- modules_tool: modules-tcl-1.147
26-
python: 3.7
27-
- modules_tool: modules-tcl-1.147
28-
python: 3.8
29-
- modules_tool: modules-tcl-1.147
30-
python: 3.9
31-
- modules_tool: modules-3.2.10
32-
python: 3.5
33-
- modules_tool: modules-3.2.10
34-
python: 3.7
35-
- modules_tool: modules-3.2.10
36-
python: 3.8
37-
- modules_tool: modules-3.2.10
38-
python: 3.9
39-
- modules_tool: modules-4.1.4
40-
python: 3.5
41-
- modules_tool: modules-4.1.4
42-
python: 3.7
43-
- modules_tool: modules-4.1.4
44-
python: 3.8
45-
- modules_tool: modules-4.1.4
46-
python: 3.9
47-
- modules_tool: Lmod-7.8.22
48-
python: 3.5
49-
- modules_tool: Lmod-7.8.22
50-
python: 3.7
51-
- modules_tool: Lmod-7.8.22
52-
python: 3.8
53-
- modules_tool: Lmod-7.8.22
54-
python: 3.9
55-
# There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set
56-
# Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7)
5739
include:
40+
# Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax)
41+
- python: 3.5
42+
modules_tool: ${{needs.setup.outputs.lmod8}}
43+
module_syntax: Lua
44+
- python: 3.5
45+
modules_tool: ${{needs.setup.outputs.lmod8}}
46+
module_syntax: Tcl
47+
- python: 3.7
48+
modules_tool: ${{needs.setup.outputs.lmod8}}
49+
module_syntax: Lua
50+
- python: 3.7
51+
modules_tool: ${{needs.setup.outputs.lmod8}}
52+
module_syntax: Tcl
53+
- python: 3.8
54+
modules_tool: ${{needs.setup.outputs.lmod8}}
55+
module_syntax: Lua
56+
- python: 3.8
57+
modules_tool: ${{needs.setup.outputs.lmod8}}
58+
module_syntax: Tcl
59+
- python: 3.9
60+
modules_tool: ${{needs.setup.outputs.lmod8}}
61+
module_syntax: Lua
62+
- python: 3.9
63+
modules_tool: ${{needs.setup.outputs.lmod8}}
64+
module_syntax: Tcl
65+
# There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set
66+
# Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7)
5867
- python: 3.6
59-
modules_tool: Lmod-8.2.9
68+
modules_tool: ${{needs.setup.outputs.lmod8}}
6069
module_syntax: Lua
6170
lc_all: C
6271
fail-fast: false

RELEASE_NOTES

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,40 @@ For more detailed information, please see the git log.
44
These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
55

66

7+
v4.3.4 (April 9th 2021)
8+
-----------------------
9+
10+
update/bugfix release
11+
12+
- various enhancements, including:
13+
- add support for filtering dependencies by using False as version (#3506)
14+
- add create_unused_dir function to create a directory which does not yet exist (#3551)
15+
- avoid running expensive 'module use' and 'module unuse' commands when using Lmod as modules tool, update $MODULEPATH directly instead (#3557, #3633)
16+
- create CUDA cache (for JIT compiled PTX code) in build dir instead of $HOME (#3569)
17+
- add "Citing" section to module files (#3596)
18+
- add support for using fallback 'arch=*' key in dependency version specified as arch->version mapping (#3600)
19+
- also check for pending change requests and mergeable_state in check_pr_eligible_to_merge (#3604)
20+
- ignore undismissed 'changes requested' review if there is an 'approved' review by the same user (#3607, #3608)
21+
- sort output of 'eb --search' in natural order (respecting numbers) (#3609)
22+
- enhance 'eb' command to ensure that easybuild.main can be imported before settling on python* command to use (#3610)
23+
- add --env-for-shebang configuration option to define the env command to use for shebangs (#3613)
24+
- add templates for architecture independent Python wheels (#3618)
25+
- mention easyblocks PR in gist when uploading test report for it + fix clean_gists.py script (#3622)
26+
- also accept regular expression value for --accept-eula-for (#3630)
27+
- update validate_github_token function to accept GitHub token in new format (#3632)
28+
- various bug fixes, including:
29+
- fix $BLAS_LIB_MT for OpenBLAS, ensure -lpthread is included (#3584)
30+
- use '--opt=val' for passing settings from config file to option parser to avoid error for values starting with '-' or '--' (#3594)
31+
- avoid raised exception when getting output from interactive command in run_cmd_qa (#3599)
32+
- add option to write file from file-like object and use in download_file (#3614)
33+
- make sure that path to eb is always found by tests (#3617)
34+
- other changes:
35+
- add pick_default_branch function to clean up duplicate code in tools/github.py (#3592)
36+
- refactor the CI configuration to use inclusion instead of exclusion (#3616)
37+
- use develop branch when testing push access in --check-github (#3629)
38+
- deprecate --accept-eula, rename to --accept-eula-for (#3630)
39+
40+
741
v4.3.3 (February 23rd 2021)
842
---------------------------
943

@@ -17,7 +51,7 @@ update/bugfix release
1751
- detect 'SYSTEM' toolchain as special case in easystack files (#3543)
1852
- enhance extract_cmd function to use 'cp -a' for shell scripts (.sh) (#3545)
1953
- allow use of alternate envvar(s) to $HOME for user modules (#3558)
20-
- use https://sources/easybuild.io as fallback source URL (#3572, #3576)
54+
- use https://sources.easybuild.io as fallback source URL (#3572, #3576)
2155
- add toolchain definition for iibff toolchain (#3574)
2256
- add %(cuda_cc_space_sep)s and %(cuda_cc_semicolon_sep)s templates (#3578)
2357
- add support for intel-compiler toolchain (>= 2021.x versions, oneAPI) (#3581, #3582)
@@ -34,6 +68,7 @@ update/bugfix release
3468
- other changes:
3569
- rename EasyBlock._skip_step to EasyBlock.skip_step, to make it part of the public API (#3561)
3670
- make symlinking of posix_c.so to posix.so in test suite configuration conditional (#3570)
71+
- use 'main' rather than 'master' branch in GitHub integration functionality (#3589)
3772

3873

3974
v4.3.2 (December 10th 2020)

easybuild/base/generaloption.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1376,8 +1376,7 @@ def parseconfigfiles(self):
13761376
configfile_values[opt_dest] = newval
13771377
else:
13781378
configfile_cmdline_dest.append(opt_dest)
1379-
configfile_cmdline.append("--%s" % opt_name)
1380-
configfile_cmdline.append(val)
1379+
configfile_cmdline.append("--%s=%s" % (opt_name, val))
13811380

13821381
# reparse
13831382
self.log.debug('parseconfigfiles: going to parse options through cmdline %s' % configfile_cmdline)

easybuild/framework/easyblock.py

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1034,6 +1034,27 @@ def make_dir(self, dir_name, clean, dontcreateinstalldir=False):
10341034

10351035
mkdir(dir_name, parents=True)
10361036

1037+
def set_up_cuda_cache(self):
1038+
"""Set up CUDA PTX cache."""
1039+
1040+
cuda_cache_maxsize = build_option('cuda_cache_maxsize')
1041+
if cuda_cache_maxsize is None:
1042+
cuda_cache_maxsize = 1 * 1024 # 1 GiB default value
1043+
else:
1044+
cuda_cache_maxsize = int(cuda_cache_maxsize)
1045+
1046+
if cuda_cache_maxsize == 0:
1047+
self.log.info("Disabling CUDA PTX cache since cache size was set to zero")
1048+
env.setvar('CUDA_CACHE_DISABLE', '1')
1049+
else:
1050+
cuda_cache_dir = build_option('cuda_cache_dir')
1051+
if not cuda_cache_dir:
1052+
cuda_cache_dir = os.path.join(self.builddir, 'eb-cuda-cache')
1053+
self.log.info("Enabling CUDA PTX cache of size %s MiB at %s", cuda_cache_maxsize, cuda_cache_dir)
1054+
env.setvar('CUDA_CACHE_DISABLE', '0')
1055+
env.setvar('CUDA_CACHE_PATH', cuda_cache_dir)
1056+
env.setvar('CUDA_CACHE_MAXSIZE', str(cuda_cache_maxsize * 1024 * 1024))
1057+
10371058
#
10381059
# MODULE UTILITY FUNCTIONS
10391060
#
@@ -1655,8 +1676,8 @@ def check_accepted_eula(self, name=None, more_info=None):
16551676
if name is None:
16561677
name = self.name
16571678

1658-
accepted_eulas = build_option('accept_eula') or []
1659-
if self.cfg['accept_eula'] or name in accepted_eulas:
1679+
accepted_eulas = build_option('accept_eula_for') or []
1680+
if self.cfg['accept_eula'] or name in accepted_eulas or any(re.match(x, name) for x in accepted_eulas):
16601681
self.log.info("EULA for %s is accepted", name)
16611682
else:
16621683
error_lines = [
@@ -1667,7 +1688,7 @@ def check_accepted_eula(self, name=None, more_info=None):
16671688

16681689
error_lines.extend([
16691690
"You should either:",
1670-
"- add --accept-eula=%(name)s to the 'eb' command;",
1691+
"- add --accept-eula-for=%(name)s to the 'eb' command;",
16711692
"- update your EasyBuild configuration to always accept the EULA for %(name)s;",
16721693
"- add 'accept_eula = True' to the easyconfig file you are using;",
16731694
'',
@@ -2163,6 +2184,10 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True):
21632184
self.log.info("Loading extra modules: %s", extra_modules)
21642185
self.modules_tool.load(extra_modules)
21652186

2187+
# Setup CUDA cache if required. If we don't do this, CUDA will use the $HOME for its cache files
2188+
if get_software_root('CUDA') or get_software_root('CUDAcore'):
2189+
self.set_up_cuda_cache()
2190+
21662191
# guess directory to start configure/build/install process in, and move there
21672192
if start_dir:
21682193
self.guess_start_dir()
@@ -2367,7 +2392,7 @@ def fix_shebang(self):
23672392
if isinstance(fix_shebang_for, string_type):
23682393
fix_shebang_for = [fix_shebang_for]
23692394

2370-
shebang = '#!/usr/bin/env %s' % lang
2395+
shebang = '#!%s %s' % (build_option('env_for_shebang'), lang)
23712396
for glob_pattern in fix_shebang_for:
23722397
paths = glob.glob(os.path.join(self.installdir, glob_pattern))
23732398
self.log.info("Fixing '%s' shebang to '%s' for files that match '%s': %s",

easybuild/framework/easyconfig/default.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,7 @@
194194

195195
# MODULES documentation easyconfig parameters
196196
# (docurls is part of MANDATORY)
197+
'citing': [None, "Free-form text that describes how the software should be cited in publications", MODULES],
197198
'docpaths': [None, "List of paths for documentation relative to installation directory", MODULES],
198199
'examples': [None, "Free-form text with examples on using the software", MODULES],
199200
'site_contacts': [None, "String/list of strings with site contacts for the software", MODULES],

easybuild/framework/easyconfig/easyconfig.py

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -702,10 +702,13 @@ def parse(self):
702702
# parse dependency specifications
703703
# it's important that templating is still disabled at this stage!
704704
self.log.info("Parsing dependency specifications...")
705-
self['dependencies'] = [self._parse_dependency(dep) for dep in self['dependencies']]
706-
self['hiddendependencies'] = [
707-
self._parse_dependency(dep, hidden=True) for dep in self['hiddendependencies']
708-
]
705+
706+
def remove_false_versions(deps):
707+
return [dep for dep in deps if not (isinstance(dep, dict) and dep['version'] is False)]
708+
709+
self['dependencies'] = remove_false_versions(self._parse_dependency(dep) for dep in self['dependencies'])
710+
self['hiddendependencies'] = remove_false_versions(self._parse_dependency(dep, hidden=True) for dep in
711+
self['hiddendependencies'])
709712

710713
# need to take into account that builddependencies may need to be iterated over,
711714
# i.e. when the value is a list of lists of tuples
@@ -715,7 +718,7 @@ def parse(self):
715718
builddeps = [[self._parse_dependency(dep, build_only=True) for dep in x] for x in builddeps]
716719
else:
717720
builddeps = [self._parse_dependency(dep, build_only=True) for dep in builddeps]
718-
self['builddependencies'] = builddeps
721+
self['builddependencies'] = remove_false_versions(builddeps)
719722

720723
# keep track of parsed multi deps, they'll come in handy during sanity check & module steps...
721724
self.multi_deps = self.get_parsed_multi_deps()

easybuild/framework/easyconfig/templates.py

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,19 @@
155155
('SOURCE_%s' % suffix, '%(name)s-%(version)s.' + ext, "Source .%s bundle" % ext),
156156
('SOURCELOWER_%s' % suffix, '%(namelower)s-%(version)s.' + ext, "Source .%s bundle with lowercase name" % ext),
157157
]
158+
for pyver in ('py2.py3', 'py2', 'py3'):
159+
if pyver == 'py2.py3':
160+
desc = 'Python 2 & Python 3'
161+
name_infix = ''
162+
else:
163+
desc = 'Python ' + pyver[-1]
164+
name_infix = pyver.upper() + '_'
165+
TEMPLATE_CONSTANTS += [
166+
('SOURCE_%sWHL' % name_infix, '%%(name)s-%%(version)s-%s-none-any.whl' % pyver,
167+
'Generic (non-compiled) %s wheel package' % desc),
168+
('SOURCELOWER_%sWHL' % name_infix, '%%(namelower)s-%%(version)s-%s-none-any.whl' % pyver,
169+
'Generic (non-compiled) %s wheel package with lowercase name' % desc),
170+
]
158171

159172
# TODO derived config templates
160173
# versionmajor, versionminor, versionmajorminor (eg '.'.join(version.split('.')[:2])) )

0 commit comments

Comments
 (0)