Skip to content

Commit ed0d6ae

Browse files
boegelboegel
committed
Merge branch 'develop' into create_unused_dir
2 parents 2a15533 + 7c3c13f commit ed0d6ae

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+2080
-395
lines changed

‎.github/workflows/eb_command.yml‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
2+
name: Tests for the 'eb' command
3+
on: [push, pull_request]
4+
jobs:
5+
test-eb:
6+
runs-on: ubuntu-18.04
7+
strategy:
8+
matrix:
9+
python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9]
10+
fail-fast: false
11+
steps:
12+
- uses: actions/checkout@v2
13+
14+
- name: set up Python
15+
uses: actions/setup-python@v2
16+
with:
17+
python-version: ${{matrix.python}}
18+
architecture: x64
19+
20+
- name: install OS & Python packages
21+
run: |
22+
# check Python version
23+
python -V
24+
# update to latest pip, check version
25+
pip install --upgrade pip
26+
pip --version
27+
# install packages required for modules tool
28+
sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
29+
# fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
30+
# needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
31+
if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
32+
sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
33+
fi
34+
35+
- name: install modules tool
36+
run: |
37+
# avoid downloading modules tool sources into easybuild-framework dir
38+
cd $HOME
39+
export INSTALL_DEP=$GITHUB_WORKSPACE/easybuild/scripts/install_eb_dep.sh
40+
# install Lmod
41+
source $INSTALL_DEP Lmod-8.4.26 $HOME
42+
# changes in environment are not passed to other steps, so need to create files...
43+
echo $MOD_INIT > mod_init
44+
echo $PATH > path
45+
if [ ! -z $MODULESHOME ]; then echo $MODULESHOME > moduleshome; fi
46+
47+
- name: install EasyBuild framework
48+
run: |
49+
# install from source distribution tarball, to test release as published on PyPI
50+
python setup.py sdist
51+
ls dist
52+
export PREFIX=/tmp/$USER/$GITHUB_SHA
53+
pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
54+
55+
- name: run tests for 'eb' command
56+
env:
57+
EB_VERBOSE: 1
58+
run: |
59+
# run tests *outside* of checked out easybuild-framework directory,
60+
# to ensure we're testing installed version (see previous step)
61+
cd $HOME
62+
# initialize environment for modules tool
63+
if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi
64+
source $(cat $HOME/mod_init); type module
65+
# make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that);
66+
# also pick up changes to $PATH set by sourcing $MOD_INIT
67+
export PREFIX=/tmp/$USER/$GITHUB_SHA
68+
export PATH=$PREFIX/bin:$(cat $HOME/path)
69+
export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH
70+
# run --version, capture (verbose) output
71+
eb --version | tee eb_version.out 2>&1
72+
# determine active Python version
73+
pymajver=$(python -c 'import sys; print(sys.version_info[0])')
74+
pymajminver=$(python -c 'import sys; print(".".join(str(x) for x in sys.version_info[:2]))')
75+
# check patterns in verbose output
76+
for pattern in "^>> Considering .python.\.\.\." "^>> .python. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> 'python' is able to import 'easybuild.main', so retaining it" "^>> Selected Python command: python \(.*/bin/python\)" "^This is EasyBuild 4\.[0-9.]\+"; do
77+
echo "Looking for pattern \"${pattern}\" in eb_version.out..."
78+
grep "$pattern" eb_version.out
79+
done
80+
# also check when specifying Python command via $EB_PYTHON
81+
for eb_python in "python${pymajver}" "python${pymajminver}"; do
82+
export EB_PYTHON="${eb_python}"
83+
eb --version | tee eb_version.out 2>&1
84+
for pattern in "^>> Considering .${eb_python}.\.\.\." "^>> .${eb_python}. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> '${eb_python}' is able to import 'easybuild.main', so retaining it" "^>> Selected Python command: ${eb_python} \(.*/bin/${eb_python}\)" "^This is EasyBuild 4\.[0-9.]\+"; do
85+
echo "Looking for pattern \"${pattern}\" in eb_version.out..."
86+
grep "$pattern" eb_version.out
87+
done
88+
done

‎.github/workflows/unit_tests.yml‎

Lines changed: 74 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -2,55 +2,72 @@
22
name: EasyBuild framework unit tests
33
on: [push, pull_request]
44
jobs:
5+
setup:
6+
runs-on: ubuntu-latest
7+
outputs:
8+
lmod7: Lmod-7.8.22
9+
lmod8: Lmod-8.4.27
10+
modulesTcl: modules-tcl-1.147
11+
modules3: modules-3.2.10
12+
modules4: modules-4.1.4
13+
steps:
14+
- run: "true"
515
build:
16+
needs: setup
617
runs-on: ubuntu-18.04
718
strategy:
819
matrix:
9-
python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9]
10-
modules_tool: [Lmod-7.8.22, Lmod-8.2.9, modules-tcl-1.147, modules-3.2.10, modules-4.1.4]
20+
python: [2.7, 3.6]
21+
modules_tool:
22+
# use variables defined by 'setup' job above, see also
23+
# https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context
24+
- ${{needs.setup.outputs.lmod7}}
25+
- ${{needs.setup.outputs.lmod8}}
26+
- ${{needs.setup.outputs.modulesTcl}}
27+
- ${{needs.setup.outputs.modules3}}
28+
- ${{needs.setup.outputs.modules4}}
1129
module_syntax: [Lua, Tcl]
12-
# exclude some configuration for non-Lmod modules tool:
13-
# - don't test with Lua module syntax (only supported in Lmod)
14-
# - exclude Python 3.x versions other than 3.6, to limit test configurations
30+
lc_all: [""]
31+
# don't test with Lua module syntax (only supported in Lmod)
1532
exclude:
16-
- modules_tool: modules-tcl-1.147
33+
- modules_tool: ${{needs.setup.outputs.modulesTcl}}
34+
module_syntax: Lua
35+
- modules_tool: ${{needs.setup.outputs.modules3}}
36+
module_syntax: Lua
37+
- modules_tool: ${{needs.setup.outputs.modules4}}
38+
module_syntax: Lua
39+
include:
40+
# Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax)
41+
- python: 3.5
42+
modules_tool: ${{needs.setup.outputs.lmod8}}
1743
module_syntax: Lua
18-
- modules_tool: modules-3.2.10
44+
- python: 3.5
45+
modules_tool: ${{needs.setup.outputs.lmod8}}
46+
module_syntax: Tcl
47+
- python: 3.7
48+
modules_tool: ${{needs.setup.outputs.lmod8}}
1949
module_syntax: Lua
20-
- modules_tool: modules-4.1.4
50+
- python: 3.7
51+
modules_tool: ${{needs.setup.outputs.lmod8}}
52+
module_syntax: Tcl
53+
- python: 3.8
54+
modules_tool: ${{needs.setup.outputs.lmod8}}
2155
module_syntax: Lua
22-
- modules_tool: modules-tcl-1.147
23-
python: 3.5
24-
- modules_tool: modules-tcl-1.147
25-
python: 3.7
26-
- modules_tool: modules-tcl-1.147
27-
python: 3.8
28-
- modules_tool: modules-tcl-1.147
29-
python: 3.9
30-
- modules_tool: modules-3.2.10
31-
python: 3.5
32-
- modules_tool: modules-3.2.10
33-
python: 3.7
34-
- modules_tool: modules-3.2.10
35-
python: 3.8
36-
- modules_tool: modules-3.2.10
37-
python: 3.9
38-
- modules_tool: modules-4.1.4
39-
python: 3.5
40-
- modules_tool: modules-4.1.4
41-
python: 3.7
42-
- modules_tool: modules-4.1.4
43-
python: 3.8
44-
- modules_tool: modules-4.1.4
45-
python: 3.9
46-
- modules_tool: Lmod-7.8.22
47-
python: 3.5
48-
- modules_tool: Lmod-7.8.22
49-
python: 3.7
50-
- modules_tool: Lmod-7.8.22
51-
python: 3.8
52-
- modules_tool: Lmod-7.8.22
53-
python: 3.9
56+
- python: 3.8
57+
modules_tool: ${{needs.setup.outputs.lmod8}}
58+
module_syntax: Tcl
59+
- python: 3.9
60+
modules_tool: ${{needs.setup.outputs.lmod8}}
61+
module_syntax: Lua
62+
- python: 3.9
63+
modules_tool: ${{needs.setup.outputs.lmod8}}
64+
module_syntax: Tcl
65+
# There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set
66+
# Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7)
67+
- python: 3.6
68+
modules_tool: ${{needs.setup.outputs.lmod8}}
69+
module_syntax: Lua
70+
lc_all: C
5471
fail-fast: false
5572
steps:
5673
- uses: actions/checkout@v2
@@ -69,7 +86,10 @@ jobs:
6986
# for modules tool
7087
sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
7188
# fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
72-
sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
89+
# needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
90+
if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
91+
sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
92+
fi
7393
# for GitPython, python-hglib
7494
sudo apt-get install git mercurial
7595
# dep for GC3Pie
@@ -89,12 +109,20 @@ jobs:
89109
# see https://github.com/<username>/easybuild-framework/settings/secrets
90110
GITHUB_TOKEN: ${{secrets.TEST_GITHUB_TOKEN}}
91111
run: |
92-
if [ ! -z $GITHUB_TOKEN ]; then
93-
if [ "x${{matrix.python}}" == 'x2.6' ];
112+
# don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools,
113+
# to avoid hitting GitHub rate limit;
114+
# tests that require a GitHub token are skipped automatically when no GitHub token is available
115+
if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]]; then
116+
if [ ! -z $GITHUB_TOKEN ]; then
117+
if [ "x${{matrix.python}}" == 'x2.6' ];
94118
then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())";
95119
else SET_KEYRING="import keyrings; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
96-
fi;
97-
python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
120+
fi;
121+
python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
122+
fi
123+
echo "GitHub token installed!"
124+
else
125+
echo "Installation of GitHub token skipped!"
98126
fi
99127
100128
- name: install modules tool
@@ -129,6 +157,7 @@ jobs:
129157
EB_VERBOSE: 1
130158
EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}}
131159
TEST_EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}}
160+
LC_ALL: ${{matrix.lc_all}}
132161
run: |
133162
# run tests *outside* of checked out easybuild-framework directory,
134163
# to ensure we're testing installed version (see previous step)

‎RELEASE_NOTES‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,39 @@ For more detailed information, please see the git log.
33

44
These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
55

6+
7+
v4.3.3 (February 23rd 2021)
8+
---------------------------
9+
10+
update/bugfix release
11+
12+
- various enhancements, including:
13+
- advise PR labels in --review-pr and add support for --add-pr-labels (#3177)
14+
- add support for using customized HTTP headers in download_file (#3472, #3583)
15+
- also take toolchain dependencies into account when defining template values (#3541, #3560)
16+
- add support for --accept-eula configuration option + 'accept_eula' easyconfig parameter (#3535, #3536, #3546)
17+
- detect 'SYSTEM' toolchain as special case in easystack files (#3543)
18+
- enhance extract_cmd function to use 'cp -a' for shell scripts (.sh) (#3545)
19+
- allow use of alternate envvar(s) to $HOME for user modules (#3558)
20+
- use https://sources.easybuild.io as fallback source URL (#3572, #3576)
21+
- add toolchain definition for iibff toolchain (#3574)
22+
- add %(cuda_cc_space_sep)s and %(cuda_cc_semicolon_sep)s templates (#3578)
23+
- add support for intel-compiler toolchain (>= 2021.x versions, oneAPI) (#3581, #3582)
24+
- various bug fixes, including:
25+
- add --init and --recursive options to 'git submodule update' command that is used when creating source tarball for specific commit (#3537)
26+
- filter out duplicate paths in RPATH wrapper script (#3538)
27+
- don't clean up imported modules after verifying imports of included Python modules (#3544)
28+
- avoid no-op changes to $LD_* environment variables in ModulesTool (#3553)
29+
- fix UTF-8 encoding errors when running EasyBuild with Python 3.0.x-3.6.x (#3565)
30+
- create lib64 symlink as a relative symlink (#3566)
31+
- don't reuse variable name in the loop to fix adding extra compiler flags via toolchainopts (#3571)
32+
- symlink 'lib' to 'lib64' if it doesn't exist (#3580)
33+
- include %(mpi_cmd_prefix)s and %(cuda_*)s templates in output of --avail-easyconfig-templates (#3586)
34+
- other changes:
35+
- rename EasyBlock._skip_step to EasyBlock.skip_step, to make it part of the public API (#3561)
36+
- make symlinking of posix_c.so to posix.so in test suite configuration conditional (#3570)
37+
38+
639
v4.3.2 (December 10th 2020)
740
---------------------------
841

‎easybuild/base/fancylogger.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -580,6 +580,8 @@ def logToFile(filename, enable=True, filehandler=None, name=None, max_bytes=MAX_
580580
'maxBytes': max_bytes,
581581
'backupCount': backup_count,
582582
}
583+
if sys.version_info[0] >= 3:
584+
handleropts['encoding'] = 'utf-8'
583585
# logging to a file is going to create the file later on, so let's try to be helpful and create the path if needed
584586
directory = os.path.dirname(filename)
585587
if not os.path.exists(directory):

‎easybuild/base/generaloption.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1376,8 +1376,7 @@ def parseconfigfiles(self):
13761376
configfile_values[opt_dest] = newval
13771377
else:
13781378
configfile_cmdline_dest.append(opt_dest)
1379-
configfile_cmdline.append("--%s" % opt_name)
1380-
configfile_cmdline.append(val)
1379+
configfile_cmdline.append("--%s=%s" % (opt_name, val))
13811380

13821381
# reparse
13831382
self.log.debug('parseconfigfiles: going to parse options through cmdline %s' % configfile_cmdline)

‎easybuild/base/optcomplete.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -594,9 +594,8 @@ def autocomplete(parser, arg_completer=None, opt_completer=None, subcmd_complete
594594
if isinstance(debugfn, logging.Logger):
595595
debugfn.debug(txt)
596596
else:
597-
f = open(debugfn, 'a')
598-
f.write(txt)
599-
f.close()
597+
with open(debugfn, 'a') as fh:
598+
fh.write(txt)
600599

601600
# Exit with error code (we do not let the caller continue on purpose, this
602601
# is a run for completions only.)

0 commit comments

Comments
 (0)