Skip to content

Commit e149177

Browse files
boegelboegel
authored
Merge pull request #4370 from easybuilders/4.8.x
release EasyBuild v4.8.2
2 parents 14c273a + c93633c commit e149177

File tree

20 files changed

+290
-85
lines changed

20 files changed

+290
-85
lines changed

‎.github/workflows/unit_tests.yml‎

Lines changed: 33 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -34,47 +34,25 @@ jobs:
3434
- ${{needs.setup.outputs.modulesTcl}}
3535
- ${{needs.setup.outputs.modules3}}
3636
- ${{needs.setup.outputs.modules4}}
37-
module_syntax: [Lua, Tcl]
3837
lc_all: [""]
39-
# don't test with Lua module syntax (only supported in Lmod)
40-
exclude:
41-
- modules_tool: ${{needs.setup.outputs.modulesTcl}}
42-
module_syntax: Lua
43-
- modules_tool: ${{needs.setup.outputs.modules3}}
44-
module_syntax: Lua
45-
- modules_tool: ${{needs.setup.outputs.modules4}}
46-
module_syntax: Lua
4738
include:
48-
# Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax)
39+
# Test different Python 3 versions with Lmod 8.x
4940
- python: 3.5
5041
modules_tool: ${{needs.setup.outputs.lmod8}}
51-
module_syntax: Lua
5242
- python: 3.7
5343
modules_tool: ${{needs.setup.outputs.lmod8}}
54-
module_syntax: Lua
5544
- python: 3.8
5645
modules_tool: ${{needs.setup.outputs.lmod8}}
57-
module_syntax: Lua
58-
- python: 3.8
59-
modules_tool: ${{needs.setup.outputs.lmod8}}
60-
module_syntax: Tcl
6146
- python: 3.9
6247
modules_tool: ${{needs.setup.outputs.lmod8}}
63-
module_syntax: Lua
6448
- python: '3.10'
6549
modules_tool: ${{needs.setup.outputs.lmod8}}
66-
module_syntax: Lua
67-
- python: '3.11'
68-
modules_tool: ${{needs.setup.outputs.lmod8}}
69-
module_syntax: Lua
7050
- python: '3.11'
7151
modules_tool: ${{needs.setup.outputs.lmod8}}
72-
module_syntax: Tcl
7352
# There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set
7453
# Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7)
7554
- python: 3.6
7655
modules_tool: ${{needs.setup.outputs.lmod8}}
77-
module_syntax: Lua
7856
lc_all: C
7957
fail-fast: false
8058
steps:
@@ -125,16 +103,11 @@ jobs:
125103
# and are only run after the PR gets merged
126104
GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}}
127105
run: |
128-
# don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools,
129-
# and only when testing with Lua as module syntax,
130-
# to avoid hitting GitHub rate limit;
106+
# don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools, to avoid hitting GitHub rate limit;
131107
# tests that require a GitHub token are skipped automatically when no GitHub token is available
132-
if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]] && [[ "${{matrix.module_syntax}}" == 'Lua' ]]; then
108+
if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]]; then
133109
if [ ! -z $GITHUB_TOKEN ]; then
134-
if [ "x${{matrix.python}}" == 'x2.6' ];
135-
then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())";
136-
else SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
137-
fi;
110+
SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
138111
python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
139112
fi
140113
echo "GitHub token installed!"
@@ -172,8 +145,6 @@ jobs:
172145
- name: run test suite
173146
env:
174147
EB_VERBOSE: 1
175-
EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}}
176-
TEST_EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}}
177148
LC_ALL: ${{matrix.lc_all}}
178149
run: |
179150
# run tests *outside* of checked out easybuild-framework directory,
@@ -198,19 +169,32 @@ jobs:
198169
else
199170
export EASYBUILD_MODULES_TOOL=Lmod
200171
fi
201-
export TEST_EASYBUILD_MODULES_TOOL=$EASYBUILD_MODULES_TOOL
202-
eb --show-config
203-
# gather some useful info on test system
204-
eb --show-system-info
205-
# check GitHub configuration
206-
eb --check-github --github-user=easybuild_test
207-
# create file owned by root but writable by anyone (used by test_copy_file)
208-
sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
209-
sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
210-
# run test suite
211-
python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
212-
# try and make sure output of running tests is clean (no printed messages/warnings)
213-
IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test"
214-
# '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches)
215-
PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
216-
test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1)
172+
export TEST_EASYBUILD_MODULES_TOOL=${EASYBUILD_MODULES_TOOL}
173+
174+
# Run tests with LUA and Tcl module syntax (where supported)
175+
for module_syntax in Lua Tcl; do
176+
# Only Lmod supports Lua
177+
if [[ "${module_syntax}" == "Lua" ]] && [[ "${EASYBUILD_MODULES_TOOL}" != "Lmod" ]]; then
178+
echo "Not testing with '${module_syntax}' as module syntax with '${EASYBUILD_MODULES_TOOL}' as modules tool"
179+
continue
180+
fi
181+
printf '\n\n=====================> Using $module_syntax module syntax <=====================\n\n'
182+
export EASYBUILD_MODULE_SYNTAX="${module_syntax}"
183+
export TEST_EASYBUILD_MODULE_SYNTAX="${EASYBUILD_MODULE_SYNTAX}"
184+
185+
eb --show-config
186+
# gather some useful info on test system
187+
eb --show-system-info
188+
# check GitHub configuration
189+
eb --check-github --github-user=easybuild_test
190+
# create file owned by root but writable by anyone (used by test_copy_file)
191+
sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
192+
sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
193+
# run test suite
194+
python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
195+
# try and make sure output of running tests is clean (no printed messages/warnings)
196+
IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test"
197+
# '|| true' is needed to avoid that GitHub Actions stops the job on non-zero exit of grep (i.e. when there are no matches)
198+
PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
199+
test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1)
200+
done

‎.github/workflows/unit_tests_python2.yml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
2+
name: EasyBuild framework unit tests (python2)
3+
on: [push, pull_request]
4+
5+
permissions:
6+
contents: read # to fetch code (actions/checkout)
7+
8+
concurrency:
9+
group: ${{format('{0}:{1}:{2}', github.repository, github.ref, github.workflow)}}
10+
cancel-in-progress: true
11+
12+
jobs:
13+
test_python2:
14+
runs-on: ubuntu-20.04
15+
container:
16+
# CentOS 7.9 container that already includes Lmod & co,
17+
# see https://github.com/easybuilders/easybuild-containers
18+
image: ghcr.io/easybuilders/centos-7.9-amd64
19+
steps:
20+
- uses: actions/checkout@v3
21+
22+
- name: install Python packages
23+
run: |
24+
# Python packages
25+
python2 -V
26+
python2 -m pip --version
27+
python2 -m pip install --upgrade pip
28+
python2 -m pip --version
29+
# strip out GC3Pie since installation with ancient setuptools (0.9.8) fails
30+
sed -i '/GC3Pie/d' requirements.txt
31+
python2 -m pip install -r requirements.txt
32+
# git config is required to make actual git commits (cfr. tests for GitRepository)
33+
sudo -u easybuild git config --global user.name "GitHub Actions"
34+
sudo -u easybuild git config --global user.email "[email protected]"
35+
sudo -u easybuild git config --get-regexp 'user.*'
36+
37+
- name: install GitHub token (if available)
38+
env:
39+
# token (owned by @boegelbot) with gist permissions (required for some of the tests for GitHub integration);
40+
# this token is not available in pull requests, so tests that require it are skipped in PRs,
41+
# and are only run after the PR gets merged
42+
GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}}
43+
run: |
44+
# tests that require a GitHub token are skipped automatically when no GitHub token is available
45+
if [ ! -z $GITHUB_TOKEN ]; then
46+
sudo -u easybuild python2 -c "import keyring; import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring()); keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
47+
echo "GitHub token installed!"
48+
else
49+
echo "Installation of GitHub token skipped!"
50+
fi
51+
52+
- name: install sources
53+
run: |
54+
# install from source distribution tarball, to test release as published on PyPI
55+
python2 setup.py sdist
56+
ls dist
57+
export PREFIX=/tmp/$USER/$GITHUB_SHA
58+
python2 -m pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
59+
60+
- name: run test suite
61+
run: |
62+
# run tests *outside* of checked out easybuild-framework directory,
63+
# to ensure we're testing installed version (see previous step)
64+
cd $HOME
65+
# make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that)
66+
export PREFIX=/tmp/$USER/$GITHUB_SHA
67+
ENV_CMDS="export PATH=$PREFIX/bin:$PATH; export PYTHONPATH=$PREFIX/lib/python2.7/site-packages:$PYTHONPATH"
68+
ENV_CMDS="${ENV_CMDS}; export EB_VERBOSE=1; export EB_PYTHON=python2; export TEST_EASYBUILD_SILENCE_DEPRECATION_WARNINGS=python2"
69+
# run EasyBuild command via (non-root) easybuild user + login shell
70+
sudo -u easybuild bash -l -c "${ENV_CMDS}; module --version; eb --version"
71+
# show active EasyBuild configuration
72+
sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --show-config"
73+
# gather some useful info on test system
74+
sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --show-system-info"
75+
# check GitHub configuration
76+
sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --check-github --github-user=easybuild_test"
77+
# create file owned by root but writable by anyone (used by test_copy_file)
78+
sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
79+
sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
80+
# run test suite (via easybuild user + login shell)
81+
sudo -u easybuild bash -l -c "${ENV_CMDS}; python2 -O -m test.framework.suite"

‎RELEASE_NOTES‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,25 @@ For more detailed information, please see the git log.
44
These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
55

66

7+
v4.8.2 (29 October 2023)
8+
------------------------
9+
10+
update/bugfix release
11+
12+
- various enhancements, including:
13+
- add support for `%(sysroot)s` template value (#4359)
14+
- add `dependency_names` method to `EasyConfig` class to get set of names of (direct) dependencies (#4360)
15+
- various bug fixes, including:
16+
- add CI workflow to run unit tests with Python 2 (again) (#4333)
17+
- fix typo in help message for `--silence-hook-trigger` (#4343)
18+
- include major version (`*majver`) templates in auto-generated documentation (#4347)
19+
- reset `tempfile.tempdir` to `None` to avoid that tmpdir path gets progressively deeper with each easystack item (#4350)
20+
- fix `findPythonDeps.py` script when called with an (absolute or relative) path to an easyconfig instead of a filename (#4365)
21+
- fix broken test for `reasons_for_closing`, which fails because commit status of easyconfigs PR is no longer available (#4366)
22+
- other changes:
23+
- reduce number of CI jobs by testing for Lua and Tcl module syntax in a single CI job (#4192)
24+
25+
726
v4.8.1 (11 September 2023)
827
--------------------------
928

‎easybuild/framework/easyconfig/easyconfig.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1139,6 +1139,15 @@ def dependencies(self, build_only=False):
11391139

11401140
return retained_deps
11411141

1142+
def dependency_names(self, build_only=False):
1143+
"""
1144+
Return a set of names of all (direct) dependencies after filtering.
1145+
Iterable builddependencies are flattened when not iterating.
1146+
1147+
:param build_only: only return build dependencies, discard others
1148+
"""
1149+
return {dep['name'] for dep in self.dependencies(build_only=build_only) if dep['name']}
1150+
11421151
def builddependencies(self):
11431152
"""
11441153
Return a flat list of the parsed build dependencies

‎easybuild/framework/easyconfig/templates.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,9 +77,9 @@
7777
('installdir', "Installation directory"),
7878
('start_dir', "Directory in which the build process begins"),
7979
]
80-
# software names for which to define <pref>ver and <pref>shortver templates
80+
# software names for which to define <pref>ver, <pref>majver and <pref>shortver templates
8181
TEMPLATE_SOFTWARE_VERSIONS = [
82-
# software name, prefix for *ver and *shortver
82+
# software name, prefix for *ver, *majver and *shortver
8383
('CUDA', 'cuda'),
8484
('CUDAcore', 'cuda'),
8585
('Java', 'java'),
@@ -90,6 +90,8 @@
9090
# template values which are only generated dynamically
9191
TEMPLATE_NAMES_DYNAMIC = [
9292
('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"),
93+
('sysroot', "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include"
94+
"as specify by the --sysroot configuration option"),
9395
('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"),
9496
('cuda_compute_capabilities', "Comma-separated list of CUDA compute capabilities, as specified via "
9597
"--cuda-compute-capabilities configuration option or via cuda_compute_capabilities easyconfig parameter"),
@@ -201,6 +203,9 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None)
201203
# set 'arch' for system architecture based on 'machine' (4th) element of platform.uname() return value
202204
template_values['arch'] = platform.uname()[4]
203205

206+
# set 'sysroot' template based on 'sysroot' configuration option, using empty string as fallback
207+
template_values['sysroot'] = build_option('sysroot') or ''
208+
204209
# step 1: add TEMPLATE_NAMES_EASYCONFIG
205210
for name in TEMPLATE_NAMES_EASYCONFIG:
206211
if name in ignore:
@@ -427,6 +432,7 @@ def template_documentation():
427432
# step 2: add *ver/*shortver templates for software listed in TEMPLATE_SOFTWARE_VERSIONS
428433
doc.append("Template names/values for (short) software versions")
429434
for name, pref in TEMPLATE_SOFTWARE_VERSIONS:
435+
doc.append("%s%%(%smajver)s: major version for %s" % (indent_l1, pref, name))
430436
doc.append("%s%%(%sshortver)s: short version for %s (<major>.<minor>)" % (indent_l1, pref, name))
431437
doc.append("%s%%(%sver)s: full version for %s" % (indent_l1, pref, name))
432438

‎easybuild/main.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import os
4343
import stat
4444
import sys
45+
import tempfile
4546
import traceback
4647

4748
# IMPORTANT this has to be the first easybuild import as it customises the logging
@@ -254,8 +255,9 @@ def process_easystack(easystack_path, args, logfile, testing, init_session_state
254255
easyconfig._easyconfigs_cache.clear()
255256
easyconfig._easyconfig_files_cache.clear()
256257

257-
# restore environment
258+
# restore environment and reset tempdir (to avoid tmpdir path getting progressively longer)
258259
restore_env(init_env)
260+
tempfile.tempdir = None
259261

260262
# If EasyConfig specific arguments were supplied in EasyStack file
261263
# merge arguments with original command line args

‎easybuild/scripts/findPythonDeps.py‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,11 @@ def run_cmd(arguments, action_desc, capture_stderr=True, **kwargs):
5555
extra_args['universal_newlines'] = True
5656
stderr = subprocess.STDOUT if capture_stderr else subprocess.PIPE
5757
p = subprocess.Popen(arguments, stdout=subprocess.PIPE, stderr=stderr, **extra_args)
58-
out, _ = p.communicate()
58+
out, err = p.communicate()
5959
if p.returncode != 0:
60-
raise RuntimeError('Failed to %s: %s' % (action_desc, out))
60+
if err:
61+
err = "\nSTDERR:\n" + err
62+
raise RuntimeError('Failed to %s: %s%s' % (action_desc, out, err))
6163
return out
6264

6365

@@ -171,20 +173,23 @@ def print_deps(package, verbose):
171173
capture_stderr=False,
172174
action_desc='Get missing dependencies'
173175
)
176+
excluded_dep = '(%s)' % os.path.basename(args.ec)
174177
missing_deps = [dep for dep in missing_dep_out.split('\n')
175-
if dep.startswith('*') and '(%s)' % args.ec not in dep
178+
if dep.startswith('*') and excluded_dep not in dep
176179
]
177180
if missing_deps:
178181
print('You need to install all modules on which %s depends first!' % args.ec)
179182
print('\n\t'.join(['Missing:'] + missing_deps))
180183
sys.exit(1)
181184

185+
# If the --ec argument is a (relative) existing path make it absolute so we can find it after the chdir
186+
ec_arg = os.path.abspath(args.ec) if os.path.exists(args.ec) else args.ec
182187
with temporary_directory() as tmp_dir:
183188
old_dir = os.getcwd()
184189
os.chdir(tmp_dir)
185190
if args.verbose:
186191
print('Running EasyBuild to get build environment')
187-
run_cmd(['eb', args.ec, '--dump-env', '--force'], action_desc='Dump build environment')
192+
run_cmd(['eb', ec_arg, '--dump-env', '--force'], action_desc='Dump build environment')
188193
os.chdir(old_dir)
189194

190195
cmd = "source %s/*.env && python %s '%s'" % (tmp_dir, sys.argv[0], args.package)

‎easybuild/toolchains/compiler/gcc.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ class Gcc(Compiler):
5050

5151
COMPILER_FAMILY = TC_CONSTANT_GCC
5252
COMPILER_UNIQUE_OPTS = {
53-
'loop': (False, "Automatic loop parallellisation"),
53+
'loop': (False, "Automatic loop parallelisation"),
5454
'f2c': (False, "Generate code compatible with f2c and f77"),
5555
'lto': (False, "Enable Link Time Optimization"),
5656
}

0 commit comments

Comments
 (0)