Skip to content

Commit 9c31fd5

Browse files
boegelboegel
committed
Merge branch 'develop' into 5.0.x
2 parents 7e899c3 + d450a80 commit 9c31fd5

File tree

11 files changed

+167
-59
lines changed

11 files changed

+167
-59
lines changed

‎.github/workflows/unit_tests.yml‎

Lines changed: 32 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -34,44 +34,23 @@ jobs:
3434
- ${{needs.setup.outputs.modulesTcl}}
3535
- ${{needs.setup.outputs.modules3}}
3636
- ${{needs.setup.outputs.modules4}}
37-
module_syntax: [Lua, Tcl]
3837
lc_all: [""]
39-
# don't test with Lua module syntax (only supported in Lmod)
40-
exclude:
41-
- modules_tool: ${{needs.setup.outputs.modulesTcl}}
42-
module_syntax: Lua
43-
- modules_tool: ${{needs.setup.outputs.modules3}}
44-
module_syntax: Lua
45-
- modules_tool: ${{needs.setup.outputs.modules4}}
46-
module_syntax: Lua
4738
include:
4839
# Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax)
4940
- python: 3.7
5041
modules_tool: ${{needs.setup.outputs.lmod8}}
51-
module_syntax: Lua
5242
- python: 3.8
5343
modules_tool: ${{needs.setup.outputs.lmod8}}
54-
module_syntax: Lua
55-
- python: 3.8
56-
modules_tool: ${{needs.setup.outputs.lmod8}}
57-
module_syntax: Tcl
5844
- python: 3.9
5945
modules_tool: ${{needs.setup.outputs.lmod8}}
60-
module_syntax: Lua
6146
- python: '3.10'
6247
modules_tool: ${{needs.setup.outputs.lmod8}}
63-
module_syntax: Lua
64-
- python: '3.11'
65-
modules_tool: ${{needs.setup.outputs.lmod8}}
66-
module_syntax: Lua
6748
- python: '3.11'
6849
modules_tool: ${{needs.setup.outputs.lmod8}}
69-
module_syntax: Tcl
7050
# There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set
7151
# Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7)
7252
- python: 3.6
7353
modules_tool: ${{needs.setup.outputs.lmod8}}
74-
module_syntax: Lua
7554
lc_all: C
7655
fail-fast: false
7756
steps:
@@ -122,16 +101,11 @@ jobs:
122101
# and are only run after the PR gets merged
123102
GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}}
124103
run: |
125-
# don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools,
126-
# and only when testing with Lua as module syntax,
127-
# to avoid hitting GitHub rate limit;
104+
# don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools, to avoid hitting GitHub rate limit;
128105
# tests that require a GitHub token are skipped automatically when no GitHub token is available
129-
if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]] && [[ "${{matrix.module_syntax}}" == 'Lua' ]]; then
106+
if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]]; then
130107
if [ ! -z $GITHUB_TOKEN ]; then
131-
if [ "x${{matrix.python}}" == 'x2.6' ];
132-
then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())";
133-
else SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
134-
fi;
108+
SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
135109
python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
136110
fi
137111
echo "GitHub token installed!"
@@ -169,8 +143,6 @@ jobs:
169143
- name: run test suite
170144
env:
171145
EB_VERBOSE: 1
172-
EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}}
173-
TEST_EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}}
174146
LC_ALL: ${{matrix.lc_all}}
175147
run: |
176148
# run tests *outside* of checked out easybuild-framework directory,
@@ -195,19 +167,32 @@ jobs:
195167
else
196168
export EASYBUILD_MODULES_TOOL=Lmod
197169
fi
198-
export TEST_EASYBUILD_MODULES_TOOL=$EASYBUILD_MODULES_TOOL
199-
eb --show-config
200-
# gather some useful info on test system
201-
eb --show-system-info
202-
# check GitHub configuration
203-
eb --check-github --github-user=easybuild_test
204-
# create file owned by root but writable by anyone (used by test_copy_file)
205-
sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
206-
sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
207-
# run test suite
208-
python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
209-
# try and make sure output of running tests is clean (no printed messages/warnings)
210-
IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test"
211-
# '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches)
212-
PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
213-
test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1)
170+
export TEST_EASYBUILD_MODULES_TOOL=${EASYBUILD_MODULES_TOOL}
171+
172+
# Run tests with LUA and Tcl module syntax (where supported)
173+
for module_syntax in Lua Tcl; do
174+
# Only Lmod supports Lua
175+
if [[ "${module_syntax}" == "Lua" ]] && [[ "${EASYBUILD_MODULES_TOOL}" != "Lmod" ]]; then
176+
echo "Not testing with '${module_syntax}' as module syntax with '${EASYBUILD_MODULES_TOOL}' as modules tool"
177+
continue
178+
fi
179+
printf '\n\n=====================> Using $module_syntax module syntax <=====================\n\n'
180+
export EASYBUILD_MODULE_SYNTAX="${module_syntax}"
181+
export TEST_EASYBUILD_MODULE_SYNTAX="${EASYBUILD_MODULE_SYNTAX}"
182+
183+
eb --show-config
184+
# gather some useful info on test system
185+
eb --show-system-info
186+
# check GitHub configuration
187+
eb --check-github --github-user=easybuild_test
188+
# create file owned by root but writable by anyone (used by test_copy_file)
189+
sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
190+
sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
191+
# run test suite
192+
python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
193+
# try and make sure output of running tests is clean (no printed messages/warnings)
194+
IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test"
195+
# '|| true' is needed to avoid that GitHub Actions stops the job on non-zero exit of grep (i.e. when there are no matches)
196+
PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
197+
test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1)
198+
done

‎.github/workflows/unit_tests_python2.yml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
2+
name: EasyBuild framework unit tests (python2)
3+
on: [push, pull_request]
4+
5+
permissions:
6+
contents: read # to fetch code (actions/checkout)
7+
8+
concurrency:
9+
group: ${{format('{0}:{1}:{2}', github.repository, github.ref, github.workflow)}}
10+
cancel-in-progress: true
11+
12+
jobs:
13+
test_python2:
14+
runs-on: ubuntu-20.04
15+
container:
16+
# CentOS 7.9 container that already includes Lmod & co,
17+
# see https://github.com/easybuilders/easybuild-containers
18+
image: ghcr.io/easybuilders/centos-7.9-amd64
19+
steps:
20+
- uses: actions/checkout@v3
21+
22+
- name: install Python packages
23+
run: |
24+
# Python packages
25+
python2 -V
26+
python2 -m pip --version
27+
python2 -m pip install --upgrade pip
28+
python2 -m pip --version
29+
# strip out GC3Pie since installation with ancient setuptools (0.9.8) fails
30+
sed -i '/GC3Pie/d' requirements.txt
31+
python2 -m pip install -r requirements.txt
32+
# git config is required to make actual git commits (cfr. tests for GitRepository)
33+
sudo -u easybuild git config --global user.name "GitHub Actions"
34+
sudo -u easybuild git config --global user.email "[email protected]"
35+
sudo -u easybuild git config --get-regexp 'user.*'
36+
37+
- name: install GitHub token (if available)
38+
env:
39+
# token (owned by @boegelbot) with gist permissions (required for some of the tests for GitHub integration);
40+
# this token is not available in pull requests, so tests that require it are skipped in PRs,
41+
# and are only run after the PR gets merged
42+
GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}}
43+
run: |
44+
# tests that require a GitHub token are skipped automatically when no GitHub token is available
45+
if [ ! -z $GITHUB_TOKEN ]; then
46+
sudo -u easybuild python2 -c "import keyring; import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring()); keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
47+
echo "GitHub token installed!"
48+
else
49+
echo "Installation of GitHub token skipped!"
50+
fi
51+
52+
- name: install sources
53+
run: |
54+
# install from source distribution tarball, to test release as published on PyPI
55+
python2 setup.py sdist
56+
ls dist
57+
export PREFIX=/tmp/$USER/$GITHUB_SHA
58+
python2 -m pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
59+
60+
- name: run test suite
61+
run: |
62+
# run tests *outside* of checked out easybuild-framework directory,
63+
# to ensure we're testing installed version (see previous step)
64+
cd $HOME
65+
# make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that)
66+
export PREFIX=/tmp/$USER/$GITHUB_SHA
67+
ENV_CMDS="export PATH=$PREFIX/bin:$PATH; export PYTHONPATH=$PREFIX/lib/python2.7/site-packages:$PYTHONPATH"
68+
ENV_CMDS="${ENV_CMDS}; export EB_VERBOSE=1; export EB_PYTHON=python2; export TEST_EASYBUILD_SILENCE_DEPRECATION_WARNINGS=python2"
69+
# run EasyBuild command via (non-root) easybuild user + login shell
70+
sudo -u easybuild bash -l -c "${ENV_CMDS}; module --version; eb --version"
71+
# show active EasyBuild configuration
72+
sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --show-config"
73+
# gather some useful info on test system
74+
sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --show-system-info"
75+
# check GitHub configuration
76+
sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --check-github --github-user=easybuild_test"
77+
# create file owned by root but writable by anyone (used by test_copy_file)
78+
sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
79+
sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
80+
# run test suite (via easybuild user + login shell)
81+
sudo -u easybuild bash -l -c "${ENV_CMDS}; python2 -O -m test.framework.suite"

‎easybuild/framework/easyconfig/templates.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,9 +76,9 @@
7676
('installdir', "Installation directory"),
7777
('start_dir', "Directory in which the build process begins"),
7878
]
79-
# software names for which to define <pref>ver and <pref>shortver templates
79+
# software names for which to define <pref>ver, <pref>majver and <pref>shortver templates
8080
TEMPLATE_SOFTWARE_VERSIONS = [
81-
# software name, prefix for *ver and *shortver
81+
# software name, prefix for *ver, *majver and *shortver
8282
('CUDA', 'cuda'),
8383
('CUDAcore', 'cuda'),
8484
('Java', 'java'),
@@ -423,6 +423,7 @@ def template_documentation():
423423
# step 2: add *ver/*shortver templates for software listed in TEMPLATE_SOFTWARE_VERSIONS
424424
doc.append("Template names/values for (short) software versions")
425425
for name, pref in TEMPLATE_SOFTWARE_VERSIONS:
426+
doc.append("%s%%(%smajver)s: major version for %s" % (indent_l1, pref, name))
426427
doc.append("%s%%(%sshortver)s: short version for %s (<major>.<minor>)" % (indent_l1, pref, name))
427428
doc.append("%s%%(%sver)s: full version for %s" % (indent_l1, pref, name))
428429

‎easybuild/main.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import os
4343
import stat
4444
import sys
45+
import tempfile
4546
import traceback
4647

4748
# IMPORTANT this has to be the first easybuild import as it customises the logging
@@ -253,8 +254,9 @@ def process_easystack(easystack_path, args, logfile, testing, init_session_state
253254
easyconfig._easyconfigs_cache.clear()
254255
easyconfig._easyconfig_files_cache.clear()
255256

256-
# restore environment
257+
# restore environment and reset tempdir (to avoid tmpdir path getting progressively longer)
257258
restore_env(init_env)
259+
tempfile.tempdir = None
258260

259261
# If EasyConfig specific arguments were supplied in EasyStack file
260262
# merge arguments with original command line args

‎easybuild/tools/docs.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -441,6 +441,7 @@ def avail_easyconfig_templates_txt():
441441
# step 2: add SOFTWARE_VERSIONS
442442
doc.append('Template names/values for (short) software versions')
443443
for name, pref in TEMPLATE_SOFTWARE_VERSIONS:
444+
doc.append("%s%%(%smajver)s: major version for %s" % (INDENT_4SPACES, pref, name))
444445
doc.append("%s%%(%sshortver)s: short version for %s (<major>.<minor>)" % (INDENT_4SPACES, pref, name))
445446
doc.append("%s%%(%sver)s: full version for %s" % (INDENT_4SPACES, pref, name))
446447
doc.append('')
@@ -495,8 +496,10 @@ def avail_easyconfig_templates_rst():
495496
ver = []
496497
ver_desc = []
497498
for name, pref in TEMPLATE_SOFTWARE_VERSIONS:
499+
ver.append('``%%(%smajver)s``' % pref)
498500
ver.append('``%%(%sshortver)s``' % pref)
499501
ver.append('``%%(%sver)s``' % pref)
502+
ver_desc.append('major version for %s' % name)
500503
ver_desc.append('short version for %s (<major>.<minor>)' % name)
501504
ver_desc.append('full version for %s' % name)
502505
table_values = [ver, ver_desc]
@@ -558,8 +561,10 @@ def avail_easyconfig_templates_md():
558561
ver = []
559562
ver_desc = []
560563
for name, pref in TEMPLATE_SOFTWARE_VERSIONS:
564+
ver.append('``%%(%smajver)s``' % pref)
561565
ver.append('``%%(%sshortver)s``' % pref)
562566
ver.append('``%%(%sver)s``' % pref)
567+
ver_desc.append('major version for %s' % name)
563568
ver_desc.append('short version for %s (``<major>.<minor>``)' % name)
564569
ver_desc.append('full version for %s' % name)
565570
table_values = [ver, ver_desc]

‎test/framework/docs.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -785,6 +785,7 @@ def test_avail_easyconfig_templates(self):
785785
r"^Template names/values derived from easyconfig instance",
786786
r"^\s+%\(version_major\)s: Major version",
787787
r"^Template names/values for \(short\) software versions",
788+
r"^\s+%\(pymajver\)s: major version for Python",
788789
r"^\s+%\(pyshortver\)s: short version for Python \(<major>\.<minor>\)",
789790
r"^Template constants that can be used in easyconfigs",
790791
r"^\s+SOURCE_TAR_GZ: Source \.tar\.gz bundle \(%\(name\)s-%\(version\)s.tar.gz\)",

‎test/framework/easyblock.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1527,6 +1527,7 @@ def test_fetch_sources(self):
15271527

15281528
def test_download_instructions(self):
15291529
"""Test use of download_instructions easyconfig parameter."""
1530+
15301531
orig_test_ec = '\n'.join([
15311532
"easyblock = 'ConfigureMake'",
15321533
"name = 'software_with_missing_sources'",

‎test/framework/easyconfig.py‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1294,7 +1294,7 @@ def test_templating_doc(self):
12941294
# expected length: 1 per constant and 2 extra per constantgroup (title + empty line in between)
12951295
temps = [
12961296
easyconfig.templates.TEMPLATE_NAMES_EASYCONFIG,
1297-
easyconfig.templates.TEMPLATE_SOFTWARE_VERSIONS * 2,
1297+
easyconfig.templates.TEMPLATE_SOFTWARE_VERSIONS * 3,
12981298
easyconfig.templates.TEMPLATE_NAMES_CONFIG,
12991299
easyconfig.templates.TEMPLATE_NAMES_LOWER,
13001300
easyconfig.templates.TEMPLATE_NAMES_EASYBLOCK_RUN_STEP,
@@ -2140,12 +2140,15 @@ def eval_quoted_string(quoted_val, val):
21402140
Helper function to sanity check we can use the quoted string in Python contexts.
21412141
Returns the evaluated (i.e. unquoted) string
21422142
"""
2143-
globals = dict()
2143+
scope = dict()
21442144
try:
2145-
exec('res = %s' % quoted_val, globals)
2146-
except Exception as e: # pylint: disable=broad-except
2147-
self.fail('Failed to evaluate %s (from %s): %s' % (quoted_val, val, e))
2148-
return globals['res']
2145+
# this is needlessly complicated because we can't use 'exec' here without potentially running
2146+
# into a SyntaxError bug in old Python 2.7 versions (for example when running the tests in CentOS 7.9)
2147+
# cfr. https://stackoverflow.com/questions/4484872/why-doesnt-exec-work-in-a-function-with-a-subfunction
2148+
eval(compile('res = %s' % quoted_val, '<string>', 'exec'), dict(), scope)
2149+
except Exception as err: # pylint: disable=broad-except
2150+
self.fail('Failed to evaluate %s (from %s): %s' % (quoted_val, val, err))
2151+
return scope['res']
21492152

21502153
def assertEqual_unquoted(quoted_val, val):
21512154
"""Assert that evaluating the quoted_val yields the val"""

0 commit comments

Comments
 (0)