Skip to content

Commit 05ab036

Browse files
boegelboegel
authored
Merge pull request #3720 from easybuilders/4.4.x
release EasyBuild v4.4.0
2 parents 74a32f8 + 5f2813d commit 05ab036

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

75 files changed

+3600
-683
lines changed

‎.github/workflows/bootstrap_script.yml‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
2+
name: test EasyBuild bootstrap script
3+
on: [push, pull_request]
4+
jobs:
5+
setup:
6+
runs-on: ubuntu-latest
7+
outputs:
8+
lmod7: Lmod-7.8.22
9+
lmod8: Lmod-8.4.27
10+
modulesTcl: modules-tcl-1.147
11+
modules3: modules-3.2.10
12+
modules4: modules-4.1.4
13+
steps:
14+
- run: "true"
15+
build:
16+
needs: setup
17+
runs-on: ubuntu-18.04
18+
strategy:
19+
matrix:
20+
# Don't run for Python 3.8, 3.9 , people should just use `pip install easybuild`
21+
python: [2.7, 3.6, 3.7]
22+
modules_tool:
23+
# use variables defined by 'setup' job above, see also
24+
# https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context
25+
- ${{needs.setup.outputs.lmod7}}
26+
- ${{needs.setup.outputs.lmod8}}
27+
module_syntax: [Lua]
28+
lc_all: [""]
29+
include:
30+
# also test with module tools other than Lmod (only Tcl syntax)
31+
- modules_tool: ${{needs.setup.outputs.modulesTcl}}
32+
module_syntax: Tcl
33+
python: 2.7
34+
- modules_tool: ${{needs.setup.outputs.modulesTcl}}
35+
module_syntax: Tcl
36+
python: 3.6
37+
- modules_tool: ${{needs.setup.outputs.modules3}}
38+
module_syntax: Tcl
39+
python: 2.7
40+
- modules_tool: ${{needs.setup.outputs.modules3}}
41+
module_syntax: Tcl
42+
python: 3.6
43+
- modules_tool: ${{needs.setup.outputs.modules4}}
44+
module_syntax: Tcl
45+
python: 2.7
46+
- modules_tool: ${{needs.setup.outputs.modules4}}
47+
module_syntax: Tcl
48+
python: 3.6
49+
# There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set
50+
# Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7)
51+
- python: 3.6
52+
modules_tool: ${{needs.setup.outputs.lmod8}}
53+
module_syntax: Lua
54+
lc_all: C
55+
fail-fast: false
56+
steps:
57+
- uses: actions/checkout@v2
58+
59+
- name: set up Python
60+
uses: actions/setup-python@v2
61+
with:
62+
python-version: ${{matrix.python}}
63+
architecture: x64
64+
65+
- name: install OS & Python packages
66+
run: |
67+
# disable apt-get update, we don't really need it,
68+
# and it does more harm than good (it's fairly expensive, and it results in flaky test runs)
69+
# sudo apt-get update
70+
# for modules tool
71+
sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
72+
# fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
73+
# needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
74+
if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
75+
sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
76+
fi
77+
78+
- name: install modules tool
79+
run: |
80+
# avoid downloading modules tool sources into easybuild-framework dir
81+
cd $HOME
82+
export INSTALL_DEP=$GITHUB_WORKSPACE/easybuild/scripts/install_eb_dep.sh
83+
# install Lmod
84+
source $INSTALL_DEP ${{matrix.modules_tool}} $HOME
85+
# changes in environment are not passed to other steps, so need to create files...
86+
echo $MOD_INIT > mod_init
87+
echo $PATH > path
88+
if [ ! -z $MODULESHOME ]; then echo $MODULESHOME > moduleshome; fi
89+
90+
- name: test bootstrap script
91+
run: |
92+
# (re)initialize environment for modules tool
93+
if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi
94+
source $(cat $HOME/mod_init); type module
95+
# also pick up changes to $PATH set by sourcing $HOME/mod_init
96+
export PATH=$(cat $HOME/path)
97+
98+
# define $EASYBUILD_MODULES_TOOL only for oldest module tools
99+
# (for Lmod and EnvironmentModules 4.x the bootstrap script should correctly auto-detect the modules tool)
100+
if [[ ${{matrix.modules_tool}} =~ ^modules-tcl- ]]; then
101+
export EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl
102+
elif [[ ${{matrix.modules_tool}} =~ ^modules-3 ]]; then
103+
export EASYBUILD_MODULES_TOOL=EnvironmentModulesC
104+
fi
105+
106+
# version and SHA256 checksum are hardcoded below to avoid forgetting to update the version in the script along with contents
107+
EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
108+
EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
109+
EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
110+
EB_BOOTSTRAP_EXPECTED="20210106.01 c2d93de0dd91123eb4f51cfc16d1f5efb80f1d238b3d6cd100994086887a1ae0"
111+
test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
112+
113+
# test bootstrap script
114+
export PREFIX=/tmp/$USER/$GITHUB_SHA/eb_bootstrap
115+
python easybuild/scripts/bootstrap_eb.py $PREFIX
116+
# unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module
117+
unset PYTHONPATH
118+
# simple sanity check on bootstrapped EasyBuild module
119+
module use $PREFIX/modules/all
120+
module load EasyBuild
121+
eb --version

‎.github/workflows/unit_tests.yml‎

Lines changed: 0 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -197,36 +197,3 @@ jobs:
197197
# '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches)
198198
PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
199199
test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite\n${PRINTED_MSG}" && exit 1)
200-
201-
- name: test bootstrap script
202-
run: |
203-
# (re)initialize environment for modules tool
204-
if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi
205-
source $(cat $HOME/mod_init); type module
206-
# also pick up changes to $PATH set by sourcing $HOME/mod_init
207-
export PATH=$(cat $HOME/path)
208-
209-
# define $EASYBUILD_MODULES_TOOL only for oldest module tools
210-
# (for Lmod and EnvironmentModules 4.x the bootstrap script should correctly auto-detect the modules tool)
211-
if [[ ${{matrix.modules_tool}} =~ ^modules-tcl- ]]; then
212-
export EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl
213-
elif [[ ${{matrix.modules_tool}} =~ ^modules-3 ]]; then
214-
export EASYBUILD_MODULES_TOOL=EnvironmentModulesC
215-
fi
216-
217-
# version and SHA256 checksum are hardcoded below to avoid forgetting to update the version in the script along with contents
218-
EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
219-
EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
220-
EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
221-
EB_BOOTSTRAP_EXPECTED="20210106.01 c2d93de0dd91123eb4f51cfc16d1f5efb80f1d238b3d6cd100994086887a1ae0"
222-
test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
223-
224-
# test bootstrap script
225-
export PREFIX=/tmp/$USER/$GITHUB_SHA/eb_bootstrap
226-
python easybuild/scripts/bootstrap_eb.py $PREFIX
227-
# unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module
228-
unset PYTHONPATH
229-
# simple sanity check on bootstrapped EasyBuild module
230-
module use $PREFIX/modules/all
231-
module load EasyBuild
232-
eb --version

‎RELEASE_NOTES‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,68 @@ For more detailed information, please see the git log.
44
These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
55

66

7+
v4.4.0 (June 2nd 2021)
8+
----------------------
9+
10+
feature release
11+
12+
- various enhancements, including:
13+
- enhance apply_regex_substitutions to allow specifying action to take in case there are no matches (#3440)
14+
- performance improvements for easyconfig parsing and eb startup (#3555)
15+
- add support for downloading easyconfigs from multiple PRs with --from-pr (#3605, #3707, #3708)
16+
- add support for prepending custom library paths in RPATH section via --rpath-override-dirs (#3650)
17+
- allow amending easyconfig parameters which are not the default (#3651)
18+
- update HierarchicalMNS for Intel OneAPI compilers (#3653)
19+
- add support for --sanity-check-only (#3655)
20+
- add support for running commands asynchronously via run_cmd + complete_cmd functions (#3656)
21+
- add support for using oneAPI versions of 'intel' toolchain components (#3665)
22+
- add toolchain definition for gofbf (foss with FlexiBLAS rather than OpenBLAS) (#3666)
23+
- add support for using ARCH constant in easyconfig files (#3670)
24+
- honor keyboard interrupt in 'eb' command (#3674)
25+
- add toolchain definition for Fujitsu toolchain (#3677, #3704, #3712, #3713, #3714, #3717)
26+
- extend sanity check step to check whether specific libraries are not linked into installed binaries/libraries (#3678)
27+
- via banned-linked-shared-libs and required-linked-shared-libs EasyBuild configuration options
28+
- via banned_linked_shared_libs and required_linked_shared_libs methods in toolchain support
29+
- via banned_linked_shared_libs and required_linked_shared_libs methods in easyblock
30+
- via banned_linked_shared_libs and required_linked_shared_libs easyconfig parameters
31+
- add locate_solib function to locate Linux shared libraries (#3682)
32+
- add system agnostic function to locate shared libraries (#3683)
33+
- add update_build_option function to update specific build options after initializing the EasyBuild configuration (#3684)
34+
- allow opting out of recursively unloaded of modules via recursive_module_unload easyconfig parameter (#3689)
35+
- check for correct version values when parsing easystack file (#3693)
36+
- run post-install commands specified for a specific extension (#3696)
37+
- add support for --skip-extensions (#3702)
38+
- include PR title in output produced by --merge-pr (#3706)
39+
- various bug fixes, including:
40+
- avoid metadata greedy behaviour when probing for external module metadata (mostly relevant for integration with Cray Programming Environment) (#3559)
41+
- catch problems early on if --github-user is not specified for --new-pr & co (#3644)
42+
- re-enable write permissions when installing with read-only-installdir (#3649)
43+
- also run sanity check for extensions when using --module-only (#3655)
44+
- improve logging when failing to load class from exts_classmap (#3657)
45+
- fix use of --module-only on existing installations without write permissions (#3659)
46+
- correct help text for subdir-user-modules (#3660)
47+
- avoid picking up easyblocks outside of sandbox in framework tests (#3680)
48+
- use unload/load in ModuleGeneratorLua.swap_module, since 'swap' is not supported by Lmod (#3685)
49+
- update HierarchicalMNS to also return 'Toolchain/<name>/<version>' as $MODULEPATH extension for cpe* Cray toolchains (#3686)
50+
- make EasyConfigParser.get_config_dict return a copy rather than a reference (#3692)
51+
- make sure that $TAPE is unset when using piped tar (#3698)
52+
- fix extending message for changed files in new_pr_from_branch (#3699)
53+
- enhance sched_getaffinity function to avoid early crash when counting available cores on systems with more than 1024 cores (#3701)
54+
- correctly strip extension from filename in extract_cmd and back_up_file functions (#3705)
55+
- other changes:
56+
- deprecate adding a non-existing path to $MODULEPATH (#3637)
57+
- bump cryptography requirement from 3.2.1 to 3.3.2 (#3643, #3648)
58+
- test bootstrap script in separate workflow, and limit test configurations a bit (#3646)
59+
- update setup.py to indicate compatibility with Python 3.8 and 3.9 (#3647)
60+
- replace log_error parameter of which() by on_error (#3661, #3664)
61+
- don't skip sanity check for --module-only --rebuild (#3645)
62+
- move disable_templating function into the EasyConfig class (#3668)
63+
- pin GitPython version for Python<3.6, don't test bootstrap script against Python 3.8/3.9 (#3675)
64+
- tweak foss toolchain definition to switch from OpenBLAS to FlexiBLAS in foss/2021a (#3679)
65+
- suggest missing SSH key when not able to read from remote repository in --check-github (#3681)
66+
- drop support for Python 2.6 (#3715)
67+
68+
769
v4.3.4 (April 9th 2021)
870
-----------------------
971

‎contrib/hooks/hpc2n_hooks.py‎

Lines changed: 19 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -162,21 +162,19 @@ def pre_module_hook(self, *args, **kwargs):
162162
self.log.info("[pre-module hook] Set I_MPI_PMI_LIBRARY in impi module")
163163
# Must be done this way, updating self.cfg['modextravars']
164164
# directly doesn't work due to templating.
165-
en_templ = self.cfg.enable_templating
166-
self.cfg.enable_templating = False
167-
shlib_ext = get_shared_lib_ext()
168-
pmix_root = get_software_root('PMIx')
169-
if pmix_root:
170-
mpi_type = 'pmix_v3'
171-
self.cfg['modextravars'].update({
172-
'I_MPI_PMI_LIBRARY': os.path.join(pmix_root, "lib", "libpmi." + shlib_ext)
173-
})
174-
self.cfg['modextravars'].update({'SLURM_MPI_TYPE': mpi_type})
175-
# Unfortunately UCX doesn't yet work for unknown reasons. Make sure it is off.
176-
self.cfg['modextravars'].update({'SLURM_PMIX_DIRECT_CONN_UCX': 'false'})
177-
else:
178-
self.cfg['modextravars'].update({'I_MPI_PMI_LIBRARY': "/lap/slurm/lib/libpmi.so"})
179-
self.cfg.enable_templating = en_templ
165+
with self.cfg.disable_templating():
166+
shlib_ext = get_shared_lib_ext()
167+
pmix_root = get_software_root('PMIx')
168+
if pmix_root:
169+
mpi_type = 'pmix_v3'
170+
self.cfg['modextravars'].update({
171+
'I_MPI_PMI_LIBRARY': os.path.join(pmix_root, "lib", "libpmi." + shlib_ext)
172+
})
173+
self.cfg['modextravars'].update({'SLURM_MPI_TYPE': mpi_type})
174+
# Unfortunately UCX doesn't yet work for unknown reasons. Make sure it is off.
175+
self.cfg['modextravars'].update({'SLURM_PMIX_DIRECT_CONN_UCX': 'false'})
176+
else:
177+
self.cfg['modextravars'].update({'I_MPI_PMI_LIBRARY': "/lap/slurm/lib/libpmi.so"})
180178

181179
if self.name == 'OpenBLAS':
182180
self.log.info("[pre-module hook] Set OMP_NUM_THREADS=1 in OpenBLAS module")
@@ -197,18 +195,14 @@ def pre_module_hook(self, *args, **kwargs):
197195
self.log.info("[pre-module hook] Set SLURM_MPI_TYPE=%s in OpenMPI module" % mpi_type)
198196
# Must be done this way, updating self.cfg['modextravars']
199197
# directly doesn't work due to templating.
200-
en_templ = self.cfg.enable_templating
201-
self.cfg.enable_templating = False
202-
self.cfg['modextravars'].update({'SLURM_MPI_TYPE': mpi_type})
203-
# Unfortunately UCX doesn't yet work for unknown reasons. Make sure it is off.
204-
self.cfg['modextravars'].update({'SLURM_PMIX_DIRECT_CONN_UCX': 'false'})
205-
self.cfg.enable_templating = en_templ
198+
with self.cfg.disable_templating():
199+
self.cfg['modextravars'].update({'SLURM_MPI_TYPE': mpi_type})
200+
# Unfortunately UCX doesn't yet work for unknown reasons. Make sure it is off.
201+
self.cfg['modextravars'].update({'SLURM_PMIX_DIRECT_CONN_UCX': 'false'})
206202

207203
if self.name == 'PMIx':
208204
# This is a, hopefully, temporary workaround for https://github.com/pmix/pmix/issues/1114
209205
if LooseVersion(self.version) > LooseVersion('2') and LooseVersion(self.version) < LooseVersion('3'):
210206
self.log.info("[pre-module hook] Set PMIX_MCA_gds=^ds21 in PMIx module")
211-
en_templ = self.cfg.enable_templating
212-
self.cfg.enable_templating = False
213-
self.cfg['modextravars'].update({'PMIX_MCA_gds': '^ds21'})
214-
self.cfg.enable_templating = en_templ
207+
with self.cfg.disable_templating():
208+
self.cfg['modextravars'].update({'PMIX_MCA_gds': '^ds21'})

‎easybuild/base/generaloption.py‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,13 +40,30 @@
4040
from functools import reduce
4141
from optparse import Option, OptionGroup, OptionParser, OptionValueError, Values
4242
from optparse import SUPPRESS_HELP as nohelp # supported in optparse of python v2.4
43-
from optparse import gettext as _gettext # this is gettext.gettext normally
4443

4544
from easybuild.base.fancylogger import getLogger, setroot, setLogLevel, getDetailsLogLevels
4645
from easybuild.base.optcomplete import autocomplete, CompleterOption
4746
from easybuild.tools.py2vs3 import StringIO, configparser, string_type
4847
from easybuild.tools.utilities import mk_rst_table, nub, shell_quote
4948

49+
try:
50+
import gettext
51+
eb_translation = None
52+
53+
def get_translation():
54+
global eb_translation
55+
if not eb_translation:
56+
# Finding a translation is expensive, so do only once
57+
domain = gettext.textdomain()
58+
eb_translation = gettext.translation(domain, gettext.bindtextdomain(domain), fallback=True)
59+
return eb_translation
60+
61+
def _gettext(message):
62+
return get_translation().gettext(message)
63+
except ImportError:
64+
def _gettext(message):
65+
return message
66+
5067

5168
HELP_OUTPUT_FORMATS = ['', 'rst', 'short', 'config']
5269

0 commit comments

Comments
 (0)