Skip to content

Commit f25817b

Browse files
migueldiascostamigueldiascosta
committed
Merge branch 'develop' into missing_patches
2 parents 78ecc14 + 23071bd commit f25817b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

91 files changed

+5143
-1148
lines changed

‎.github/workflows/bootstrap_script.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ jobs:
107107
EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
108108
EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
109109
EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
110-
EB_BOOTSTRAP_EXPECTED="20210618.01 e5d477d717c6d3648ba2027ab735713ba5804fbf52f4b4adcca0bc1379b44618"
110+
EB_BOOTSTRAP_EXPECTED="20210715.01 784dd29063d941be2d8b70e4c2eec12a9afe360f3ef8f753dcb518abf43ca7de"
111111
test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
112112
113113
# test bootstrap script

‎.github/workflows/container_tests.yml‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
2+
name: Tests for container support
3+
on: [push, pull_request]
4+
jobs:
5+
build:
6+
# stick to Ubuntu 18.04, where we can still easily install yum via 'apt-get install'
7+
runs-on: ubuntu-18.04
8+
strategy:
9+
matrix:
10+
python: [2.7, 3.6]
11+
fail-fast: false
12+
steps:
13+
- uses: actions/checkout@v2
14+
15+
- name: set up Python
16+
uses: actions/setup-python@v2
17+
with:
18+
python-version: ${{matrix.python}}
19+
architecture: x64
20+
21+
- name: install OS & Python packages
22+
run: |
23+
# for building Singularity images
24+
sudo apt-get install rpm
25+
sudo apt-get install yum
26+
# for modules tool
27+
sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
28+
# fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
29+
# needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
30+
if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
31+
sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
32+
fi
33+
34+
- name: install Lmod
35+
run: |
36+
# avoid downloading modules tool sources into easybuild-framework dir
37+
cd $HOME
38+
export INSTALL_DEP=$GITHUB_WORKSPACE/easybuild/scripts/install_eb_dep.sh
39+
# install Lmod
40+
source $INSTALL_DEP Lmod-8.4.27 $HOME
41+
# changes in environment are not passed to other steps, so need to create files...
42+
echo $MOD_INIT > mod_init
43+
echo $PATH > path
44+
if [ ! -z $MODULESHOME ]; then echo $MODULESHOME > moduleshome; fi
45+
46+
# see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181
47+
- name: install Singularity
48+
run: |
49+
# install alien, which can be used to convert RPMs to Debian packages
50+
sudo apt-get install alien
51+
alien --version
52+
# determine latest version of Singularity available in EPEL, and download RPM
53+
singularity_rpm=$(curl -sL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/s/ | grep singularity | sed 's/.*singularity/singularity/g' | sed 's/rpm.*/rpm/g')
54+
curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/s/${singularity_rpm}
55+
# convert Singularity RPM to Debian package, and install it
56+
sudo alien -d ${singularity_rpm}
57+
sudo apt install ./singularity*.deb
58+
singularity --version
59+
60+
- name: install sources
61+
run: |
62+
# install from source distribution tarball, to test release as published on PyPI
63+
python setup.py sdist
64+
ls dist
65+
export PREFIX=/tmp/$USER/$GITHUB_SHA
66+
pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
67+
pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/develop.tar.gz
68+
69+
- name: run test
70+
run: |
71+
# run tests *outside* of checked out easybuild-framework directory,
72+
# to ensure we're testing installed version (see previous step)
73+
cd $HOME
74+
# initialize environment for modules tool
75+
if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi
76+
source $(cat $HOME/mod_init); type module
77+
# make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that);
78+
# also pick up changes to $PATH set by sourcing $MOD_INIT
79+
export PREFIX=/tmp/$USER/$GITHUB_SHA
80+
export PATH=$PREFIX/bin:$(cat $HOME/path)
81+
export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH
82+
eb --version
83+
# create $HOME/.rpmmacros, see also https://github.com/apptainer/singularity/issues/241
84+
echo '%_var /var' > $HOME/.rpmmacros
85+
echo '%_dbpath %{_var}/lib/rpm' >> $HOME/.rpmmacros
86+
# build CentOS 7 container image for bzip2 1.0.8 using EasyBuild;
87+
# see https://docs.easybuild.io/en/latest/Containers.html
88+
curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/develop/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb
89+
export EASYBUILD_CONTAINERPATH=$PWD
90+
export EASYBUILD_CONTAINER_CONFIG='bootstrap=yum,osversion=7'
91+
eb bzip2-1.0.8.eb --containerize --experimental --container-build-image
92+
singularity exec bzip2-1.0.8.sif command -v bzip2 | grep '/app/software/bzip2/1.0.8/bin/bzip2' || (echo "Path to bzip2 '$which_bzip2' is not correct" && exit 1)
93+
singularity exec bzip2-1.0.8.sif bzip2 --help

‎.github/workflows/eb_command.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ jobs:
66
runs-on: ubuntu-18.04
77
strategy:
88
matrix:
9-
python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9]
9+
python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10']
1010
fail-fast: false
1111
steps:
1212
- uses: actions/checkout@v2

‎.github/workflows/linting.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ jobs:
77
- uses: actions/checkout@v2
88

99
- name: set up Python
10-
uses: actions/setup-python@v1
10+
uses: actions/setup-python@v2
1111
with:
1212
python-version: 3.8
1313

‎.github/workflows/unit_tests.yml‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,12 @@ jobs:
6262
- python: 3.9
6363
modules_tool: ${{needs.setup.outputs.lmod8}}
6464
module_syntax: Tcl
65+
- python: '3.10'
66+
modules_tool: ${{needs.setup.outputs.lmod8}}
67+
module_syntax: Lua
68+
- python: '3.10'
69+
modules_tool: ${{needs.setup.outputs.lmod8}}
70+
module_syntax: Tcl
6571
# There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set
6672
# Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7)
6773
- python: 3.6
@@ -110,13 +116,14 @@ jobs:
110116
GITHUB_TOKEN: ${{secrets.TEST_GITHUB_TOKEN}}
111117
run: |
112118
# don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools,
119+
# and only when testing with Lua as module syntax,
113120
# to avoid hitting GitHub rate limit;
114121
# tests that require a GitHub token are skipped automatically when no GitHub token is available
115-
if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]]; then
122+
if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]] && [[ "${{matrix.modules_syntax}}" == 'Lua' ]]; then
116123
if [ ! -z $GITHUB_TOKEN ]; then
117124
if [ "x${{matrix.python}}" == 'x2.6' ];
118125
then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())";
119-
else SET_KEYRING="import keyrings; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
126+
else SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
120127
fi;
121128
python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
122129
fi
@@ -193,7 +200,7 @@ jobs:
193200
# run test suite
194201
python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
195202
# try and make sure output of running tests is clean (no printed messages/warnings)
196-
IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.5|from cryptography.*default_backend|CryptographyDeprecationWarning: Python 2"
203+
IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.5|from cryptography.*default_backend|CryptographyDeprecationWarning: Python 2|from cryptography.utils import int_from_bytes"
197204
# '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches)
198205
PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
199206
test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite\n${PRINTED_MSG}" && exit 1)

‎RELEASE_NOTES‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,172 @@ For more detailed information, please see the git log.
44
These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
55

66

7+
v4.5.3 (February 11th 2022)
8+
---------------------------
9+
10+
update/bugfix release
11+
12+
- various enhancements, including:
13+
- also check for git in --check-eb-deps (#3954)
14+
- add end2end test for 'eb --containerize' (#3958)
15+
- various bug fixes, including:
16+
- take into account that patch files can also be zipped when checking filename extension for patches (#3936)
17+
- initialize BACKUP_PKG_URL with empty string in install_eb_dep.sh script (#3939)
18+
- fix get_os_name and get_os_version to avoid reporting UNKNOWN in output of eb --show-system-info (#3942)
19+
- consistently use actions/setup-python@v2 in CI workflows (#3944)
20+
- switch to using pip3 for installing EasyBuild in container recipes generated by EasyBuild (#3945)
21+
- specify easybuild.io as EasyBuild homepage in setup.py (#3947)
22+
- avoid crash in get_os_version on modern SLES-based OSs (#3955)
23+
- other changes:
24+
- indicate compatibility with Python 3.10 in setup.py (#3940)
25+
26+
27+
v4.5.2 (January 24th 2022)
28+
--------------------------
29+
30+
update/bugfix release
31+
32+
- various enhancements, including:
33+
- automatically prepend value for env-for-shebang configuration setting with sysroot (#3919)
34+
- enhance update_build_option to return original value + add support for update_build_options (#3923)
35+
- fix compatibility with Python 3.10 (#3926, #3933)
36+
- also take /etc/os-release into account in get_os_name and get_os_version (#3930)
37+
- enhance get_cpu_architecture and get_cpu_family to be aware of RISC-V (#3931)
38+
- various bug fixes, including:
39+
- relax pattern checks in test_toy_exts_parallel (#3921)
40+
- use sources.easybuild.io as backup URL for downloading Environment Modules sources to run test suite (#3928)
41+
- make intelfftw toolchain component aware of GCCcore (#3929)
42+
- other changes:
43+
- deprecate use of (actual) patch files that don't have a filename ending with .patch (#3920)
44+
- remove version restriction for keyring in requirements.txt to test with latest version (#3925)
45+
- simplify requirements.txt by removing Python 2.6 support (#3927)
46+
- only run GitHub tests when testing with Lua module syntax, to avoid hitting GitHub rate limit when running tests (#3938)
47+
48+
49+
v4.5.1 (December 13th 2021)
50+
---------------------------
51+
52+
update/bugfix release
53+
54+
- various enhancements, including:
55+
- also dump environment to reprod directory (#3374)
56+
- determine which extensions can be skipped in parallel (if --parallel-extensions-install is enabled) (#3890)
57+
- fall back to sequential installation for extensions with unknown dependencies when using --parallel-extensions-install (#3906)
58+
- allow oversubscription in sanity check for OpenMPI-based toolchains (#3909)
59+
- various bug fixes, including:
60+
- don't try to ensure absolute path for path part of repositorypath (#3893, #3899)
61+
- fix typo in EULA agreement error message (#3894)
62+
- only remove lock if it was created in the same EasyBuild session (not if it existed already) (#3889)
63+
- introduce EasyBlock.post_init method to correctly define builddir variable when build-in-installdir mode is enabled in easyconfig or easyblock (#3900)
64+
- also show download progress bar when using --inject-checksums (#3905)
65+
- pick up custom extract_cmd specified for extension (#3907)
66+
- make test_run_cmd_async more robust against fluke failures (#3908)
67+
68+
69+
v4.5.0 (October 29th 2021)
70+
--------------------------
71+
72+
feature release
73+
74+
- various enhancements, including:
75+
- add --review-pr-max and --review-pr-filter options to limit easyconfigs used by --review-pr + retain order of easyconfigs being compared with (#3754)
76+
- use Rich (if available) to show progress bars when installing easyconfigs (#3823, #3826, #3833, #3835, #3844, #3864, #3867, #3882)
77+
- see also https://docs.easybuild.io/en/latest/Progress_bars.html
78+
- add support for checking required/optional EasyBuild dependencies via 'eb --check-eb-deps' (#3829)
79+
- add support for collecting GPU info (via nvidia-smi), and include it in --show-system-info and test report (#3851)
80+
- added support for --insecure-download configuration option (#3859)
81+
- make intelfftw toolchain component aware of imkl-FFTW module (#3861)
82+
- make sure the contrib/hooks tree is included in the source tarball for easybuild-framework (#3862)
83+
- add check_async_cmd function to facilitate checking on asynchronously running commands (#3865, #3881)
84+
- add initial/experimental support for installing extensions in parallel (#3667, #3878)
85+
- see also https://docs.easybuild.io/en/latest/Installing_extensions_in_parallel.html
86+
- filter out duplicate paths added to module files, and print warning when they occur (#3770, #3874)
87+
- various bug fixes, including:
88+
- ensure that path configuration options have absolute path values (#3832)
89+
- fix broken test by taking into account changed error raised by Python 3.9.7 when copying directory via shutil.copyfile (#3840)
90+
- ensure newer location of CUDA stubs is taken into account by RPATH filter (#3850)
91+
- replace 'which' by 'command -v' in 'eb' wrapper script to avoid dependency on 'which' command (#3852)
92+
- refactor EasyBlock to decouple collecting of information on extension source/patch files from downloading them (#3860)
93+
- in this context, the EasyBlock.fetch_extension_sources method was deprecated, and replaced by EasyBlock.collect_exts_file_info
94+
- fix library paths to add to $LDFLAGS for intel-compilers toolchain component (#3866)
95+
- remove '--depth 1' from git clone when 'commit' specified (#3871, #3872)
96+
- make sure correct include directory is used for FlexiBLAS toolchain component (#3875)
97+
- explictly disable rebase when pulling develop branch to create a merge commit, since not specifying how to reconcile divergent branches is an error with Git 2.33.1 and newer (#3879)
98+
- tweak test_http_header_fields_urlpat to download from sources.easybuild.io rather than https://ftp.gnu.org (#3883)
99+
- other changes:
100+
- change copy_file function to raise an error when trying to copy non-existing file (#3836, #3855, #3877)
101+
- only print the hook messages if EasyBuild is running in debug mode (#3843)
102+
- deprecate old toolchain versions (pre-2019a common toolchains) (#3876, #3884)
103+
- see also https://docs.easybuild.io/en/latest/Deprecated-easyconfigs.html#deprecated-toolchains
104+
105+
106+
v4.4.2 (September 7th 2021)
107+
---------------------------
108+
109+
update/bugfix release
110+
111+
- various enhancements, including:
112+
- add per-extension timing in output produced by eb command (#3734)
113+
- add definition for new toolchain nvpsmpic (NVHPC + ParaStationMPI + CUDA) (#3736)
114+
- include list of missing libraries in warning about missing FFTW libraries in imkl toolchain component (#3776)
115+
- check for recursive symlinks by default before copying a folder (#3784)
116+
- add --filter-ecs configuration option to filter out easyconfigs from set of easyconfigs to install (#3796)
117+
- check type of source_tmpl value for extensions, ensure it's a string value (not a list) (#3799)
118+
- also define $BLAS_SHARED_LIBS & co in build environment (analogous to $BLAS_STATIC_LIBS) (#3800)
119+
- report use of --ignore-test-failure in success message in output (#3806)
120+
- add get_cuda_cc_template_value method to EasyConfig class (#3807)
121+
- add support for fix_bash_shebang_for (#3808)
122+
- pick up $MODULES_CMD to facilitate using Environment Modules 4.x as modules tool (#3816)
123+
- use more sensible branch name for creating easyblocks PR with --new-pr (#3817)
124+
- various bug fixes, including:
125+
- remove Python 2.6 from list of supported Python versions in setup.py (#3767)
126+
- don't add directory that doesn't include any files to $PATH or $LD_LIBRARY_PATH (#3769)
127+
- make logdir writable also when --stop/--fetch is used and --read-only-installdir is enabled (#3771)
128+
- fix forgotten renaming of 'l' to 'char' __init__.py that is created for included Python modules (#3773)
129+
- fix verify_imports by deleting all imported modules before re-importing them one by one (#3780)
130+
- fix ignore_test_failure not set for Extension instances (#3782)
131+
- update iompi toolchain to intel-compiler subtoolchain for oneAPI versions (>= iompi 2020.12) (#3785)
132+
- don't parse patch files as easyconfigs when searching for where patch file is used (#3786)
133+
- make sure git clone with a tag argument actually downloads a tag (#3795)
134+
- fix CI by excluding GC3Pie 2.6.7 (which is broken with Python 2) and improve error reporting for option parsing (#3798)
135+
- correctly resolve templates for patches in extensions when uploading to GitHub (#3805)
136+
- add --easystack to ignored options when submitting job (#3813)
137+
- other changes:
138+
- speed up tests by caching checked paths in set_tmpdir + less test cases for test_compiler_dependent_optarch (#3802)
139+
- speed up set_parallel method in EasyBlock class (#3812)
140+
141+
142+
v4.4.1 (July 6th 2021)
143+
----------------------
144+
145+
update/bugfix release
146+
147+
- various enhancements, including:
148+
- enhance detection of patch files supplied to 'eb' command with better error messages (#3709)
149+
- add per-step timing information (#3716)
150+
- add module-write hook (#3728)
151+
- add ignore-test-failure configuration option to ignore failing test step (#3732)
152+
- add toolchain definition for nvompic (NVHPC + OpenMPI) (#3735)
153+
- warn about generic milestone in --review-pr and --merge-pr (#3751)
154+
- various bug fixes, including:
155+
- don't override COMPILER_MODULE_NAME, inherited from Ffmpi, in Fujitsu toolchain definition (#3721)
156+
- avoid overwritting pr_nr in post_pr_test_report for reports with --include-easyblocks-from-pr (#3724, #3726)
157+
- fix crash in get_config_dict when copying modules that were imported in easyconfig file (like 'import os') (#3729)
158+
- parse C standard flags in CFLAGS for Fujitsu compiler (#3731)
159+
- fix error message for --use-ccache (#3733)
160+
- error out when passing a list to run_cmd* to avoid running wrong command unintended, unless shell=True is used (#3737)
161+
- minor fixes to output of test reports when using multiple PRs (#3741)
162+
- fix location for modules installed with intel-compilers toolchain in HierarchicalMNS by always checking toolchain compiler name against template map (#3745)
163+
- avoid checking msg attribute of GitCommandError (#3756)
164+
- consider sources provided as dict in EasyBlock.check_checksums_for (#3758)
165+
- don't make changes to software installation directory when using --sanity-check-only (#3761)
166+
- honor specified easyblock for extensions (#3762)
167+
- other changes:
168+
- make sure that tests requiring a github token have 'github' in the test name so that they can be easily filtered (#3730)
169+
- deprecate EasyBuild bootstrap script (#3742)
170+
- use temporary download folder in test_http_header_fields_urlpat (#3755)
171+
172+
7173
v4.4.0 (June 2nd 2021)
8174
----------------------
9175

‎easybuild/base/frozendict.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,10 @@
2121
It can be used as a drop-in replacement for dictionaries where immutability is desired.
2222
"""
2323
import operator
24-
from collections import Mapping
2524
from functools import reduce
2625

2726
from easybuild.base import fancylogger
27+
from easybuild.tools.py2vs3 import Mapping
2828

2929

3030
# minor adjustments:

0 commit comments

Comments
 (0)