Merge pull request #3588 from easybuilders/4.3.x

release EasyBuild v4.3.3

Author: boegel

.github/workflows/unit_tests.yml

@@ -9,6 +9,7 @@ jobs:
         python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9]
         modules_tool: [Lmod-7.8.22, Lmod-8.2.9, modules-tcl-1.147, modules-3.2.10, modules-4.1.4]
         module_syntax: [Lua, Tcl]
+        lc_all: [""]
         # exclude some configuration for non-Lmod modules tool:
         # - don't test with Lua module syntax (only supported in Lmod)
         # - exclude Python 3.x versions other than 3.6, to limit test configurations
@@ -51,6 +52,13 @@ jobs:
             python: 3.8
           - modules_tool: Lmod-7.8.22
             python: 3.9
+        # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set
+        # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7)
+        include:
+          - python: 3.6
+            modules_tool: Lmod-8.2.9
+            module_syntax: Lua
+            lc_all: C
       fail-fast: false
     steps:
     - uses: actions/checkout@v2
@@ -69,7 +77,10 @@ jobs:
         # for modules tool
         sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
         # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
-        sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
+        # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
+        if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
+            sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
+        fi
         # for GitPython, python-hglib
         sudo apt-get install git mercurial
         # dep for GC3Pie
@@ -89,12 +100,20 @@ jobs:
         # see https://github.com/<username>/easybuild-framework/settings/secrets
         GITHUB_TOKEN: ${{secrets.TEST_GITHUB_TOKEN}}
       run: |
-        if [ ! -z $GITHUB_TOKEN ]; then
-          if [ "x${{matrix.python}}" == 'x2.6' ];
+        # don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools,
+        # to avoid hitting GitHub rate limit;
+        # tests that require a GitHub token are skipped automatically when no GitHub token is available
+        if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]]; then
+          if [ ! -z $GITHUB_TOKEN ]; then
+            if [ "x${{matrix.python}}" == 'x2.6' ];
               then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())";
               else SET_KEYRING="import keyrings; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
-          fi;
-          python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
+            fi;
+            python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
+          fi
+          echo "GitHub token installed!"
+        else
+          echo "Installation of GitHub token skipped!"
         fi
 
     - name: install modules tool
@@ -129,6 +148,7 @@ jobs:
         EB_VERBOSE: 1
         EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}}
         TEST_EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}}
+        LC_ALL: ${{matrix.lc_all}}
       run: |
           # run tests *outside* of checked out easybuild-framework directory,
           # to ensure we're testing installed version (see previous step)
@@ -189,7 +209,7 @@ jobs:
           EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
           EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
           EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
-          EB_BOOTSTRAP_EXPECTED="20200820.01 d490d229a18bd5eaa717bb8d5684d754729143d5e995e35a40c84d03ffb1de50"
+          EB_BOOTSTRAP_EXPECTED="20210106.01 c2d93de0dd91123eb4f51cfc16d1f5efb80f1d238b3d6cd100994086887a1ae0"
           test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
 
           # test bootstrap script

.travis.yml

@@ -89,7 +89,7 @@ script:
     - EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
     - EB_BOOTSTRAP_SHA256SUM=$(sha256sum $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
     - EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
-    - EB_BOOTSTRAP_EXPECTED="20200820.01 d490d229a18bd5eaa717bb8d5684d754729143d5e995e35a40c84d03ffb1de50"
+    - EB_BOOTSTRAP_EXPECTED="20210106.01 c2d93de0dd91123eb4f51cfc16d1f5efb80f1d238b3d6cd100994086887a1ae0"
     - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
     # test bootstrap script
     - python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap

RELEASE_NOTES

@@ -3,6 +3,39 @@ For more detailed information, please see the git log.
 
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
+
+v4.3.3 (February 23rd 2021)
+---------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - advise PR labels in --review-pr and add support for --add-pr-labels (#3177)
+  - add support for using customized HTTP headers in download_file (#3472, #3583)
+  - also take toolchain dependencies into account when defining template values (#3541, #3560)
+  - add support for --accept-eula configuration option + 'accept_eula' easyconfig parameter (#3535, #3536, #3546)
+  - detect 'SYSTEM' toolchain as special case in easystack files (#3543)
+  - enhance extract_cmd function to use 'cp -a' for shell scripts (.sh) (#3545)
+  - allow use of alternate envvar(s) to $HOME for user modules (#3558)
+  - use https://sources/easybuild.io as fallback source URL (#3572, #3576)
+  - add toolchain definition for iibff toolchain (#3574)
+  - add %(cuda_cc_space_sep)s and %(cuda_cc_semicolon_sep)s templates (#3578)
+  - add support for intel-compiler toolchain (>= 2021.x versions, oneAPI) (#3581, #3582)
+- various bug fixes, including:
+  - add --init and --recursive options to 'git submodule update' command that is used when creating source tarball for specific commit (#3537)
+  - filter out duplicate paths in RPATH wrapper script (#3538)
+  - don't clean up imported modules after verifying imports of included Python modules (#3544)
+  - avoid no-op changes to $LD_* environment variables in ModulesTool (#3553)
+  - fix UTF-8 encoding errors when running EasyBuild with Python 3.0.x-3.6.x (#3565)
+  - create lib64 symlink as a relative symlink (#3566)
+  - don't reuse variable name in the loop to fix adding extra compiler flags via toolchainopts (#3571)
+  - symlink 'lib' to 'lib64' if it doesn't exist (#3580)
+  - include %(mpi_cmd_prefix)s and %(cuda_*)s templates in output of --avail-easyconfig-templates (#3586)
+- other changes:
+  - rename EasyBlock._skip_step to EasyBlock.skip_step, to make it part of the public API (#3561)
+  - make symlinking of posix_c.so to posix.so in test suite configuration conditional (#3570)
+
+
 v4.3.2 (December 10th 2020)
 ---------------------------
 

easybuild/__init__.py

@@ -1,5 +1,5 @@
 ##
-# Copyright 2011-2020 Ghent University
+# Copyright 2011-2021 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/base/exceptions.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2015-2020 Ghent University
+# Copyright 2015-2021 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/base/fancylogger.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2011-2020 Ghent University
+# Copyright 2011-2021 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -580,6 +580,8 @@ def logToFile(filename, enable=True, filehandler=None, name=None, max_bytes=MAX_
         'maxBytes': max_bytes,
         'backupCount': backup_count,
     }
+    if sys.version_info[0] >= 3:
+        handleropts['encoding'] = 'utf-8'
     # logging to a file is going to create the file later on, so let's try to be helpful and create the path if needed
     directory = os.path.dirname(filename)
     if not os.path.exists(directory):

easybuild/base/generaloption.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2011-2020 Ghent University
+# Copyright 2011-2021 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/base/optcomplete.py

@@ -594,9 +594,8 @@ def autocomplete(parser, arg_completer=None, opt_completer=None, subcmd_complete
         if isinstance(debugfn, logging.Logger):
             debugfn.debug(txt)
         else:
-            f = open(debugfn, 'a')
-            f.write(txt)
-            f.close()
+            with open(debugfn, 'a') as fh:
+                fh.write(txt)
 
     # Exit with error code (we do not let the caller continue on purpose, this
     # is a run for completions only.)

easybuild/base/testing.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2014-2020 Ghent University
+# Copyright 2014-2021 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/__init__.py

@@ -1,5 +1,5 @@
 ##
-# Copyright 2009-2020 Ghent University
+# Copyright 2009-2021 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),