Merge branch 'develop' into review_pr_labels

Author: boegel

.github/workflows/unit_tests.yml

@@ -9,6 +9,7 @@ jobs:
         python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9]
         modules_tool: [Lmod-7.8.22, Lmod-8.2.9, modules-tcl-1.147, modules-3.2.10, modules-4.1.4]
         module_syntax: [Lua, Tcl]
+        lc_all: [""]
         # exclude some configuration for non-Lmod modules tool:
         # - don't test with Lua module syntax (only supported in Lmod)
         # - exclude Python 3.x versions other than 3.6, to limit test configurations
@@ -51,6 +52,13 @@ jobs:
             python: 3.8
           - modules_tool: Lmod-7.8.22
             python: 3.9
+        # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set
+        # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7)
+        include:
+          - python: 3.6
+            modules_tool: Lmod-8.2.9
+            module_syntax: Lua
+            lc_all: C
       fail-fast: false
     steps:
     - uses: actions/checkout@v2
@@ -69,7 +77,10 @@ jobs:
         # for modules tool
         sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
         # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
-        sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
+        # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
+        if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
+            sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
+        fi
         # for GitPython, python-hglib
         sudo apt-get install git mercurial
         # dep for GC3Pie
@@ -129,7 +140,11 @@ jobs:
         EB_VERBOSE: 1
         EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}}
         TEST_EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}}
+        LC_ALL: ${{matrix.lc_all}}
       run: |
+          # run tests *outside* of checked out easybuild-framework directory,
+          # to ensure we're testing installed version (see previous step)
+          cd $HOME
           # initialize environment for modules tool
           if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi
           source $(cat $HOME/mod_init); type module
@@ -161,15 +176,12 @@ jobs:
           # run test suite
           python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
           # try and make sure output of running tests is clean (no printed messages/warnings)
-          IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.5|from cryptography.*default_backend"
+          IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.5|from cryptography.*default_backend|CryptographyDeprecationWarning: Python 2"
           # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches)
           PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
           test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite\n${PRINTED_MSG}" && exit 1)
 
     - name: test bootstrap script
-      # skip testing of bootstrap script with Python 3.9,
-      # until an EasyBuild release that is compatible with Python 3.9 is available
-      if: ${{ matrix.python != 3.9 }}
       run: |
           # (re)initialize environment for modules tool
           if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi
@@ -189,7 +201,7 @@ jobs:
           EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
           EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
           EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
-          EB_BOOTSTRAP_EXPECTED="20200820.01 d490d229a18bd5eaa717bb8d5684d754729143d5e995e35a40c84d03ffb1de50"
+          EB_BOOTSTRAP_EXPECTED="20210106.01 c2d93de0dd91123eb4f51cfc16d1f5efb80f1d238b3d6cd100994086887a1ae0"
           test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
 
           # test bootstrap script

.travis.yml

@@ -89,7 +89,7 @@ script:
     - EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
     - EB_BOOTSTRAP_SHA256SUM=$(sha256sum $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
     - EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
-    - EB_BOOTSTRAP_EXPECTED="20200820.01 d490d229a18bd5eaa717bb8d5684d754729143d5e995e35a40c84d03ffb1de50"
+    - EB_BOOTSTRAP_EXPECTED="20210106.01 c2d93de0dd91123eb4f51cfc16d1f5efb80f1d238b3d6cd100994086887a1ae0"
     - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
     # test bootstrap script
     - python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap

MANIFEST.in

@@ -4,7 +4,7 @@ include optcomplete.bash
 recursive-include etc *
 recursive-include easybuild *py
 recursive-include easybuild/scripts *
-recursive-include test *py *eb
+recursive-include test *py *eb *yaml
 recursive-include test/framework/modules *
 recursive-include test/framework/sandbox/sources *
 include CONTRIBUTING.md

RELEASE_NOTES

@@ -3,6 +3,41 @@ For more detailed information, please see the git log.
 
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
+v4.3.2 (December 10th 2020)
+---------------------------
+
+update/bugfix release
+
+- add (experimental) support for specifying easyconfig files via an "easystack" file (#3479, #3511, #3515, #3517, #3520, #3521)
+  - see also https://easybuild.readthedocs.io/en/latest/Easystack-files.html
+- add definition for new 'gobff' toolchain using BLIS and LibFLAME (#3505)
+- various enhancements, including:
+  - add support for toolchain options like 'extra_cxxflags' to specify extra compiler options (#2193)
+  - fix combination of --copy-ec and --from-pr (#3482)
+  - enhance copy_files function: support single file target, error on empty input list, support verbose mode (#3483)
+  - cache result of fetch_files_from_pr function (mainly to speed up framework test suite) (#3484)
+  - add locate_files function to filetools module (#3485)
+  - add support for %(module_name)s template value (#3497)
+  - clarify input format for --cuda-compute-capabilities in 'eb --help' output (#3509)
+  - add support for skiping unit tests (test step) via --skip-test-step (#3524)
+- various bug fixes, including:
+  - also ignore vsc.* imports coming from from pkg_resources/__init__.py (setuptools) in fake vsc namespace (#3491)
+  - don't pass username in github_api_get_request when no GitHub token is available (#3494)
+  - also inject -rpath options for all entries in $LIBRARY_PATH in RPATH wrappers (#3495)
+  - avoid TypeError being raised by list_toolchains (#3499)
+  - check if PR is already merged in --merge-pr (#3502)
+  - graciously handle wrong PR id in fetch_pr_data (#3503)
+  - fix regression in apply_regex_substitutions: also accept list of paths to patch (#3507)
+  - update installation procedure for EasyBuild in generated Singularity container recipes (#3510)
+  - fix GitHub Actions workflow for test suite: run outside of repo checkout + also test bootstrap script with Python 3.9 (#3518)
+  - bump cryptography from 2.9.2 to 3.2 for Python 2 in requirements.txt (#3519)
+  - fix 'eb --help=rst' when running with Python 3 (#3525)
+- other changes:
+  - exclude test configurations with Lmod 7 and Python 3, except for Python 3.6 (#3496)
+  - significantly speed up parsing of easyconfig files by only extracting comments from an easyconfig file when they're actually needed (#3498)
+  - don't include file/ldd/readelf commands run during RPATH sanity check in --trace output (#3508)
+
+
 v4.3.1 (October 29th 2020)
 --------------------------
 

easybuild/__init__.py

@@ -1,5 +1,5 @@
 ##
-# Copyright 2011-2020 Ghent University
+# Copyright 2011-2021 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/base/exceptions.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2015-2020 Ghent University
+# Copyright 2015-2021 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/base/fancylogger.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2011-2020 Ghent University
+# Copyright 2011-2021 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -580,6 +580,8 @@ def logToFile(filename, enable=True, filehandler=None, name=None, max_bytes=MAX_
         'maxBytes': max_bytes,
         'backupCount': backup_count,
     }
+    if sys.version_info[0] >= 3:
+        handleropts['encoding'] = 'utf-8'
     # logging to a file is going to create the file later on, so let's try to be helpful and create the path if needed
     directory = os.path.dirname(filename)
     if not os.path.exists(directory):

easybuild/base/generaloption.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2011-2020 Ghent University
+# Copyright 2011-2021 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/base/optcomplete.py

@@ -594,9 +594,8 @@ def autocomplete(parser, arg_completer=None, opt_completer=None, subcmd_complete
         if isinstance(debugfn, logging.Logger):
             debugfn.debug(txt)
         else:
-            f = open(debugfn, 'a')
-            f.write(txt)
-            f.close()
+            with open(debugfn, 'a') as fh:
+                fh.write(txt)
 
     # Exit with error code (we do not let the caller continue on purpose, this
     # is a run for completions only.)

easybuild/base/testing.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2014-2020 Ghent University
+# Copyright 2014-2021 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),