Merge branch 'develop' into easystack_easyconfigs

Author: boegel

.github/workflows/bootstrap_script.yml

@@ -3,7 +3,7 @@ name: test EasyBuild bootstrap script
 on: [push, pull_request]
 jobs:
   setup:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
         lmod7: Lmod-7.8.22
         lmod8: Lmod-8.7.6
@@ -14,7 +14,7 @@ jobs:
       - run: "true"
   build:
     needs: setup
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         # Don't run for Python 3.8, 3.9 , people should just use `pip install easybuild`
@@ -64,11 +64,11 @@ jobs:
 
     - name: install OS & Python packages
       run: |
-        # disable apt-get update, we don't really need it,
+        # disable apt update, we don't really need it,
         # and it does more harm than good (it's fairly expensive, and it results in flaky test runs)
-        # sudo apt-get update
+        # sudo apt update
         # for modules tool
-        sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
+        sudo apt install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
         # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
         # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
         if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then

.github/workflows/container_tests.yml

@@ -3,7 +3,7 @@ name: Tests for container support
 on: [push, pull_request]
 jobs:
   build:
-    # stick to Ubuntu 18.04, where we can still easily install yum via 'apt-get install'
+    # stick to Ubuntu 18.04, where we can still easily install yum via 'apt install'
     runs-on: ubuntu-18.04
     strategy:
       matrix:
@@ -21,12 +21,12 @@ jobs:
     - name: install OS & Python packages
       run: |
         # ensure package list is up to date to avoid 404's for old packages
-        sudo apt-get update -yqq
+        sudo apt update -yqq
         # for building Singularity images
-        sudo apt-get install rpm
-        sudo apt-get install yum
+        sudo apt install rpm
+        sudo apt install yum
         # for modules tool
-        sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
+        sudo apt install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
         # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
         # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
         if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
@@ -49,7 +49,7 @@ jobs:
     - name: install Singularity
       run: |
         # install alien, which can be used to convert RPMs to Debian packages
-        sudo apt-get install alien
+        sudo apt install alien
         alien --version
         # determine latest version of Singularity available in EPEL, and download RPM
         singularity_rpm=$(curl -sL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/s/ | grep singularity | sed 's/.*singularity/singularity/g' | sed 's/rpm.*/rpm/g')

.github/workflows/eb_command.yml

@@ -3,7 +3,7 @@ name: Tests for the 'eb' command
 on: [push, pull_request]
 jobs:
   test-eb:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10']
@@ -25,7 +25,7 @@ jobs:
         pip install --upgrade pip
         pip --version
         # install packages required for modules tool
-        sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
+        sudo apt install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
         # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
         # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
         if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
@@ -77,6 +77,10 @@ jobs:
               echo "Looking for pattern \"${pattern}\" in eb_version.out..."
               grep "$pattern" eb_version.out
           done
+          if grep -q "Considering ''" eb_version.out; then
+              echo '`eb` did wrongly consider an empty command'
+              false
+          fi
           # also check when specifying Python command via $EB_PYTHON
           for eb_python in "python${pymajver}" "python${pymajminver}"; do
               export EB_PYTHON="${eb_python}"

.github/workflows/linting.yml

@@ -2,7 +2,7 @@ name: Static Analysis
 on: [push, pull_request]
 jobs:
   python-linting:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10']

.github/workflows/unit_tests.yml

@@ -3,7 +3,7 @@ name: EasyBuild framework unit tests
 on: [push, pull_request]
 jobs:
   setup:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
         lmod7: Lmod-7.8.22
         lmod8: Lmod-8.7.6
@@ -14,7 +14,7 @@ jobs:
       - run: "true"
   build:
     needs: setup
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         python: [2.7, 3.6]
@@ -86,20 +86,20 @@ jobs:
 
     - name: install OS & Python packages
       run: |
-        # disable apt-get update, we don't really need it,
+        # disable apt update, we don't really need it,
         # and it does more harm than good (it's fairly expensive, and it results in flaky test runs)
-        # sudo apt-get update
+        # sudo apt update
         # for modules tool
-        sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
+        sudo apt install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
         # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
         # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
         if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
             sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
         fi
         # for GitPython, python-hglib
-        sudo apt-get install git mercurial
+        sudo apt install git mercurial
         # dep for GC3Pie
-        sudo apt-get install time
+        sudo apt install time
         # Python packages
         pip --version
         pip install --upgrade pip

RELEASE_NOTES

@@ -4,6 +4,29 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.6.1 (September 12th 2022)
+----------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - add script to find dependencies of Python packages (#3839)
+  - add 'ai' default module class (#4053)
+- various bug fixes, including:
+  - fix code style issues reported by recent flake8 linter (#4049)
+  - stick to autopep8 < 1.7.0 with Python 2.7 (#4055)
+  - ensure we call EasyBlock.patch_step for postinstallpatches (#4063)
+  - fix leaked handles in set_columns, complete_cmd, run_cmd_qa, det_terminal_size functions + tests (#4066)
+  - fix `quote_str` when string with both ' and " ends with a double quote (#4068)
+  - fix type-checking of patches to allow dict values + correctly handle patches specified as dict values in --new-pr (#4070)
+  - relax toolchain test by accepting both -march=native (x86_64) and -mcpu=native (aarch64) (#4071)
+- other changes:
+  - run python in the same process as `eb` wrapper script by using `exec` (#4048)
+  - add get_linked_libs_raw function, and use it from both check_linked_shared_libs and sanity_check_rpath (#4051)
+  - update CI workflows (except container tests) to use Ubuntu 20.04, since Ubuntu 18.04 is deprecated (#4064)
+  - use SYSTEM constant for dependency that uses system toolchain in dumped easyconfig (#4069)
+
+
 v4.6.0 (July 8th 2022)
 ----------------------
 

easybuild/base/generaloption.py

@@ -43,7 +43,7 @@
 
 from easybuild.base.fancylogger import getLogger, setroot, setLogLevel, getDetailsLogLevels
 from easybuild.base.optcomplete import autocomplete, CompleterOption
-from easybuild.tools.py2vs3 import StringIO, configparser, string_type
+from easybuild.tools.py2vs3 import StringIO, configparser, string_type, subprocess_popen_text
 from easybuild.tools.utilities import mk_rst_table, nub, shell_quote
 
 try:
@@ -80,7 +80,9 @@ def set_columns(cols=None):
         stty = '/usr/bin/stty'
         if os.path.exists(stty):
             try:
-                cols = int(os.popen('%s size 2>/dev/null' % stty).read().strip().split(' ')[1])
+                with open(os.devnull, 'w') as devnull:
+                    proc = subprocess_popen_text([stty, "size"], stderr=devnull)
+                    cols = int(proc.communicate()[0].strip().split(' ')[1])
             except (AttributeError, IndexError, OSError, ValueError):
                 # do nothing
                 pass

easybuild/framework/easyblock.py

@@ -96,7 +96,8 @@
 from easybuild.tools.package.utilities import package
 from easybuild.tools.py2vs3 import extract_method_name, string_type
 from easybuild.tools.repository.repository import init_repository
-from easybuild.tools.systemtools import check_linked_shared_libs, det_parallelism, get_shared_lib_ext, use_group
+from easybuild.tools.systemtools import check_linked_shared_libs, det_parallelism, get_linked_libs_raw
+from easybuild.tools.systemtools import get_shared_lib_ext, use_group
 from easybuild.tools.utilities import INDENT_4SPACES, get_class_for, nub, quote_str
 from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg
 from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION
@@ -2374,33 +2375,34 @@ def check_checksums_for(self, ent, sub='', source_cnt=None):
             checksum_issues.append(msg)
 
         for fn, checksum in zip(sources + patches, checksums):
+
+            # a checksum may be specified as a dictionary which maps filename to actual checksum
+            # for example when different source files are used for different CPU architectures
             if isinstance(checksum, dict):
-                # sources entry may be a dictionary rather than just a string value with filename
-                if isinstance(fn, dict):
-                    filename = fn['filename']
-                else:
-                    filename = fn
-                checksum = checksum.get(filename)
-
-            # take into account that we may encounter a tuple of valid SHA256 checksums
-            # (see https://github.com/easybuilders/easybuild-framework/pull/2958)
-            if isinstance(checksum, tuple):
-                # 1st tuple item may indicate checksum type, must be SHA256 or else it's blatently ignored here
-                if len(checksum) == 2 and checksum[0] == CHECKSUM_TYPE_SHA256:
-                    valid_checksums = (checksum[1],)
-                else:
-                    valid_checksums = checksum
+                checksums_to_check = checksum.values()
             else:
-                valid_checksums = (checksum,)
-
-            non_sha256_checksums = [c for c in valid_checksums if not is_sha256_checksum(c)]
-            if non_sha256_checksums:
-                if all(c is None for c in non_sha256_checksums):
-                    print_warning("Found %d None checksum value(s), please make sure this is intended!" %
-                                  len(non_sha256_checksums))
+                checksums_to_check = [checksum]
+
+            for checksum in checksums_to_check:
+                # take into account that we may encounter a tuple of valid SHA256 checksums
+                # (see https://github.com/easybuilders/easybuild-framework/pull/2958)
+                if isinstance(checksum, tuple):
+                    # 1st tuple item may indicate checksum type, must be SHA256 or else it's blatently ignored here
+                    if len(checksum) == 2 and checksum[0] == CHECKSUM_TYPE_SHA256:
+                        valid_checksums = (checksum[1],)
+                    else:
+                        valid_checksums = checksum
                 else:
-                    msg = "Non-SHA256 checksum(s) found for %s: %s" % (fn, valid_checksums)
-                    checksum_issues.append(msg)
+                    valid_checksums = (checksum,)
+
+                non_sha256_checksums = [c for c in valid_checksums if not is_sha256_checksum(c)]
+                if non_sha256_checksums:
+                    if all(c is None for c in non_sha256_checksums):
+                        print_warning("Found %d None checksum value(s), please make sure this is intended!" %
+                                      len(non_sha256_checksums))
+                    else:
+                        msg = "Non-SHA256 checksum(s) found for %s: %s" % (fn, valid_checksums)
+                        checksum_issues.append(msg)
 
         return checksum_issues
 
@@ -2875,7 +2877,9 @@ def apply_post_install_patches(self, patches=None):
 
         self.log.debug("Post-install patches to apply: %s", patches)
         if patches:
-            self.patch_step(beginpath=self.installdir, patches=patches)
+            # self may be inherited from the Bundle easyblock and that patch_step is a no-op
+            # To allow postinstallpatches for Bundle, and derived, easyblocks we directly call EasyBlock.patch_step
+            EasyBlock.patch_step(self, beginpath=self.installdir, patches=patches)
 
     def post_install_step(self):
         """
@@ -2994,24 +2998,15 @@ def sanity_check_rpath(self, rpath_dirs=None):
                 for path in [os.path.join(dirpath, x) for x in os.listdir(dirpath)]:
                     self.log.debug("Sanity checking RPATH for %s", path)
 
-                    out, ec = run_cmd("file %s" % path, simple=False, trace=False)
-                    if ec:
-                        fail_msg = "Failed to run 'file %s': %s" % (path, out)
-                        self.log.warning(fail_msg)
-                        fails.append(fail_msg)
-
-                    # only run ldd/readelf on dynamically linked executables/libraries
-                    # example output:
-                    # ELF 64-bit LSB executable, x86-64, version 1 (SYSV), dynamically linked (uses shared libs), ...
-                    # ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, not stripped
-                    if "dynamically linked" in out:
+                    out = get_linked_libs_raw(path)
+
+                    if out is None:
+                        msg = "Failed to determine dynamically linked libraries for %s, "
+                        msg += "so skipping it in RPATH sanity check"
+                        self.log.debug(msg, path)
+                    else:
                         # check whether all required libraries are found via 'ldd'
-                        out, ec = run_cmd("ldd %s" % path, simple=False, trace=False)
-                        if ec:
-                            fail_msg = "Failed to run 'ldd %s': %s" % (path, out)
-                            self.log.warning(fail_msg)
-                            fails.append(fail_msg)
-                        elif not_found_regex.search(out):
+                        if not_found_regex.search(out):
                             fail_msg = "One or more required libraries not found for %s: %s" % (path, out)
                             self.log.warning(fail_msg)
                             fails.append(fail_msg)
@@ -3030,9 +3025,6 @@ def sanity_check_rpath(self, rpath_dirs=None):
                             fails.append(fail_msg)
                         else:
                             self.log.debug("Output of 'readelf -d %s' checked, looks OK", path)
-
-                    else:
-                        self.log.debug("%s is not dynamically linked, so skipping it in RPATH sanity check", path)
             else:
                 self.log.debug("Not sanity checking files in non-existing directory %s", dirpath)
 

easybuild/framework/easyconfig/easyconfig.py

@@ -50,7 +50,7 @@
 import easybuild.tools.filetools as filetools
 from easybuild.base import fancylogger
 from easybuild.framework.easyconfig import MANDATORY
-from easybuild.framework.easyconfig.constants import EXTERNAL_MODULE_MARKER
+from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS, EXTERNAL_MODULE_MARKER
 from easybuild.framework.easyconfig.default import DEFAULT_CONFIG
 from easybuild.framework.easyconfig.format.convert import Dependency
 from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS
@@ -1589,7 +1589,7 @@ def _parse_dependency(self, dep, hidden=False, build_only=False):
 
         # (true) boolean value simply indicates that a system toolchain is used
         elif isinstance(tc_spec, bool) and tc_spec:
-            tc = {'name': SYSTEM_TOOLCHAIN_NAME, 'version': ''}
+            tc = EASYCONFIG_CONSTANTS['SYSTEM'][0]
 
         # two-element list/tuple value indicates custom toolchain specification
         elif isinstance(tc_spec, (list, tuple,)):

easybuild/framework/easyconfig/format/one.py

@@ -76,16 +76,19 @@ def dump_dependency(dep, toolchain, toolchain_hierarchy=None):
     else:
         # minimal spec: (name, version)
         tup = (dep['name'], dep['version'])
+        res = None
         if all(dep['toolchain'] != subtoolchain for subtoolchain in toolchain_hierarchy):
             if dep[SYSTEM_TOOLCHAIN_NAME]:
-                tup += (dep['versionsuffix'], True)
+                # use SYSTEM constant to indicate that system toolchain should be used for this dependency
+                res = re.sub(r'\)$', ', SYSTEM)', str(tup + (dep['versionsuffix'],)))
             else:
                 tup += (dep['versionsuffix'], (dep['toolchain']['name'], dep['toolchain']['version']))
 
         elif dep['versionsuffix']:
             tup += (dep['versionsuffix'],)
 
-        res = str(tup)
+        if res is None:
+            res = str(tup)
     return res