Merge branch 'develop' into tests_python2

Author: boegel

.github/workflows/end2end.yml

@@ -0,0 +1,61 @@
+name: End-to-end test of EasyBuild in different distros
+on: [push, pull_request]
+jobs:
+  build_publish:
+    name: End-to-end test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        container:
+          - centos-7.9
+          - centos-8.5
+          - fedora-36
+          - opensuse-15.4
+          - rockylinux-8.8
+          - rockylinux-9.2
+          - ubuntu-20.04
+          - ubuntu-22.04
+      fail-fast: false
+    container:
+      image: ghcr.io/easybuilders/${{ matrix.container }}-amd64
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: download and unpack easyblocks and easyconfigs repositories
+        run: |
+          cd $HOME
+          for pkg in easyblocks easyconfigs; do
+              curl -OL https://github.com/easybuilders/easybuild-${pkg}/archive/develop.tar.gz
+              tar xfz develop.tar.gz
+              rm -f develop.tar.gz
+          done
+
+      - name: Set up environment
+        shell: bash
+        run: |
+          # collect environment variables to be set in subsequent steps in script that can be sourced
+          echo "export PATH=$PWD:$PATH" > /tmp/eb_env
+          echo "export PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop" >> /tmp/eb_env
+
+      - name: Run commands to check test environment
+        shell: bash
+        run: |
+          cmds=(
+            "whoami"
+            "pwd"
+            "env | sort"
+            "eb --version"
+            "eb --show-system-info"
+            "eb --check-eb-deps"
+            "eb --show-config"
+          )
+          for cmd in "${cmds[@]}"; do
+              echo ">>> $cmd"
+              sudo -u easybuild bash -l -c "source /tmp/eb_env; $cmd"
+          done
+
+      - name: End-to-end test of installing bzip2 with EasyBuild
+        shell: bash
+        run: |
+          sudo -u easybuild bash -l -c "source /tmp/eb_env; eb bzip2-1.0.8.eb --trace --robot"

RELEASE_NOTES

@@ -4,6 +4,34 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.8.1 (11 September 2023)
+--------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - add end-to-end test for running EasyBuild in different Linux distros using containers (#3968)
+  - suggest default title in `--review-pr` (#4287)
+  - add `build_and_install_loop` hooks to run before and after the install loop for individual easyconfigs (#4304)
+  - implement support for `cancel_hook` and `fail_hook` (#4315, #4325)
+  - add postiter hook to the list of steps so the corresponding hook can be used (#4316)
+  - add `run_shell_cmd` hook (#4323)
+  - add `build_info_msg` easyconfig parameter to print message during installation of an easyconfig (#4324)
+  - add `--silence-hook-trigger` configuration option to supress printing of debug message every time a hook is triggered (#4329)
+  - add support for using fine grained Github tokens (#4332)
+  - add definitions for ifbf and iofbf toolchain (#4337)
+  - add support for submodule filtering and specifying extra Git configuration in `git_config` (#4338, #4339)
+- various bug fixes, including:
+  - improve error when checksum dict has no entry for a file (#4150)
+  - avoid error being logged when `checksums.json` is not found (#4261)
+  - don't fail in `mkdir` if path gets created while processing it (#4300, #4328)
+  - ignore request for external module (meta)data when no modules tool is active (#4308)
+  - use sys.executable to obtain path to `python` command in tests, rather than assuming that `python` command is available in `$PATH` (#4309)
+  - fix `test_add_and_remove_module_path` by replacing string comparison of paths by checking whether they point to the same path (since symlinks may cause trouble) (#4312)
+  - enhance `Toolchain.get_flag` to handle lists (#4319)
+  - only add extensions in module file if there are extensions (#4331)
+
+
 v4.8.0 (7 July 2023)
 --------------------
 

easybuild/framework/easyblock.py

@@ -395,13 +395,13 @@ def get_checksums_from_json(self, always_read=False):
         :param always_read: always read the checksums.json file, even if it has been read before
         """
         if always_read or self.json_checksums is None:
-            try:
-                path = self.obtain_file("checksums.json", no_download=True)
+            path = self.obtain_file("checksums.json", no_download=True, warning_only=True)
+            if path is not None:
                 self.log.info("Loading checksums from file %s", path)
                 json_txt = read_file(path)
                 self.json_checksums = json.loads(json_txt)
-            # if the file can't be found, return an empty dict
-            except EasyBuildError:
+            else:
+                # if the file can't be found, return an empty dict
                 self.json_checksums = {}
 
         return self.json_checksums
@@ -736,7 +736,8 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
         return exts_sources
 
     def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False,
-                    git_config=None, no_download=False, download_instructions=None, alt_location=None):
+                    git_config=None, no_download=False, download_instructions=None, alt_location=None,
+                    warning_only=False):
         """
         Locate the file with the given name
         - searches in different subdirectories of source path
@@ -789,7 +790,13 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                     return fullpath
 
             except IOError as err:
-                raise EasyBuildError("Downloading file %s from url %s to %s failed: %s", filename, url, fullpath, err)
+                if not warning_only:
+                    raise EasyBuildError("Downloading file %s "
+                                         "from url %s to %s failed: %s", filename, url, fullpath, err)
+                else:
+                    self.log.warning("Downloading file %s "
+                                     "from url %s to %s failed: %s", filename, url, fullpath, err)
+                    return None
 
         else:
             # try and find file in various locations
@@ -866,8 +873,13 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                     self.dry_run_msg("  * %s (MISSING)", filename)
                     return filename
                 else:
-                    raise EasyBuildError("Couldn't find file %s anywhere, and downloading it is disabled... "
+                    if not warning_only:
+                        raise EasyBuildError("Couldn't find file %s anywhere, and downloading it is disabled... "
+                                             "Paths attempted (in order): %s ", filename, ', '.join(failedpaths))
+                    else:
+                        self.log.warning("Couldn't find file %s anywhere, and downloading it is disabled... "
                                          "Paths attempted (in order): %s ", filename, ', '.join(failedpaths))
+                        return None
             elif git_config:
                 return get_source_tarball_from_git(filename, targetdir, git_config)
             else:
@@ -959,7 +971,11 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                         error_msg += "and downloading it didn't work either... "
                         error_msg += "Paths attempted (in order): %s " % failedpaths_msg
 
-                    raise EasyBuildError(error_msg, filename)
+                    if not warning_only:
+                        raise EasyBuildError(error_msg, filename)
+                    else:
+                        self.log.warning(error_msg, filename)
+                        return None
 
     #
     # GETTER/SETTER UTILITY FUNCTIONS

easybuild/toolchains/ifbf.py

@@ -0,0 +1,44 @@
+##
+# Copyright 2012-2023 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+"""
+EasyBuild support for ifbf toolchain (includes Intel compilers, FlexiBLAS, and FFTW).
+
+Authors:
+
+* Sebastian Achilles (Juelich Supercomputing Centre)
+"""
+
+from easybuild.toolchains.intel_compilers import IntelCompilersToolchain
+from easybuild.toolchains.fft.fftw import Fftw
+from easybuild.toolchains.linalg.flexiblas import FlexiBLAS
+
+
+class Ifbf(IntelCompilersToolchain, FlexiBLAS, Fftw):
+    """
+    Compiler toolchain with Intel compilers, FlexiBLAS, and FFTW
+    """
+    NAME = 'ifbf'
+    SUBTOOLCHAIN = IntelCompilersToolchain.NAME
+    OPTIONAL = True

easybuild/toolchains/iofbf.py

@@ -0,0 +1,47 @@
+##
+# Copyright 2012-2023 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+"""
+EasyBuild support for iofbf toolchain (includes Intel compilers, OpenMPI,
+FlexiBLAS, LAPACK, ScaLAPACK and FFTW).
+
+Authors:
+
+* Sebastian Achilles (Juelich Supercomputing Centre)
+"""
+
+from easybuild.toolchains.iompi import Iompi
+from easybuild.toolchains.ifbf import Ifbf
+from easybuild.toolchains.fft.fftw import Fftw
+from easybuild.toolchains.linalg.flexiblas import FlexiBLAS
+from easybuild.toolchains.linalg.scalapack import ScaLAPACK
+
+
+class Iofbf(Iompi, FlexiBLAS, ScaLAPACK, Fftw):
+    """
+    Compiler toolchain with Intel compilers (icc/ifort), OpenMPI,
+    FlexiBLAS, LAPACK, ScaLAPACK and FFTW.
+    """
+    NAME = 'iofbf'
+    SUBTOOLCHAIN = [Iompi.NAME, Ifbf.NAME]

easybuild/tools/config.py

@@ -301,6 +301,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'sequential',
         'set_default_module',
         'set_gid_bit',
+        'silence_hook_trigger',
         'skip_extensions',
         'skip_test_cases',
         'skip_test_step',

easybuild/tools/filetools.py

@@ -1264,14 +1264,11 @@ def verify_checksum(path, checksums):
 
     for checksum in checksums:
         if isinstance(checksum, dict):
-            if filename in checksum:
+            try:
                 # Set this to a string-type checksum
                 checksum = checksum[filename]
-            elif build_option('enforce_checksums'):
-                raise EasyBuildError("Missing checksum for %s", filename)
-            else:
-                # Set to None and allow to fail elsewhere
-                checksum = None
+            except KeyError:
+                raise EasyBuildError("Missing checksum for %s in %s", filename, checksum)
 
         if isinstance(checksum, string_type):
             # if no checksum type is specified, it is assumed to be MD5 (32 characters) or SHA256 (64 characters)
@@ -1301,7 +1298,8 @@ def verify_checksum(path, checksums):
                 # no matching checksums
                 return False
         else:
-            raise EasyBuildError("Invalid checksum spec '%s', should be a string (MD5) or 2-tuple (type, value).",
+            raise EasyBuildError("Invalid checksum spec '%s': should be a string (MD5 or SHA256), "
+                                 "2-tuple (type, value), or tuple of alternative checksum specs.",
                                  checksum)
 
         actual_checksum = compute_checksum(path, typ)
@@ -2619,6 +2617,8 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
     recursive = git_config.pop('recursive', False)
     clone_into = git_config.pop('clone_into', False)
     keep_git_dir = git_config.pop('keep_git_dir', False)
+    extra_config_params = git_config.pop('extra_config_params', None)
+    recurse_submodules = git_config.pop('recurse_submodules', None)
 
     # input validation of git_config dict
     if git_config:
@@ -2644,7 +2644,11 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
     targetpath = os.path.join(targetdir, filename)
 
     # compose 'git clone' command, and run it
-    clone_cmd = ['git', 'clone']
+    if extra_config_params:
+        git_cmd = 'git ' + ' '.join(['-c %s' % param for param in extra_config_params])
+    else:
+        git_cmd = 'git'
+    clone_cmd = [git_cmd, 'clone']
 
     if not keep_git_dir and not commit:
         # Speed up cloning by only fetching the most recent commit, not the whole history
@@ -2655,6 +2659,8 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
         clone_cmd.extend(['--branch', tag])
         if recursive:
             clone_cmd.append('--recursive')
+        if recurse_submodules:
+            clone_cmd.extend(["--recurse-submodules='%s'" % pat for pat in recurse_submodules])
     else:
         # checkout is done separately below for specific commits
         clone_cmd.append('--no-checkout')
@@ -2674,16 +2680,21 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
 
     # if a specific commit is asked for, check it out
     if commit:
-        checkout_cmd = ['git', 'checkout', commit]
-        if recursive:
-            checkout_cmd.extend(['&&', 'git', 'submodule', 'update', '--init', '--recursive'])
+        checkout_cmd = [git_cmd, 'checkout', commit]
+
+        if recursive or recurse_submodules:
+            checkout_cmd.extend(['&&', git_cmd, 'submodule', 'update', '--init'])
+            if recursive:
+                checkout_cmd.append('--recursive')
+            if recurse_submodules:
+                checkout_cmd.extend(["--recurse-submodules='%s'" % pat for pat in recurse_submodules])
 
         run.run_cmd(' '.join(checkout_cmd), log_all=True, simple=True, regexp=False, path=repo_name)
 
     elif not build_option('extended_dry_run'):
         # If we wanted to get a tag make sure we actually got a tag and not a branch with the same name
         # This doesn't make sense in dry-run mode as we don't have anything to check
-        cmd = 'git describe --exact-match --tags HEAD'
+        cmd = '%s describe --exact-match --tags HEAD' % git_cmd
         # Note: Disable logging to also disable the error handling in run_cmd
         (out, ec) = run.run_cmd(cmd, log_ok=False, log_all=False, regexp=False, path=repo_name)
         if ec != 0 or tag not in out.splitlines():
@@ -2696,13 +2707,16 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
                 # make the repo unshallow first;
                 # this is equivalent with 'git fetch -unshallow' in Git 1.8.3+
                 # (first fetch seems to do nothing, unclear why)
-                cmds.append('git fetch --depth=2147483647 && git fetch --depth=2147483647')
+                cmds.append('%s fetch --depth=2147483647 && git fetch --depth=2147483647' % git_cmd)
 
-            cmds.append('git checkout refs/tags/' + tag)
+            cmds.append('%s checkout refs/tags/' % git_cmd + tag)
             # Clean all untracked files, e.g. from left-over submodules
-            cmds.append('git clean --force -d -x')
+            cmds.append('%s clean --force -d -x' % git_cmd)
             if recursive:
-                cmds.append('git submodule update --init --recursive')
+                cmds.append('%s submodule update --init --recursive' % git_cmd)
+            elif recurse_submodules:
+                cmds.append('%s submodule update --init ' % git_cmd)
+                cmds[-1] += ' '.join(["--recurse-submodules='%s'" % pat for pat in recurse_submodules])
             for cmd in cmds:
                 run.run_cmd(cmd, log_all=True, simple=True, regexp=False, path=repo_name)
 

easybuild/tools/github.py

@@ -2250,15 +2250,18 @@ def install_github_token(github_user, silent=False):
 def validate_github_token(token, github_user):
     """
     Check GitHub token:
-    * see if it conforms expectations (only [a-f]+[0-9] characters, length of 40)
-    * see if it can be used for authenticated access
+   * see if it conforms expectations (classic GitHub token with only [0-9a-f] characters
+     and length of 40 starting with 'ghp_', or fine-grained GitHub token with only
+     alphanumeric ([a-zA-Z0-9]) characters + '_' and length of 93 starting with 'github_pat_'),
+    * see if it can be used for authenticated access.
     """
     # cfr. https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/
     token_regex = re.compile('^ghp_[a-zA-Z0-9]{36}$')
     token_regex_old_format = re.compile('^[0-9a-f]{40}$')
+    # https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/about-authentication-to-github#githubs-token-formats
+    token_regex_fine_grained = re.compile('github_pat_[a-zA-Z0-9_]{82}')
 
-    # token should be 40 characters long, and only contain characters in [0-9a-f]
-    sanity_check = bool(token_regex.match(token))
+    sanity_check = bool(token_regex.match(token)) or bool(token_regex_fine_grained.match(token))
     if sanity_check:
         _log.info("Sanity check on token passed")
     else: