Merge branch 'develop' into gha_ci_reduce

Author: boegel

.github/workflows/end2end.yml

@@ -0,0 +1,61 @@
+name: End-to-end test of EasyBuild in different distros
+on: [push, pull_request]
+jobs:
+  build_publish:
+    name: End-to-end test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        container:
+          - centos-7.9
+          - centos-8.5
+          - fedora-36
+          - opensuse-15.4
+          - rockylinux-8.8
+          - rockylinux-9.2
+          - ubuntu-20.04
+          - ubuntu-22.04
+      fail-fast: false
+    container:
+      image: ghcr.io/easybuilders/${{ matrix.container }}-amd64
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: download and unpack easyblocks and easyconfigs repositories
+        run: |
+          cd $HOME
+          for pkg in easyblocks easyconfigs; do
+              curl -OL https://github.com/easybuilders/easybuild-${pkg}/archive/develop.tar.gz
+              tar xfz develop.tar.gz
+              rm -f develop.tar.gz
+          done
+
+      - name: Set up environment
+        shell: bash
+        run: |
+          # collect environment variables to be set in subsequent steps in script that can be sourced
+          echo "export PATH=$PWD:$PATH" > /tmp/eb_env
+          echo "export PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop" >> /tmp/eb_env
+
+      - name: Run commands to check test environment
+        shell: bash
+        run: |
+          cmds=(
+            "whoami"
+            "pwd"
+            "env | sort"
+            "eb --version"
+            "eb --show-system-info"
+            "eb --check-eb-deps"
+            "eb --show-config"
+          )
+          for cmd in "${cmds[@]}"; do
+              echo ">>> $cmd"
+              sudo -u easybuild bash -l -c "source /tmp/eb_env; $cmd"
+          done
+
+      - name: End-to-end test of installing bzip2 with EasyBuild
+        shell: bash
+        run: |
+          sudo -u easybuild bash -l -c "source /tmp/eb_env; eb bzip2-1.0.8.eb --trace --robot"

.github/workflows/unit_tests_python2.yml

@@ -0,0 +1,81 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: EasyBuild framework unit tests (python2)
+on: [push, pull_request]
+
+permissions:
+  contents: read # to fetch code (actions/checkout)
+
+concurrency:
+  group: ${{format('{0}:{1}:{2}', github.repository, github.ref, github.workflow)}}
+  cancel-in-progress: true
+
+jobs:
+  test_python2:
+    runs-on: ubuntu-20.04
+    container:
+      # CentOS 7.9 container that already includes Lmod & co,
+      # see https://github.com/easybuilders/easybuild-containers
+      image: ghcr.io/easybuilders/centos-7.9-amd64
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: install Python packages
+      run: |
+        # Python packages
+        python2 -V
+        python2 -m pip --version
+        python2 -m pip install --upgrade pip
+        python2 -m pip --version
+        # strip out GC3Pie since installation with ancient setuptools (0.9.8) fails
+        sed -i '/GC3Pie/d' requirements.txt
+        python2 -m pip install -r requirements.txt
+        # git config is required to make actual git commits (cfr. tests for GitRepository)
+        sudo -u easybuild git config --global user.name "GitHub Actions"
+        sudo -u easybuild git config --global user.email "[email protected]"
+        sudo -u easybuild git config --get-regexp 'user.*'
+
+    - name: install GitHub token (if available)
+      env:
+        # token (owned by @boegelbot) with gist permissions (required for some of the tests for GitHub integration);
+        # this token is not available in pull requests, so tests that require it are skipped in PRs,
+        # and are only run after the PR gets merged
+        GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}}
+      run: |
+        # tests that require a GitHub token are skipped automatically when no GitHub token is available
+        if [ ! -z $GITHUB_TOKEN ]; then
+          sudo -u easybuild python2 -c "import keyring; import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring()); keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
+          echo "GitHub token installed!"
+        else
+          echo "Installation of GitHub token skipped!"
+        fi
+
+    - name: install sources
+      run: |
+          # install from source distribution tarball, to test release as published on PyPI
+          python2 setup.py sdist
+          ls dist
+          export PREFIX=/tmp/$USER/$GITHUB_SHA
+          python2 -m pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
+
+    - name: run test suite
+      run: |
+          # run tests *outside* of checked out easybuild-framework directory,
+          # to ensure we're testing installed version (see previous step)
+          cd $HOME
+          # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that)
+          export PREFIX=/tmp/$USER/$GITHUB_SHA
+          ENV_CMDS="export PATH=$PREFIX/bin:$PATH; export PYTHONPATH=$PREFIX/lib/python2.7/site-packages:$PYTHONPATH"
+          ENV_CMDS="${ENV_CMDS}; export EB_VERBOSE=1; export EB_PYTHON=python2; export TEST_EASYBUILD_SILENCE_DEPRECATION_WARNINGS=python2"
+          # run EasyBuild command via (non-root) easybuild user + login shell
+          sudo -u easybuild bash -l -c "${ENV_CMDS}; module --version; eb --version"
+          # show active EasyBuild configuration
+          sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --show-config"
+          # gather some useful info on test system
+          sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --show-system-info"
+          # check GitHub configuration
+          sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --check-github --github-user=easybuild_test"
+          # create file owned by root but writable by anyone (used by test_copy_file)
+          sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
+          sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
+          # run test suite (via easybuild user + login shell)
+          sudo -u easybuild bash -l -c "${ENV_CMDS}; python2 -O -m test.framework.suite"

RELEASE_NOTES

@@ -4,6 +4,34 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.8.1 (11 September 2023)
+--------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - add end-to-end test for running EasyBuild in different Linux distros using containers (#3968)
+  - suggest default title in `--review-pr` (#4287)
+  - add `build_and_install_loop` hooks to run before and after the install loop for individual easyconfigs (#4304)
+  - implement support for `cancel_hook` and `fail_hook` (#4315, #4325)
+  - add postiter hook to the list of steps so the corresponding hook can be used (#4316)
+  - add `run_shell_cmd` hook (#4323)
+  - add `build_info_msg` easyconfig parameter to print message during installation of an easyconfig (#4324)
+  - add `--silence-hook-trigger` configuration option to supress printing of debug message every time a hook is triggered (#4329)
+  - add support for using fine grained Github tokens (#4332)
+  - add definitions for ifbf and iofbf toolchain (#4337)
+  - add support for submodule filtering and specifying extra Git configuration in `git_config` (#4338, #4339)
+- various bug fixes, including:
+  - improve error when checksum dict has no entry for a file (#4150)
+  - avoid error being logged when `checksums.json` is not found (#4261)
+  - don't fail in `mkdir` if path gets created while processing it (#4300, #4328)
+  - ignore request for external module (meta)data when no modules tool is active (#4308)
+  - use sys.executable to obtain path to `python` command in tests, rather than assuming that `python` command is available in `$PATH` (#4309)
+  - fix `test_add_and_remove_module_path` by replacing string comparison of paths by checking whether they point to the same path (since symlinks may cause trouble) (#4312)
+  - enhance `Toolchain.get_flag` to handle lists (#4319)
+  - only add extensions in module file if there are extensions (#4331)
+
+
 v4.8.0 (7 July 2023)
 --------------------
 

easybuild/framework/easyblock.py

@@ -395,13 +395,13 @@ def get_checksums_from_json(self, always_read=False):
         :param always_read: always read the checksums.json file, even if it has been read before
         """
         if always_read or self.json_checksums is None:
-            try:
-                path = self.obtain_file("checksums.json", no_download=True)
+            path = self.obtain_file("checksums.json", no_download=True, warning_only=True)
+            if path is not None:
                 self.log.info("Loading checksums from file %s", path)
                 json_txt = read_file(path)
                 self.json_checksums = json.loads(json_txt)
-            # if the file can't be found, return an empty dict
-            except EasyBuildError:
+            else:
+                # if the file can't be found, return an empty dict
                 self.json_checksums = {}
 
         return self.json_checksums
@@ -736,7 +736,8 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
         return exts_sources
 
     def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False,
-                    git_config=None, no_download=False, download_instructions=None, alt_location=None):
+                    git_config=None, no_download=False, download_instructions=None, alt_location=None,
+                    warning_only=False):
         """
         Locate the file with the given name
         - searches in different subdirectories of source path
@@ -789,7 +790,13 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                     return fullpath
 
             except IOError as err:
-                raise EasyBuildError("Downloading file %s from url %s to %s failed: %s", filename, url, fullpath, err)
+                if not warning_only:
+                    raise EasyBuildError("Downloading file %s "
+                                         "from url %s to %s failed: %s", filename, url, fullpath, err)
+                else:
+                    self.log.warning("Downloading file %s "
+                                     "from url %s to %s failed: %s", filename, url, fullpath, err)
+                    return None
 
         else:
             # try and find file in various locations
@@ -866,8 +873,13 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                     self.dry_run_msg("  * %s (MISSING)", filename)
                     return filename
                 else:
-                    raise EasyBuildError("Couldn't find file %s anywhere, and downloading it is disabled... "
+                    if not warning_only:
+                        raise EasyBuildError("Couldn't find file %s anywhere, and downloading it is disabled... "
+                                             "Paths attempted (in order): %s ", filename, ', '.join(failedpaths))
+                    else:
+                        self.log.warning("Couldn't find file %s anywhere, and downloading it is disabled... "
                                          "Paths attempted (in order): %s ", filename, ', '.join(failedpaths))
+                        return None
             elif git_config:
                 return get_source_tarball_from_git(filename, targetdir, git_config)
             else:
@@ -959,7 +971,11 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                         error_msg += "and downloading it didn't work either... "
                         error_msg += "Paths attempted (in order): %s " % failedpaths_msg
 
-                    raise EasyBuildError(error_msg, filename)
+                    if not warning_only:
+                        raise EasyBuildError(error_msg, filename)
+                    else:
+                        self.log.warning(error_msg, filename)
+                        return None
 
     #
     # GETTER/SETTER UTILITY FUNCTIONS
@@ -4174,6 +4190,10 @@ def build_and_install_one(ecdict, init_env):
         dry_run_msg('', silent=silent)
     print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent)
 
+    if ecdict['ec']['build_info_msg']:
+        msg = "This easyconfig provides the following build information:\n\n%s\n"
+        print_msg(msg % ecdict['ec']['build_info_msg'], log=_log, silent=silent)
+
     if dry_run:
         # print note on interpreting dry run output (argument is reference to location of dry run messages)
         print_dry_run_note('below', silent=silent)

easybuild/framework/easyconfig/default.py

@@ -228,6 +228,8 @@
     'buildstats': [None, "A list of dicts with build statistics", OTHER],
     'deprecated': [False, "String specifying reason why this easyconfig file is deprecated "
                           "and will be archived in the next major release of EasyBuild", OTHER],
+    'build_info_msg': [None, "String with information to be printed to stdout and logged during the building "
+                             "of the easyconfig", OTHER],
 }
 
 

easybuild/framework/easyconfig/easyconfig.py

@@ -73,7 +73,7 @@
 from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX
 from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version
 from easybuild.tools.module_naming_scheme.utilities import det_hidden_modname, is_valid_module_name
-from easybuild.tools.modules import modules_tool
+from easybuild.tools.modules import modules_tool, NoModulesTool
 from easybuild.tools.py2vs3 import OrderedDict, create_base_metaclass, string_type
 from easybuild.tools.systemtools import check_os_dependency, pick_dep_version
 from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME, is_system_toolchain
@@ -1306,6 +1306,9 @@ def probe_external_module_metadata(self, mod_name, existing_metadata=None):
         :param existing_metadata: already available metadata for this external module (if any)
         """
         res = {}
+        if isinstance(self.modules_tool, NoModulesTool):
+            self.log.debug('Ignoring request for external module data for %s as no modules tool is active', mod_name)
+            return res
 
         if existing_metadata is None:
             existing_metadata = {}

easybuild/framework/easyconfig/tools.py

@@ -60,7 +60,7 @@
 from easybuild.tools.filetools import find_easyconfigs, is_patch_file, locate_files
 from easybuild.tools.filetools import read_file, resolve_path, which, write_file
 from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO
-from easybuild.tools.github import det_pr_labels, download_repo, fetch_easyconfigs_from_pr, fetch_pr_data
+from easybuild.tools.github import det_pr_labels, det_pr_title, download_repo, fetch_easyconfigs_from_pr, fetch_pr_data
 from easybuild.tools.github import fetch_files_from_pr
 from easybuild.tools.multidiff import multidiff
 from easybuild.tools.py2vs3 import OrderedDict
@@ -561,6 +561,11 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop', testing=False
             lines.extend(['', "This PR is associated with a generic '.x' milestone, "
                               "it should be associated to the next release milestone once merged"])
 
+        default_new_title = det_pr_title([ec['ec'] for ec in ecs])
+        if default_new_title != pr_data['title']:
+            lines.extend(['', "If this PR contains only new easyconfigs and has not been edited from the default, "
+                              "then the title should be: %s" % default_new_title])
+
     return '\n'.join(lines)