Merge branch 'develop' into fix-checksum-check

Author: boegel

.github/workflows/container_tests.yml

@@ -14,13 +14,13 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python: [2.7, 3.7]
+        python: [3.7]
       fail-fast: false
     steps:
     - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: ${{matrix.python}}
         architecture: x64

.github/workflows/container_tests_apptainer.yml

@@ -14,14 +14,14 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python: [2.7, 3.7]
+        python: [3.7]
         apptainer: [1.0.0, 1.1.7]
       fail-fast: false
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
         python-version: ${{matrix.python}}
         architecture: x64

.github/workflows/eb_command.yml

@@ -14,13 +14,13 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
+        python: [3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
       fail-fast: false
     steps:
     - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: ${{matrix.python}}
         architecture: x64

.github/workflows/end2end.yml

@@ -0,0 +1,61 @@
+name: End-to-end test of EasyBuild in different distros
+on: [push, pull_request]
+jobs:
+  build_publish:
+    name: End-to-end test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        container:
+          - centos-7.9
+          - centos-8.5
+          - fedora-36
+          - opensuse-15.4
+          - rockylinux-8.8
+          - rockylinux-9.2
+          - ubuntu-20.04
+          - ubuntu-22.04
+      fail-fast: false
+    container:
+      image: ghcr.io/easybuilders/${{ matrix.container }}-amd64
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: download and unpack easyblocks and easyconfigs repositories
+        run: |
+          cd $HOME
+          for pkg in easyblocks easyconfigs; do
+              curl -OL https://github.com/easybuilders/easybuild-${pkg}/archive/develop.tar.gz
+              tar xfz develop.tar.gz
+              rm -f develop.tar.gz
+          done
+
+      - name: Set up environment
+        shell: bash
+        run: |
+          # collect environment variables to be set in subsequent steps in script that can be sourced
+          echo "export PATH=$PWD:$PATH" > /tmp/eb_env
+          echo "export PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop" >> /tmp/eb_env
+
+      - name: Run commands to check test environment
+        shell: bash
+        run: |
+          cmds=(
+            "whoami"
+            "pwd"
+            "env | sort"
+            "eb --version"
+            "eb --show-system-info"
+            "eb --check-eb-deps"
+            "eb --show-config"
+          )
+          for cmd in "${cmds[@]}"; do
+              echo ">>> $cmd"
+              sudo -u easybuild bash -l -c "source /tmp/eb_env; $cmd"
+          done
+
+      - name: End-to-end test of installing bzip2 with EasyBuild
+        shell: bash
+        run: |
+          sudo -u easybuild bash -l -c "source /tmp/eb_env; eb bzip2-1.0.8.eb --trace --robot"

.github/workflows/linting.yml

@@ -13,13 +13,13 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
+        python-version: [3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
 
     steps:
     - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 

.github/workflows/unit_tests.yml

@@ -25,7 +25,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python: [2.7, 3.6]
+        python: [3.6]
         modules_tool:
           # use variables defined by 'setup' job above, see also
           # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context
@@ -81,7 +81,7 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: ${{matrix.python}}
         architecture: x64
@@ -207,10 +207,6 @@ jobs:
           # create file owned by root but writable by anyone (used by test_copy_file)
           sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
           sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
-          # silence deprecation warning when using Python 2, since it breaks a bunch of tests
-          if [ "${{matrix.python}}" == '2.7' ]; then
-            export TEST_EASYBUILD_SILENCE_DEPRECATION_WARNINGS=python2
-          fi
           # run test suite
           python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
           # try and make sure output of running tests is clean (no printed messages/warnings)

RELEASE_NOTES

@@ -4,6 +4,43 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.8.0 (7 July 2023)
+--------------------
+
+feature release
+
+- various enhancements, including:
+  - use version suffixes in PR title instead of Python versions when using --new-pr (#4253 + #4286)
+  - add script to find updated ECs for installed module (#4271)
+- various bug fixes, including:
+  - fix regex for extracting glibc version from output of '`ldd --version`' in Gentoo Linux (#4290)
+  - allow versionsuffix to be set to None in `det_full_ec_version` (#4292)
+- other changes:
+  - stop running tests with Python 2.7 since it is no longer supported in GitHub Action (#4267)
+  - replace imp.load_source by leveraging importlib.util.module_from_spec when using Python 3 (#4280)
+
+
+v4.7.2 (27 May 2023)
+--------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - extend (experimental) support for generating container images with Apptainer (#3975)
+  - add default download timeout (#4226)
+  - make the RPATH section check with `readelf -d` in sanity check optional  (#4249)
+- various bug fixes, including:
+  - add `make_extension_string` and `_make_extension_list` methods to `EasyBlock` class, so easyblocks can customize them (#3697)
+  - fix typo in condition guarding installation of GitHub token in CI workflow to run unit tests + fix broken tests related to GitHub integration features (#4216)
+  - fix --check-github as gist url now includes username (#4231)
+  - switch to Ubuntu 22.04 in GitHub Actions workflow to run container tests (#4238)
+  - use "/usr/bin/env python" in test_run_cmd_script (#4241)
+- other changes:
+  - print potential errors after running a command only once (#4065)
+  - change github_* template descriptions to be somewhat clearer (#4251)
+  - remove workflow to test bootstrap script (#4255)
+
+
 v4.7.1 (March 20th 2023)
 ------------------------
 

easybuild/framework/easyblock.py

@@ -395,13 +395,13 @@ def get_checksums_from_json(self, always_read=False):
         :param always_read: always read the checksums.json file, even if it has been read before
         """
         if always_read or self.json_checksums is None:
-            try:
-                path = self.obtain_file("checksums.json", no_download=True)
+            path = self.obtain_file("checksums.json", no_download=True, warning_only=True)
+            if path is not None:
                 self.log.info("Loading checksums from file %s", path)
                 json_txt = read_file(path)
                 self.json_checksums = json.loads(json_txt)
-            # if the file can't be found, return an empty dict
-            except EasyBuildError:
+            else:
+                # if the file can't be found, return an empty dict
                 self.json_checksums = {}
 
         return self.json_checksums
@@ -736,7 +736,8 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
         return exts_sources
 
     def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False,
-                    git_config=None, no_download=False, download_instructions=None, alt_location=None):
+                    git_config=None, no_download=False, download_instructions=None, alt_location=None,
+                    warning_only=False):
         """
         Locate the file with the given name
         - searches in different subdirectories of source path
@@ -789,7 +790,13 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                     return fullpath
 
             except IOError as err:
-                raise EasyBuildError("Downloading file %s from url %s to %s failed: %s", filename, url, fullpath, err)
+                if not warning_only:
+                    raise EasyBuildError("Downloading file %s "
+                                         "from url %s to %s failed: %s", filename, url, fullpath, err)
+                else:
+                    self.log.warning("Downloading file %s "
+                                     "from url %s to %s failed: %s", filename, url, fullpath, err)
+                    return None
 
         else:
             # try and find file in various locations
@@ -866,8 +873,13 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                     self.dry_run_msg("  * %s (MISSING)", filename)
                     return filename
                 else:
-                    raise EasyBuildError("Couldn't find file %s anywhere, and downloading it is disabled... "
+                    if not warning_only:
+                        raise EasyBuildError("Couldn't find file %s anywhere, and downloading it is disabled... "
+                                             "Paths attempted (in order): %s ", filename, ', '.join(failedpaths))
+                    else:
+                        self.log.warning("Couldn't find file %s anywhere, and downloading it is disabled... "
                                          "Paths attempted (in order): %s ", filename, ', '.join(failedpaths))
+                        return None
             elif git_config:
                 return get_source_tarball_from_git(filename, targetdir, git_config)
             else:
@@ -959,7 +971,11 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                         error_msg += "and downloading it didn't work either... "
                         error_msg += "Paths attempted (in order): %s " % failedpaths_msg
 
-                    raise EasyBuildError(error_msg, filename)
+                    if not warning_only:
+                        raise EasyBuildError(error_msg, filename)
+                    else:
+                        self.log.warning(error_msg, filename)
+                        return None
 
     #
     # GETTER/SETTER UTILITY FUNCTIONS
@@ -3087,7 +3103,7 @@ def _sanity_check_step_multi_deps(self, *args, **kwargs):
         self.cfg['builddependencies'] = builddeps
         self.cfg.iterating = False
 
-    def sanity_check_rpath(self, rpath_dirs=None):
+    def sanity_check_rpath(self, rpath_dirs=None, check_readelf_rpath=True):
         """Sanity check binaries/libraries w.r.t. RPATH linking."""
 
         self.log.info("Checking RPATH linkage for binaries/libraries...")
@@ -3152,17 +3168,21 @@ def sanity_check_rpath(self, rpath_dirs=None):
                             self.log.debug("Output of 'ldd %s' checked, looks OK", path)
 
                         # check whether RPATH section in 'readelf -d' output is there
-                        out, ec = run_cmd("readelf -d %s" % path, simple=False, trace=False)
-                        if ec:
-                            fail_msg = "Failed to run 'readelf %s': %s" % (path, out)
-                            self.log.warning(fail_msg)
-                            fails.append(fail_msg)
-                        elif not readelf_rpath_regex.search(out):
-                            fail_msg = "No '(RPATH)' found in 'readelf -d' output for %s: %s" % (path, out)
-                            self.log.warning(fail_msg)
-                            fails.append(fail_msg)
+                        if check_readelf_rpath:
+                            fail_msg = None
+                            out, ec = run_cmd("readelf -d %s" % path, simple=False, trace=False)
+                            if ec:
+                                fail_msg = "Failed to run 'readelf %s': %s" % (path, out)
+                            elif not readelf_rpath_regex.search(out):
+                                fail_msg = "No '(RPATH)' found in 'readelf -d' output for %s: %s" % (path, out)
+
+                            if fail_msg:
+                                self.log.warning(fail_msg)
+                                fails.append(fail_msg)
+                            else:
+                                self.log.debug("Output of 'readelf -d %s' checked, looks OK", path)
                         else:
-                            self.log.debug("Output of 'readelf -d %s' checked, looks OK", path)
+                            self.log.debug("Skipping the RPATH section check with 'readelf -d', as requested")
             else:
                 self.log.debug("Not sanity checking files in non-existing directory %s", dirpath)
 
@@ -4170,6 +4190,10 @@ def build_and_install_one(ecdict, init_env):
         dry_run_msg('', silent=silent)
     print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent)
 
+    if ecdict['ec']['build_info_msg']:
+        msg = "This easyconfig provides the following build information:\n\n%s\n"
+        print_msg(msg % ecdict['ec']['build_info_msg'], log=_log, silent=silent)
+
     if dry_run:
         # print note on interpreting dry run output (argument is reference to location of dry run messages)
         print_dry_run_note('below', silent=silent)

easybuild/framework/easyconfig/default.py

@@ -228,6 +228,8 @@
     'buildstats': [None, "A list of dicts with build statistics", OTHER],
     'deprecated': [False, "String specifying reason why this easyconfig file is deprecated "
                           "and will be archived in the next major release of EasyBuild", OTHER],
+    'build_info_msg': [None, "String with information to be printed to stdout and logged during the building "
+                             "of the easyconfig", OTHER],
 }
 
 

easybuild/framework/easyconfig/easyconfig.py

@@ -73,7 +73,7 @@
 from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX
 from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version
 from easybuild.tools.module_naming_scheme.utilities import det_hidden_modname, is_valid_module_name
-from easybuild.tools.modules import modules_tool
+from easybuild.tools.modules import modules_tool, NoModulesTool
 from easybuild.tools.py2vs3 import OrderedDict, create_base_metaclass, string_type
 from easybuild.tools.systemtools import check_os_dependency, pick_dep_version
 from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME, is_system_toolchain
@@ -1306,6 +1306,9 @@ def probe_external_module_metadata(self, mod_name, existing_metadata=None):
         :param existing_metadata: already available metadata for this external module (if any)
         """
         res = {}
+        if isinstance(self.modules_tool, NoModulesTool):
+            self.log.debug('Ignoring request for external module data for %s as no modules tool is active', mod_name)
+            return res
 
         if existing_metadata is None:
             existing_metadata = {}