Merge branch 'easybuilders:develop' into patch-1

Author: Flamefire

.github/workflows/container_tests.yml

@@ -97,7 +97,7 @@ jobs:
           # see https://docs.easybuild.io/en/latest/Containers.html
           curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/develop/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb
           export EASYBUILD_CONTAINERPATH=$PWD
-          export EASYBUILD_CONTAINER_CONFIG='bootstrap=docker,from=ghcr.io/easybuilders/centos-7.9-python3-amd64'
+          export EASYBUILD_CONTAINER_CONFIG='bootstrap=docker,from=ghcr.io/easybuilders/rockylinux-8.10-amd64'
           eb bzip2-1.0.8.eb --containerize --experimental --container-build-image
           singularity exec bzip2-1.0.8.sif command -v bzip2 | grep '/app/software/bzip2/1.0.8/bin/bzip2' || (echo "Path to bzip2 '$which_bzip2' is not correct" && exit 1)
           singularity exec bzip2-1.0.8.sif bzip2 --help

.github/workflows/container_tests_apptainer.yml

@@ -91,7 +91,7 @@ jobs:
           # see https://docs.easybuild.io/en/latest/Containers.html
           curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/develop/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb
           export EASYBUILD_CONTAINERPATH=$PWD
-          export EASYBUILD_CONTAINER_CONFIG='bootstrap=docker,from=ghcr.io/easybuilders/centos-7.9-python3-amd64'
+          export EASYBUILD_CONTAINER_CONFIG='bootstrap=docker,from=ghcr.io/easybuilders/rockylinux-8.10-amd64'
           export EASYBUILD_CONTAINER_TYPE='apptainer'
           eb bzip2-1.0.8.eb --containerize --experimental --container-build-image
           apptainer exec bzip2-1.0.8.sif command -v bzip2 | grep '/app/software/bzip2/1.0.8/bin/bzip2' || (echo "Path to bzip2 '$which_bzip2' is not correct" && exit 1)

RELEASE_NOTES

@@ -4,9 +4,62 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://docs.easybuild.io/release-notes .
 
 
+v5.1.0 (26 May 2025)
+--------------------
+
+feature release
+
+- various enhancements, including:
+  - add support for data installations (#4474, #4873, #4874)
+  - allow specifying location for RPATH wrapper scripts via `rpath_wrappers_dir` (#4596)
+  - add a CUDA device code sanity check (#4692)
+  - add support for check_readelf_rpath easyconfig parameter to optionally skip RPATH checks (#4768)
+  - add support for using environment variables in value used in modextravars (#4855)
+  - print summary after the build in trace output (#4861, #4875)
+  - avoid leaking keys by mistake with `--upload-test-report` (#4877)
+  - obtain PR/commit diff via GitHub API rather than downloading `*.diff` file via github.com (#4878)
+  - support options for patch command (#4886)
+  - replace full trace message for extension check command with simple pass/fail message (#4892)
+- various bug fixes, including:
+  - also pass `rpath_include_dirs` when preparing build environment for extensions (#4596)
+  - fix `check_checksums` when `nosource: True` is used (#4806)
+  - fix help string of findPythonDeps.py script (#4821)
+  - take into account `job-output-dir` option in Slurm job backend (#4842)
+  - fix unbound variable in error case in `build_and_install_software` (#4843)
+  - avoid failure when only some passed easyconfigs exist (#4847)
+  - restore original value for non-list easyconfig parameter values that are considered for iterating over (#4848)
+  - add '-' before 'DMKL_ILP64' in $CFLAGS (#4850)
+  - enhance RPATH sanity check to skip anything whose absolute path resolves to outside the install dir (#4854)
+  - use new `ModEnvVarType.STRICT_PATH_WITH_FILES` with `CMAKE_LIBRARY_PATH` environment variable (#4858)
+  - fix `is_patch_for` for patch dicts (#4865)
+  - update fake module for each extension installed (#4868, #4888, #4895)
+  - implement exponential backoff in `download_file` (#4870, #4880)
+  - use `develop` branch for PRs that target removed `5.0.x` branch in `fetch_files_from_pr` (#4879)
+  - also ignore errors raised during test step when `--ignore-test-failure` is used (#4881)
+  - fix download progress bar (#4885)
+  - fix `--dep-graph` by using `graphviz` Python package (#4891)
+- fixes for test suite:
+  - fix `test_github_preview_pr` which got broken because there's no more easyconfigs for bzip2 1.0.6 in easyconfigs repo (#4827)
+  - fix tests after removal of 5.0.x branch (#4830)
+  - fix missed message in trivial equal-asserts (#4831)
+  - fix wrong variable name used for Python version in linting CI (#4839)
+  - fix failing GitHub integration CI tests (#4841)
+  - fix testsuite badge in README (#4845)
+- other changes:
+  - go back to using `develop` branch rather than `5.0.x` branch in GitHub Actions workflows (#4820)
+  - remove source tarball for Python 3.7.2 from test sources (#4828)
+  - delete .coveragerc (#4833)
+  - remove Python 2 constructs (#4834)
+  - remove superflous assignment in `_sanity_check_step` (#4851)
+  - replace deprecation warning about reproducible tarballs on Python older than 3.9 with a regular warning (#4852)
+  - move `EasyBlock.expand_module_search_path` into `ModuleEnvironmentVariable.expand_paths` (#4859)
+
+
 v5.0.0 (18 March 2025)
 ----------------------
 
+major release (includes breaking changes)
+
 - remove support for Python 2.7 and 3.5 (#4229, #4270, #4306, #4473, #4477, #4476, #4478, #4524, #4607, #4756, #4810, #4811) 
   - also run unit test suite with Python 3.12 + 3.13 (#4484, #4674)
 - changed defaults in EasyBuild configuration:

easybuild/framework/easyblock.py

@@ -2289,11 +2289,12 @@ def fake_module_environment(self, extra_modules=None, with_build_deps=False):
 
         fake_mod_data = self.load_fake_module(purge=True, extra_modules=extra_modules)
 
-        yield
-
-        # cleanup (unload fake module, remove fake module dir)
-        if fake_mod_data:
-            self.clean_up_fake_module(fake_mod_data)
+        try:
+            yield
+        finally:
+            # cleanup (unload fake module, remove fake module dir)
+            if fake_mod_data:
+                self.clean_up_fake_module(fake_mod_data)
 
     def guess_start_dir(self):
         """

easybuild/tools/output.py

@@ -48,13 +48,19 @@
     pass
 
 
+COLOR_BLUE = 'blue'
+COLOR_CYAN = 'cyan'
 COLOR_GREEN = 'green'
+COLOR_PURPLE = 'purple'
 COLOR_RED = 'red'
 COLOR_YELLOW = 'yellow'
 
 # map known colors to ANSII color codes
 KNOWN_COLORS = {
+    COLOR_BLUE: '\033[0;34m',
+    COLOR_CYAN: '\033[0;36m',
     COLOR_GREEN: '\033[0;32m',
+    COLOR_PURPLE: '\033[0;35m',
     COLOR_RED: '\033[0;31m',
     COLOR_YELLOW: '\033[1;33m',
 }

easybuild/tools/run.py

@@ -424,6 +424,8 @@ def to_cmd_str(cmd):
         kwargs = {
             'interactive': interactive,
             'work_dir': work_dir,
+            'fail_on_error': fail_on_error,
+            'hidden': hidden,
         }
         hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs=kwargs)
         if hook_res:
@@ -589,6 +591,19 @@ def to_cmd_str(cmd):
                             work_dir=work_dir, out_file=cmd_out_fp, err_file=cmd_err_fp, cmd_sh=cmd_sh,
                             thread_id=thread_id, task_id=task_id)
 
+    if with_hooks:
+        run_hook_kwargs = {
+            'exit_code': res.exit_code,
+            'interactive': interactive,
+            'output': res.output,
+            'stderr': res.stderr,
+            'work_dir': res.work_dir,
+            'shell_cmd_result': res,
+            'fail_on_error': fail_on_error,
+            'hidden': hidden,
+        }
+        run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs)
+
     # always log command output
     cmd_name = cmd_str.split(' ')[0]
     if split_stderr:
@@ -618,16 +633,6 @@ def to_cmd_str(cmd):
         except OSError as err:
             raise EasyBuildError(f"Failed to return to {initial_work_dir} after executing command `{cmd_str}`: {err}")
 
-    if with_hooks:
-        run_hook_kwargs = {
-            'exit_code': res.exit_code,
-            'interactive': interactive,
-            'output': res.output,
-            'stderr': res.stderr,
-            'work_dir': res.work_dir,
-        }
-        run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs)
-
     if not hidden:
         time_since_start = time_str_since(start_time)
         trace_msg(f"command completed: exit {res.exit_code}, ran in {time_since_start}")

easybuild/tools/systemtools.py

@@ -56,11 +56,20 @@
 # pkg_resources is provided by the setuptools Python package,
 # which we really want to keep as an *optional* dependency
 try:
-    import pkg_resources
+    # catch & ignore deprecation warning when importing pkg_resources produced by setuptools
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", UserWarning)
+        import pkg_resources
     HAVE_PKG_RESOURCES = True
 except ImportError:
     HAVE_PKG_RESOURCES = False
 
+# importlib.metadata only available in Python 3.10+ (which we take into account when using it)
+try:
+    import importlib.metadata
+except ImportError:
+    pass
+
 try:
     # only needed on macOS, may not be available on Linux
     import ctypes.macholib.dyld
@@ -1467,17 +1476,38 @@ def det_pypkg_version(pkg_name, imported_pkg, import_name=None):
 
     version = None
 
-    if HAVE_PKG_RESOURCES:
+    # prefer using importlib.metadata, since pkg_resources is deprecated since setuptools v68.0.0
+    # and is scheduled to be removed in November 2025; see also https://github.com/pypa/setuptools/pull/5007
+
+    raised_error = None
+
+    # figure out which function to use to determine module/package version,
+    # and which error may be raised if the name is unknown
+    if check_python_version() >= (3, 10):
+
+        def _get_version(name):
+            return importlib.metadata.version(name)
+
+        raised_error = importlib.metadata.PackageNotFoundError
+
+    elif HAVE_PKG_RESOURCES:
+
+        def _get_version(name):
+            return pkg_resources.get_distribution(name).version
+
+        raised_error = pkg_resources.DistributionNotFound
+
+    if raised_error is not None:
         if import_name:
             try:
-                version = pkg_resources.get_distribution(import_name).version
-            except pkg_resources.DistributionNotFound as err:
+                version = _get_version(import_name)
+            except raised_error as err:
                 _log.debug("%s Python package not found: %s", import_name, err)
 
         if version is None:
             try:
-                version = pkg_resources.get_distribution(pkg_name).version
-            except pkg_resources.DistributionNotFound as err:
+                version = _get_version(pkg_name)
+            except raised_error as err:
                 _log.debug("%s Python package not found: %s", pkg_name, err)
 
     if version is None:

easybuild/tools/version.py

@@ -45,7 +45,7 @@
 # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
 #   UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
 # This causes problems further up the dependency chain...
-VERSION = LooseVersion('5.0.1.dev0')
+VERSION = LooseVersion('5.1.1.dev0')
 UNKNOWN = 'UNKNOWN'
 UNKNOWN_EASYBLOCKS_VERSION = '0.0.UNKNOWN.EASYBLOCKS'
 

test/framework/output.py

@@ -130,10 +130,13 @@ def test_colorize(self):
         Test colorize function
         """
         if HAVE_RICH:
-            for color in ('green', 'red', 'yellow'):
+            for color in ('blue', 'cyan', 'green', 'purple', 'red', 'yellow'):
                 self.assertEqual(colorize('test', color), '[bold %s]test[/bold %s]' % (color, color))
         else:
+            self.assertEqual(colorize('test', 'blue'), '\x1b[0;34mtest\x1b[0m')
+            self.assertEqual(colorize('test', 'cyan'), '\x1b[0;36mtest\x1b[0m')
             self.assertEqual(colorize('test', 'green'), '\x1b[0;32mtest\x1b[0m')
+            self.assertEqual(colorize('test', 'purple'), '\x1b[0;35mtest\x1b[0m')
             self.assertEqual(colorize('test', 'red'), '\x1b[0;31mtest\x1b[0m')
             self.assertEqual(colorize('test', 'yellow'), '\x1b[1;33mtest\x1b[0m')
 

test/framework/run.py

@@ -2068,7 +2068,9 @@ def pre_run_shell_cmd_hook(cmd, *args, **kwargs):
                     print("pre-run hook '%s' in %s" % (cmd, work_dir))
                     import sys
                     sys.stderr.write('pre-run hook done\\n')
-                if not cmd.startswith('echo'):
+                print("command is allowed to fail: %s" % kwargs.get('fail_on_error', 'NOT AVAILABLE'))
+                print("command is hidden: %s" % kwargs.get('hidden', 'NOT AVAILABLE'))
+                if cmd != 'false' and not cmd.startswith('echo'):
                     cmds = cmd.split(';')
                     return '; '.join(cmds[:-1] + ["echo " + cmds[-1].lstrip()])
 
@@ -2081,6 +2083,8 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
                 else:
                     msg = "post-run hook '%s'" % cmd
                 msg += " (exit code: %s, output: '%s')" % (exit_code, output)
+                msg += "\\ncommand was allowed to fail: %s" % kwargs.get('fail_on_error', 'NOT AVAILABLE')
+                msg += "\\ncommand was hidden: %s" % kwargs.get('hidden', 'NOT AVAILABLE')
                 print(msg)
         """)
         write_file(hooks_file, hooks_file_txt)
@@ -2095,7 +2099,11 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
 
         expected_stdout = '\n'.join([
             f"pre-run hook 'make' in {cwd}",
+            "command is allowed to fail: True",
+            "command is hidden: False",
             "post-run hook 'echo make' (exit code: 0, output: 'make\n')",
+            "command was allowed to fail: True",
+            "command was hidden: False",
             '',
         ])
         self.assertEqual(stdout, expected_stdout)
@@ -2109,6 +2117,8 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
 
         expected_stdout = '\n'.join([
             "pre-run hook 'make' in %s" % cwd,
+            "command is allowed to fail: True",
+            "command is hidden: False",
             '  running shell command "echo make"',
             '  (in %s)' % cwd,
             '',
@@ -2126,6 +2136,24 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
         regex = re.compile('>> running shell command:\n\techo make', re.M)
         self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout))
 
+        with self.mocked_stdout_stderr():
+            # run_shell_cmd will raise RunShellCmdError which we don't care about here,
+            # we just want to verify that the post_run_shell_cmd_hook has run
+            try:
+                run_shell_cmd("false")
+            except RunShellCmdError:
+                pass
+            stdout = self.get_stdout()
+
+        expected_end = '\n'.join([
+            '',
+            "post-run hook 'false' (exit code: 1, output: '')",
+            "command was allowed to fail: True",
+            "command was hidden: False",
+            '',
+        ])
+        self.assertTrue(stdout.endswith(expected_end), f"Stdout should end with '{expected_end}': {stdout}")
+
     def test_run_shell_cmd_delete_cwd(self):
         """
         Test commands that destroy directories inside initial working directory