Merge branch 'develop' into finalpath-fix

Author: Flamefire

RELEASE_NOTES

@@ -4,6 +4,42 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://docs.easybuild.io/release-notes .
 
 
+v5.1.1 (6 July 2025)
+--------------------
+
+update/bugfix release
+
+- enhancements:
+  - don't allow using `--pr-commit-msg` when only adding new files with `--new-pr` (unless `--force` is used) (#4498)
+  - update `install-EasyBuild-develop.sh` script to allow installation without forking (#4899)
+  - add blue, cyan, purple as known colors that can be used in colorize function (#4907)
+  - trigger `post_run_shell_cmd_hook` before raising error if shell command failed + pass down full `RunShellCmdResult` instance (#4908)
+  - also pass value of `fail_on_error` and `hidden` options of `run_shell_cmd` call down to pre/post `run_shell_cmd` hook (#4911)
+  - add configuration option to ignore unversioned (0.0.0) Python packages (#4912)
+  - add `required` option to `EasyConfig.get_cuda_cc_template_value` method (#4913)
+  - add total iteration count to trace output (#4919, #4952)
+  - add support for pre/post `easyblock` hook (#4923, #4938)
+  - catch `HTTPException` when doing GitHub API request (#4926)
+- bug fixes:
+  - fix reporting of method name in `EasyBlock.run_step` (#4920)
+  - fix handling of broken symlinks in `filetools.remove` (#4921)
+  - keep symlinks when copying build dirs of failed installations (#4933)
+  - also copy patches of extensions to easybuild subdirectory of installation directory (#4939, #4946)
+  - also copy easyblocks of bundle components to reprod dir (#4944)
+  - take into account that forked repository may not have same name as origin `--sync-pr-with-develop` (#4947)
+- enhancements and fixes for test suite:
+  - add support for passing valid unittest options to test suite (#3790)
+  - enhance `LooseVersion` test to add cases with leading alphabetical characters (#4901)
+  - prefer using `importlib.metadata` over the deprecated `pkg_resources` in `det_pypkg_version` + switch container tests to Rocky Linux 8.10 (#4904)
+  - fix `TypeError` for Python version comparison in `test_fetch_sources_git` (#4934)
+  - fix broken tests for GitHub integration features (#4954)
+- other changes:
+  - change value of architectures constants RISCV32 and RISCV64 (#4915)
+  - remove some superflous Python 2 checks (#4917)
+  - remove use of unsupported `source` key in patch spec (#4924)
+  - remove super() arguments from mk_tmpl_easyblock_for (#4936)
+
+
 v5.1.0 (26 May 2025)
 --------------------
 

easybuild/base/frozendict.py

@@ -52,7 +52,7 @@ def __repr__(self):
 
     def __hash__(self):
         if self.__hash is None:
-            self.__hash = reduce(operator.xor, map(hash, self.iteritems()), 0)
+            self.__hash = reduce(operator.xor, map(hash, self.items()), 0)
 
         return self.__hash
 

easybuild/framework/easyblock.py

@@ -188,6 +188,7 @@ def __init__(self, ec, logfile=None):
 
         # list of patch/source files, along with checksums
         self.patches = []
+        self.all_patches_paths = set()  # set of paths to all patches (including patches of extensions)
         self.src = []
         self.data_src = []
         self.checksums = []
@@ -311,6 +312,9 @@ def __init__(self, ec, logfile=None):
         # initialize logger
         self._init_log()
 
+        # number of iterations
+        self.iter_cnt = -1
+
         # try and use the specified group (if any)
         group_name = build_option('group')
         group_spec = self.cfg['group']
@@ -598,6 +602,7 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None):
                 patch_info['path'] = path
                 patch_info['checksum'] = self.get_checksum_for(checksums, filename=patch_info['name'], index=index)
 
+                self.all_patches_paths.add(path)
                 if extension:
                     patches.append(patch_info)
                 else:
@@ -2387,7 +2392,8 @@ def handle_iterate_opts(self):
                     self.log.debug("Iterating opt %s: %s", opt, self.iter_opts[opt])
 
             if self.iter_opts:
-                print_msg("starting iteration #%s ..." % self.iter_idx, log=self.log, silent=self.silent)
+                print_msg(f"starting iteration {self.iter_idx + 1}/{self.iter_cnt} ...", log=self.log,
+                          silent=self.silent)
                 self.log.info("Current iteration index: %s", self.iter_idx)
 
             # pop first element from all iterative easyconfig parameters as next value to use
@@ -2429,7 +2435,7 @@ def det_iter_cnt(self):
 
         # we need to take into account that builddependencies is always a list
         # we're only iterating over it if it's a list of lists
-        builddeps = self.cfg['builddependencies']
+        builddeps = self.cfg.get_ref('builddependencies')
         if all(isinstance(x, list) for x in builddeps):
             iter_opt_counts.append(len(builddeps))
 
@@ -4789,7 +4795,8 @@ def run_all_steps(self, run_test_cases):
         if self.cfg['stop'] == 'cfg':
             return True
 
-        steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt())
+        self.iter_cnt = self.det_iter_cnt()
+        steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.iter_cnt)
 
         # figure out how many steps will actually be run (not be skipped)
         step_cnt = 0
@@ -4922,7 +4929,7 @@ def copy_build_dirs_logs_failed_install(application_log, silent, app, easyconfig
             msg = f"Build directory of failed installation copied to {build_dirs_path}"
 
             def operation(src, dest):
-                copy_dir(src, dest, dirs_exist_ok=True)
+                copy_dir(src, dest, dirs_exist_ok=True, symlinks=True)
 
             operation_args.append((operation, [app.builddir], build_dirs_path, msg))
 
@@ -5032,7 +5039,9 @@ def build_and_install_one(ecdict, init_env):
             except EasyBuildError as err:
                 _log.warning("Failed to create build environment dump for easyconfig %s: %s", reprod_spec, err)
 
-            # also add any extension easyblocks used during the build for reproducibility
+            # also add any component/extension easyblocks used during the build for reproducibility
+            if hasattr(app, 'comp_instances'):
+                copy_easyblocks_for_reprod([comp for cfg, comp in app.comp_instances], reprod_dir)
             if app.ext_instances:
                 copy_easyblocks_for_reprod(app.ext_instances, reprod_dir)
             # If not already done remove the granted write permissions if we did so
@@ -5119,15 +5128,13 @@ def ensure_writable_log_dir(log_dir):
                     block = det_full_ec_version(app.cfg) + ".block"
                     repo.add_easyconfig(ecdict['original_spec'], app.name, block, buildstats, currentbuildstats)
                 repo.add_easyconfig(spec, app.name, det_full_ec_version(app.cfg), buildstats, currentbuildstats)
-                for patch in app.patches:
-                    repo.add_patch(patch['path'], app.name)
+                for patch_path in app.all_patches_paths:
+                    repo.add_patch(patch_path, app.name)
                 repo.commit("Built %s" % app.full_mod_name)
                 del repo
             except EasyBuildError as err:
                 _log.warning("Unable to commit easyconfig to repository: %s", err)
 
-        run_hook(EASYBLOCK, hooks, post_step_hook=True, args=[app])
-
         # cleanup logs
         app.close_log()
 
@@ -5144,10 +5151,10 @@ def ensure_writable_log_dir(log_dir):
                 _log.debug("Copied easyconfig file %s to %s", spec, newspec)
 
                 # copy patches
-                for patch in app.patches:
-                    target = os.path.join(new_log_dir, os.path.basename(patch['path']))
-                    copy_file(patch['path'], target)
-                    _log.debug("Copied patch %s to %s", patch['path'], target)
+                for patch_path in app.all_patches_paths:
+                    target = os.path.join(new_log_dir, os.path.basename(patch_path))
+                    copy_file(patch_path, target)
+                    _log.debug("Copied patch %s to %s", patch_path, target)
 
                 if build_option('read_only_installdir') and not app.cfg['stop']:
                     # take away user write permissions (again)
@@ -5201,6 +5208,8 @@ def ensure_writable_log_dir(log_dir):
     if not success:
         copy_build_dirs_logs_failed_install(application_log, silent, app, ecdict['ec'])
 
+    run_hook(EASYBLOCK, hooks, post_step_hook=True, args=[app])
+
     del app
 
     return (success, application_log, error_msg, exit_code)

easybuild/framework/easyconfig/default.py

@@ -84,6 +84,7 @@
     'toolchainopts': [None, 'Extra options for compilers', TOOLCHAIN],
 
     # BUILD easyconfig parameters
+    'amdgcn_capabilities': [[], "List of AMDGCN capabilities to build with (if supported)", BUILD],
     'banned_linked_shared_libs': [[], "List of shared libraries (names, file names, or paths) which are not allowed "
                                       "to be linked in any installed binary/library", BUILD],
     'bitbucket_account': ['%(namelower)s', "Bitbucket account name to be used to resolve template values in source"

easybuild/framework/easyconfig/easyconfig.py

@@ -1992,6 +1992,28 @@ def get_cuda_cc_template_value(self, key, required=True):
             error_msg = "%s is not a template value based on --cuda-compute-capabilities/cuda_compute_capabilities"
             raise EasyBuildError(error_msg, key)
 
+    def get_amdgcn_cc_template_value(self, key, required=True):
+        """
+        Get template value based on --amdgcn-capabilities EasyBuild configuration option
+        and amdgcn_capabilities easyconfig parameter.
+        Returns user-friendly error message in case neither are defined,
+        or if an unknown key is used.
+        """
+        if key.startswith('amdgcn_') and any(x == key for x in TEMPLATE_NAMES_DYNAMIC):
+            try:
+                return self.template_values[key]
+            except KeyError:
+                if not required:
+                    self.log.debug(f'Key {key} not found in template values, returning empty value')
+                    return ''
+                error_msg = "Template value '%s' is not defined!\n"
+                error_msg += "Make sure that either the --amdgcn-capabilities EasyBuild configuration "
+                error_msg += "option is set, or that the amdgcn_capabilities easyconfig parameter is defined."
+                raise EasyBuildError(error_msg, key)
+        else:
+            error_msg = "%s is not a template value based on --amdgcn-capabilities/amdgcn_capabilities"
+            raise EasyBuildError(error_msg, key)
+
 
 def det_installversion(version, toolchain_name, toolchain_version, prefix, suffix):
     """Deprecated 'det_installversion' function, to determine exact install version, based on supplied parameters."""

easybuild/framework/easyconfig/templates.py

@@ -88,6 +88,11 @@
 # template values which are only generated dynamically
 TEMPLATE_NAMES_DYNAMIC = {
     'arch': 'System architecture (e.g. x86_64, aarch64, ppc64le, ...)',
+    'amdgcn_capabilities': "Comma-separated list of AMDGCN capabilities, as specified via "
+                           "--amdgcn-capabilities configuration option or "
+                           "via amdgcn_capabilities easyconfig parameter",
+    'amdgcn_cc_space_sep': "Space-separated list of AMDGCN capabilities",
+    'amdgcn_cc_semicolon_sep': "Semicolon-separated list of AMDGCN capabilities",
     'cuda_compute_capabilities': "Comma-separated list of CUDA compute capabilities, as specified via "
                                  "--cuda-compute-capabilities configuration option or "
                                  "via cuda_compute_capabilities easyconfig parameter",
@@ -478,6 +483,14 @@ def template_constant_dict(config, ignore=None, toolchain=None):
         template_values['cuda_sm_comma_sep'] = ','.join(sm_values)
         template_values['cuda_sm_space_sep'] = ' '.join(sm_values)
 
+    # step 7. AMDGCN capabilities
+    #         Use the commandline / easybuild config option if given, else use the value from the EC (as a default)
+    amdgcn_cc = build_option('amdgcn_capabilities') or config.get('amdgcn_capabilities')
+    if amdgcn_cc:
+        template_values['amdgcn_capabilities'] = ','.join(amdgcn_cc)
+        template_values['amdgcn_cc_space_sep'] = ' '.join(amdgcn_cc)
+        template_values['amdgcn_cc_semicolon_sep'] = ';'.join(amdgcn_cc)
+
     unknown_names = []
     for key in template_values:
         if not (key in common_template_names or key in TEMPLATE_NAMES_DYNAMIC):

easybuild/scripts/install-EasyBuild-develop.sh

@@ -13,6 +13,7 @@ print_usage()
     echo "Usage: $0 <github_username> <install_dir>"
     echo
     echo "    github_username:     username on GitHub for which the EasyBuild repositories should be cloned"
+    echo "                         Use 'easybuilders' if you don't want to use a fork"
     echo
     echo "    install_dir:         directory where all the EasyBuild files will be installed"
     echo
@@ -28,11 +29,13 @@ github_clone_branch()
     echo "=== Cloning ${GITHUB_USERNAME}/${REPO} ..."
     git clone --branch "${BRANCH}" "[email protected]:${GITHUB_USERNAME}/${REPO}.git"
 
-    echo "=== Adding and fetching EasyBuilders GitHub repository @ easybuilders/${REPO} ..."
-    cd "${REPO}"
-    git remote add "github_easybuilders" "[email protected]:easybuilders/${REPO}.git"
-    git fetch github_easybuilders
-    git branch --set-upstream-to "github_easybuilders/${BRANCH}" "${BRANCH}"
+    if [[ $GITHUB_USERNAME != "easybuilders" ]]; then
+        echo "=== Adding and fetching EasyBuilders GitHub repository @ easybuilders/${REPO} ..."
+        cd "${REPO}"
+        git remote add "github_easybuilders" "[email protected]:easybuilders/${REPO}.git"
+        git fetch github_easybuilders
+        git branch --set-upstream-to "github_easybuilders/${BRANCH}" "${BRANCH}"
+    fi
 }
 
 # Print the content of the module

easybuild/scripts/mk_tmpl_easyblock_for.py

@@ -131,7 +131,7 @@ class %(class_name)s(%(parent)s):
 
     def __init__(self, *args, **kwargs):
         \"\"\"Initialisation of custom class variables for %(name)s.\"\"\"
-        super(%(class_name)s, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
         self.example = None
 
@@ -154,7 +154,7 @@ def configure_step(self):
         run_shell_cmd(cmd)
 
         # complete configuration with configure_method of parent
-        super(%(class_name)s, self).configure_step()
+        super().configure_step()
 
     def build_step(self):
         \"\"\"Custom build procedure for %(name)s.\"\"\"
@@ -191,12 +191,12 @@ def sanity_check_step(self):
             'dirs': ['dir1', 'dir2'],
         }
 
-        super(%(class_name)s, self).sanity_check_step(custom_paths=custom_paths)
+        super().sanity_check_step(custom_paths=custom_paths)
 
     def make_module_req_guess(self):
         \"\"\"Custom guesses for environment variables (PATH, ...) for %(name)s.\"\"\"
 
-        guesses = super(%(class_name)s, self).make_module_req_guess()
+        guesses = super().make_module_req_guess()
 
         guesses.update({
             'VARIABLE': ['value1', 'value2'],
@@ -207,7 +207,7 @@ def make_module_req_guess(self):
     def make_module_extra(self):
         \"\"\"Custom extra module file entries for %(name)s.\"\"\"
 
-        txt = super(%(class_name)s, self).make_module_extra()
+        txt = super().make_module_extra()
 
         txt += self.module_generator.set_environment("VARIABLE", 'value')
         txt += self.module_generator.prepend_paths("PATH_VAR", ['path1', 'path2'])

easybuild/tools/config.py

@@ -228,6 +228,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
 BUILD_OPTIONS_CMDLINE = {
     None: [
         'aggregate_regtest',
+        'amdgcn_capabilities',
         'backup_modules',
         'banned_linked_shared_libs',
         'checksum_priority',

easybuild/tools/github.py

@@ -1127,6 +1127,14 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_
 
     # figure out commit message to use
     if commit_msg:
+        if pr_target_repo == GITHUB_EASYCONFIGS_REPO and all(file_info['new']) and not paths['files_to_delete']:
+            msg = "When only adding new easyconfigs usually a PR commit msg (--pr-commit-msg) should not be used, as "
+            msg += "the PR title will be automatically generated."
+            if build_option('force'):
+                print_msg(msg)
+                print_msg("Using the specified --pr-commit-msg as the force build option was specified.")
+            else:
+                raise EasyBuildError(msg)
         cnt = len(file_info['paths_in_repo'])
         _log.debug("Using specified commit message for all %d new/modified files at once: %s", cnt, commit_msg)
     elif pr_target_repo == GITHUB_EASYCONFIGS_REPO and all(file_info['new']) and not paths['files_to_delete']:
@@ -2167,7 +2175,7 @@ def new_pr(paths, ecs, title=None, descr=None, commit_msg=None):
                        pr_metadata=(file_info, deleted_paths, diff_stat), commit_msg=commit_msg)
 
 
-def det_account_branch_for_pr(pr_id, github_user=None, pr_target_repo=None):
+def det_account_repo_branch_for_pr(pr_id, github_user=None, pr_target_repo=None):
     """Determine account & branch corresponding to pull request with specified id."""
 
     if github_user is None:
@@ -2184,10 +2192,19 @@ def det_account_branch_for_pr(pr_id, github_user=None, pr_target_repo=None):
 
     # branch that corresponds with PR is supplied in form <account>:<branch_label>
     account = pr_data['head']['label'].split(':')[0]
+    repo = pr_data['head']['repo']['name']
     branch = ':'.join(pr_data['head']['label'].split(':')[1:])
     github_target = '%s/%s' % (pr_target_account, pr_target_repo)
     print_msg("Determined branch name corresponding to %s PR #%s: %s" % (github_target, pr_id, branch), log=_log)
 
+    return account, repo, branch
+
+
+def det_account_branch_for_pr(pr_id, github_user=None, pr_target_repo=None):
+    """Deprecated version of `det_account_repo_branch_for_pr`"""
+    _log.deprecated("`det_account_branch_for_pr` is deprecated, use `det_account_repo_branch_for_pr` instead", "6.0")
+    account, _, branch = det_account_repo_branch_for_pr(pr_id, github_user=github_user, pr_target_repo=pr_target_repo)
+
     return account, branch
 
 
@@ -2285,7 +2302,7 @@ def update_pr(pr_id, paths, ecs, commit_msg=None):
             exit_code=EasyBuildExit.OPTION_ERROR
         )
 
-    github_account, branch_name = det_account_branch_for_pr(pr_id, pr_target_repo=pr_target_repo)
+    github_account, _, branch_name = det_account_repo_branch_for_pr(pr_id, pr_target_repo=pr_target_repo)
 
     update_branch(branch_name, paths, ecs, github_account=github_account, commit_msg=commit_msg)
 
@@ -2877,18 +2894,18 @@ def sync_pr_with_develop(pr_id):
     target_account = build_option('pr_target_account')
     target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO
 
-    pr_account, pr_branch = det_account_branch_for_pr(pr_id)
+    pr_account, pr_repo, pr_branch = det_account_repo_branch_for_pr(pr_id)
 
     # initialize repository
     git_working_dir = tempfile.mkdtemp(prefix='git-working-dir')
     git_repo = init_repo(git_working_dir, target_repo)
 
-    setup_repo(git_repo, pr_account, target_repo, pr_branch)
+    setup_repo(git_repo, pr_account, pr_repo, pr_branch)
 
     sync_with_develop(git_repo, pr_branch, target_account, target_repo)
 
     # push updated branch back to GitHub (unless we're doing a dry run)
-    return push_branch_to_github(git_repo, pr_account, target_repo, pr_branch)
+    return push_branch_to_github(git_repo, pr_account, pr_repo, pr_branch)
 
 
 def sync_branch_with_develop(branch_name):