Merge branch 'develop' into 20250813131209_new_pr_bundle

Author: Flamefire

easybuild/easyblocks/a/amber.py

@@ -88,35 +88,37 @@ def extract_step(self):
     def patch_step(self, *args, **kwargs):
         """Patch Amber using 'update_amber' tool, prior to applying listed patch files (if any)."""
 
-        # figure out which Python command to use to run the update_amber script;
-        # by default it uses 'python', but this may not be available (on CentOS 8 for example);
-        # note that the dependencies are not loaded yet at this point, so we're at the mercy of the OS here...
-        python_cmd = None
-        for cand_python_cmd in ['python', 'python3', 'python2']:
-            if which(cand_python_cmd):
-                python_cmd = cand_python_cmd
-                break
-
-        if python_cmd is None:
-            raise EasyBuildError("No suitable Python command found to run update_amber script!")
-
-        if self.cfg['patchlevels'] == "latest":
-            cmd = "%s ./update_amber --update" % python_cmd
-            # Run as many times as specified. It is the responsibility
-            # of the easyconfig author to get this right, especially if
-            # he or she selects "latest". (Note: "latest" is not
-            # recommended for this reason and others.)
-            for _ in range(self.cfg['patchruns']):
-                run_shell_cmd(cmd)
-        else:
-            for (tree, patch_level) in zip(['AmberTools', 'Amber'], self.cfg['patchlevels']):
-                if patch_level == 0:
-                    continue
-                cmd = "%s ./update_amber --update-to %s/%s" % (python_cmd, tree, patch_level)
+        # Use the update_amber script if patchlevels is defined - if not then the easyconfig should apply the patches
+        if self.cfg['patchlevels']:
+            # figure out which Python command to use to run the update_amber script;
+            # by default it uses 'python', but this may not be available (on CentOS 8 for example);
+            # note that the dependencies are not loaded yet at this point, so we're at the mercy of the OS here...
+            python_cmd = None
+            for cand_python_cmd in ['python', 'python3', 'python2']:
+                if which(cand_python_cmd):
+                    python_cmd = cand_python_cmd
+                    break
+
+            if python_cmd is None:
+                raise EasyBuildError("No suitable Python command found to run update_amber script!")
+
+            if self.cfg['patchlevels'] == "latest":
+                cmd = "%s ./update_amber --update" % python_cmd
                 # Run as many times as specified. It is the responsibility
-                # of the easyconfig author to get this right.
+                # of the easyconfig author to get this right, especially if
+                # he or she selects "latest". (Note: "latest" is not
+                # recommended for this reason and others.)
                 for _ in range(self.cfg['patchruns']):
                     run_shell_cmd(cmd)
+            else:
+                for (tree, patch_level) in zip(['AmberTools', 'Amber'], self.cfg['patchlevels']):
+                    if patch_level == 0:
+                        continue
+                    cmd = "%s ./update_amber --update-to %s/%s" % (python_cmd, tree, patch_level)
+                    # Run as many times as specified. It is the responsibility
+                    # of the easyconfig author to get this right.
+                    for _ in range(self.cfg['patchruns']):
+                        run_shell_cmd(cmd)
 
         super().patch_step(*args, **kwargs)
 

easybuild/easyblocks/e/elpa.py

@@ -206,7 +206,8 @@ def configure_step(self):
             self.cfg.update('configopts', '--with-NVIDIA-GPU-compute-capability=sm_%s' % cuda_cc_string)
             self.log.info("Enabling nvidia GPU support for compute capability: %s", cuda_cc_string)
             # There is a dedicated kernel for sm80, but only from version 2021.11.001 onwards
-            if float(cuda_cc) >= 8.0 and LooseVersion(self.version) >= LooseVersion('2021.11.001'):
+            # Trying to use these kernels for GPUs newer than sm80 will fail ELPA configure
+            if float(cuda_cc) == 8.0 and LooseVersion(self.version) >= LooseVersion('2021.11.001'):
                 self.cfg.update('configopts', '--enable-nvidia-sm80-gpu')
 
         # From v2022.05.001 onwards, the config complains if CPP is not set, resulting in non-zero exit of configure

easybuild/easyblocks/g/ghc.py

@@ -37,7 +37,7 @@ class EB_GHC(ConfigureMake):
     Support for building and installing applications with configure/make/make install
     """
 
-    def build_step(self, verbose=False):
+    def build_step(self, *args, **kwargs):
         """
         Support for a binary 6.12.x installation. Starting there,
         later GHC versions are build from source and thus require
@@ -46,4 +46,4 @@ def build_step(self, verbose=False):
         if LooseVersion(self.version) < LooseVersion("7.0"):
             pass
         else:
-            super().build_step(verbose=verbose)
+            super().build_step(*args, **kwargs)

easybuild/easyblocks/generic/buildenv.py

@@ -28,11 +28,14 @@
 
 @author: Alan O'Cais (Juelich Supercomputing Centre)
 """
+import glob
 import os
 import stat
 
 from easybuild.easyblocks.generic.bundle import Bundle
-from easybuild.tools.filetools import adjust_permissions, copy_dir
+from easybuild.framework.easyconfig import CUSTOM
+from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.filetools import adjust_permissions, apply_regex_substitutions, copy_dir
 from easybuild.tools.toolchain.toolchain import RPATH_WRAPPERS_SUBDIR
 
 
@@ -41,6 +44,15 @@ class BuildEnv(Bundle):
     Build environment of toolchain: only generate module file
     """
 
+    @staticmethod
+    def extra_options():
+        """Add extra easyconfig parameters for Boost."""
+        extra_vars = {
+            'python_executable': ['python3', "Python executable to use for the wrappers (use None to use path to "
+                                  "Python executable used by EasyBuild).", CUSTOM],
+        }
+        return Bundle.extra_options(extra_vars)
+
     def prepare_step(self, *args, **kwargs):
         """
         Custom prepare step for buildenv: export rpath wrappers if they are being used
@@ -60,7 +72,16 @@ def install_step(self, *args, **kwargs):
         wrappers_dir = os.path.join(self.rpath_wrappers_dir, RPATH_WRAPPERS_SUBDIR)
         if os.path.exists(wrappers_dir):
             self.rpath_wrappers_dir = os.path.join(self.installdir, 'bin')
-            copy_dir(wrappers_dir, os.path.join(self.rpath_wrappers_dir, RPATH_WRAPPERS_SUBDIR))
+            rpath_wrappers_path = os.path.join(self.rpath_wrappers_dir, RPATH_WRAPPERS_SUBDIR)
+            copy_dir(wrappers_dir, rpath_wrappers_path)
+            py_exe = self.cfg['python_executable']
+            if py_exe is not None:
+                if not isinstance(py_exe, str) or not py_exe:
+                    raise EasyBuildError(f"python_executable should be None or non-empty string, got {py_exe}")
+                wrapper_files = list(filter(os.path.isfile, glob.glob(os.path.join(rpath_wrappers_path, '*', '*'))))
+                # replace path to Python executable with python_executable in the wrappers,
+                # as this is the executable that runs EasyBuild and may be unavailable when using the buildenv wrappers.
+                apply_regex_substitutions(wrapper_files, [(r'^PYTHON_EXE=.*$', f'PYTHON_EXE={py_exe}')], backup=False)
             # Make sure wrappers are readable/executable by everyone
             adjust_permissions(
                 self.rpath_wrappers_dir,

easybuild/easyblocks/generic/bundle.py

@@ -35,17 +35,29 @@
 """
 import copy
 import os
+from datetime import datetime
 
 import easybuild.tools.environment as env
 from easybuild.framework.easyblock import EasyBlock
 from easybuild.framework.easyconfig import CUSTOM
-from easybuild.framework.easyconfig.default import DEFAULT_CONFIG
+from easybuild.framework.easyconfig.default import get_easyconfig_parameter_default
+from easybuild.framework.easyconfig.default import is_easyconfig_parameter_default_value
 from easybuild.framework.easyconfig.easyconfig import get_easyblock_class
 from easybuild.tools.build_log import EasyBuildError, print_msg
 from easybuild.tools.config import build_option
 from easybuild.tools.hooks import TEST_STEP
 from easybuild.tools.modules import get_software_root, get_software_version
-from easybuild.tools.utilities import nub
+from easybuild.tools.utilities import nub, time2str
+
+
+# Description and step name run during component installation
+COMPONENT_INSTALL_STEPS = [
+    ('patching', 'patch'),
+    ('configuring', 'configure'),
+    ('building', 'build'),
+    ('testing', 'test'),
+    ('installing', 'install'),
+]
 
 
 class Bundle(EasyBlock):
@@ -125,24 +137,6 @@ def __init__(self, *args, **kwargs):
                 if len(comp) == 3:
                     comp_specs = comp[2]
 
-                comp_cfg = self.cfg.copy()
-
-                comp_cfg['name'] = comp_name
-                comp_cfg['version'] = comp_version
-
-                # The copy above may include unexpected settings for common values.
-                # In particular for a Pythonbundle we have seen a component inheriting
-                #  runtest = True
-                # which is not a valid value for many easyblocks.
-                # Reset runtest to the original default, if people want the test step
-                # they can set it explicitly, in default_component_specs or by the component easyblock
-                if comp_cfg._config['runtest'] != DEFAULT_CONFIG["runtest"]:
-                    self.log.warning(
-                        "Resetting runtest to default value for component easyblock "
-                        f"(from {comp_cfg._config['runtest']})."
-                        )
-                    comp_cfg._config['runtest'] = DEFAULT_CONFIG["runtest"]
-
                 # determine easyblock to use for this component
                 # - if an easyblock is specified explicitly, that will be used
                 # - if not, a software-specific easyblock will be considered by get_easyblock_class
@@ -164,28 +158,45 @@ def __init__(self, *args, **kwargs):
                 if easyblock == 'Bundle':
                     raise EasyBuildError("The Bundle easyblock can not be used to install components in a bundle")
 
+                comp_cfg = self.cfg.copy()
                 comp_cfg.easyblock = easyblock_class
 
                 # make sure that extra easyconfig parameters are known, so they can be set
                 extra_opts = comp_cfg.easyblock.extra_options()
                 comp_cfg.extend_params(copy.deepcopy(extra_opts))
 
-                comp_cfg.generate_template_values()
+                # The copy above may include unexpected settings for common values.
+                # In particular for a Pythonbundle we have seen a component inheriting
+                #  runtest = True
+                # which is not a valid value for many easyblocks.
+                # Reset runtest to the original default, if people want the test step
+                # they can set it explicitly, in default_component_specs or by the component easyblock
+                if not is_easyconfig_parameter_default_value('runtest', comp_cfg.get('runtest', resolve=False)):
+                    self.log.warning(
+                        "Resetting runtest to default value for component easyblock "
+                        f"(from {comp_cfg.get('runtest', resolve=False)})."
+                        )
+                    comp_cfg['runtest'] = get_easyconfig_parameter_default('runtest')
 
-                # do not inherit easyblock to use from parent
-                # (since that would result in an infinite loop in install_step)
-                comp_cfg['easyblock'] = None
+                # Reset others to their default value
+                # Inheriting easyblock would lead to an infinite loop in the install step
+                for var in ('easyblock',
+                            'sources', 'source_urls', 'checksums',
+                            'patches', 'postinstallpatches',
+                            'modextravars', 'modextrapaths'):
+                    comp_cfg[var] = copy.deepcopy(get_easyconfig_parameter_default(var))
 
-                # reset list of sources/source_urls/checksums
-                comp_cfg['sources'] = comp_cfg['source_urls'] = comp_cfg['checksums'] = []
-                comp_cfg['patches'] = comp_cfg['postinstallpatches'] = []
+                comp_cfg['name'] = comp_name
+                comp_cfg['version'] = comp_version
 
                 for key in self.cfg['default_component_specs']:
                     comp_cfg[key] = self.cfg['default_component_specs'][key]
 
                 for key in comp_specs:
                     comp_cfg[key] = comp_specs[key]
 
+                comp_cfg.generate_template_values()
+
                 # Don't require that all template values can be resolved at this point but still resolve them.
                 # This is important to ensure that template values like %(name)s and %(version)s
                 # are correctly resolved with the component name/version before values are copied over to self.cfg
@@ -274,14 +285,39 @@ def build_step(self):
         """Do nothing."""
         pass
 
+    def _install_component(self, comp):
+        """Run the installation steps for a single component"""
+        # run relevant steps
+        for descr, step_name in COMPONENT_INSTALL_STEPS:
+            if step_name in comp.cfg['skipsteps']:
+                comp.log.info("Skipping '%s' step for component %s v%s", step_name, comp.name, comp.version)
+            elif build_option('skip_test_step') and step_name == TEST_STEP:
+                comp.log.info("Skipping %s step for component %s v%s, as requested via skip-test-step", step_name,
+                              comp.name, comp.version)
+            else:
+                msg = f'   {descr} component {comp.name}...'
+                if self.dry_run:
+                    self.dry_run_msg("%s [DRY RUN]\n", msg)
+                else:
+                    print_msg(msg, log=self.log, silent=self.silent)
+                start_time = datetime.now()
+                try:
+                    comp.run_step(step_name, [lambda x: getattr(x, '%s_step' % step_name)])
+                finally:
+                    if not self.dry_run:
+                        step_duration = datetime.now() - start_time
+                        if step_duration.total_seconds() >= 1:
+                            print_msg("   ... (took %s)", time2str(step_duration), log=self.log, silent=self.silent)
+                        elif self.logdebug or build_option('trace'):
+                            print_msg("   ... (took < 1 sec)", log=self.log, silent=self.silent)
+
     def install_step(self):
         """Install components, if specified."""
         comp_cnt = len(self.cfg['components'])
         for idx, (cfg, comp) in enumerate(self.comp_instances):
-
             print_msg("installing bundle component %s v%s (%d/%d)..." %
-                      (cfg['name'], cfg['version'], idx + 1, comp_cnt))
-            self.log.info("Installing component %s v%s using easyblock %s", cfg['name'], cfg['version'], cfg.easyblock)
+                      (comp.name, comp.version, idx + 1, comp_cnt))
+            self.log.info("Installing component %s v%s using easyblock %s", comp.name, comp.version, cfg.easyblock)
 
             # correct build/install dirs
             comp.builddir = self.builddir
@@ -326,18 +362,10 @@ def install_step(self):
                 comp.src[-1]['finalpath'] = comp.cfg['start_dir']
 
             # check if sanity checks are enabled for the component
-            if self.cfg['sanity_check_all_components'] or comp.cfg['name'] in self.cfg['sanity_check_components']:
+            if self.cfg['sanity_check_all_components'] or comp.name in self.cfg['sanity_check_components']:
                 self.comp_cfgs_sanity_check.append(comp)
 
-            # run relevant steps
-            for step_name in ['patch', 'configure', 'build', 'test', 'install']:
-                if step_name in cfg['skipsteps']:
-                    comp.log.info("Skipping '%s' step for component %s v%s", step_name, cfg['name'], cfg['version'])
-                elif build_option('skip_test_step') and step_name == TEST_STEP:
-                    comp.log.info("Skipping %s step for component %s v%s, as requested via skip-test-step", step_name,
-                                  cfg['name'], cfg['version'])
-                else:
-                    comp.run_step(step_name, [lambda x: getattr(x, '%s_step' % step_name)])
+            self._install_component(comp)
 
             if comp.make_module_req_guess.__qualname__ != 'EasyBlock.make_module_req_guess':
                 depr_msg = f"Easyblock used to install component {comp.name} still uses make_module_req_guess"
@@ -390,13 +418,13 @@ def make_module_step(self, *args, **kwargs):
         as this is done in the generic EasyBlock while creating
         the module file already.
         """
-        for cfg, comp in self.comp_instances:
-            self.log.info("Gathering module paths for component %s v%s", cfg['name'], cfg['version'])
+        for _, comp in self.comp_instances:
+            self.log.info("Gathering module paths for component %s v%s", comp.name, comp.version)
 
             # take into account that easyblock used for component may not be migrated yet to module_load_environment
             if comp.make_module_req_guess.__qualname__ != 'EasyBlock.make_module_req_guess':
 
-                depr_msg = f"Easyblock used to install component {cfg['name']} still uses make_module_req_guess"
+                depr_msg = f"Easyblock used to install component {comp.name} still uses make_module_req_guess"
                 self.log.deprecated(depr_msg, '6.0')
 
                 reqs = comp.make_module_req_guess()
@@ -412,7 +440,7 @@ def make_module_step(self, *args, **kwargs):
                             setattr(self.module_load_environment, key, value)
                 except AttributeError:
                     raise EasyBuildError("Cannot process module requirements of bundle component %s v%s",
-                                         cfg['name'], cfg['version'])
+                                         comp.name, comp.version)
             else:
                 # Explicit call required as adding step to 'install_step' is not sufficient
                 # for module-only build. Set fake arg to True, as module components should
@@ -453,9 +481,23 @@ def sanity_check_step(self, *args, **kwargs):
 
         # run sanity checks for specific components
         cnt = len(self.comp_cfgs_sanity_check)
-        for idx, comp in enumerate(self.comp_cfgs_sanity_check):
-            comp_name, comp_ver = comp.cfg['name'], comp.cfg['version']
-            print_msg("sanity checking bundle component %s v%s (%i/%i)...", comp_name, comp_ver, idx + 1, cnt)
-            self.log.info("Starting sanity check step for component %s v%s", comp_name, comp_ver)
-
-            comp.run_step('sanity_check', [lambda x: x.sanity_check_step])
+        if cnt > 0:
+            if self.sanity_check_module_loaded:
+                loaded_module = False
+            else:
+                self.sanity_check_load_module(extension=kwargs.get('extension', False),
+                                              extra_modules=kwargs.get('extra_modules', None))
+                loaded_module = self.sanity_check_module_loaded
+            for idx, comp in enumerate(self.comp_cfgs_sanity_check):
+                print_msg("sanity checking bundle component %s v%s (%i/%i)...", comp.name, comp.version, idx + 1, cnt)
+                self.log.info("Starting sanity check step for component %s v%s", comp.name, comp.version)
+
+                # Avoid loading the module in components again
+                comp.sanity_check_module_loaded = True
+                comp.run_step('sanity_check', [lambda x: x.sanity_check_step])
+                comp.sanity_check_module_loaded = False
+            if loaded_module:
+                if self.fake_mod_data:
+                    self.clean_up_fake_module(self.fake_mod_data)
+                    self.fake_mod_data = None
+                self.sanity_check_module_loaded = False