Merge branch 'develop' into 20250828122810_new_pr_bundle

Author: boegel

easybuild/easyblocks/generic/bundle.py

@@ -35,6 +35,7 @@
 """
 import copy
 import os
+from datetime import datetime
 
 import easybuild.tools.environment as env
 from easybuild.framework.easyblock import EasyBlock
@@ -45,7 +46,17 @@
 from easybuild.tools.config import build_option
 from easybuild.tools.hooks import TEST_STEP
 from easybuild.tools.modules import get_software_root, get_software_version
-from easybuild.tools.utilities import nub
+from easybuild.tools.utilities import nub, time2str
+
+
+# Description and step name run during component installation
+COMPONENT_INSTALL_STEPS = [
+    ('patching', 'patch'),
+    ('configuring', 'configure'),
+    ('building', 'build'),
+    ('testing', 'test'),
+    ('installing', 'install'),
+]
 
 
 class Bundle(EasyBlock):
@@ -265,14 +276,39 @@ def build_step(self):
         """Do nothing."""
         pass
 
+    def _install_component(self, comp):
+        """Run the installation steps for a single component"""
+        # run relevant steps
+        for descr, step_name in COMPONENT_INSTALL_STEPS:
+            if step_name in comp.cfg['skipsteps']:
+                comp.log.info("Skipping '%s' step for component %s v%s", step_name, comp.name, comp.version)
+            elif build_option('skip_test_step') and step_name == TEST_STEP:
+                comp.log.info("Skipping %s step for component %s v%s, as requested via skip-test-step", step_name,
+                              comp.name, comp.version)
+            else:
+                msg = f'   {descr} component {comp.name}...'
+                if self.dry_run:
+                    self.dry_run_msg("%s [DRY RUN]\n", msg)
+                else:
+                    print_msg(msg, log=self.log, silent=self.silent)
+                start_time = datetime.now()
+                try:
+                    comp.run_step(step_name, [lambda x: getattr(x, '%s_step' % step_name)])
+                finally:
+                    if not self.dry_run:
+                        step_duration = datetime.now() - start_time
+                        if step_duration.total_seconds() >= 1:
+                            print_msg("   ... (took %s)", time2str(step_duration), log=self.log, silent=self.silent)
+                        elif self.logdebug or build_option('trace'):
+                            print_msg("   ... (took < 1 sec)", log=self.log, silent=self.silent)
+
     def install_step(self):
         """Install components, if specified."""
         comp_cnt = len(self.cfg['components'])
         for idx, (cfg, comp) in enumerate(self.comp_instances):
-
             print_msg("installing bundle component %s v%s (%d/%d)..." %
-                      (cfg['name'], cfg['version'], idx + 1, comp_cnt))
-            self.log.info("Installing component %s v%s using easyblock %s", cfg['name'], cfg['version'], cfg.easyblock)
+                      (comp.name, comp.version, idx + 1, comp_cnt))
+            self.log.info("Installing component %s v%s using easyblock %s", comp.name, comp.version, cfg.easyblock)
 
             # correct build/install dirs
             comp.builddir = self.builddir
@@ -317,18 +353,10 @@ def install_step(self):
                 comp.src[-1]['finalpath'] = comp.cfg['start_dir']
 
             # check if sanity checks are enabled for the component
-            if self.cfg['sanity_check_all_components'] or comp.cfg['name'] in self.cfg['sanity_check_components']:
+            if self.cfg['sanity_check_all_components'] or comp.name in self.cfg['sanity_check_components']:
                 self.comp_cfgs_sanity_check.append(comp)
 
-            # run relevant steps
-            for step_name in ['patch', 'configure', 'build', 'test', 'install']:
-                if step_name in cfg['skipsteps']:
-                    comp.log.info("Skipping '%s' step for component %s v%s", step_name, cfg['name'], cfg['version'])
-                elif build_option('skip_test_step') and step_name == TEST_STEP:
-                    comp.log.info("Skipping %s step for component %s v%s, as requested via skip-test-step", step_name,
-                                  cfg['name'], cfg['version'])
-                else:
-                    comp.run_step(step_name, [lambda x: getattr(x, '%s_step' % step_name)])
+            self._install_component(comp)
 
             if comp.make_module_req_guess.__qualname__ != 'EasyBlock.make_module_req_guess':
                 depr_msg = f"Easyblock used to install component {comp.name} still uses make_module_req_guess"
@@ -381,13 +409,13 @@ def make_module_step(self, *args, **kwargs):
         as this is done in the generic EasyBlock while creating
         the module file already.
         """
-        for cfg, comp in self.comp_instances:
-            self.log.info("Gathering module paths for component %s v%s", cfg['name'], cfg['version'])
+        for _, comp in self.comp_instances:
+            self.log.info("Gathering module paths for component %s v%s", comp.name, comp.version)
 
             # take into account that easyblock used for component may not be migrated yet to module_load_environment
             if comp.make_module_req_guess.__qualname__ != 'EasyBlock.make_module_req_guess':
 
-                depr_msg = f"Easyblock used to install component {cfg['name']} still uses make_module_req_guess"
+                depr_msg = f"Easyblock used to install component {comp.name} still uses make_module_req_guess"
                 self.log.deprecated(depr_msg, '6.0')
 
                 reqs = comp.make_module_req_guess()
@@ -403,7 +431,7 @@ def make_module_step(self, *args, **kwargs):
                             setattr(self.module_load_environment, key, value)
                 except AttributeError:
                     raise EasyBuildError("Cannot process module requirements of bundle component %s v%s",
-                                         cfg['name'], cfg['version'])
+                                         comp.name, comp.version)
             else:
                 # Explicit call required as adding step to 'install_step' is not sufficient
                 # for module-only build. Set fake arg to True, as module components should

easybuild/easyblocks/generic/cmdcp.py

@@ -67,7 +67,9 @@ def build_step(self):
             # determine command to use
             # find (first) regex match, then complete matching command template
             cmd = None
-            for pattern, regex_cmd in self.cfg['cmds_map']:
+            for pattern, regex_cmd in self.cfg.get_ref('cmds_map'):
+                pattern = pattern % self.cfg.template_values
+                regex_cmd = regex_cmd % self.cfg.template_values
                 try:
                     regex = re.compile(pattern)
                 except re.error as err:
@@ -78,6 +80,6 @@ def build_step(self):
                     break
             if cmd is None:
                 raise EasyBuildError("No match for %s in %s, don't know which command to use.",
-                                     src, self.cfg['cmds_map'])
+                                     src, self.cfg.get_ref('cmds_map'))
 
             run_shell_cmd(cmd)

easybuild/easyblocks/h/hpl.py

@@ -30,12 +30,16 @@
 @author: Kenneth Hoste (Ghent University)
 @author: Pieter De Baets (Ghent University)
 @author: Jens Timmerman (Ghent University)
+@author: Davide Grassano (CECAM - EPFL)
 """
 
+import re
 import os
 
+import easybuild.tools.toolchain as toolchain
 from easybuild.easyblocks.generic.configuremake import ConfigureMake
 from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.config import build_option
 from easybuild.tools.filetools import change_dir, copy_file, mkdir, remove_file, symlink
 from easybuild.tools.run import run_shell_cmd
 
@@ -104,10 +108,74 @@ def build_step(self, topdir=None):
         # C compilers flags
         extra_makeopts += "CCFLAGS='$(HPL_DEFS) %s' " % os.getenv('CFLAGS')
 
+        comp_fam = self.toolchain.comp_family()
+        if comp_fam in [toolchain.INTELCOMP]:
+            # Explicitly disable optimization, since Intel compilers apply some default
+            # level not shown on the command line.
+            # This breaks the result comparison, resulting in all tests failing residual checks.
+            # See https://github.com/easybuilders/easybuild-easyconfigs/pull/23704#issuecomment-3202392904
+            extra_makeopts += 'CCNOOPT=\'$(HPL_DEFS) -O0\' '
+
         # set options and build
         self.cfg.update('buildopts', extra_makeopts)
         super().build_step()
 
+    def test_step(self):
+        """Test by running xhpl"""
+        srcdir = os.path.join(self.cfg['start_dir'], 'bin', 'UNKNOWN')
+        change_dir(srcdir)
+
+        pre_cmd = ""
+        post_cmd = ""
+
+        # xhpl needs atleast 4 processes to run the test suite
+        req_cpus = 4
+
+        mpi_fam = self.toolchain.mpi_family()
+        if mpi_fam is None:
+            self.report_test_failure("Toolchain does not include an MPI implementation, cannot run tests")
+
+        parallel = self.cfg.parallel
+        if not build_option('mpi_tests'):
+            self.log.info("MPI tests disabled from buildoption. Setting parallel to 1")
+            parallel = 1
+
+        if parallel < req_cpus:
+            self.log.info("Running tests with 1 oversubscribed process")
+
+            pin_str = ','.join(["0"] * req_cpus)
+            if mpi_fam in [toolchain.INTELMPI]:
+                pre_cmd = f"I_MPI_PIN_PROCESSOR_LIST=\"{pin_str}\" I_MPI_PIN=on "
+            elif mpi_fam in [toolchain.OPENMPI]:
+                post_cmd = f"--cpu-set {pin_str}"
+            elif mpi_fam in [toolchain.MPICH]:
+                post_cmd = f"-bind-to user:{pin_str}"
+            else:
+                self.report_test_failure("Don't know how to oversubscribe for `%s` MPI family" % mpi_fam)
+
+        cmd = self.toolchain.mpi_cmd_for(f'{post_cmd} ./xhpl', req_cpus)
+        cmd = f'{pre_cmd} {cmd}'
+        res = run_shell_cmd(cmd)
+        out = res.output
+
+        passed_rgx = re.compile(r'(\d+) tests completed and passed')
+        failed_rgx = re.compile(r'(\d+) tests completed and failed')
+
+        nfailed = 0
+        passed_mch = passed_rgx.search(out)
+        failed_mch = failed_rgx.search(out)
+        if passed_mch:
+            npassed = int(passed_mch.group(1))
+            self.log.info("%d tests passed residual checks in xhpl output" % npassed)
+        else:
+            self.report_test_failure("Could not find test results in output of xhpl")
+
+        if failed_mch:
+            nfailed = int(failed_mch.group(1))
+
+        if nfailed > 0:
+            self.report_test_failure("%d tests failed residual checks in xhpl output" % nfailed)
+
     def install_step(self):
         """
         Install by copying files to install dir
@@ -119,14 +187,15 @@ def install_step(self):
             srcfile = os.path.join(srcdir, filename)
             copy_file(srcfile, destdir)
 
-    def sanity_check_step(self):
+    def sanity_check_step(self, **kwargs):
         """
         Custom sanity check for HPL
         """
 
-        custom_paths = {
+        # Allow subclasses to set own custom paths
+        kwargs.setdefault('custom_paths', {
             'files': ["bin/xhpl"],
             'dirs': []
-        }
+        })
 
-        super().sanity_check_step(custom_paths)
+        super().sanity_check_step(**kwargs)

easybuild/easyblocks/l/lammps.py

@@ -40,6 +40,7 @@
 import easybuild.tools.environment as env
 import easybuild.tools.toolchain as toolchain
 from easybuild.base import fancylogger
+from easybuild.easyblocks.python import set_py_env_vars
 from easybuild.framework.easyconfig import CUSTOM, MANDATORY
 from easybuild.tools.build_log import EasyBuildError, print_warning, print_msg
 from easybuild.tools.config import build_option
@@ -646,6 +647,7 @@ def install_step(self):
                 'site_packages': site_packages,
             }
 
+            set_py_env_vars(self.log)
             run_shell_cmd(cmd)
 
     def sanity_check_step(self, *args, **kwargs):

easybuild/easyblocks/w/wps.py

@@ -231,6 +231,7 @@ def configure_step(self):
             'DM_CC': os.getenv('MPICC'),
             'FC': os.getenv('MPIF90'),
             'CC': os.getenv('MPICC'),
+            'CPP': 'cpp -P -traditional',
         }
         if self.toolchain.options.get('openmp', None):
             comps.update({'LDFLAGS': '%s %s' % (self.toolchain.get_flag('openmp'), os.environ['LDFLAGS'])})