Merge branch 'develop' into fake-module-per-extension

Author: lexming

easybuild/framework/easyblock.py

@@ -239,6 +239,9 @@ def __init__(self, ec, logfile=None):
         # list of locations to include in RPATH used by toolchain
         self.rpath_include_dirs = []
 
+        # directory to export RPATH wrappers to
+        self.rpath_wrappers_dir = None
+
         # logging
         self.log = None
         self.logfile = logfile
@@ -2097,7 +2100,9 @@ def install_extensions_sequential(self, install=True):
                 # don't reload modules for toolchain, there is no need since they will be loaded already;
                 # the (fake) module for the parent software gets loaded before installing extensions
                 ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False,
-                                      rpath_filter_dirs=self.rpath_filter_dirs)
+                                      rpath_filter_dirs=self.rpath_filter_dirs,
+                                      rpath_include_dirs=self.rpath_include_dirs,
+                                      rpath_wrappers_dir=self.rpath_wrappers_dir)
 
             # actual installation of the extension
             if install and not self.dry_run:
@@ -2259,7 +2264,9 @@ def update_exts_progress_bar_helper(running_exts, progress_size):
                     # don't reload modules for toolchain, there is no need since they will be loaded already;
                     # the (fake) module for the parent software gets loaded before installing extensions
                     ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False,
-                                          rpath_filter_dirs=self.rpath_filter_dirs)
+                                          rpath_filter_dirs=self.rpath_filter_dirs,
+                                          rpath_include_dirs=self.rpath_include_dirs,
+                                          rpath_wrappers_dir=self.rpath_wrappers_dir)
                     if install and not self.dry_run:
                         with self.fake_module_environment(with_build_deps=True):
                             ext.install_extension_substep("pre_install_extension")
@@ -2797,7 +2804,8 @@ def check_checksums(self):
                 # take into account that extension may be a 2-tuple with just name/version
                 ext_opts = ext[2] if len(ext) == 3 else {}
                 # only a single source per extension is supported (see source_tmpl)
-                res = self.check_checksums_for(ext_opts, sub="of extension %s" % ext_name, source_cnt=1)
+                source_cnt = 1 if not ext_opts.get('nosource') else 0
+                res = self.check_checksums_for(ext_opts, sub="of extension %s" % ext_name, source_cnt=source_cnt)
                 checksum_issues.extend(res)
 
         return checksum_issues
@@ -2908,6 +2916,14 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True):
             '$ORIGIN/../lib64',
         ])
 
+        # Location to store RPATH wrappers
+        if self.rpath_wrappers_dir is not None:
+            # Verify the path given is absolute
+            if os.path.isabs(self.rpath_wrappers_dir):
+                _log.info(f"Using {self.rpath_wrappers_dir} to store/use RPATH wrappers")
+            else:
+                raise EasyBuildError(f"Path used for rpath_wrappers_dir is not an absolute path: {path}")
+
         if self.iter_idx > 0:
             # reset toolchain for iterative runs before preparing it again
             self.toolchain.reset()
@@ -2923,9 +2939,11 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True):
                 self.modules_tool.prepend_module_path(full_mod_path)
 
         # prepare toolchain: load toolchain module and dependencies, set up build environment
-        self.toolchain.prepare(self.cfg['onlytcmod'], deps=self.cfg.dependencies(), silent=self.silent,
-                               loadmod=load_tc_deps_modules, rpath_filter_dirs=self.rpath_filter_dirs,
-                               rpath_include_dirs=self.rpath_include_dirs)
+        self.toolchain.prepare(onlymod=self.cfg['onlytcmod'], deps=self.cfg.dependencies(),
+                               silent=self.silent, loadmod=load_tc_deps_modules,
+                               rpath_filter_dirs=self.rpath_filter_dirs,
+                               rpath_include_dirs=self.rpath_include_dirs,
+                               rpath_wrappers_dir=self.rpath_wrappers_dir)
 
         # keep track of environment variables that were tweaked and need to be restored after environment got reset
         # $TMPDIR may be tweaked for OpenMPI 2.x, which doesn't like long $TMPDIR paths...
@@ -3395,6 +3413,14 @@ def sanity_check_rpath(self, rpath_dirs=None, check_readelf_rpath=True):
                 self.log.debug(f"Sanity checking RPATH for files in {dirpath}")
 
                 for path in [os.path.join(dirpath, x) for x in os.listdir(dirpath)]:
+                    # skip the check for any symlinks that resolve to outside the installation directory
+                    if not is_parent_path(self.installdir, path):
+                        realpath = os.path.realpath(path)
+                        msg = (f"Skipping RPATH sanity check for {path}, since its absolute path {realpath} resolves to"
+                               f" outside the installation directory {self.installdir}")
+                        self.log.info(msg)
+                        continue
+
                     self.log.debug(f"Sanity checking RPATH for {path}")
 
                     out = get_linked_libs_raw(path)

easybuild/main.py

@@ -111,7 +111,26 @@ def find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=
     return [(ec_file, generated)]
 
 
-def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
+def summary(ecs_with_res):
+    """
+    Compose summary of the build:
+    * [S] for a successful build
+    * [F] for a failed build
+    * [-] for a build that didn’t run
+
+    :param ecs_with_res: list of tuples (ec, ec_res), ec is an EasyConfig object, and ec_res is a dict of the result
+    """
+    summary_fmt = "   * [{}] {}"
+    success_map = {True: 'S', False: 'F', None: '-'}
+    lines = ["Summary:"]
+    lines.extend([
+        summary_fmt.format(success_map[ec_res.get('success', False)], ec['full_mod_name'])
+        for ec, ec_res in ecs_with_res
+    ])
+    return '\n'.join(lines)
+
+
+def build_and_install_software(ecs, init_session_state, exit_on_failure=True, testing=False):
     """
     Build and install software for all provided parsed easyconfig files.
 
@@ -126,7 +145,7 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
 
     start_progress_bar(STATUS_BAR, size=len(ecs))
 
-    res = []
+    ecs_with_res = []
     ec_results = []
     failed_cnt = 0
 
@@ -171,14 +190,17 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
                 write_file(test_report_fp, test_report_txt['full'])
                 adjust_permissions(parent_dir, stat.S_IWUSR, add=False, recursive=False)
 
+        ecs_with_res.append((ec, ec_res))
+
         if not ec_res['success'] and exit_on_failure:
+            ecs_in_res = [res[0] for res in ecs_with_res]
+            ecs_without_res = [(ec, {'success': None}) for ec in ecs if ec not in ecs_in_res]
+            print_msg(summary(ecs_with_res + ecs_without_res), log=_log, silent=testing)
             error = ec_res['err']
             if isinstance(error, EasyBuildError):
                 error = EasyBuildError(test_msg, exit_code=error.exit_code)
             raise error
 
-        res.append((ec, ec_res))
-
         if failed_cnt:
             # if installations failed: indicate th
             status_label = ' (%s): ' % colorize('%s failed!' % failed_cnt, COLOR_RED)
@@ -192,7 +214,7 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
 
     stop_progress_bar(STATUS_BAR)
 
-    return res
+    return ecs_with_res
 
 
 def run_contrib_style_checks(ecs, check_contrib, check_style):
@@ -563,11 +585,11 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
 
         with rich_live_cm():
             run_hook(PRE_PREF + BUILD_AND_INSTALL_LOOP, hooks, args=[ordered_ecs])
-            ecs_with_res = build_and_install_software(ordered_ecs, init_session_state,
-                                                      exit_on_failure=exit_on_failure)
+            ecs_with_res = build_and_install_software(
+                ordered_ecs, init_session_state, exit_on_failure=exit_on_failure, testing=testing)
             run_hook(POST_PREF + BUILD_AND_INSTALL_LOOP, hooks, args=[ecs_with_res])
     else:
-        ecs_with_res = [(ec, {}) for ec in ordered_ecs]
+        ecs_with_res = [(ec, {'success': None}) for ec in ordered_ecs]
 
     correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)])
     overall_success = correct_builds_cnt == len(ordered_ecs)
@@ -585,6 +607,8 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
         print_msg(test_report_msg)
 
     print_msg(success_msg, log=_log, silent=testing)
+    if ecs_with_res:
+        print_msg(summary(ecs_with_res), log=_log, silent=testing)
 
     # cleanup and spec files
     for ec in easyconfigs:

easybuild/scripts/rpath_wrapper_template.sh.in

@@ -42,6 +42,7 @@ function log {
 
 # command name
 CMD=`basename $0`
+TOPDIR=`dirname $0`
 
 log "found CMD: $CMD | original command: %(orig_cmd)s | orig args: '$(echo \"$@\")'"
 

easybuild/tools/toolchain/toolchain.py

@@ -62,7 +62,7 @@
 from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_warning
 from easybuild.tools.config import build_option, install_path
 from easybuild.tools.environment import setvar
-from easybuild.tools.filetools import adjust_permissions, find_eb_script, read_file, which, write_file
+from easybuild.tools.filetools import adjust_permissions, copy_file, find_eb_script, mkdir, read_file, which, write_file
 from easybuild.tools.module_generator import dependencies_for
 from easybuild.tools.modules import get_software_root, get_software_root_env_var_name
 from easybuild.tools.modules import get_software_version, get_software_version_env_var_name
@@ -839,7 +839,7 @@ def reset(self):
         self.variables_init()
 
     def prepare(self, onlymod=None, deps=None, silent=False, loadmod=True,
-                rpath_filter_dirs=None, rpath_include_dirs=None):
+                rpath_filter_dirs=None, rpath_include_dirs=None, rpath_wrappers_dir=None):
         """
         Prepare a set of environment parameters based on name/version of toolchain
         - load modules for toolchain and dependencies
@@ -853,6 +853,7 @@ def prepare(self, onlymod=None, deps=None, silent=False, loadmod=True,
         :param loadmod: whether or not to (re)load the toolchain module, and the modules for the dependencies
         :param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...)
         :param rpath_include_dirs: extra directories to include in RPATH
+        :param rpath_wrappers_dir: directory in which to create RPATH wrappers
         """
 
         # take into account --sysroot configuration setting
@@ -906,7 +907,11 @@ def prepare(self, onlymod=None, deps=None, silent=False, loadmod=True,
 
         if build_option('rpath'):
             if self.options.get('rpath', True):
-                self.prepare_rpath_wrappers(rpath_filter_dirs, rpath_include_dirs)
+                self.prepare_rpath_wrappers(
+                    rpath_filter_dirs=rpath_filter_dirs,
+                    rpath_include_dirs=rpath_include_dirs,
+                    rpath_wrappers_dir=rpath_wrappers_dir
+                    )
                 self.use_rpath = True
             else:
                 self.log.info("Not putting RPATH wrappers in place, disabled via 'rpath' toolchain option")
@@ -975,11 +980,13 @@ def is_rpath_wrapper(path):
         # need to use binary mode to read the file, since it may be an actual compiler command (which is a binary file)
         return b'rpath_args.py $CMD' in read_file(path, mode='rb')
 
-    def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None):
+    def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None, rpath_wrappers_dir=None):
         """
         Put RPATH wrapper script in place for compiler and linker commands
 
         :param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...)
+        :param rpath_include_dirs: extra directories to include in RPATH
+        :param rpath_wrappers_dir: directory in which to create RPATH wrappers (tmpdir is created if None)
         """
         if get_os_type() == LINUX:
             self.log.info("Putting RPATH wrappers in place...")
@@ -989,6 +996,11 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None
         if rpath_filter_dirs is None:
             rpath_filter_dirs = []
 
+        # only enable logging by RPATH wrapper scripts in debug mode
+        enable_wrapper_log = build_option('debug')
+
+        copy_rpath_args_py = False
+
         # always include filter for 'stubs' library directory,
         # cfr. https://github.com/easybuilders/easybuild-framework/issues/2683
         # (since CUDA 11.something the stubs are in $EBROOTCUDA/stubs/lib64)
@@ -997,13 +1009,35 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None
             if lib_stubs_pattern not in rpath_filter_dirs:
                 rpath_filter_dirs.append(lib_stubs_pattern)
 
-        # directory where all wrappers will be placed
-        wrappers_dir = os.path.join(tempfile.mkdtemp(), RPATH_WRAPPERS_SUBDIR)
+        # directory where all RPATH wrapper script will be placed;
+        if rpath_wrappers_dir is None:
+            wrappers_dir = tempfile.mkdtemp()
+        else:
+            wrappers_dir = rpath_wrappers_dir
+            # disable logging in RPATH wrapper scripts when they may be exported for use outside of EasyBuild
+            enable_wrapper_log = False
+            # copy rpath_args.py script to sit alongside RPATH wrapper scripts
+            copy_rpath_args_py = True
+
+        # it's important to honor RPATH_WRAPPERS_SUBDIR, see is_rpath_wrapper method
+        wrappers_dir = os.path.join(wrappers_dir, RPATH_WRAPPERS_SUBDIR)
+        mkdir(wrappers_dir, parents=True)
 
         # must also wrap compilers commands, required e.g. for Clang ('gcc' on OS X)?
         c_comps, fortran_comps = self.compilers()
 
         rpath_args_py = find_eb_script('rpath_args.py')
+
+        # copy rpath_args.py script along RPATH wrappers, if desired
+        if copy_rpath_args_py:
+            copy_file(rpath_args_py, wrappers_dir)
+            # use path for %(rpath_args)s template value relative to location of the RPATH wrapper script,
+            # to avoid that the RPATH wrapper scripts rely on a script that's located elsewhere;
+            # that's mostly important when RPATH wrapper scripts are retained to be used outside of EasyBuild;
+            # we assume that each RPATH wrapper script is created in a separate subdirectory (see wrapper_dir below);
+            # ${TOPDIR} is defined in template for RPATH wrapper scripts, refers to parent dir of RPATH wrapper script
+            rpath_args_py = os.path.join('${TOPDIR}', '..', os.path.basename(rpath_args_py))
+
         rpath_wrapper_template = find_eb_script('rpath_wrapper_template.sh.in')
 
         # figure out list of patterns to use in rpath filter
@@ -1042,11 +1076,11 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None
 
                 # make *very* sure we don't wrap around ourselves and create a fork bomb...
                 if os.path.exists(cmd_wrapper) and os.path.exists(orig_cmd) and os.path.samefile(orig_cmd, cmd_wrapper):
-                    raise EasyBuildError("Refusing the create a fork bomb, which(%s) == %s", cmd, orig_cmd)
+                    raise EasyBuildError("Refusing to create a fork bomb, which(%s) == %s", cmd, orig_cmd)
 
                 # enable debug mode in wrapper script by specifying location for log file
-                if build_option('debug'):
-                    rpath_wrapper_log = os.path.join(tempfile.gettempdir(), 'rpath_wrapper_%s.log' % cmd)
+                if enable_wrapper_log:
+                    rpath_wrapper_log = os.path.join(tempfile.gettempdir(), f'rpath_wrapper_{cmd}.log')
                 else:
                     rpath_wrapper_log = '/dev/null'
 
@@ -1060,7 +1094,15 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None
                     'rpath_wrapper_log': rpath_wrapper_log,
                     'wrapper_dir': wrapper_dir,
                 }
-                write_file(cmd_wrapper, cmd_wrapper_txt)
+
+                # it may be the case that the wrapper already exists if the user provides a fixed location to store
+                # the RPATH wrappers, in this case the wrappers will be overwritten as they do not yet appear in the
+                # PATH (`which(cmd)` does not "see" them). Warn that they will be overwritten.
+                if os.path.exists(cmd_wrapper):
+                    _log.warning(f"Overwriting existing RPATH wrapper {cmd_wrapper}")
+                    write_file(cmd_wrapper, cmd_wrapper_txt, always_overwrite=True)
+                else:
+                    write_file(cmd_wrapper, cmd_wrapper_txt)
                 adjust_permissions(cmd_wrapper, stat.S_IXUSR)
 
                 # prepend location to this wrapper to $PATH

test/framework/easyconfig.py

@@ -4197,10 +4197,10 @@ def test_check_sha256_checksums(self):
         # re-test with right checksum in place
         toy_sha256 = '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc'
         test_ec_txt = checksums_regex.sub('checksums = ["%s"]' % toy_sha256, toy_ec_txt)
-        test_ec_txt = re.sub(r'patches = \[(.|\n)*\]', '', test_ec_txt)
+        passing_test_ec_txt = re.sub(r'patches = \[(.|\n)*\]', '', test_ec_txt)
 
         test_ec = os.path.join(self.test_prefix, 'toy-0.0-ok.eb')
-        write_file(test_ec, test_ec_txt)
+        write_file(test_ec, passing_test_ec_txt)
         ecs, _ = parse_easyconfigs([(test_ec, False)])
         ecs = [ec['ec'] for ec in ecs]
 
@@ -4236,6 +4236,15 @@ def test_check_sha256_checksums(self):
         regex = re.compile(r"Non-SHA256 checksum\(s\) found for toy-0.0.tar.gz:.*not_really_a_sha256_checksum")
         self.assertTrue(regex.match(res[0]), "Pattern '%s' found in: %s" % (regex.pattern, res[0]))
 
+        # Extension with nosource: True
+        test_ec_txt = passing_test_ec_txt + "exts_list = [('bar', '0.0', { 'nosource': True })]"
+        toy_sha256 = '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc'
+        test_ec = os.path.join(self.test_prefix, 'toy-0.0-nosource.eb')
+        write_file(test_ec, test_ec_txt)
+        ecs, _ = parse_easyconfigs([(test_ec, False)])
+        ecs = [ec['ec'] for ec in ecs]
+        self.assertEqual(check_sha256_checksums(ecs), [])
+
     def test_deprecated(self):
         """Test use of 'deprecated' easyconfig parameter."""
         topdir = os.path.dirname(os.path.abspath(__file__))