Merge pull request #47 from boegel/export_rpath_wrappers

remove `--rpath-wrapper-dir` configuration option + copy `rpath_args.py` script to avoid that RPATH wrapper scripts rely on external script

Author: ocaisa

easybuild/framework/easyblock.py

@@ -2902,13 +2902,12 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True):
         ])
 
         # Location to store RPATH wrappers
-        self.rpath_wrappers_dir = build_option('rpath_wrappers_dir')
         if self.rpath_wrappers_dir is not None:
             # Verify the path given is absolute
             if os.path.isabs(self.rpath_wrappers_dir):
-                _log.debug("Using %s to store/use RPATH wrappers" % self.rpath_wrappers_dir)
+                _log.info(f"Using {self.rpath_wrappers_dir} to store/use RPATH wrappers")
             else:
-                raise EasyBuildError("Path used for rpath_wrappers_dir is not an absolute path: %s", path)
+                raise EasyBuildError(f"Path used for rpath_wrappers_dir is not an absolute path: {path}")
 
         if self.iter_idx > 0:
             # reset toolchain for iterative runs before preparing it again
@@ -2925,9 +2924,11 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True):
                 self.modules_tool.prepend_module_path(full_mod_path)
 
         # prepare toolchain: load toolchain module and dependencies, set up build environment
-        self.toolchain.prepare(self.cfg['onlytcmod'], deps=self.cfg.dependencies(), silent=self.silent,
-                               loadmod=load_tc_deps_modules, rpath_filter_dirs=self.rpath_filter_dirs,
-                               rpath_include_dirs=self.rpath_include_dirs, rpath_wrappers_dir=self.rpath_wrappers_dir)
+        self.toolchain.prepare(onlymod=self.cfg['onlytcmod'], deps=self.cfg.dependencies(),
+                               silent=self.silent, loadmod=load_tc_deps_modules,
+                               rpath_filter_dirs=self.rpath_filter_dirs,
+                               rpath_include_dirs=self.rpath_include_dirs,
+                               rpath_wrappers_dir=self.rpath_wrappers_dir)
 
         # keep track of environment variables that were tweaked and need to be restored after environment got reset
         # $TMPDIR may be tweaked for OpenMPI 2.x, which doesn't like long $TMPDIR paths...

easybuild/scripts/rpath_wrapper_template.sh.in

@@ -42,6 +42,7 @@ function log {
 
 # command name
 CMD=`basename $0`
+TOPDIR=`dirname $0`
 
 log "found CMD: $CMD | original command: %(orig_cmd)s | orig args: '$(echo \"$@\")'"
 

easybuild/tools/config.py

@@ -277,7 +277,6 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'regtest_output_dir',
         'rpath_filter',
         'rpath_override_dirs',
-        'rpath_wrappers_dir',
         'required_linked_shared_libs',
         'search_path_cpp_headers',
         'search_path_linker',

easybuild/tools/options.py

@@ -524,8 +524,6 @@ def override_options(self):
             'rpath-filter': ("List of regex patterns to use for filtering out RPATH paths", 'strlist', 'store', None),
             'rpath-override-dirs': ("Path(s) to be prepended when linking with RPATH (string, colon-separated)",
                                     None, 'store', None),
-            'rpath-wrappers-dir': ("Absolute path to directory to use for RPATH wrappers creation/use",
-                                   None, 'store', None),
             'sanity-check-only': ("Only run sanity check (module is expected to be installed already",
                                   None, 'store_true', False),
             'set-default-module': ("Set the generated module as default", None, 'store_true', False),

easybuild/tools/toolchain/toolchain.py

@@ -62,7 +62,7 @@
 from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_warning
 from easybuild.tools.config import build_option, install_path
 from easybuild.tools.environment import setvar
-from easybuild.tools.filetools import adjust_permissions, find_eb_script, read_file, which, write_file
+from easybuild.tools.filetools import adjust_permissions, copy_file, find_eb_script, mkdir, read_file, which, write_file
 from easybuild.tools.module_generator import dependencies_for
 from easybuild.tools.modules import get_software_root, get_software_root_env_var_name
 from easybuild.tools.modules import get_software_version, get_software_version_env_var_name
@@ -985,6 +985,8 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None
         Put RPATH wrapper script in place for compiler and linker commands
 
         :param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...)
+        :param rpath_include_dirs: extra directories to include in RPATH
+        :param rpath_wrappers_dir: directory in which to create RPATH wrappers (tmpdir is created if None)
         """
         if get_os_type() == LINUX:
             self.log.info("Putting RPATH wrappers in place...")
@@ -994,6 +996,11 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None
         if rpath_filter_dirs is None:
             rpath_filter_dirs = []
 
+        # only enable logging by RPATH wrapper scripts in debug mode
+        enable_wrapper_log = build_option('debug')
+
+        copy_rpath_args_py = False
+
         # always include filter for 'stubs' library directory,
         # cfr. https://github.com/easybuilders/easybuild-framework/issues/2683
         # (since CUDA 11.something the stubs are in $EBROOTCUDA/stubs/lib64)
@@ -1002,19 +1009,35 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None
             if lib_stubs_pattern not in rpath_filter_dirs:
                 rpath_filter_dirs.append(lib_stubs_pattern)
 
-        # directory where all wrappers will be placed
-        disable_wrapper_log = False
+        # directory where all RPATH wrapper script will be placed;
         if rpath_wrappers_dir is None:
-            wrappers_dir = os.path.join(tempfile.mkdtemp(), RPATH_WRAPPERS_SUBDIR)
+            wrappers_dir = tempfile.mkdtemp()
         else:
-            wrappers_dir = os.path.join(rpath_wrappers_dir, RPATH_WRAPPERS_SUBDIR)
-            # No logging when we may be exporting wrappers
-            disable_wrapper_log = True
+            wrappers_dir = rpath_wrappers_dir
+            # disable logging in RPATH wrapper scripts when they may be exported for use outside of EasyBuild
+            enable_wrapper_log = False
+            # copy rpath_args.py script to sit alongside RPATH wrapper scripts
+            copy_rpath_args_py = True
+
+        # it's important to honor RPATH_WRAPPERS_SUBDIR, see is_rpath_wrapper method
+        wrappers_dir = os.path.join(wrappers_dir, RPATH_WRAPPERS_SUBDIR)
+        mkdir(wrappers_dir, parents=True)
 
         # must also wrap compilers commands, required e.g. for Clang ('gcc' on OS X)?
         c_comps, fortran_comps = self.compilers()
 
         rpath_args_py = find_eb_script('rpath_args.py')
+
+        # copy rpath_args.py script along RPATH wrappers, if desired
+        if copy_rpath_args_py:
+            copy_file(rpath_args_py, wrappers_dir)
+            # use path for %(rpath_args)s template value relative to location of the RPATH wrapper script,
+            # to avoid that the RPATH wrapper scripts rely on a script that's located elsewhere;
+            # that's mostly important when RPATH wrapper scripts are retained to be used outside of EasyBuild;
+            # we assume that each RPATH wrapper script is created in a separate subdirectory (see wrapper_dir below);
+            # ${TOPDIR} is defined in template for RPATH wrapper scripts, refers to parent dir of RPATH wrapper script
+            rpath_args_py = os.path.join('${TOPDIR}', '..', os.path.basename(rpath_args_py))
+
         rpath_wrapper_template = find_eb_script('rpath_wrapper_template.sh.in')
 
         # figure out list of patterns to use in rpath filter
@@ -1056,8 +1079,8 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None
                     raise EasyBuildError("Refusing to create a fork bomb, which(%s) == %s", cmd, orig_cmd)
 
                 # enable debug mode in wrapper script by specifying location for log file
-                if build_option('debug') and not disable_wrapper_log:
-                    rpath_wrapper_log = os.path.join(tempfile.gettempdir(), 'rpath_wrapper_%s.log' % cmd)
+                if enable_wrapper_log:
+                    rpath_wrapper_log = os.path.join(tempfile.gettempdir(), f'rpath_wrapper_{cmd}.log')
                 else:
                     rpath_wrapper_log = '/dev/null'
 

test/framework/toolchain.py

@@ -3153,7 +3153,8 @@ def test_toolchain_prepare_rpath_external(self):
         tc = self.get_toolchain('gompi', version='2018a')
         tc.set_options({'rpath': True})
         # allow the underlying toolchain to be in a prepared state (which may include rpath wrapping)
-        tc.prepare(rpath_wrappers_dir=target_wrapper_dir)
+        with self.mocked_stdout_stderr():
+            tc.prepare(rpath_wrappers_dir=target_wrapper_dir)
 
         # check that wrapper was created
         target_wrapper = os.path.join(target_wrapper_dir, RPATH_WRAPPERS_SUBDIR, 'gxx_wrapper', 'g++')