Merge pull request #4051 from boegel/get_linked_libs_raw

add get_linked_libs_raw function, and use it from both check_linked_shared_libs and sanity_check_rpath

Author: Micket

easybuild/framework/easyblock.py

@@ -96,7 +96,8 @@
 from easybuild.tools.package.utilities import package
 from easybuild.tools.py2vs3 import extract_method_name, string_type
 from easybuild.tools.repository.repository import init_repository
-from easybuild.tools.systemtools import check_linked_shared_libs, det_parallelism, get_shared_lib_ext, use_group
+from easybuild.tools.systemtools import check_linked_shared_libs, det_parallelism, get_linked_libs_raw
+from easybuild.tools.systemtools import get_shared_lib_ext, use_group
 from easybuild.tools.utilities import INDENT_4SPACES, get_class_for, nub, quote_str
 from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg
 from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION
@@ -2994,24 +2995,15 @@ def sanity_check_rpath(self, rpath_dirs=None):
                 for path in [os.path.join(dirpath, x) for x in os.listdir(dirpath)]:
                     self.log.debug("Sanity checking RPATH for %s", path)
 
-                    out, ec = run_cmd("file %s" % path, simple=False, trace=False)
-                    if ec:
-                        fail_msg = "Failed to run 'file %s': %s" % (path, out)
-                        self.log.warning(fail_msg)
-                        fails.append(fail_msg)
-
-                    # only run ldd/readelf on dynamically linked executables/libraries
-                    # example output:
-                    # ELF 64-bit LSB executable, x86-64, version 1 (SYSV), dynamically linked (uses shared libs), ...
-                    # ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, not stripped
-                    if "dynamically linked" in out:
+                    out = get_linked_libs_raw(path)
+
+                    if out is None:
+                        msg = "Failed to determine dynamically linked libraries for %s, "
+                        msg += "so skipping it in RPATH sanity check"
+                        self.log.debug(msg, path)
+                    else:
                         # check whether all required libraries are found via 'ldd'
-                        out, ec = run_cmd("ldd %s" % path, simple=False, trace=False)
-                        if ec:
-                            fail_msg = "Failed to run 'ldd %s': %s" % (path, out)
-                            self.log.warning(fail_msg)
-                            fails.append(fail_msg)
-                        elif not_found_regex.search(out):
+                        if not_found_regex.search(out):
                             fail_msg = "One or more required libraries not found for %s: %s" % (path, out)
                             self.log.warning(fail_msg)
                             fails.append(fail_msg)
@@ -3030,9 +3022,6 @@ def sanity_check_rpath(self, rpath_dirs=None):
                             fails.append(fail_msg)
                         else:
                             self.log.debug("Output of 'readelf -d %s' checked, looks OK", path)
-
-                    else:
-                        self.log.debug("%s is not dynamically linked, so skipping it in RPATH sanity check", path)
             else:
                 self.log.debug("Not sanity checking files in non-existing directory %s", dirpath)
 

easybuild/tools/systemtools.py

@@ -935,31 +935,16 @@ def get_glibc_version():
     return glibc_ver
 
 
-def check_linked_shared_libs(path, required_patterns=None, banned_patterns=None):
+def get_linked_libs_raw(path):
     """
-    Check for (lack of) patterns in linked shared libraries for binary/library at specified path.
-    Uses 'ldd' on Linux and 'otool -L' on macOS to determine linked shared libraries.
-
-    Returns True or False for dynamically linked binaries and shared libraries to indicate
-    whether all patterns match and antipatterns don't match.
-
-    Returns None if given path is not a dynamically linked binary or library.
+    Get raw output from command that reports linked libraries for dynamically linked executables/libraries,
+    or None for other types of files.
     """
-    if required_patterns is None:
-        required_regexs = []
-    else:
-        required_regexs = [re.compile(p) if isinstance(p, string_type) else p for p in required_patterns]
 
-    if banned_patterns is None:
-        banned_regexs = []
-    else:
-        banned_regexs = [re.compile(p) if isinstance(p, string_type) else p for p in banned_patterns]
-
-    # resolve symbolic links (unless they're broken)
-    if os.path.islink(path) and os.path.exists(path):
-        path = os.path.realpath(path)
-
-    file_cmd_out, _ = run_cmd("file %s" % path, simple=False, trace=False)
+    file_cmd_out, ec = run_cmd("file %s" % path, simple=False, trace=False)
+    if ec:
+        fail_msg = "Failed to run 'file %s': %s" % (path, file_cmd_out)
+        _log.warning(fail_msg)
 
     os_type = get_os_type()
 
@@ -970,9 +955,7 @@ def check_linked_shared_libs(path, required_patterns=None, banned_patterns=None)
         # example output for shared libraries:
         #   /lib64/libc-2.17.so: ELF 64-bit LSB shared object, x86-64, ..., dynamically linked (uses shared libs), ...
         if "dynamically linked" in file_cmd_out:
-            # determine linked libraries via 'ldd', but take into account that 'ldd' may fail for strange reasons,
-            # like printing 'not a dynamic executable' when not enough memory is available
-            # (see also https://bugzilla.redhat.com/show_bug.cgi?id=1817111)
+            # determine linked libraries via 'ldd'
             linked_libs_cmd = "ldd %s" % path
         else:
             return None
@@ -990,11 +973,46 @@ def check_linked_shared_libs(path, required_patterns=None, banned_patterns=None)
     else:
         raise EasyBuildError("Unknown OS type: %s", os_type)
 
+    # take into account that 'ldd' may fail for strange reasons,
+    # like printing 'not a dynamic executable' when not enough memory is available
+    # (see also https://bugzilla.redhat.com/show_bug.cgi?id=1817111)
     out, ec = run_cmd(linked_libs_cmd, simple=False, trace=False, log_ok=False, log_all=False)
     if ec == 0:
         linked_libs_out = out
     else:
-        print_warning("Determining linked libraries for %s via '%s' failed! Output: '%s'", path, linked_libs_cmd, out)
+        fail_msg = "Determining linked libraries for %s via '%s' failed! Output: '%s'" % (path, linked_libs_cmd, out)
+        print_warning(fail_msg)
+        linked_libs_out = None
+
+    return linked_libs_out
+
+
+def check_linked_shared_libs(path, required_patterns=None, banned_patterns=None):
+    """
+    Check for (lack of) patterns in linked shared libraries for binary/library at specified path.
+    Uses 'ldd' on Linux and 'otool -L' on macOS to determine linked shared libraries.
+
+    Returns True or False for dynamically linked binaries and shared libraries to indicate
+    whether all patterns match and antipatterns don't match.
+
+    Returns None if given path is not a dynamically linked binary or library.
+    """
+    if required_patterns is None:
+        required_regexs = []
+    else:
+        required_regexs = [re.compile(p) if isinstance(p, string_type) else p for p in required_patterns]
+
+    if banned_patterns is None:
+        banned_regexs = []
+    else:
+        banned_regexs = [re.compile(p) if isinstance(p, string_type) else p for p in banned_patterns]
+
+    # resolve symbolic links (unless they're broken)
+    if os.path.islink(path) and os.path.exists(path):
+        path = os.path.realpath(path)
+
+    linked_libs_out = get_linked_libs_raw(path)
+    if linked_libs_out is None:
         return None
 
     found_banned_patterns = []