fix filtering of duplicate paths for a particular environment variable

easybuild/framework/easyblock.py

@@ -109,7 +109,7 @@
 from easybuild.tools.repository.repository import init_repository
 from easybuild.tools.systemtools import check_linked_shared_libs, det_parallelism, get_linked_libs_raw
 from easybuild.tools.systemtools import get_shared_lib_ext, pick_system_specific_value, use_group
-from easybuild.tools.utilities import INDENT_4SPACES, get_class_for, nub, quote_str
+from easybuild.tools.utilities import INDENT_4SPACES, get_class_for, quote_str
 from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg
 from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION
 
@@ -126,17 +126,6 @@
 _log = fancylogger.getLogger('easyblock')
 
 
-class LibSymlink(Enum):
-    """
-    Possible states for symlinking of lib/lib64 subdirectories:
-    - UNKNOWN: has not been determined yet
-    - LIB_TO_LIB64: 'lib' is a symlink to 'lib64'
-    - LIB64_TO_LIB: 'lib64' is a symlink to 'lib'
-    - NEITHER: neither 'lib' is a symlink to 'lib64', nor 'lib64' is a symlink to 'lib'
-    - """
-    LIB_TO_LIB64, LIB64_TO_LIB, NEITHER = range(3)
-
-
 class EasyBlock(object):
     """Generic support for building and installing software, base class for actual easyblocks."""
 
@@ -225,9 +214,6 @@ def __init__(self, ec, logfile=None):
         # determine install subdirectory, based on module name
         self.install_subdir = None
 
-        # track status of symlink between library directories
-        self._install_lib_symlink = None
-
         # indicates whether build should be performed in installation dir
         self.build_in_installdir = self.cfg['buildininstalldir']
 
@@ -311,13 +297,6 @@ def __init__(self, ec, logfile=None):
 
         self.log.info("Init completed for application name %s version %s" % (self.name, self.version))
 
-    @property
-    def install_lib_symlink(self):
-        """Return symlink state of lib/lib64 folders"""
-        if self._install_lib_symlink is None:
-            self.check_install_lib_symlink()
-        return self._install_lib_symlink
-
     def post_init(self):
         """
         Run post-initialization tasks.
@@ -1694,15 +1673,47 @@ def make_module_req(self):
         for env_var, search_paths in env_var_requirements.items():
             if self.dry_run:
                 self.dry_run_msg(f" ${env_var}:{', '.join(search_paths)}")
-                # Don't expand globs or do any filtering for dry run
-                mod_req_paths = search_paths
+                # Don't expand globs or do any filtering for dry run,
+                # convert to regular list that can be subscripted below
+                mod_req_paths = list(search_paths)
             else:
                 mod_req_paths = []
                 for path in search_paths:
                     mod_req_paths.extend(self.expand_module_search_path(path, path_type=search_paths.type))
 
             if mod_req_paths:
-                mod_req_paths = nub(mod_req_paths)  # remove duplicates
+                # find duplicate paths (taking into account possible symlinks)
+                dup_paths = []
+                # always retain first entry
+                retained_paths = [mod_req_paths[0]]
+                full_retained_paths = [os.path.join(self.installdir, retained_paths[0])]
+
+                for path in mod_req_paths[1:]:
+                    full_path = os.path.join(self.installdir, path)
+                    # retain all paths in dry run mode (since then paths may not exist)
+                    if self.dry_run:
+                        retained_paths.append(path)
+                    elif os.path.exists(full_path) and any(os.path.samefile(full_path, p) for p in full_retained_paths):
+                        dup_paths.append(path)
+                    else:
+                        retained_paths.append(path)
+                        full_retained_paths = [os.path.join(self.installdir, p) for p in retained_paths]
+
+                if dup_paths:
+                    self.log.info(f"Filtering out duplicate paths for ${env_var}: {dup_paths}")
+                    mod_req_paths = retained_paths
+                    self.log.info(f"Retained paths for ${env_var}: {mod_req_paths}")
+                else:
+                    self.log.info(f"No duplicate paths found for ${env_var}: {mod_req_paths}")
+
+                # for $CMAKE_LIBRARY_PATH, only retain 'lib64' if it's standalone (*not* a symlink to 'lib')
+                if env_var == 'CMAKE_LIBRARY_PATH' and 'lib64' in mod_req_paths:
+                    full_lib = os.path.join(self.installdir, 'lib')
+                    full_lib64 = os.path.join(self.installdir, 'lib64')
+                    if os.path.exists(full_lib64) and os.path.exists(full_lib):
+                        if os.path.samefile(full_lib64, full_lib):
+                            mod_req_paths.remove('lib64')
+
                 mod_lines.append(self.module_generator.prepend_paths(env_var, mod_req_paths))
 
         if self.dry_run:
@@ -1732,15 +1743,6 @@ def expand_module_search_path(self, search_path, path_type=ModEnvVarType.PATH_WI
             tentative_path = os.path.relpath(abs_path, start=self.installdir)
             tentative_path = '' if tentative_path == '.' else tentative_path  # use empty string instead of dot
 
-            # avoid duplicate entries between symlinked library dirs
-            tent_path_sep = tentative_path + os.path.sep
-            if self.install_lib_symlink == LibSymlink.LIB64_TO_LIB and tent_path_sep.startswith('lib64' + os.path.sep):
-                self.log.debug("Discarded search path to symlinked lib64 directory: %s", tentative_path)
-                continue
-            if self.install_lib_symlink == LibSymlink.LIB_TO_LIB64 and tent_path_sep.startswith('lib' + os.path.sep):
-                self.log.debug("Discarded search path to symlinked lib directory: %s", tentative_path)
-                continue
-
             check_dir_files = path_type in (ModEnvVarType.PATH_WITH_FILES, ModEnvVarType.PATH_WITH_TOP_FILES)
             if os.path.isdir(abs_path) and check_dir_files:
                 # only retain paths to directories that contain at least one file
@@ -1753,18 +1755,6 @@ def expand_module_search_path(self, search_path, path_type=ModEnvVarType.PATH_WI
 
         return retained_search_paths
 
-    def check_install_lib_symlink(self):
-        """Update the symlink state between library directories in installation prefix"""
-        lib_dir = os.path.join(self.installdir, 'lib')
-        lib64_dir = os.path.join(self.installdir, 'lib64')
-
-        self._install_lib_symlink = LibSymlink.NEITHER
-        if os.path.exists(lib_dir) and os.path.exists(lib64_dir):
-            if os.path.islink(lib_dir) and os.path.samefile(lib_dir, lib64_dir):
-                self._install_lib_symlink = LibSymlink.LIB_TO_LIB64
-            elif os.path.islink(lib64_dir) and os.path.samefile(lib_dir, lib64_dir):
-                self._install_lib_symlink = LibSymlink.LIB64_TO_LIB
-
     def make_module_req_guess(self):
         """
         A dictionary of common search path variables to be loaded by environment modules
@@ -3218,9 +3208,6 @@ def post_install_step(self):
             # create *relative* 'lib' symlink to 'lib64';
             symlink('lib64', lib_dir, use_abspath_source=False)
 
-        # refresh symlink state in install_lib_symlink class variable
-        self.check_install_lib_symlink()
-
         self.run_post_install_commands()
         self.apply_post_install_patches()
         self.print_post_install_messages()

test/framework/easyblock.py

@@ -42,7 +42,7 @@
 
 import easybuild.tools.systemtools as st
 from easybuild.base import fancylogger
-from easybuild.framework.easyblock import EasyBlock, LibSymlink, get_easyblock_instance
+from easybuild.framework.easyblock import EasyBlock, get_easyblock_instance
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.framework.easyconfig.easyconfig import EasyConfig
 from easybuild.framework.easyconfig.tools import avail_easyblocks, process_easyconfig
@@ -52,7 +52,7 @@
 from easybuild.tools.config import get_module_syntax, update_build_option
 from easybuild.tools.environment import modify_env
 from easybuild.tools.filetools import change_dir, copy_dir, copy_file, mkdir, read_file, remove_dir, remove_file
-from easybuild.tools.filetools import verify_checksum, write_file
+from easybuild.tools.filetools import symlink, verify_checksum, write_file
 from easybuild.tools.module_generator import module_generator
 from easybuild.tools.modules import EnvironmentModules, Lmod, ModEnvVarType, reset_module_caches
 from easybuild.tools.version import get_git_revision, this_is_easybuild
@@ -439,7 +439,6 @@ def test_make_module_req(self):
         for path in ('bin', ('bin', 'testdir'), 'sbin', 'share', ('share', 'man'), 'lib', 'lib64'):
             path_components = (path, ) if isinstance(path, str) else path
             os.mkdir(os.path.join(eb.installdir, *path_components))
-        eb.check_install_lib_symlink()
 
         write_file(os.path.join(eb.installdir, 'foo.jar'), 'foo.jar')
         write_file(os.path.join(eb.installdir, 'bla.jar'), 'bla.jar')
@@ -505,7 +504,6 @@ def test_make_module_req(self):
         write_file(os.path.join(eb.installdir, 'lib', 'libfoo.so'), 'test')
         shutil.rmtree(os.path.join(eb.installdir, 'lib64'))
         os.symlink('lib', os.path.join(eb.installdir, 'lib64'))
-        eb.check_install_lib_symlink()
         with eb.module_generator.start_module_creation():
             guess = eb.make_module_req()
         if get_module_syntax() == 'Tcl':
@@ -616,17 +614,54 @@ def test_make_module_req(self):
         self.assertEqual(list(eb.module_load_environment), ['PATH', 'LD_LIBRARY_PATH', 'NONPATH'])
 
         if get_module_syntax() == 'Tcl':
-            self.assertTrue(re.match(r"^\nprepend-path\s+PATH\s+\$root/bin\n$", txt, re.M))
-            self.assertFalse(re.match(r"^\nprepend-path\s+NONPATH\s+\$root/non_path\n$", txt, re.M))
+            self.assertTrue(re.search(r"^prepend-path\s+PATH\s+\$root/bin$", txt, re.M))
+            self.assertFalse(re.search(r"^prepend-path\s+NONPATH\s+\$root/non_path$", txt, re.M))
         elif get_module_syntax() == 'Lua':
-            self.assertTrue(re.match(r'^\nprepend_path\("PATH", pathJoin\(root, "bin"\)\)\n$', txt, re.M))
-            self.assertFalse(re.match(r'^\nprepend_path\("NONPATH", pathJoin\(root, "non_path"\)\)\n$', txt, re.M))
+            self.assertTrue(re.search(r'^prepend_path\("PATH", pathJoin\(root, "bin"\)\)$', txt, re.M))
+            self.assertFalse(re.search(r'^prepend_path\("NONPATH", pathJoin\(root, "non_path"\)\)$', txt, re.M))
         else:
             self.fail("Unknown module syntax: %s" % get_module_syntax())
 
         logtxt = read_file(eb.logfile)
         self.assertTrue(re.search(r"WARNING Non-path variables found in module load env.*NONPATH", logtxt, re.M))
 
+        delattr(eb.module_load_environment, 'NONPATH')
+
+        # make sure that entries that symlink to another directory are retained;
+        # the test case inspired by the directory structure for old imkl versions (like 2020.4)
+        remove_dir(eb.installdir)
+
+        # lib/ symlinked to libraries/
+        real_libdir = os.path.join(eb.installdir, 'libraries')
+        mkdir(real_libdir, parents=True)
+        symlink(real_libdir, os.path.join(eb.installdir, 'lib'))
+
+        # lib/intel64/ symlinked to lib/intel64_lin/
+        mkdir(os.path.join(eb.installdir, 'lib', 'intel64_lin'), parents=True)
+        symlink(os.path.join(eb.installdir, 'lib', 'intel64_lin'), os.path.join(eb.installdir, 'lib', 'intel64'))
+
+        # library file present in lib/intel64
+        write_file(os.path.join(eb.installdir, 'lib', 'intel64', 'libfoo.so'), 'libfoo.so')
+
+        # lib64/ symlinked to lib/
+        symlink(os.path.join(eb.installdir, 'lib'), os.path.join(eb.installdir, 'lib64'))
+
+        eb.module_load_environment.LD_LIBRARY_PATH = [os.path.join('lib', 'intel64')]
+        eb.module_load_environment.LIBRARY_PATH = eb.module_load_environment.LD_LIBRARY_PATH
+        with eb.module_generator.start_module_creation():
+            txt = eb.make_module_req()
+
+        if get_module_syntax() == 'Tcl':
+            self.assertTrue(re.search(r"^prepend-path\s+LD_LIBRARY_PATH\s+\$root/lib/intel64$", txt, re.M))
+            self.assertTrue(re.search(r"^prepend-path\s+LIBRARY_PATH\s+\$root/lib/intel64\n$", txt, re.M))
+        elif get_module_syntax() == 'Lua':
+            self.assertTrue(re.search(r'^prepend_path\("LD_LIBRARY_PATH", pathJoin\(root, "lib/intel64"\)\)$',
+                                      txt, re.M))
+            self.assertTrue(re.search(r'^prepend_path\("LIBRARY_PATH", pathJoin\(root, "lib/intel64"\)\)$',
+                                      txt, re.M))
+        else:
+            self.fail("Unknown module syntax: %s" % get_module_syntax())
+
         # cleanup
         eb.close_log()
         os.remove(eb.logfile)
@@ -3215,9 +3250,6 @@ def test_expand_module_search_path(self):
         write_file(os.path.join(eb.installdir, 'dir_full_subdirs', 'subdir1', 'file12.txt'), 'test file 1.2')
         write_file(os.path.join(eb.installdir, 'dir_full_subdirs', 'subdir2', 'file21.txt'), 'test file 2.1')
 
-        eb.check_install_lib_symlink()
-        self.assertEqual(eb.install_lib_symlink, LibSymlink.NEITHER)
-
         self.assertEqual(test_emsp("nonexistent", ModEnvVarType.PATH), [])
         self.assertEqual(test_emsp("nonexistent", ModEnvVarType.PATH_WITH_FILES), [])
         self.assertEqual(test_emsp("nonexistent", ModEnvVarType.PATH_WITH_TOP_FILES), [])
@@ -3252,13 +3284,8 @@ def test_expand_module_search_path(self):
         self.assertEqual(test_emsp("nonexistent/*", ModEnvVarType.PATH_WITH_FILES), [])
         self.assertEqual(test_emsp("nonexistent/*", ModEnvVarType.PATH_WITH_TOP_FILES), [])
 
-        # state of install_lib_symlink should not have changed
-        self.assertEqual(eb.install_lib_symlink, LibSymlink.NEITHER)
-
         # test just one lib directory
         os.mkdir(os.path.join(eb.installdir, "lib"))
-        eb.check_install_lib_symlink()
-        self.assertEqual(eb.install_lib_symlink, LibSymlink.NEITHER)
         self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), ["lib"])
         self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), [])
         self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), [])
@@ -3269,8 +3296,6 @@ def test_expand_module_search_path(self):
 
         # test both lib and lib64 directories
         os.mkdir(os.path.join(eb.installdir, "lib64"))
-        eb.check_install_lib_symlink()
-        self.assertEqual(eb.install_lib_symlink, LibSymlink.NEITHER)
         self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH)), ["lib", "lib64"])
         self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES), ["lib"])
         self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib"])
@@ -3282,35 +3307,31 @@ def test_expand_module_search_path(self):
         # test lib64 symlinked to lib
         remove_dir(os.path.join(eb.installdir, "lib64"))
         os.symlink("lib", os.path.join(eb.installdir, "lib64"))
-        eb.check_install_lib_symlink()
-        self.assertEqual(eb.install_lib_symlink, LibSymlink.LIB64_TO_LIB)
         self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), ["lib"])
         self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), ["lib"])
         self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib"])
-        self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH), [])
-        self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_FILES), [])
-        self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_TOP_FILES), [])
-        self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH), ["lib"])
-        self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES), ["lib"])
-        self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib"])
+        self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH), ["lib64"])
+        self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_FILES), ["lib64"])
+        self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib64"])
+        self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH)), ["lib", "lib64"])
+        self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES)), ["lib", "lib64"])
+        self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES)), ["lib", "lib64"])
 
         # test lib symlinked to lib64
         remove_dir(os.path.join(eb.installdir, "lib"))
         remove_file(os.path.join(eb.installdir, "lib64"))
         os.mkdir(os.path.join(eb.installdir, "lib64"))
         write_file(os.path.join(eb.installdir, "lib64", "libtest.so"), "not actually a lib")
         os.symlink("lib64", os.path.join(eb.installdir, "lib"))
-        eb.check_install_lib_symlink()
-        self.assertEqual(eb.install_lib_symlink, LibSymlink.LIB_TO_LIB64)
-        self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), [])
-        self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), [])
-        self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), [])
+        self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), ["lib"])
+        self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), ["lib"])
+        self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib"])
         self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH), ["lib64"])
         self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_FILES), ["lib64"])
         self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib64"])
-        self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH), ["lib64"])
-        self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES), ["lib64"])
-        self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib64"])
+        self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH)), ["lib", "lib64"])
+        self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES)), ["lib", "lib64"])
+        self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES)), ["lib", "lib64"])
 
 
 def suite():