Merge branch '5.0.x' into parallel_prop

Author: boegel

easybuild/easyblocks/g/gromacs.py

@@ -32,25 +32,27 @@
 @author: Guilherme Peretti-Pezzi (CSCS)
 @author: Oliver Stueker (Compute Canada/ACENET)
 @author: Davide Vanzo (Vanderbilt University)
+@author: Alex Domingo (Vrije Universiteit Brussel)
 """
 import glob
 import os
 import re
 import shutil
-from easybuild.tools import LooseVersion
 
 import easybuild.tools.environment as env
 import easybuild.tools.toolchain as toolchain
-from easybuild.easyblocks.generic.configuremake import ConfigureMake
 from easybuild.easyblocks.generic.cmakemake import CMakeMake
+from easybuild.easyblocks.generic.configuremake import ConfigureMake
 from easybuild.framework.easyconfig import CUSTOM
+from easybuild.tools import LooseVersion
 from easybuild.tools.build_log import EasyBuildError, print_warning
 from easybuild.tools.config import build_option
 from easybuild.tools.filetools import copy_dir, find_backup_name_candidate, remove_dir, which
 from easybuild.tools.modules import get_software_libdir, get_software_root, get_software_version
 from easybuild.tools.run import run_shell_cmd
+from easybuild.tools.systemtools import X86_64, get_cpu_architecture, get_cpu_features, get_shared_lib_ext
 from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC
-from easybuild.tools.systemtools import X86_64, get_cpu_architecture, get_shared_lib_ext, get_cpu_features
+from easybuild.tools.utilities import nub
 from easybuild.tools.version import VERBOSE_VERSION as EASYBUILD_VERSION
 
 
@@ -79,12 +81,15 @@ def extra_options():
     def __init__(self, *args, **kwargs):
         """Initialize GROMACS-specific variables."""
         super(EB_GROMACS, self).__init__(*args, **kwargs)
-        self.lib_subdir = ''
+
+        self.lib_subdirs = ''
         self.pre_env = ''
         self.cfg['build_shared_libs'] = self.cfg.get('build_shared_libs', False)
+
         if LooseVersion(self.version) >= LooseVersion('2019'):
             # Building the gmxapi interface requires shared libraries
             self.cfg['build_shared_libs'] = True
+
         if self.cfg['build_shared_libs']:
             self.libext = get_shared_lib_ext()
         else:
@@ -142,6 +147,7 @@ def get_gromacs_arch(self):
 
         return res
 
+    @property
     def is_double_precision_cuda_build(self):
         """Check if the current build step involves double precision and CUDA"""
         cuda = get_software_root('CUDA')
@@ -461,15 +467,15 @@ def build_step(self):
         iteration is for double precision
         """
 
-        if self.is_double_precision_cuda_build():
+        if self.is_double_precision_cuda_build:
             self.log.info("skipping build step")
         else:
             super(EB_GROMACS, self).build_step()
 
     def test_step(self):
         """Run the basic tests (but not necessarily the full regression tests) using make check"""
 
-        if self.is_double_precision_cuda_build():
+        if self.is_double_precision_cuda_build:
             self.log.info("skipping test step")
         else:
             # allow to escape testing by setting runtest to False
@@ -525,7 +531,7 @@ def install_step(self):
         Custom install step for GROMACS; figure out where libraries were installed to.
         """
         # Skipping if CUDA is enabled and the current iteration is double precision
-        if self.is_double_precision_cuda_build():
+        if self.is_double_precision_cuda_build:
             self.log.info("skipping install step")
         else:
             # run 'make install' in parallel since it involves more compilation
@@ -548,48 +554,55 @@ def extensions_step(self, fetch=False):
             super(EB_GROMACS, self).extensions_step(fetch)
             self.cfg['runtest'] = orig_runtest
 
-    def get_lib_subdir(self):
-        # the GROMACS libraries get installed in different locations (deeper subdirectory),
-        # depending on the platform;
-        # this is determined by the GNUInstallDirs CMake module;
-        # rather than trying to replicate the logic, we just figure out where the library was placed
+    def get_lib_subdirs(self):
+        """
+        Return list of relative paths to sub-directories that contain GROMACS libraries
+
+        The GROMACS libraries get installed in different locations (deeper subdirectory),
+        depending on the platform;
+        this is determined by the GNUInstallDirs CMake module;
+        rather than trying to replicate the logic, we just figure out where the library was placed
+        """
 
         if LooseVersion(self.version) < LooseVersion('5.0'):
-            libname = 'libgmx*.%s' % self.libext
+            libname = f'libgmx*.{self.libext}'
         else:
-            libname = 'libgromacs*.%s' % self.libext
-        lib_subdir = None
-        for libdir in ['lib', 'lib64']:
-            if os.path.exists(os.path.join(self.installdir, libdir)):
-                for subdir in [libdir, os.path.join(libdir, '*')]:
-                    libpaths = glob.glob(os.path.join(self.installdir, subdir, libname))
-                    if libpaths:
-                        lib_subdir = os.path.dirname(libpaths[0])[len(self.installdir) + 1:]
-                        self.log.info("Found lib subdirectory that contains %s: %s", libname, lib_subdir)
-                        break
-        if not lib_subdir:
-            raise EasyBuildError("Failed to determine lib subdirectory in %s", self.installdir)
-
-        return lib_subdir
-
-    def make_module_req_guess(self):
+            libname = f'libgromacs*.{self.libext}'
+
+        lib_subdirs = []
+        real_installdir = os.path.realpath(self.installdir)
+        for lib_path in glob.glob(os.path.join(real_installdir, '**', libname)):
+            lib_relpath = os.path.realpath(lib_path)  # avoid symlinks
+            lib_relpath = lib_relpath[len(real_installdir) + 1:]  # relative path from installdir
+            subdir = lib_relpath.split(os.sep)[0:-1]
+            lib_subdirs.append(os.path.join(*subdir))
+
+        if len(lib_subdirs) == 0:
+            raise EasyBuildError(f"Failed to determine sub-directory with {libname} in {self.installdir}")
+
+        # remove duplicates, 'libname' pattern can match symlinks to actual library file
+        lib_subdirs = nub(lib_subdirs)
+        self.log.info(f"Found sub-directories that contain {libname}: {', '.join(lib_subdirs)}")
+
+        return lib_subdirs
+
+    def make_module_step(self, *args, **kwargs):
         """Custom library subdirectories for GROMACS."""
-        guesses = super(EB_GROMACS, self).make_module_req_guess()
-        if not self.lib_subdir:
+        if not self.lib_subdirs:
             try:
-                self.lib_subdir = self.get_lib_subdir()
+                self.lib_subdirs = self.get_lib_subdirs()
             except EasyBuildError as error:
                 if build_option('force') and build_option('module_only'):
-                    self.log.info("No lib subdirectory directory found in installation: %s", error)
+                    self.log.info(f"No sub-directory with GROMACS libraries found in installation: {error}")
                     self.log.info("You are forcing module creation for a non-existent installation!")
                 else:
                     raise error
-        guesses.update({
-            'LD_LIBRARY_PATH': [self.lib_subdir],
-            'LIBRARY_PATH': [self.lib_subdir],
-            'PKG_CONFIG_PATH': [os.path.join(self.lib_subdir, 'pkgconfig')],
-        })
-        return guesses
+
+        self.module_load_environment.LD_LIBRARY_PATH = self.lib_subdirs
+        self.module_load_environment.LIBRARY_PATH = self.lib_subdirs
+        self.module_load_environment.PKG_CONFIG_PATH = [os.path.join(ld, 'pkgconfig') for ld in self.lib_subdirs]
+
+        return super().make_module_step(*args, **kwargs)
 
     def sanity_check_step(self):
         """Custom sanity check for GROMACS."""
@@ -660,21 +673,19 @@ def sanity_check_step(self):
         if dsuff:
             suffixes.extend([dsuff])
 
-        lib_files.extend([
-            'lib%s%s.%s' % (x, suff, self.libext) for x in libnames + mpi_libnames for suff in suffixes
-        ])
+        lib_files.extend([f'lib{x}{suff}.{self.libext}' for x in libnames + mpi_libnames for suff in suffixes])
         bin_files.extend([b + suff for b in bins + mpi_bins for suff in suffixes])
 
-        if not self.lib_subdir:
-            self.lib_subdir = self.get_lib_subdir()
+        if not self.lib_subdirs:
+            self.lib_subdirs = self.get_lib_subdirs()
 
         # pkgconfig dir not available for earlier versions, exact version to use here is unclear
         if LooseVersion(self.version) >= LooseVersion('4.6'):
-            dirs.append(os.path.join(self.lib_subdir, 'pkgconfig'))
+            dirs.extend([os.path.join(ld, 'pkgconfig') for ld in self.lib_subdirs])
 
         custom_paths = {
             'files': [os.path.join('bin', b) for b in bin_files] +
-            [os.path.join(self.lib_subdir, lib) for lib in lib_files],
+            [os.path.join(libdir, lib) for libdir in self.lib_subdirs for lib in lib_files],
             'dirs': dirs,
         }
         super(EB_GROMACS, self).sanity_check_step(custom_paths=custom_paths)

easybuild/easyblocks/generic/pythonpackage.py

@@ -474,7 +474,6 @@ def __init__(self, *args, **kwargs):
         self.pylibdir = UNKNOWN
         self.all_pylibdirs = [UNKNOWN]
 
-        self.py_installopts = []
         self.install_cmd_output = ''
 
         # make sure there's no site.cfg in $HOME, because setup.py will find it and use it
@@ -499,8 +498,6 @@ def __init__(self, *args, **kwargs):
         # figure out whether this Python package is being installed for multiple Python versions
         self.multi_python = 'Python' in self.cfg['multi_deps']
 
-        # determine install command
-        self.use_setup_py = False
         self.determine_install_command()
 
         # avoid that pip (ab)uses $HOME/.cache/pip
@@ -522,7 +519,9 @@ def determine_install_command(self):
         """
         Determine install command to use.
         """
+        self.py_installopts = []
         if self.cfg.get('use_pip', True) or self.cfg.get('use_pip_editable', False):
+            self.use_setup_py = False
             self.install_cmd = PIP_INSTALL_CMD
 
             pip_verbose = self.cfg.get('pip_verbose', None)
@@ -551,8 +550,8 @@ def determine_install_command(self):
         else:
             self.use_setup_py = True
             self.install_cmd = SETUP_PY_INSTALL_CMD
-            install_target = self.cfg.get_ref('install_target')
 
+            install_target = self.cfg.get_ref('install_target')
             if install_target == EASY_INSTALL_TARGET:
                 self.install_cmd += " %(loc)s"
                 self.py_installopts.append('--no-deps')

easybuild/easyblocks/i/icc.py

@@ -177,7 +177,7 @@ def sanity_check_step(self):
         }
 
         # make very sure that expected 'compilers_and_libraries_<VERSION>/linux' subdir is there for recent versions,
-        # since we rely on it being there in make_module_req_guess
+        # since we rely on it being there in module_load_environment
         if self.comp_libs_subdir:
             custom_paths['dirs'].append(self.comp_libs_subdir)
 

easybuild/easyblocks/i/ifort.py

@@ -82,7 +82,7 @@ def sanity_check_step(self):
         }
 
         # make very sure that expected 'compilers_and_libraries_<VERSION>/linux' subdir is there for recent versions,
-        # since we rely on it being there in make_module_req_guess
+        # since we rely on it being there in module_load_environment
         if self.comp_libs_subdir:
             custom_paths['dirs'].append(self.comp_libs_subdir)
 

easybuild/easyblocks/n/neuron.py

@@ -27,6 +27,7 @@
 
 @author: Kenneth Hoste (Ghent University)
 @author: Maxime Boissonneault (Universite Laval, Compute Canada)
+@author: Alex Domingo (Vrije Universiteit Brussel)
 """
 import os
 import re