Merge branch 'develop' into 5.0.x

Author: boegel

easybuild/framework/easyblock.py

@@ -1730,9 +1730,15 @@ def _make_extension_list(self):
 
         Each entry should be a (name, version) tuple or just (name, ) if no version exists
         """
-        # We need only name and version, so don't resolve templates
         # Each extension in exts_list is either a string or a list/tuple with name, version as first entries
-        return [(ext, ) if isinstance(ext, str) else ext[:2] for ext in self.cfg.get_ref('exts_list')]
+        # As name can be a templated value we must resolve templates
+        exts_list = []
+        for ext in self.cfg.get_ref('exts_list'):
+            if isinstance(ext, str):
+                exts_list.append((resolve_template(ext, self.cfg.template_values), ))
+            else:
+                exts_list.append((resolve_template(ext[0], self.cfg.template_values), ext[1]))
+        return exts_list
 
     def make_extension_string(self, name_version_sep='-', ext_sep=', ', sort=True):
         """
@@ -4644,8 +4650,14 @@ def inject_checksums(ecs, checksum_type):
     """
     def make_list_lines(values, indent_level):
         """Make lines for list of values."""
+        def to_str(s):
+            if isinstance(s, string_type):
+                return "'%s'" % s
+            else:
+                return str(s)
+
         line_indent = INDENT_4SPACES * indent_level
-        return [line_indent + "'%s'," % x for x in values]
+        return [line_indent + to_str(x) + ',' for x in values]
 
     def make_checksum_lines(checksums, indent_level):
         """Make lines for list of checksums."""

easybuild/framework/easyconfig/tools.py

@@ -465,7 +465,7 @@ def find_related_easyconfigs(path, ec):
     if len(parsed_version) >= 2:
         version_patterns.append(r'%s\.%s\.\w+' % tuple(parsed_version[:2]))  # major/minor version match
     if parsed_version != parsed_version[0]:
-        version_patterns.append(r'%s\.[\d-]+\.\w+' % parsed_version[0])  # major version match
+        version_patterns.append(r'%s\.[\d-]+(\.\w+)*' % parsed_version[0])  # major version match
     version_patterns.append(r'[\w.]+')  # any version
 
     regexes = []

easybuild/framework/easyconfig/tweak.py

@@ -90,15 +90,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
     tweaked_ecs_path, tweaked_ecs_deps_path = None, None
     if targetdirs is not None:
         tweaked_ecs_path, tweaked_ecs_deps_path = targetdirs
-    # make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble)
-    toolchains = nub(['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs])
-    if len(toolchains) > 1:
-        raise EasyBuildError("Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s",
-                             toolchains)
-    # Toolchain is unique, let's store it
-    source_toolchain = easyconfigs[-1]['ec']['toolchain']
     modifying_toolchains_or_deps = False
-    target_toolchain = {}
     src_to_dst_tc_mapping = {}
     revert_to_regex = False
 
@@ -116,6 +108,16 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
                 revert_to_regex = True
 
         if not revert_to_regex:
+            # make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble)
+            toolchains = nub(['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs])
+            if len(toolchains) > 1:
+                raise EasyBuildError("Multiple toolchains featured in easyconfigs, "
+                                     "--try-X not supported in that case: %s",
+                                     toolchains)
+            # Toolchain is unique, let's store it
+            source_toolchain = easyconfigs[-1]['ec']['toolchain']
+            target_toolchain = {}
+
             # we're doing something that involves the toolchain hierarchy;
             # obtain full dependency graph for specified easyconfigs;
             # easyconfigs will be ordered 'top-to-bottom' (toolchains and dependencies appearing first)

easybuild/toolchains/compiler/intel_compilers.py

@@ -109,8 +109,9 @@ def set_variables(self):
             self.options.options_map['loose'] = ['fp-model fast']
             # fp-model fast=2 gives "warning: overriding '-ffp-model=fast=2' option with '-ffp-model=fast'"
             self.options.options_map['veryloose'] = ['fp-model fast']
-            # recommended in porting guide
-            self.options.options_map['openmp'] = ['fiopenmp']
+            # recommended in porting guide: qopenmp, unlike fiopenmp, works for both classic and oneapi compilers
+            # https://www.intel.com/content/www/us/en/developer/articles/guide/porting-guide-for-ifort-to-ifx.html
+            self.options.options_map['openmp'] = ['qopenmp']
 
             # -xSSE2 is not supported by Intel oneAPI compilers,
             # so use -march=x86-64 -mtune=generic when using optarch=GENERIC

easybuild/toolchains/linalg/flexiblas.py

@@ -70,6 +70,7 @@ class FlexiBLAS(LinAlg):
     """
     BLAS_MODULE_NAME = ['FlexiBLAS']
     BLAS_LIB = ['flexiblas']
+    BLAS_LIB_MT = ['flexiblas']
     BLAS_INCLUDE_DIR = [os.path.join('include', 'flexiblas')]
     BLAS_FAMILY = TC_CONSTANT_FLEXIBLAS
 

easybuild/tools/config.py

@@ -240,6 +240,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'job_polling_interval',
         'job_target_resource',
         'locks_dir',
+        'module_cache_suffix',
         'modules_footer',
         'modules_header',
         'mpi_cmd_template',

easybuild/tools/loose_version.py

@@ -81,6 +81,9 @@ def _cmp(self, other):
     def __eq__(self, other):
         return self._cmp(other) == 0
 
+    def __ne__(self, other):
+        return self._cmp(other) != 0
+
     def __lt__(self, other):
         return self._cmp(other) < 0
 

easybuild/tools/module_generator.py

@@ -48,7 +48,7 @@
 from easybuild.tools.config import build_option, get_module_syntax, install_path
 from easybuild.tools.filetools import convert_name, mkdir, read_file, remove_file, resolve_path, symlink, write_file
 from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, EnvironmentModulesC, Lmod, modules_tool
-from easybuild.tools.utilities import get_subclasses, quote_str
+from easybuild.tools.utilities import get_subclasses, nub, quote_str
 
 
 _log = fancylogger.getLogger('module_generator', fname=False)
@@ -666,7 +666,7 @@ def _generate_help_text(self):
         if multi_deps:
             compatible_modules_txt = '\n'.join([
                 "This module is compatible with the following modules, one of each line is required:",
-            ] + ['* %s' % d for d in multi_deps])
+            ] + ['* %s' % d for d in nub(multi_deps)])
             lines.extend(self._generate_section("Compatible modules", compatible_modules_txt))
 
         # Extensions (if any)

easybuild/tools/modules.py

@@ -1323,13 +1323,82 @@ class EnvironmentModules(EnvironmentModulesTcl):
     MAX_VERSION = None
     VERSION_REGEXP = r'^Modules\s+Release\s+(?P<version>\d\S*)\s'
 
+    def __init__(self, *args, **kwargs):
+        """Constructor, set Environment Modules-specific class variable values."""
+        # ensure in-depth modulepath search (MODULES_AVAIL_INDEPTH has been introduced in v4.3)
+        setvar('MODULES_AVAIL_INDEPTH', '1', verbose=False)
+        # match against module name start (MODULES_SEARCH_MATCH has been introduced in v4.3)
+        setvar('MODULES_SEARCH_MATCH', 'starts_with', verbose=False)
+        # ensure no debug message (MODULES_VERBOSITY has been introduced in v4.3)
+        setvar('MODULES_VERBOSITY', 'normal', verbose=False)
+        # make module search case sensitive (search is case insensitive by default since v5.0)
+        setvar('MODULES_ICASE', 'never', verbose=False)
+        # disable extended default (introduced in v4.4 and enabled by default in v5.0)
+        setvar('MODULES_EXTENDED_DEFAULT', '0', verbose=False)
+        # hard disable output redirection, output messages are expected on stderr
+        setvar('MODULES_REDIRECT_OUTPUT', '0', verbose=False)
+        # make sure modulefile cache is ignored (cache mechanism supported since v5.3)
+        setvar('MODULES_IGNORE_CACHE', '1', verbose=False)
+        # ensure only module names are returned on avail (MODULES_AVAIL_TERSE_OUTPUT added in v4.7)
+        setvar('MODULES_AVAIL_TERSE_OUTPUT', '', verbose=False)
+        # ensure only module names are returned on list (MODULES_LIST_TERSE_OUTPUT added in v4.7)
+        setvar('MODULES_LIST_TERSE_OUTPUT', '', verbose=False)
+
+        super(EnvironmentModules, self).__init__(*args, **kwargs)
+
+    def check_module_function(self, allow_mismatch=False, regex=None):
+        """Check whether selected module tool matches 'module' function definition."""
+        # Modules 5.1.0+: module command is called from _module_raw shell function
+        # Modules 4.2.0..5.0.1: module command is called from _module_raw shell function if it has
+        #   been initialized in an interactive shell session (i.e., a session attached to a tty)
+        if self.testing:
+            if '_module_raw' in os.environ:
+                out, ec = os.environ['_module_raw'], 0
+            else:
+                out, ec = None, 1
+        else:
+            cmd = "type _module_raw"
+            out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False, force_in_dry_run=True, trace=False)
+
+        if regex is None:
+            regex = r".*%s" % os.path.basename(self.cmd)
+        mod_cmd_re = re.compile(regex, re.M)
+
+        if ec == 0 and mod_cmd_re.search(out):
+            self.log.debug("Found pattern '%s' in defined '_module_raw' function." % mod_cmd_re.pattern)
+        else:
+            self.log.debug("Pattern '%s' not found in '_module_raw' function, falling back to 'module' function",
+                           mod_cmd_re.pattern)
+            super(EnvironmentModules, self).check_module_function(allow_mismatch, regex)
+
     def check_module_output(self, cmd, stdout, stderr):
         """Check output of 'module' command, see if if is potentially invalid."""
         if "_mlstatus = False" in stdout:
             raise EasyBuildError("Failed module command detected: %s (stdout: %s, stderr: %s)", cmd, stdout, stderr)
         else:
             self.log.debug("No errors detected when running module command '%s'", cmd)
 
+    def get_setenv_value_from_modulefile(self, mod_name, var_name):
+        """
+        Get value for specific 'setenv' statement from module file for the specified module.
+
+        :param mod_name: module name
+        :param var_name: name of the variable being set for which value should be returned
+        """
+        # Tcl-based module tools produce "module show" output with setenv statements like:
+        # "setenv		 GCC_PATH /opt/gcc/8.3.0"
+        # "setenv		 VAR {some text}
+        # - line starts with 'setenv'
+        # - whitespace (spaces & tabs) around variable name
+        # - curly braces around value if it contain spaces
+        value = super(EnvironmentModules, self).get_setenv_value_from_modulefile(mod_name=mod_name,
+                                                                                 var_name=var_name)
+
+        if value:
+            value = value.strip('{}')
+
+        return value
+
 
 class Lmod(ModulesTool):
     """Interface to Lmod."""
@@ -1340,7 +1409,6 @@ class Lmod(ModulesTool):
     DEPR_VERSION = '7.0.0'
     REQ_VERSION_DEPENDS_ON = '7.6.1'
     VERSION_REGEXP = r"^Modules\s+based\s+on\s+Lua:\s+Version\s+(?P<version>\d\S*)\s"
-    USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.lmod.d', '.cache')
 
     SHOW_HIDDEN_OPTION = '--show-hidden'
 
@@ -1356,7 +1424,14 @@ def __init__(self, *args, **kwargs):
         setvar('LMOD_EXTENDED_DEFAULT', 'no', verbose=False)
 
         super(Lmod, self).__init__(*args, **kwargs)
-        self.supports_depends_on = StrictVersion(self.version) >= StrictVersion(self.REQ_VERSION_DEPENDS_ON)
+        version = StrictVersion(self.version)
+
+        self.supports_depends_on = version >= self.REQ_VERSION_DEPENDS_ON
+        # See https://lmod.readthedocs.io/en/latest/125_personal_spider_cache.html
+        if version >= '8.7.12':
+            self.USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.cache', 'lmod')
+        else:
+            self.USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.lmod.d', '.cache')
 
     def check_module_function(self, *args, **kwargs):
         """Check whether selected module tool matches 'module' function definition."""
@@ -1433,7 +1508,8 @@ def update(self):
                 # don't actually update local cache when testing, just return the cache contents
                 return stdout
             else:
-                cache_fp = os.path.join(self.USER_CACHE_DIR, 'moduleT.lua')
+                suffix = build_option('module_cache_suffix') or ''
+                cache_fp = os.path.join(self.USER_CACHE_DIR, 'moduleT%s.lua' % suffix)
                 self.log.debug("Updating Lmod spider cache %s with output from '%s'", cache_fp, cmd)
                 cache_dir = os.path.dirname(cache_fp)
                 if not os.path.exists(cache_dir):

easybuild/tools/options.py

@@ -455,6 +455,9 @@ def override_options(self):
                                   "environment variable and its value separated by a colon (':')",
                                   None, 'store', DEFAULT_MINIMAL_BUILD_ENV),
             'minimal-toolchains': ("Use minimal toolchain when resolving dependencies", None, 'store_true', False),
+            'module-cache-suffix': ("Suffix to add to the cache file name (before the extension) "
+                                    "when updating the modules tool cache",
+                                    None, 'store', None),
             'module-only': ("Only generate module file(s); skip all steps except for %s" % ', '.join(MODULE_ONLY_STEPS),
                             None, 'store_true', False),
             'modules-tool-version-check': ("Check version of modules tool being used", None, 'store_true', True),
@@ -1725,11 +1728,13 @@ def process_software_build_specs(options):
             })
 
     # provide both toolchain and toolchain_name/toolchain_version keys
-    if 'toolchain_name' in build_specs:
+    try:
         build_specs['toolchain'] = {
             'name': build_specs['toolchain_name'],
-            'version': build_specs.get('toolchain_version', None),
+            'version': build_specs['toolchain_version'],
         }
+    except KeyError:
+        pass  # Don't set toolchain key if we don't have both keys
 
     # process --amend and --try-amend
     if options.amend or options.try_amend: