Merge branch 'develop' into depr_py26

Author: boegel

.github/workflows/unit_tests.yml

@@ -6,12 +6,12 @@ jobs:
     runs-on: ubuntu-18.04
     strategy:
       matrix:
-        python: [2.7, 3.5, 3.6, 3.7]
-        modules_tool: [Lmod-6.6.3, Lmod-7.8.22, Lmod-8.1.14, modules-tcl-1.147, modules-3.2.10, modules-4.1.4]
+        python: [2.7, 3.5, 3.6, 3.7, 3.8]
+        modules_tool: [Lmod-7.8.22, Lmod-8.2.3, modules-tcl-1.147, modules-3.2.10, modules-4.1.4]
         module_syntax: [Lua, Tcl]
         # exclude some configuration for non-Lmod modules tool:
         # - don't test with Lua module syntax (only supported in Lmod)
-        # - don't test with Python 3.5 and 3.7 (only with 2.7 and 3.6), to limit test configurations
+        # - exclude Python 3.x versions other than 3.6, to limit test configurations
         exclude:
           - modules_tool: modules-tcl-1.147
             module_syntax: Lua
@@ -23,14 +23,20 @@ jobs:
             python: 3.5
           - modules_tool: modules-tcl-1.147
             python: 3.7
+          - modules_tool: modules-tcl-1.147
+            python: 3.8
           - modules_tool: modules-3.2.10
             python: 3.5
           - modules_tool: modules-3.2.10
             python: 3.7
+          - modules_tool: modules-3.2.10
+            python: 3.8
           - modules_tool: modules-4.1.4
             python: 3.5
           - modules_tool: modules-4.1.4
             python: 3.7
+          - modules_tool: modules-4.1.4
+            python: 3.8
       fail-fast: false
     steps:
     - uses: actions/checkout@v1

.travis.yml

@@ -1,15 +1,13 @@
 language: python
-python: 2.6
-dist: trusty
+python: 2.7
+dist: xenial
 env:
   matrix:
     # purposely specifying slowest builds first, to gain time overall
-    - LMOD_VERSION=6.5.1
-    - LMOD_VERSION=6.5.1 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
     - LMOD_VERSION=7.8.22
     - LMOD_VERSION=7.8.22 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
-    - LMOD_VERSION=8.0.9
-    - LMOD_VERSION=8.0.9 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
+    - LMOD_VERSION=8.2.3
+    - LMOD_VERSION=8.2.3 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
     - ENV_MOD_VERSION=3.2.10 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesC TEST_EASYBUILD_MODULE_SYNTAX=Tcl
     - ENV_MOD_TCL_VERSION=1.147 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl TEST_EASYBUILD_MODULE_SYNTAX=Tcl
     - ENV_MOD_VERSION=4.1.4 TEST_EASYBUILD_MODULE_SYNTAX=Tcl TEST_EASYBUILD_MODULES_TOOL=EnvironmentModules  # Tmod 4.1.4 is used in RHEL8
@@ -19,43 +17,34 @@ matrix:
   # mark build as finished as soon as job has failed
   fast_finish: true
   include:
-    # also test default configuration with Python 2.7
+    # also test default configuration with Python 2.6
+    - python: 2.6
+      env: LMOD_VERSION=7.8.22
+      dist: trusty
+    # single configuration to test with Lmod 6 and Python 2.7
     - python: 2.7
-      env: LMOD_VERSION=6.6.3
-      dist: xenial
+      env: LMOD_VERSION=6.5.1 TEST_EASYBUILD_SILENCE_DEPRECATION_WARNINGS=Lmod6
     # also test with Python 3.6
-    - python: 3.6
-      env: LMOD_VERSION=6.6.3
-      dist: xenial
-    - python: 3.6
-      env: LMOD_VERSION=6.5.1 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
-      dist: xenial
     - python: 3.6
       env: LMOD_VERSION=7.8.22
-      dist: xenial
     - python: 3.6
       env: LMOD_VERSION=7.8.22 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
-      dist: xenial
     - python: 3.6
-      env: LMOD_VERSION=8.0.9
-      dist: xenial
+      env: LMOD_VERSION=8.2.3
     - python: 3.6
-      env: LMOD_VERSION=8.0.9 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
-      dist: xenial
+      env: LMOD_VERSION=8.2.3 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
     - python: 3.6
       env: ENV_MOD_VERSION=3.2.10 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesC TEST_EASYBUILD_MODULE_SYNTAX=Tcl
-      dist: xenial
     - python: 3.6
       env: ENV_MOD_TCL_VERSION=1.147 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl TEST_EASYBUILD_MODULE_SYNTAX=Tcl
-      dist: xenial
     - python: 3.6
       env: ENV_MOD_VERSION=4.1.4 TEST_EASYBUILD_MODULE_SYNTAX=Tcl TEST_EASYBUILD_MODULES_TOOL=EnvironmentModules  # Tmod 4.1.4 is used in RHEL8
-      dist: xenial
     # also test most common configuration with Python 3.5 and 3.7
     - python: 3.5
       env: LMOD_VERSION=7.8.22
-      dist: xenial
     - python: 3.7
+      env: LMOD_VERSION=7.8.22
+    - python: 3.8
       dist: xenial
       env: LMOD_VERSION=7.8.22
 addons:

easybuild/tools/config.py

@@ -196,6 +196,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'pr_target_repo',
         'rpath_filter',
         'regtest_output_dir',
+        'silence_deprecation_warnings',
         'skip',
         'stop',
         'subdir_user_modules',

easybuild/tools/modules.py

@@ -130,6 +130,8 @@
 
 class ModulesTool(object):
     """An abstract interface to a tool that deals with modules."""
+    # name of this modules tool (used in log/warning/error messages)
+    NAME = None
     # position and optionname
     TERSE_OPTION = (0, '--terse')
     # module command to use
@@ -143,6 +145,8 @@ class ModulesTool(object):
     VERSION_OPTION = '--version'
     # minimal required version (StrictVersion; suffix rc replaced with b (and treated as beta by StrictVersion))
     REQ_VERSION = None
+    # deprecated version limit (support for versions below this version is deprecated)
+    DEPR_VERSION = None
     # maximum version allowed (StrictVersion; suffix rc replaced with b (and treated as beta by StrictVersion))
     MAX_VERSION = None
     # the regexp, should have a "version" group (multiline search)
@@ -159,7 +163,7 @@ def __init__(self, mod_paths=None, testing=False):
         # this can/should be set to True during testing
         self.testing = testing
 
-        self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
+        self.log = fancylogger.getLogger(self.NAME, fname=False)
 
         # DEPRECATED!
         self._modules = []
@@ -178,19 +182,20 @@ def __init__(self, mod_paths=None, testing=False):
 
         # only use command path in environment variable if command in not available in $PATH
         if which(self.cmd) is None and env_cmd_path is not None:
-            self.log.debug('Set command via environment variable %s: %s', self.COMMAND_ENVIRONMENT, self.cmd)
+            self.log.debug("Set %s command via environment variable %s: %s",
+                           self.NAME, self.COMMAND_ENVIRONMENT, self.cmd)
             self.cmd = env_cmd_path
 
         # check whether paths obtained via $PATH and $LMOD_CMD are different
         elif which(self.cmd) != env_cmd_path:
-            self.log.debug("Different paths found for module command '%s' via which/$PATH and $%s: %s vs %s",
-                           self.COMMAND, self.COMMAND_ENVIRONMENT, self.cmd, env_cmd_path)
+            self.log.debug("Different paths found for %s command '%s' via which/$PATH and $%s: %s vs %s",
+                           self.NAME, self.COMMAND, self.COMMAND_ENVIRONMENT, self.cmd, env_cmd_path)
 
         # make sure the module command was found
         if self.cmd is None:
-            raise EasyBuildError("No command set.")
+            raise EasyBuildError("No command set for %s", self.NAME)
         else:
-            self.log.debug('Using command %s' % self.cmd)
+            self.log.debug('Using %s command %s', self.NAME, self.cmd)
 
         # version of modules tool
         self.version = None
@@ -204,13 +209,13 @@ def __init__(self, mod_paths=None, testing=False):
 
     def buildstats(self):
         """Return tuple with data to be included in buildstats"""
-        return (self.__class__.__name__, self.cmd, self.version)
+        return (self.NAME, self.cmd, self.version)
 
     def set_and_check_version(self):
         """Get the module version, and check any requirements"""
-        if self.COMMAND in MODULE_VERSION_CACHE:
-            self.version = MODULE_VERSION_CACHE[self.COMMAND]
-            self.log.debug("Found cached version for %s: %s", self.COMMAND, self.version)
+        if self.cmd in MODULE_VERSION_CACHE:
+            self.version = MODULE_VERSION_CACHE[self.cmd]
+            self.log.debug("Found cached version for %s command %s: %s", self.NAME, self.COMMAND, self.version)
             return
 
         if self.VERSION_REGEXP is None:
@@ -223,7 +228,7 @@ def set_and_check_version(self):
             res = ver_re.search(txt)
             if res:
                 self.version = res.group('version')
-                self.log.info("Found version %s" % self.version)
+                self.log.info("Found %s version %s", self.NAME, self.version)
 
                 # make sure version is a valid StrictVersion (e.g., 5.7.3.1 is invalid),
                 # and replace 'rc' by 'b', to make StrictVersion treat it as a beta-release
@@ -233,47 +238,59 @@ def set_and_check_version(self):
 
                 self.log.info("Converted actual version to '%s'" % self.version)
             else:
-                raise EasyBuildError("Failed to determine version from option '%s' output: %s",
-                                     self.VERSION_OPTION, txt)
+                raise EasyBuildError("Failed to determine %s version from option '%s' output: %s",
+                                     self.NAME, self.VERSION_OPTION, txt)
         except (OSError) as err:
-            raise EasyBuildError("Failed to check version: %s", err)
+            raise EasyBuildError("Failed to check %s version: %s", self.NAME, err)
 
         if self.REQ_VERSION is None and self.MAX_VERSION is None:
             self.log.debug("No version requirement defined.")
 
         elif build_option('modules_tool_version_check'):
-            self.log.debug("Checking whether modules tool version '%s' meets requirements", self.version)
+            self.log.debug("Checking whether %s version %s meets requirements", self.NAME, self.version)
 
             if self.REQ_VERSION is not None:
-                self.log.debug("Required minimum version defined.")
+                self.log.debug("Required minimum %s version defined: %s", self.NAME, self.REQ_VERSION)
                 if StrictVersion(self.version) < StrictVersion(self.REQ_VERSION):
                     raise EasyBuildError("EasyBuild requires %s >= v%s, found v%s",
-                                         self.__class__.__name__, self.REQ_VERSION, self.version)
+                                         self.NAME, self.REQ_VERSION, self.version)
                 else:
-                    self.log.debug('Version %s matches requirement >= %s', self.version, self.REQ_VERSION)
+                    self.log.debug('%s version %s matches requirement >= %s', self.NAME, self.version, self.REQ_VERSION)
+
+            if self.DEPR_VERSION is not None:
+                self.log.debug("Deprecated %s version limit defined: %s", self.NAME, self.DEPR_VERSION)
+                if StrictVersion(self.version) < StrictVersion(self.DEPR_VERSION):
+                    depr_msg = "Support for %s version < %s is deprecated, " % (self.NAME, self.DEPR_VERSION)
+                    depr_msg += "found version %s" % self.version
+
+                    silence_deprecation_warnings = build_option('silence_deprecation_warnings') or []
+
+                    if self.version.startswith('6') and 'Lmod6' in silence_deprecation_warnings:
+                        self.log.warning(depr_msg)
+                    else:
+                        self.log.deprecated(depr_msg, '5.0')
 
             if self.MAX_VERSION is not None:
-                self.log.debug("Maximum allowed version defined.")
+                self.log.debug("Maximum allowed %s version defined: %s", self.NAME, self.MAX_VERSION)
                 if StrictVersion(self.version) > StrictVersion(self.MAX_VERSION):
                     raise EasyBuildError("EasyBuild requires %s <= v%s, found v%s",
-                                         self.__class__.__name__, self.MAX_VERSION, self.version)
+                                         self.NAME, self.MAX_VERSION, self.version)
                 else:
                     self.log.debug('Version %s matches requirement <= %s', self.version, self.MAX_VERSION)
         else:
             self.log.debug("Skipping modules tool version '%s' requirements check", self.version)
 
-        MODULE_VERSION_CACHE[self.COMMAND] = self.version
+        MODULE_VERSION_CACHE[self.cmd] = self.version
 
     def check_cmd_avail(self):
         """Check whether modules tool command is available."""
         cmd_path = which(self.cmd)
         if cmd_path is not None:
             self.cmd = cmd_path
-            self.log.info("Full path for module command is %s, so using it" % self.cmd)
+            self.log.info("Full path for %s command is %s, so using it", self.NAME, self.cmd)
         else:
-            mod_tool = self.__class__.__name__
             mod_tools = avail_modules_tools().keys()
-            error_msg = "%s modules tool can not be used, '%s' command is not available" % (mod_tool, self.cmd)
+            error_msg = "%s modules tool can not be used, '%s' command is not available" % (self.NAME, self.cmd)
             error_msg += "; use --modules-tool to specify a different modules tool to use (%s)" % ', '.join(mod_tools)
             raise EasyBuildError(error_msg)
 
@@ -292,7 +309,7 @@ def check_module_function(self, allow_mismatch=False, regex=None):
         if regex is None:
             regex = r".*%s" % os.path.basename(self.cmd)
         mod_cmd_re = re.compile(regex, re.M)
-        mod_details = "pattern '%s' (%s)" % (mod_cmd_re.pattern, self.__class__.__name__)
+        mod_details = "pattern '%s' (%s)" % (mod_cmd_re.pattern, self.NAME)
 
         if ec == 0:
             if mod_cmd_re.search(out):
@@ -671,7 +688,7 @@ def set_path_env_var(self, key, paths):
 
     def check_module_output(self, cmd, stdout, stderr):
         """Check output of 'module' command, see if if is potentially invalid."""
-        self.log.debug("No checking of module output implemented for %s", self.__class__.__name__)
+        self.log.debug("No checking of module output implemented for %s", self.NAME)
 
     def compose_cmd_list(self, args, opts=None):
         """
@@ -1075,6 +1092,7 @@ def update(self):
 
 class EnvironmentModulesC(ModulesTool):
     """Interface to (C) environment modules (modulecmd)."""
+    NAME = "Environment Modules v3"
     COMMAND = "modulecmd"
     REQ_VERSION = '3.2.10'
     MAX_VERSION = '3.99'
@@ -1110,6 +1128,7 @@ def update(self):
 
 class EnvironmentModulesTcl(EnvironmentModulesC):
     """Interface to (Tcl) environment modules (modulecmd.tcl)."""
+    NAME = "ancient Tcl-only Environment Modules"
     # Tcl environment modules have no --terse (yet),
     #   -t must be added after the command ('avail', 'list', etc.)
     TERSE_OPTION = (1, '-t')
@@ -1182,6 +1201,7 @@ def remove_module_path(self, path, set_mod_paths=True):
 
 class EnvironmentModules(EnvironmentModulesTcl):
     """Interface to environment modules 4.0+"""
+    NAME = "Environment Modules v4"
     COMMAND = os.path.join(os.getenv('MODULESHOME', 'MODULESHOME_NOT_DEFINED'), 'libexec', 'modulecmd.tcl')
     REQ_VERSION = '4.0.0'
     MAX_VERSION = None
@@ -1197,9 +1217,11 @@ def check_module_output(self, cmd, stdout, stderr):
 
 class Lmod(ModulesTool):
     """Interface to Lmod."""
+    NAME = "Lmod"
     COMMAND = 'lmod'
     COMMAND_ENVIRONMENT = 'LMOD_CMD'
     REQ_VERSION = '6.5.1'
+    DEPR_VERSION = '7.0.0'
     REQ_VERSION_DEPENDS_ON = '7.6.1'
     VERSION_REGEXP = r"^Modules\s+based\s+on\s+Lua:\s+Version\s+(?P<version>\d\S*)\s"
     USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.lmod.d', '.cache')

easybuild/tools/options.py

@@ -413,6 +413,7 @@ def override_options(self):
             'rpath-filter': ("List of regex patterns to use for filtering out RPATH paths", 'strlist', 'store', None),
             'set-default-module': ("Set the generated module as default", None, 'store_true', False),
             'set-gid-bit': ("Set group ID bit on newly created directories", None, 'store_true', False),
+            'silence-deprecation-warnings': ("Silence specified deprecation warnings", 'strlist', 'extend', None),
             'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False),
             'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'),
             'trace': ("Provide more information in output to stdout on progress", None, 'store_true', False, 'T'),
@@ -450,8 +451,8 @@ def config_options(self):
             'buildpath': ("Temporary build path", None, 'store', mk_full_default_path('buildpath')),
             'containerpath': ("Location where container recipe & image will be stored", None, 'store',
                               mk_full_default_path('containerpath')),
-            'external-modules-metadata': ("List of files specifying metadata for external modules (INI format)",
-                                          'strlist', 'store', None),
+            'external-modules-metadata': ("List of (glob patterns for) paths to files specifying metadata "
+                                          "for external modules (INI format)", 'strlist', 'store', None),
             'hooks': ("Location of Python module with hook implementations", 'str', 'store', None),
             'ignore-dirs': ("Directory names to ignore when searching for files/dirs",
                             'strlist', 'store', ['.git', '.svn']),
@@ -1473,14 +1474,25 @@ def parse_external_modules_metadata(cfgs):
     """
     Parse metadata for external modules.
 
-    :param cfgs: list of config files providing metadata for external modules
+    :param cfgs: list of (glob patterns for) paths to config files providing metadata for external modules
     :return: parsed metadata for external modules
     """
+    if cfgs is None:
+        cfgs = []
+
+    # expand glob patterns, and report error for faulty paths
+    paths = []
+    for cfg in cfgs:
+        res = glob.glob(cfg)
+        if res:
+            paths.extend(res)
+        else:
+            # if there are no matches, we report an error to avoid silently ignores faulty paths
+            raise EasyBuildError("Specified path for file with external modules metadata does not exist: %s", cfg)
+    cfgs = paths
 
     # use external modules metadata configuration files that are available by default, unless others are specified
     if not cfgs:
-        cfgs = []
-
         # we expect to find *external_modules_metadata.cfg files in etc/ on same level as easybuild/framework
         topdirs = [os.path.dirname(os.path.dirname(os.path.dirname(__file__)))]
 
@@ -1504,14 +1516,11 @@ def parse_external_modules_metadata(cfgs):
 
     parsed_metadata = ConfigObj()
     for cfg in cfgs:
-        if os.path.isfile(cfg):
-            _log.debug("Parsing %s with external modules metadata", cfg)
-            try:
-                parsed_metadata.merge(ConfigObj(cfg))
-            except ConfigObjError as err:
-                raise EasyBuildError("Failed to parse %s with external modules metadata: %s", cfg, err)
-        else:
-            raise EasyBuildError("Specified path for file with external modules metadata does not exist: %s", cfg)
+        _log.debug("Parsing %s with external modules metadata", cfg)
+        try:
+            parsed_metadata.merge(ConfigObj(cfg))
+        except ConfigObjError as err:
+            raise EasyBuildError("Failed to parse %s with external modules metadata: %s", cfg, err)
 
     # make sure name/version values are always lists, make sure they're equal length
     for mod, entry in parsed_metadata.items():