Merge pull request #3287 from Flamefire/less_templating

Introduce contextmanager for disabling templating and reduce resolving errors

Author: akesandgren

easybuild/framework/easyblock.py

@@ -1194,7 +1194,8 @@ def make_module_extra_extensions(self):
         lines = [self.module_extra_extensions]
 
         # set environment variable that specifies list of extensions
-        exts_list = ','.join(['%s-%s' % (ext[0], ext[1]) for ext in self.cfg['exts_list']])
+        # We need only name and version, so don't resolve templates
+        exts_list = ','.join(['-'.join(ext[:2]) for ext in self.cfg.get_ref('exts_list')])
         env_var_name = convert_name(self.name, upper=True)
         lines.append(self.module_generator.set_environment('EBEXTSLIST%s' % env_var_name, exts_list))
 
@@ -1207,7 +1208,7 @@ def make_module_footer(self):
         footer = [self.module_generator.comment("Built with EasyBuild version %s" % VERBOSE_VERSION)]
 
         # add extra stuff for extensions (if any)
-        if self.cfg['exts_list']:
+        if self.cfg.get_ref('exts_list'):
             footer.append(self.make_module_extra_extensions())
 
         # include modules footer if one is specified
@@ -1791,7 +1792,7 @@ def fetch_step(self, skip_checksums=False):
                 trace_msg(msg)
 
         # fetch extensions
-        if self.cfg['exts_list']:
+        if self.cfg.get_ref('exts_list'):
             self.exts = self.fetch_extension_sources(skip_checksums=skip_checksums)
 
         # create parent dirs in install and modules path already
@@ -2065,7 +2066,7 @@ def extensions_step(self, fetch=False):
         - find source for extensions, in 'extensions' (and 'packages' for legacy reasons)
         - run extra_extensions
         """
-        if len(self.cfg['exts_list']) == 0:
+        if not self.cfg.get_ref('exts_list'):
             self.log.debug("No extensions in exts_list")
             return
 

easybuild/framework/easyconfig/easyconfig.py

@@ -45,6 +45,7 @@
 import os
 import re
 from distutils.version import LooseVersion
+from contextlib import contextmanager
 
 import easybuild.tools.filetools as filetools
 from easybuild.base import fancylogger
@@ -383,6 +384,23 @@ def get_toolchain_hierarchy(parent_toolchain, incl_capabilities=False):
     return toolchain_hierarchy
 
 
+@contextmanager
+def disable_templating(ec):
+    """Temporarily disable templating on the given EasyConfig
+
+    Usage:
+        with disable_templating(ec):
+            # Do what you want without templating
+        # Templating set to previous value
+    """
+    old_enable_templating = ec.enable_templating
+    ec.enable_templating = False
+    try:
+        yield old_enable_templating
+    finally:
+        ec.enable_templating = old_enable_templating
+
+
 class EasyConfig(object):
     """
     Class which handles loading, reading, validation of easyconfigs
@@ -592,18 +610,15 @@ def set_keys(self, params):
         """
         # disable templating when setting easyconfig parameters
         # required to avoid problems with values that need more parsing to be done (e.g. dependencies)
-        prev_enable_templating = self.enable_templating
-        self.enable_templating = False
-
-        for key in sorted(params.keys()):
-            # validations are skipped, just set in the config
-            if key in self._config.keys():
-                self[key] = params[key]
-                self.log.info("setting easyconfig parameter %s: value %s (type: %s)", key, self[key], type(self[key]))
-            else:
-                raise EasyBuildError("Unknown easyconfig parameter: %s (value '%s')", key, params[key])
-
-        self.enable_templating = prev_enable_templating
+        with disable_templating(self):
+            for key in sorted(params.keys()):
+                # validations are skipped, just set in the config
+                if key in self._config.keys():
+                    self[key] = params[key]
+                    self.log.info("setting easyconfig parameter %s: value %s (type: %s)",
+                                  key, self[key], type(self[key]))
+                else:
+                    raise EasyBuildError("Unknown easyconfig parameter: %s (value '%s')", key, params[key])
 
     def parse(self):
         """
@@ -647,42 +662,39 @@ def parse(self):
 
         # templating is disabled when parse_hook is called to allow for easy updating of mutable easyconfig parameters
         # (see also comment in resolve_template)
-        prev_enable_templating = self.enable_templating
-        self.enable_templating = False
-
-        # if any lists of dependency versions are specified over which we should iterate,
-        # deal with them now, before calling parse hook, parsing of dependencies & iterative easyconfig parameters...
-        self.handle_multi_deps()
-
-        parse_hook_msg = None
-        if self.path:
-            parse_hook_msg = "Running %s hook for %s..." % (PARSE, os.path.basename(self.path))
-
-        # trigger parse hook
-        hooks = load_hooks(build_option('hooks'))
-        run_hook(PARSE, hooks, args=[self], msg=parse_hook_msg)
-
-        # parse dependency specifications
-        # it's important that templating is still disabled at this stage!
-        self.log.info("Parsing dependency specifications...")
-        self['dependencies'] = [self._parse_dependency(dep) for dep in self['dependencies']]
-        self['hiddendependencies'] = [self._parse_dependency(dep, hidden=True) for dep in self['hiddendependencies']]
-
-        # need to take into account that builddependencies may need to be iterated over,
-        # i.e. when the value is a list of lists of tuples
-        builddeps = self['builddependencies']
-        if builddeps and all(isinstance(x, (list, tuple)) for b in builddeps for x in b):
-            self.iterate_options.append('builddependencies')
-            builddeps = [[self._parse_dependency(dep, build_only=True) for dep in x] for x in builddeps]
-        else:
-            builddeps = [self._parse_dependency(dep, build_only=True) for dep in builddeps]
-        self['builddependencies'] = builddeps
+        with disable_templating(self):
+            # if any lists of dependency versions are specified over which we should iterate,
+            # deal with them now, before calling parse hook, parsing of dependencies & iterative easyconfig parameters
+            self.handle_multi_deps()
 
-        # keep track of parsed multi deps, they'll come in handy during sanity check & module steps...
-        self.multi_deps = self.get_parsed_multi_deps()
+            parse_hook_msg = None
+            if self.path:
+                parse_hook_msg = "Running %s hook for %s..." % (PARSE, os.path.basename(self.path))
+
+            # trigger parse hook
+            hooks = load_hooks(build_option('hooks'))
+            run_hook(PARSE, hooks, args=[self], msg=parse_hook_msg)
+
+            # parse dependency specifications
+            # it's important that templating is still disabled at this stage!
+            self.log.info("Parsing dependency specifications...")
+            self['dependencies'] = [self._parse_dependency(dep) for dep in self['dependencies']]
+            self['hiddendependencies'] = [
+                self._parse_dependency(dep, hidden=True) for dep in self['hiddendependencies']
+            ]
+
+            # need to take into account that builddependencies may need to be iterated over,
+            # i.e. when the value is a list of lists of tuples
+            builddeps = self['builddependencies']
+            if builddeps and all(isinstance(x, (list, tuple)) for b in builddeps for x in b):
+                self.iterate_options.append('builddependencies')
+                builddeps = [[self._parse_dependency(dep, build_only=True) for dep in x] for x in builddeps]
+            else:
+                builddeps = [self._parse_dependency(dep, build_only=True) for dep in builddeps]
+            self['builddependencies'] = builddeps
 
-        # restore templating
-        self.enable_templating = prev_enable_templating
+            # keep track of parsed multi deps, they'll come in handy during sanity check & module steps...
+            self.multi_deps = self.get_parsed_multi_deps()
 
         # update templating dictionary
         self.generate_template_values()
@@ -1108,63 +1120,57 @@ def dump(self, fp, always_overwrite=True, backup=False, explicit_toolchains=Fals
         :param always_overwrite: overwrite existing file at specified location without use of --force
         :param backup: create backup of existing file before overwriting it
         """
-        orig_enable_templating = self.enable_templating
-
         # templated values should be dumped unresolved
-        self.enable_templating = False
-
-        # build dict of default values
-        default_values = dict([(key, DEFAULT_CONFIG[key][0]) for key in DEFAULT_CONFIG])
-        default_values.update(dict([(key, self.extra_options[key][0]) for key in self.extra_options]))
+        with disable_templating(self):
+            # build dict of default values
+            default_values = dict([(key, DEFAULT_CONFIG[key][0]) for key in DEFAULT_CONFIG])
+            default_values.update(dict([(key, self.extra_options[key][0]) for key in self.extra_options]))
+
+            self.generate_template_values()
+            templ_const = dict([(quote_py_str(const[1]), const[0]) for const in TEMPLATE_CONSTANTS])
+
+            # create reverse map of templates, to inject template values where possible
+            # longer template values are considered first, shorter template keys get preference over longer ones
+            sorted_keys = sorted(self.template_values, key=lambda k: (len(self.template_values[k]), -len(k)),
+                                 reverse=True)
+            templ_val = OrderedDict([])
+            for key in sorted_keys:
+                # shortest template 'key' is retained in case of duplicates
+                # ('namelower' is preferred over 'github_account')
+                # only template values longer than 2 characters are retained
+                if self.template_values[key] not in templ_val and len(self.template_values[key]) > 2:
+                    templ_val[self.template_values[key]] = key
+
+            toolchain_hierarchy = None
+            if not explicit_toolchains:
+                try:
+                    toolchain_hierarchy = get_toolchain_hierarchy(self['toolchain'])
+                except EasyBuildError as err:
+                    # don't fail hard just because we can't get the hierarchy
+                    self.log.warning('Could not generate toolchain hierarchy for %s to use in easyconfig dump method, '
+                                     'error:\n%s', self['toolchain'], str(err))
 
-        self.generate_template_values()
-        templ_const = dict([(quote_py_str(const[1]), const[0]) for const in TEMPLATE_CONSTANTS])
-
-        # create reverse map of templates, to inject template values where possible
-        # longer template values are considered first, shorter template keys get preference over longer ones
-        sorted_keys = sorted(self.template_values, key=lambda k: (len(self.template_values[k]), -len(k)), reverse=True)
-        templ_val = OrderedDict([])
-        for key in sorted_keys:
-            # shortest template 'key' is retained in case of duplicates ('namelower' is preferred over 'github_account')
-            # only template values longer than 2 characters are retained
-            if self.template_values[key] not in templ_val and len(self.template_values[key]) > 2:
-                templ_val[self.template_values[key]] = key
-
-        toolchain_hierarchy = None
-        if not explicit_toolchains:
             try:
-                toolchain_hierarchy = get_toolchain_hierarchy(self['toolchain'])
-            except EasyBuildError as err:
-                # don't fail hard just because we can't get the hierarchy
-                self.log.warning('Could not generate toolchain hierarchy for %s to use in easyconfig dump method, '
-                                 'error:\n%s', self['toolchain'], str(err))
+                ectxt = self.parser.dump(self, default_values, templ_const, templ_val,
+                                         toolchain_hierarchy=toolchain_hierarchy)
+            except NotImplementedError as err:
+                raise NotImplementedError(err)
 
-        try:
-            ectxt = self.parser.dump(self, default_values, templ_const, templ_val,
-                                     toolchain_hierarchy=toolchain_hierarchy)
-        except NotImplementedError as err:
-            # need to restore enable_templating value in case this method is caught in a try/except block and ignored
-            # (the ability to dump is not a hard requirement for build success)
-            self.enable_templating = orig_enable_templating
-            raise NotImplementedError(err)
+            self.log.debug("Dumped easyconfig: %s", ectxt)
 
-        self.log.debug("Dumped easyconfig: %s", ectxt)
+            if build_option('dump_autopep8'):
+                autopep8_opts = {
+                    'aggressive': 1,  # enable non-whitespace changes, but don't be too aggressive
+                    'max_line_length': 120,
+                }
+                self.log.info("Reformatting dumped easyconfig using autopep8 (options: %s)", autopep8_opts)
+                ectxt = autopep8.fix_code(ectxt, options=autopep8_opts)
+                self.log.debug("Dumped easyconfig after autopep8 reformatting: %s", ectxt)
 
-        if build_option('dump_autopep8'):
-            autopep8_opts = {
-                'aggressive': 1,  # enable non-whitespace changes, but don't be too aggressive
-                'max_line_length': 120,
-            }
-            self.log.info("Reformatting dumped easyconfig using autopep8 (options: %s)", autopep8_opts)
-            ectxt = autopep8.fix_code(ectxt, options=autopep8_opts)
-            self.log.debug("Dumped easyconfig after autopep8 reformatting: %s", ectxt)
+            if not ectxt.endswith('\n'):
+                ectxt += '\n'
 
-        if not ectxt.endswith('\n'):
-            ectxt += '\n'
-
-        write_file(fp, ectxt, always_overwrite=always_overwrite, backup=backup, verbose=backup)
-
-        self.enable_templating = orig_enable_templating
+            write_file(fp, ectxt, always_overwrite=always_overwrite, backup=backup, verbose=backup)
 
     def _validate(self, attr, values):  # private method
         """
@@ -1473,7 +1479,7 @@ def _parse_dependency(self, dep, hidden=False, build_only=False):
 
         # (true) boolean value simply indicates that a system toolchain is used
         elif isinstance(tc_spec, bool) and tc_spec:
-                tc = {'name': SYSTEM_TOOLCHAIN_NAME, 'version': ''}
+            tc = {'name': SYSTEM_TOOLCHAIN_NAME, 'version': ''}
 
         # two-element list/tuple value indicates custom toolchain specification
         elif isinstance(tc_spec, (list, tuple,)):
@@ -1593,27 +1599,21 @@ def _generate_template_values(self, ignore=None):
 
         # step 1-3 work with easyconfig.templates constants
         # disable templating with creating dict with template values to avoid looping back to here via __getitem__
-        prev_enable_templating = self.enable_templating
-
-        self.enable_templating = False
-
-        if self.template_values is None:
-            # if no template values are set yet, initiate with a minimal set of template values;
-            # this is important for easyconfig that use %(version_minor)s to define 'toolchain',
-            # which is a pretty weird use case, but fine...
-            self.template_values = template_constant_dict(self, ignore=ignore)
-
-        self.enable_templating = prev_enable_templating
+        with disable_templating(self):
+            if self.template_values is None:
+                # if no template values are set yet, initiate with a minimal set of template values;
+                # this is important for easyconfig that use %(version_minor)s to define 'toolchain',
+                # which is a pretty weird use case, but fine...
+                self.template_values = template_constant_dict(self, ignore=ignore)
 
         # grab toolchain instance with templating support enabled,
         # which is important in case the Toolchain instance was not created yet
         toolchain = self.toolchain
 
         # get updated set of template values, now with toolchain instance
         # (which is used to define the %(mpi_cmd_prefix)s template)
-        self.enable_templating = False
-        template_values = template_constant_dict(self, ignore=ignore, toolchain=toolchain)
-        self.enable_templating = prev_enable_templating
+        with disable_templating(self):
+            template_values = template_constant_dict(self, ignore=ignore, toolchain=toolchain)
 
         # update the template_values dict
         self.template_values.update(template_values)
@@ -1656,13 +1656,8 @@ def get_ref(self, key):
         # see also comments in resolve_template
 
         # temporarily disable templating
-        prev_enable_templating = self.enable_templating
-        self.enable_templating = False
-
-        ref = self[key]
-
-        # restore previous value for 'enable_templating'
-        self.enable_templating = prev_enable_templating
+        with disable_templating(self):
+            ref = self[key]
 
         return ref
 

easybuild/framework/extension.py

@@ -127,14 +127,14 @@ def __init__(self, mself, ext, extra_params=None):
         # make sure they are merged into self.cfg so they can be queried;
         # unknown easyconfig parameters are ignored since self.options may include keys only there for extensions;
         # this allows to specify custom easyconfig parameters on a per-extension basis
-        for key in self.options:
+        for key, value in self.options.items():
             if key in self.cfg:
-                self.cfg[key] = resolve_template(self.options[key], self.cfg.template_values)
+                self.cfg[key] = value
                 self.log.debug("Customising known easyconfig parameter '%s' for extension %s/%s: %s",
-                               key, name, version, self.cfg[key])
+                               key, name, version, value)
             else:
                 self.log.debug("Skipping unknown custom easyconfig parameter '%s' for extension %s/%s: %s",
-                               key, name, version, self.options[key])
+                               key, name, version, value)
 
         self.sanity_check_fail_msgs = []
 

easybuild/tools/module_generator.py

@@ -524,7 +524,8 @@ def _generate_extension_list(self):
         """
         Generate a string with a comma-separated list of extensions.
         """
-        exts_list = self.app.cfg['exts_list']
+        # We need only name and version, so don't resolve templates
+        exts_list = self.app.cfg.get_ref('exts_list')
         extensions = ', '.join(sorted(['-'.join(ext[:2]) for ext in exts_list], key=str.lower))
 
         return extensions

test/framework/tweak.py

@@ -471,8 +471,7 @@ def test_map_easyconfig_to_target_tc_hierarchy(self):
                                                              update_build_specs={'version': new_version},
                                                              update_dep_versions=False)
         tweaked_ec = process_easyconfig(tweaked_spec)[0]
-        tweaked_dict = tweaked_ec['ec'].asdict()
-        extensions = tweaked_dict['exts_list']
+        extensions = tweaked_ec['ec']['exts_list']
         # check one extension with the same name exists and that the version has been updated
         hit_extension = 0
         for extension in extensions: