Merge pull request #3261 from ocaisa/toolchain_hierarchy_aware_dump

make EasyConfig.dump aware of toolchain hierarchy, to avoid hardcoded subtoolchains in dependencies easyconfig parameters

Author: boegel

easybuild/framework/easyblock.py

@@ -3330,7 +3330,7 @@ def reproduce_build(app, reprod_dir_root):
     reprod_dir = find_backup_name_candidate(os.path.join(reprod_dir_root, REPROD))
     reprod_spec = os.path.join(reprod_dir, ec_filename)
     try:
-        app.cfg.dump(reprod_spec)
+        app.cfg.dump(reprod_spec, explicit_toolchains=True)
         _log.info("Dumped easyconfig instance to %s", reprod_spec)
     except NotImplementedError as err:
         _log.warning("Unable to dump easyconfig instance to %s: %s", reprod_spec, err)

easybuild/framework/easyconfig/easyconfig.py

@@ -1101,7 +1101,7 @@ def all_dependencies(self):
 
         return self._all_dependencies
 
-    def dump(self, fp, always_overwrite=True, backup=False):
+    def dump(self, fp, always_overwrite=True, backup=False, explicit_toolchains=False):
         """
         Dump this easyconfig to file, with the given filename.
 
@@ -1130,8 +1130,18 @@ def dump(self, fp, always_overwrite=True, backup=False):
             if self.template_values[key] not in templ_val and len(self.template_values[key]) > 2:
                 templ_val[self.template_values[key]] = key
 
+        toolchain_hierarchy = None
+        if not explicit_toolchains:
+            try:
+                toolchain_hierarchy = get_toolchain_hierarchy(self['toolchain'])
+            except EasyBuildError as err:
+                # don't fail hard just because we can't get the hierarchy
+                self.log.warning('Could not generate toolchain hierarchy for %s to use in easyconfig dump method, '
+                                 'error:\n%s', self['toolchain'], str(err))
+
         try:
-            ectxt = self.parser.dump(self, default_values, templ_const, templ_val)
+            ectxt = self.parser.dump(self, default_values, templ_const, templ_val,
+                                     toolchain_hierarchy=toolchain_hierarchy)
         except NotImplementedError as err:
             # need to restore enable_templating value in case this method is caught in a try/except block and ignored
             # (the ability to dump is not a hard requirement for build success)

easybuild/framework/easyconfig/format/format.py

@@ -632,8 +632,8 @@ def parse(self, txt, **kwargs):
         """Parse the txt according to this format. This is highly version specific"""
         raise NotImplementedError
 
-    def dump(self, ecfg, default_values, templ_const, templ_val):
-        """Dump easyconfig according to this format. This is higly version specific"""
+    def dump(self, ecfg, default_values, templ_const, templ_val, toolchain_hierarchy=None):
+        """Dump easyconfig according to this format. This is highly version specific"""
         raise NotImplementedError
 
     def extract_comments(self, rawtxt):

easybuild/framework/easyconfig/format/one.py

@@ -65,15 +65,17 @@
 _log = fancylogger.getLogger('easyconfig.format.one', fname=False)
 
 
-def dump_dependency(dep, toolchain):
+def dump_dependency(dep, toolchain, toolchain_hierarchy=None):
     """Dump parsed dependency in tuple format"""
+    if not toolchain_hierarchy:
+        toolchain_hierarchy = [toolchain]
 
     if dep['external_module']:
         res = "(%s, EXTERNAL_MODULE)" % quote_py_str(dep['full_mod_name'])
     else:
         # minimal spec: (name, version)
         tup = (dep['name'], dep['version'])
-        if dep['toolchain'] != toolchain:
+        if all(dep['toolchain'] != subtoolchain for subtoolchain in toolchain_hierarchy):
             if dep[SYSTEM_TOOLCHAIN_NAME]:
                 tup += (dep['versionsuffix'], True)
             else:
@@ -260,7 +262,7 @@ def _find_param_with_comments(self, key, val, templ_const, templ_val):
 
         return res
 
-    def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_val):
+    def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_val, toolchain_hierarchy=None):
         """
         Determine parameters in the dumped easyconfig file which have a non-default value.
         """
@@ -279,12 +281,18 @@ def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_
                                 # the way that builddependencies are constructed with multi_deps
                                 # we just need to dump the first entry without the dependencies
                                 # that are listed in multi_deps
-                                valstr = [dump_dependency(d, ecfg['toolchain']) for d in val[0]
-                                          if d['name'] not in ecfg['multi_deps']]
+                                valstr = [
+                                    dump_dependency(d, ecfg['toolchain'], toolchain_hierarchy=toolchain_hierarchy)
+                                    for d in val[0] if d['name'] not in ecfg['multi_deps']
+                                ]
                             else:
-                                valstr = [[dump_dependency(d, ecfg['toolchain']) for d in dep] for dep in val]
+                                valstr = [
+                                    [dump_dependency(d, ecfg['toolchain'], toolchain_hierarchy=toolchain_hierarchy)
+                                     for d in dep] for dep in val
+                                ]
                         else:
-                            valstr = [dump_dependency(d, ecfg['toolchain']) for d in val]
+                            valstr = [dump_dependency(d, ecfg['toolchain'], toolchain_hierarchy=toolchain_hierarchy)
+                                      for d in val]
                     elif key == 'toolchain':
                         valstr = "{'name': '%(name)s', 'version': '%(version)s'}" % ecfg[key]
                     else:
@@ -299,20 +307,22 @@ def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_
 
         return eclines, printed_keys
 
-    def dump(self, ecfg, default_values, templ_const, templ_val):
+    def dump(self, ecfg, default_values, templ_const, templ_val, toolchain_hierarchy=None):
         """
         Dump easyconfig in format v1.
 
         :param ecfg: EasyConfig instance
         :param default_values: default values for easyconfig parameters
         :param templ_const: known template constants
         :param templ_val: known template values
+        :param toolchain_hierarchy: hierarchy of toolchains for easyconfig
         """
         # include header comments first
         dump = self.comments['header'][:]
 
         # print easyconfig parameters ordered and in groups specified above
-        params, printed_keys = self._find_defined_params(ecfg, GROUPED_PARAMS, default_values, templ_const, templ_val)
+        params, printed_keys = self._find_defined_params(ecfg, GROUPED_PARAMS, default_values, templ_const, templ_val,
+                                                         toolchain_hierarchy=toolchain_hierarchy)
         dump.extend(params)
 
         # print other easyconfig parameters at the end

easybuild/framework/easyconfig/format/yeb.py

@@ -126,7 +126,7 @@ def _inject_constants_dict(self, txt):
 
         return full_txt
 
-    def dump(self, ecfg, default_values, templ_const, templ_val):
+    def dump(self, ecfg, default_values, templ_const, templ_val, toolchain_hierarchy=None):
         """Dump parsed easyconfig in .yeb format"""
         raise NotImplementedError("Dumping of .yeb easyconfigs not supported yet")
 

easybuild/framework/easyconfig/parser.py

@@ -226,6 +226,7 @@ def get_config_dict(self, validate=True):
 
         return cfg
 
-    def dump(self, ecfg, default_values, templ_const, templ_val):
+    def dump(self, ecfg, default_values, templ_const, templ_val, toolchain_hierarchy=None):
         """Dump easyconfig in format it was parsed from."""
-        return self._formatter.dump(ecfg, default_values, templ_const, templ_val)
+        return self._formatter.dump(ecfg, default_values, templ_const, templ_val,
+                                    toolchain_hierarchy=toolchain_hierarchy)

test/framework/easyconfig.py

@@ -1816,6 +1816,59 @@ def test_dump(self):
                 if param in ec:
                     self.assertEqual(ec[param], dumped_ec[param])
 
+    def test_toolchain_hierarchy_aware_dump(self):
+        """Test that EasyConfig's dump() method is aware of the toolchain hierarchy."""
+        test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
+        build_options = {
+            'check_osdeps': False,
+            'robot_path': [test_ecs_dir],
+            'valid_module_classes': module_classes(),
+        }
+        init_config(build_options=build_options)
+        rawtxt = '\n'.join([
+            "easyblock = 'EB_foo'",
+            '',
+            "name = 'foo'",
+            "version = '0.0.1'",
+            '',
+            "toolchain = {'name': 'foss', 'version': '2018a'}",
+            '',
+            "homepage = 'http://foo.com/'",
+            'description = "foo description"',
+            '',
+            'sources = [SOURCE_TAR_GZ]',
+            'source_urls = ["http://example.com"]',
+            'checksums = ["6af6ab95ce131c2dd467d2ebc8270e9c265cc32496210b069e51d3749f335f3d"]',
+            '',
+            "dependencies = [",
+            "    ('toy', '0.0', '', ('gompi', '2018a')),",
+            "    ('bar', '1.0'),",
+            "    ('foobar/1.2.3', EXTERNAL_MODULE),",
+            "]",
+            '',
+            "foo_extra1 = 'foobar'",
+            '',
+            'moduleclass = "tools"',
+        ])
+
+        test_ec = os.path.join(self.test_prefix, 'test.eb')
+        ec = EasyConfig(None, rawtxt=rawtxt)
+        ecdict = ec.asdict()
+        ec.dump(test_ec)
+        # dict representation of EasyConfig instance should not change after dump
+        self.assertEqual(ecdict, ec.asdict())
+        ectxt = read_file(test_ec)
+        dumped_ec = EasyConfig(test_ec)
+        self.assertEqual(ecdict, dumped_ec.asdict())
+        self.assertTrue(r"'toy', '0.0')," in ectxt)
+        # test case where we ask for explicit toolchains
+        ec.dump(test_ec, explicit_toolchains=True)
+        self.assertEqual(ecdict, ec.asdict())
+        ectxt = read_file(test_ec)
+        dumped_ec = EasyConfig(test_ec)
+        self.assertEqual(ecdict, dumped_ec.asdict())
+        self.assertTrue(r"'toy', '0.0', '', ('gompi', '2018a'))," in ectxt)
+
     def test_dump_order(self):
         """Test order of easyconfig parameters in dumped easyconfig."""
         rawtxt = '\n'.join([