Merge pull request #3083 from boegel/external_modules_metadata_hmns

take into account that external modules may not be visible directly (due to module hierarchy)

Author: wpoely86

easybuild/framework/easyconfig/tools.py

@@ -125,10 +125,10 @@ def find_resolved_modules(easyconfigs, avail_modules, modtool, retain_all_deps=F
         for dep in easyconfig['dependencies']:
             dep_mod_name = dep.get('full_mod_name', ActiveMNS().det_full_module_name(dep))
 
-            # treat external modules as resolved when retain_all_deps is enabled (e.g., under --dry-run),
+            # always treat external modules as resolved,
             # since no corresponding easyconfig can be found for them
-            if retain_all_deps and dep.get('external_module', False):
-                _log.debug("Treating dependency marked as external dependency as resolved: %s", dep_mod_name)
+            if dep.get('external_module', False):
+                _log.debug("Treating dependency marked as external module as resolved: %s", dep_mod_name)
 
             elif retain_all_deps and dep_mod_name not in avail_modules:
                 # if all dependencies should be retained, include dep unless it has been already

easybuild/tools/options.py

@@ -1523,8 +1523,16 @@ def parse_external_modules_metadata(cfgs):
         except ConfigObjError as err:
             raise EasyBuildError("Failed to parse %s with external modules metadata: %s", cfg, err)
 
+    known_metadata_keys = ['name', 'prefix', 'version']
+    unknown_keys = {}
+
     # make sure name/version values are always lists, make sure they're equal length
     for mod, entry in parsed_metadata.items():
+        # make sure only known keys are used
+        for key in entry.keys():
+            if key not in known_metadata_keys:
+                unknown_keys.setdefault(mod, []).append(key)
+
         for key in ['name', 'version']:
             if isinstance(entry.get(key), string_type):
                 entry[key] = [entry[key]]
@@ -1537,6 +1545,12 @@ def parse_external_modules_metadata(cfgs):
             raise EasyBuildError("Different length for lists of names/versions in metadata for external module %s: "
                                  "names: %s; versions: %s", mod, names, versions)
 
+    if unknown_keys:
+        error_msg = "Found metadata entries with unknown keys:"
+        for mod in sorted(unknown_keys.keys()):
+            error_msg += "\n* %s: %s" % (mod, ', '.join(sorted(unknown_keys[mod])))
+        raise EasyBuildError(error_msg)
+
     _log.debug("External modules metadata: %s", parsed_metadata)
     return parsed_metadata
 

easybuild/tools/toolchain/toolchain.py

@@ -451,6 +451,16 @@ def _check_dependencies(self, dependencies):
             if dep_exists:
                 deps.append(dep)
                 self.log.devel("_check_dependencies: added toolchain dependency %s", str(dep))
+            elif dep['external_module']:
+                # external modules may be organised hierarchically,
+                # so not all modules may be directly available for loading;
+                # we assume here that the required modules are either provided by the toolchain,
+                # or are listed earlier as dependency
+                # examples from OpenHPC:
+                # - openmpi3 module provided by OpenHPC requires that gnu7, gnu8 or intel module is loaded first
+                # - fftw module provided by OpenHPC requires that compiler + MPI module are loaded first
+                self.log.info("Assuming non-visible external module %s is available", dep['full_mod_name'])
+                deps.append(dep)
             else:
                 missing_dep_mods.append(dep_mod_name)
 

test/framework/options.py

@@ -3571,6 +3571,24 @@ def test_parse_external_modules_metadata(self):
         self.assertEqual(metadata['two/2.1'], {'name': ['two'], 'version': ['2.1']})
         self.assertEqual(metadata['three/3.0'], {'name': ['three'], 'version': ['3.0']})
 
+        # check whether entries with unknown keys result in an error
+        cfg1 = os.path.join(self.test_prefix, 'broken_cfg1.cfg')
+        write_file(cfg1, "[one/1.0]\nname = one\nversion = 1.0\nfoo = bar")
+        cfg2 = os.path.join(self.test_prefix, 'cfg2.cfg')
+        write_file(cfg2, "[two/2.0]\nname = two\nversion = 2.0")
+        cfg3 = os.path.join(self.test_prefix, 'broken_cfg3.cfg')
+        write_file(cfg3, "[three/3.0]\nnaem = three\nzzz=zzz\nvresion = 3.0\naaa = aaa")
+        cfg4 = os.path.join(self.test_prefix, 'broken_cfg4.cfg')
+        write_file(cfg4, "[four/4]\nprfeix = /software/four/4")
+        broken_cfgs = [cfg1, cfg2, cfg3, cfg4]
+        error_pattern = '\n'.join([
+            r"Found metadata entries with unknown keys:",
+            r"\* four/4: prfeix",
+            r"\* one/1.0: foo",
+            r"\* three/3.0: aaa, naem, vresion, zzz",
+        ])
+        self.assertErrorRegex(EasyBuildError, error_pattern, parse_external_modules_metadata, broken_cfgs)
+
     def test_zip_logs(self):
         """Test use of --zip-logs"""
 

test/framework/toy_build.py

@@ -28,6 +28,7 @@
 @author: Kenneth Hoste (Ghent University)
 @author: Damian Alvarez (Forschungszentrum Juelich GmbH)
 """
+import copy
 import glob
 import grp
 import os
@@ -1287,6 +1288,7 @@ def test_toy_module_fulltxt(self):
 
     def test_external_dependencies(self):
         """Test specifying external (build) dependencies."""
+
         topdir = os.path.dirname(os.path.abspath(__file__))
         ectxt = read_file(os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-deps.eb'))
         toy_ec = os.path.join(self.test_prefix, 'toy-0.0-external-deps.eb')
@@ -1303,26 +1305,43 @@ def test_external_dependencies(self):
             mkdir(os.path.join(modulepath, os.path.dirname(mod)), parents=True)
             write_file(os.path.join(modulepath, mod), "#%Module")
 
-        self.reset_modulepath([modulepath, os.path.join(self.test_installpath, 'modules', 'all')])
+        installed_test_modules = os.path.join(self.test_installpath, 'modules', 'all')
+        self.reset_modulepath([modulepath, installed_test_modules])
+
+        start_env = copy.deepcopy(os.environ)
+
         self.test_toy_build(ec_file=toy_ec, versionsuffix='-external-deps', verbose=True, raise_error=True)
 
         self.modtool.load(['toy/0.0-external-deps'])
         # note build dependency is not loaded
         mods = ['intel/2018a', 'GCC/6.4.0-2.28', 'foobar/1.2.3', 'toy/0.0-external-deps']
         self.assertEqual([x['mod_name'] for x in self.modtool.list()], mods)
 
-        # check behaviour when a non-existing external (build) dependency is included
-        err_msg = "Missing modules for dependencies marked as external modules:"
+        # restore original environment (to undo 'module load' done above)
+        modify_env(os.environ, start_env, verbose=False)
 
+        # check behaviour when a non-existing external (build) dependency is included
         extraectxt = "\nbuilddependencies = [('nosuchbuilddep/0.0.0', EXTERNAL_MODULE)]"
         extraectxt += "\nversionsuffix = '-external-deps-broken1'"
         write_file(toy_ec, ectxt + extraectxt)
+
+        if isinstance(self.modtool, Lmod):
+            err_msg = r"Module command \\'module load nosuchbuilddep/0.0.0\\' failed"
+        else:
+            err_msg = r"Unable to locate a modulefile for 'nosuchbuilddep/0.0.0'"
+
         self.assertErrorRegex(EasyBuildError, err_msg, self.test_toy_build, ec_file=toy_ec,
                               raise_error=True, verbose=False)
 
         extraectxt = "\ndependencies += [('nosuchmodule/1.2.3', EXTERNAL_MODULE)]"
         extraectxt += "\nversionsuffix = '-external-deps-broken2'"
         write_file(toy_ec, ectxt + extraectxt)
+
+        if isinstance(self.modtool, Lmod):
+            err_msg = r"Module command \\'module load nosuchmodule/1.2.3\\' failed"
+        else:
+            err_msg = r"Unable to locate a modulefile for 'nosuchmodule/1.2.3'"
+
         self.assertErrorRegex(EasyBuildError, err_msg, self.test_toy_build, ec_file=toy_ec,
                               raise_error=True, verbose=False)