Merge pull request #4021 from boegel/easystack_easyconfigs

add support for easystack file that contains easyconfig filenames + implement parsing of configuration options

Author: casparvl

easybuild/framework/easystack.py

@@ -41,6 +41,8 @@
     pass
 _log = fancylogger.getLogger('easystack', fname=False)
 
+EASYSTACK_DOC_URL = 'https://docs.easybuild.io/en/latest/Easystack-files.html'
+
 
 def check_value(value, context):
     """
@@ -68,10 +70,16 @@ def __init__(self):
         self.easybuild_version = None
         self.robot = False
         self.software_list = []
+        self.easyconfigs = []  # A list of easyconfig names. May or may not include .eb extension
+        # A dict where keys are easyconfig names, values are dictionary of options that should be
+        # applied for that easyconfig
+        self.ec_opts = {}
 
     def compose_ec_filenames(self):
         """Returns a list of all easyconfig names"""
         ec_filenames = []
+
+        # entries specified via 'software' top-level key
         for sw in self.software_list:
             full_ec_version = det_full_ec_version({
                 'toolchain': {'name': sw.toolchain_name, 'version': sw.toolchain_version},
@@ -80,6 +88,10 @@ def compose_ec_filenames(self):
             })
             ec_filename = '%s-%s.eb' % (sw.name, full_ec_version)
             ec_filenames.append(ec_filename)
+
+        # entries specified via 'easyconfigs' top-level key
+        for ec in self.easyconfigs:
+            ec_filenames.append(ec)
         return ec_filenames
 
     # flags applicable to all sw (i.e. robot)
@@ -108,21 +120,89 @@ class EasyStackParser(object):
 
     @staticmethod
     def parse(filepath):
-        """Parses YAML file and assigns obtained values to SW config instances as well as general config instance"""
+        """
+        Parses YAML file and assigns obtained values to SW config instances as well as general config instance"""
         yaml_txt = read_file(filepath)
 
         try:
             easystack_raw = yaml.safe_load(yaml_txt)
         except yaml.YAMLError as err:
             raise EasyBuildError("Failed to parse %s: %s" % (filepath, err))
 
+        easystack_data = None
+        top_keys = ('easyconfigs', 'software')
+        keys_found = []
+        for key in top_keys:
+            if key in easystack_raw:
+                keys_found.append(key)
+        # For now, we don't support mixing multiple top_keys, so check that only one was defined
+        if len(keys_found) > 1:
+            keys_string = ', '.join(keys_found)
+            msg = "Specifying multiple top level keys (%s) " % keys_string
+            msg += "in one EasyStack file is currently not supported"
+            msg += ", see %s for documentation." % EASYSTACK_DOC_URL
+            raise EasyBuildError(msg)
+        elif len(keys_found) == 0:
+            msg = "Not a valid EasyStack YAML file: no 'easyconfigs' or 'software' top-level key found"
+            msg += ", see %s for documentation." % EASYSTACK_DOC_URL
+            raise EasyBuildError(msg)
+        else:
+            key = keys_found[0]
+            easystack_data = easystack_raw[key]
+
+        parse_method_name = 'parse_by_' + key
+        parse_method = getattr(EasyStackParser, 'parse_by_%s' % key, None)
+        if parse_method is None:
+            raise EasyBuildError("Easystack parse method '%s' not found!", parse_method_name)
+
+        # assign general easystack attributes
+        easybuild_version = easystack_raw.get('easybuild_version', None)
+        robot = easystack_raw.get('robot', False)
+
+        return parse_method(filepath, easystack_data, easybuild_version=easybuild_version, robot=robot)
+
+    @staticmethod
+    def parse_by_easyconfigs(filepath, easyconfigs, easybuild_version=None, robot=False):
+        """
+        Parse easystack file with 'easyconfigs' as top-level key.
+        """
+
         easystack = EasyStack()
 
-        try:
-            software = easystack_raw["software"]
-        except KeyError:
-            wrong_structure_file = "Not a valid EasyStack YAML file: no 'software' key found"
-            raise EasyBuildError(wrong_structure_file)
+        for easyconfig in easyconfigs:
+            if isinstance(easyconfig, str):
+                if not easyconfig.endswith('.eb'):
+                    easyconfig = easyconfig + '.eb'
+                easystack.easyconfigs.append(easyconfig)
+            elif isinstance(easyconfig, dict):
+                if len(easyconfig) == 1:
+                    # Get single key from dictionary 'easyconfig'
+                    easyconf_name = list(easyconfig.keys())[0]
+                    # Add easyconfig name to the list
+                    if not easyconf_name.endswith('.eb'):
+                        easyconf_name_with_eb = easyconf_name + '.eb'
+                    else:
+                        easyconf_name_with_eb = easyconf_name
+                    easystack.easyconfigs.append(easyconf_name_with_eb)
+                    # Add options to the ec_opts dict
+                    if 'options' in easyconfig[easyconf_name].keys():
+                        easystack.ec_opts[easyconf_name_with_eb] = easyconfig[easyconf_name]['options']
+                else:
+                    dict_keys = ', '.join(easyconfig.keys())
+                    msg = "Failed to parse easystack file: expected a dictionary with one key (the EasyConfig name), "
+                    msg += "instead found keys: %s" % dict_keys
+                    msg += ", see %s for documentation." % EASYSTACK_DOC_URL
+                    raise EasyBuildError(msg)
+
+        return easystack
+
+    @staticmethod
+    def parse_by_software(filepath, software, easybuild_version=None, robot=False):
+        """
+        Parse easystack file with 'software' as top-level key.
+        """
+
+        easystack = EasyStack()
 
         # assign software-specific easystack attributes
         for name in software:
@@ -224,8 +304,8 @@ def parse(filepath):
                     easystack.software_list.append(sw)
 
             # assign general easystack attributes
-            easystack.easybuild_version = easystack_raw.get('easybuild_version', None)
-            easystack.robot = easystack_raw.get('robot', False)
+            easystack.easybuild_version = easybuild_version
+            easystack.robot = robot
 
         return easystack
 
@@ -243,12 +323,16 @@ def parse_easystack(filepath):
 
     easyconfig_names = easystack.compose_ec_filenames()
 
-    general_options = easystack.get_general_options()
+    # Disabled general options for now. We weren't using them, and first want support for EasyConfig-specific options.
+    # Then, we need a method to resolve conflicts (specific options should win)
+    # general_options = easystack.get_general_options()
 
     _log.debug("EasyStack parsed. Proceeding to install these Easyconfigs: %s" % ', '.join(sorted(easyconfig_names)))
-    if len(general_options) != 0:
-        _log.debug("General options for installation are: \n%s" % str(general_options))
-    else:
-        _log.debug("No general options were specified in easystack")
-
-    return easyconfig_names, general_options
+    _log.debug("Using EasyConfig specific options based on the following dict:")
+    _log.debug(easystack.ec_opts)
+    # if len(general_options) != 0:
+    #    _log.debug("General options for installation are: \n%s" % str(general_options))
+    # else:
+    #    _log.debug("No general options were specified in easystack")
+
+    return easyconfig_names, easystack.ec_opts

easybuild/main.py

@@ -43,7 +43,7 @@
 
 # IMPORTANT this has to be the first easybuild import as it customises the logging
 #  expect missing log output when this not the case!
-from easybuild.tools.build_log import EasyBuildError, print_error, print_msg, stop_logging
+from easybuild.tools.build_log import EasyBuildError, print_error, print_msg, print_warning, stop_logging
 
 from easybuild.framework.easyblock import build_and_install_one, inject_checksums
 from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR
@@ -55,7 +55,6 @@
 from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for
 from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, run_contrib_checks, skip_available
 from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak
-from easybuild.tools.build_log import print_warning
 from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option
 from easybuild.tools.containers.common import containerize
 from easybuild.tools.docs import list_software
@@ -266,7 +265,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
         # TODO add general_options (i.e. robot) to build options
         orig_paths, general_options = parse_easystack(options.easystack)
         if general_options:
-            raise EasyBuildError("Specifying general configuration options in easystack file is not supported yet.")
+            print_warning("Specifying options in easystack files is not supported yet. They are parsed, but ignored.")
 
     # check whether packaging is supported when it's being used
     if options.package:

test/framework/easystack.py

@@ -56,6 +56,70 @@ def tearDown(self):
         easybuild.tools.build_log.EXPERIMENTAL = self.orig_experimental
         super(EasyStackTest, self).tearDown()
 
+    def test_easystack_basic(self):
+        """Test for basic easystack file."""
+        topdir = os.path.dirname(os.path.abspath(__file__))
+        test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_basic.yaml')
+
+        ec_fns, opts = parse_easystack(test_easystack)
+        expected = [
+            'binutils-2.25-GCCcore-4.9.3.eb',
+            'binutils-2.26-GCCcore-4.9.3.eb',
+            'foss-2018a.eb',
+            'toy-0.0-gompi-2018a-test.eb',
+        ]
+        self.assertEqual(sorted(ec_fns), sorted(expected))
+        self.assertEqual(opts, {})
+
+    def test_easystack_easyconfigs(self):
+        """Test for easystack file using 'easyconfigs' key."""
+        topdir = os.path.dirname(os.path.abspath(__file__))
+        test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_easyconfigs.yaml')
+
+        ec_fns, opts = parse_easystack(test_easystack)
+        expected = [
+            'binutils-2.25-GCCcore-4.9.3.eb',
+            'binutils-2.26-GCCcore-4.9.3.eb',
+            'foss-2018a.eb',
+            'toy-0.0-gompi-2018a-test.eb',
+        ]
+        self.assertEqual(sorted(ec_fns), sorted(expected))
+        self.assertEqual(opts, {})
+
+    def test_easystack_easyconfigs_with_eb_ext(self):
+        """Test for easystack file using 'easyconfigs' key, where eb extension is included in the easystack file"""
+        topdir = os.path.dirname(os.path.abspath(__file__))
+        test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_easyconfigs_with_eb_ext.yaml')
+
+        ec_fns, opts = parse_easystack(test_easystack)
+        expected = [
+            'binutils-2.25-GCCcore-4.9.3.eb',
+            'binutils-2.26-GCCcore-4.9.3.eb',
+            'foss-2018a.eb',
+            'toy-0.0-gompi-2018a-test.eb',
+        ]
+        self.assertEqual(sorted(ec_fns), sorted(expected))
+        self.assertEqual(opts, {})
+
+    def test_easystack_easyconfig_opts(self):
+        """Test an easystack file using the 'easyconfigs' key, with additonal options for some easyconfigs"""
+        topdir = os.path.dirname(os.path.abspath(__file__))
+        test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_easyconfigs_opts.yaml')
+
+        ec_fns, opts = parse_easystack(test_easystack)
+        expected = [
+            'binutils-2.25-GCCcore-4.9.3.eb',
+            'binutils-2.26-GCCcore-4.9.3.eb',
+            'foss-2018a.eb',
+            'toy-0.0-gompi-2018a-test.eb',
+        ]
+        expected_opts = {
+            'binutils-2.25-GCCcore-4.9.3.eb': {'debug': True, 'from-pr': 12345},
+            'foss-2018a.eb': {'enforce-checksums': True, 'robot': True},
+        }
+        self.assertEqual(sorted(ec_fns), sorted(expected))
+        self.assertEqual(opts, expected_opts)
+
     def test_parse_fail(self):
         """Test for clean error when easystack file fails to parse."""
         test_yml = os.path.join(self.test_prefix, 'test.yml')
@@ -120,15 +184,17 @@ def test_easystack_versions(self):
         versions = ('1.2.3', '1.2.30', '2021a', '1.2.3')
         for version in versions:
             write_file(test_easystack, tmpl_easystack_txt + ' ' + version)
-            ec_fns, _ = parse_easystack(test_easystack)
+            ec_fns, opts = parse_easystack(test_easystack)
             self.assertEqual(ec_fns, ['foo-%s.eb' % version])
+            self.assertEqual(opts, {})
 
         # multiple versions as a list
         test_easystack_txt = tmpl_easystack_txt + " [1.2.3, 3.2.1]"
         write_file(test_easystack, test_easystack_txt)
-        ec_fns, _ = parse_easystack(test_easystack)
+        ec_fns, opts = parse_easystack(test_easystack)
         expected = ['foo-1.2.3.eb', 'foo-3.2.1.eb']
         self.assertEqual(sorted(ec_fns), sorted(expected))
+        self.assertEqual(opts, {})
 
         # multiple versions listed with more info
         test_easystack_txt = '\n'.join([
@@ -139,9 +205,10 @@ def test_easystack_versions(self):
             "                 versionsuffix: -foo",
         ])
         write_file(test_easystack, test_easystack_txt)
-        ec_fns, _ = parse_easystack(test_easystack)
+        ec_fns, opts = parse_easystack(test_easystack)
         expected = ['foo-1.2.3.eb', 'foo-2021a.eb', 'foo-3.2.1-foo.eb']
         self.assertEqual(sorted(ec_fns), sorted(expected))
+        self.assertEqual(opts, {})
 
         # versions that get interpreted by YAML as float or int, single quotes required
         for version in ('1.2', '123', '3.50', '100', '2.44_01'):
@@ -152,8 +219,9 @@ def test_easystack_versions(self):
 
             # all is fine when wrapping the value in single quotes
             write_file(test_easystack, tmpl_easystack_txt + " '" + version + "'")
-            ec_fns, _ = parse_easystack(test_easystack)
+            ec_fns, opts = parse_easystack(test_easystack)
             self.assertEqual(ec_fns, ['foo-%s.eb' % version])
+            self.assertEqual(opts, {})
 
             # one rotten apple in the basket is enough
             test_easystack_txt = tmpl_easystack_txt + " [1.2.3, %s, 3.2.1]" % version
@@ -179,9 +247,10 @@ def test_easystack_versions(self):
                 "                 versionsuffix: -foo",
             ])
             write_file(test_easystack, test_easystack_txt)
-            ec_fns, _ = parse_easystack(test_easystack)
+            ec_fns, opts = parse_easystack(test_easystack)
             expected = ['foo-1.2.3.eb', 'foo-%s.eb' % version, 'foo-3.2.1-foo.eb']
             self.assertEqual(sorted(ec_fns), sorted(expected))
+            self.assertEqual(opts, {})
 
         # also check toolchain version that could be interpreted as a non-string value...
         test_easystack_txt = '\n'.join([
@@ -192,9 +261,10 @@ def test_easystack_versions(self):
             "        versions: [1.2.3, '2.3']",
         ])
         write_file(test_easystack, test_easystack_txt)
-        ec_fns, _ = parse_easystack(test_easystack)
+        ec_fns, opts = parse_easystack(test_easystack)
         expected = ['test-1.2.3-intel-2021.03.eb', 'test-2.3-intel-2021.03.eb']
         self.assertEqual(sorted(ec_fns), sorted(expected))
+        self.assertEqual(opts, {})
 
 
 def suite():

test/framework/easystacks/test_easystack_easyconfigs.yaml

@@ -0,0 +1,5 @@
+easyconfigs:
+    - binutils-2.25-GCCcore-4.9.3
+    - binutils-2.26-GCCcore-4.9.3
+    - foss-2018a
+    - toy-0.0-gompi-2018a-test

test/framework/easystacks/test_easystack_easyconfigs_opts.yaml

@@ -0,0 +1,13 @@
+easyconfigs:
+    - binutils-2.25-GCCcore-4.9.3:
+        options: {
+          'debug': True,
+          'from-pr': 12345,
+        }
+    - binutils-2.26-GCCcore-4.9.3
+    - foss-2018a:
+        options: {
+          'enforce-checksums': True,
+          'robot': True,
+        }
+    - toy-0.0-gompi-2018a-test

test/framework/easystacks/test_easystack_easyconfigs_with_eb_ext.yaml

@@ -0,0 +1,5 @@
+easyconfigs:
+    - binutils-2.25-GCCcore-4.9.3.eb
+    - binutils-2.26-GCCcore-4.9.3.eb
+    - foss-2018a.eb
+    - toy-0.0-gompi-2018a-test.eb