add support for easystack file that contains easyconfig filenames

Author: boegel

easybuild/framework/easystack.py

@@ -64,14 +64,19 @@ def check_value(value, context):
 class EasyStack(object):
     """One class instance per easystack. General options + list of all SoftwareSpecs instances"""
 
-    def __init__(self):
+    def __init__(self, easyconfigs=None):
         self.easybuild_version = None
         self.robot = False
         self.software_list = []
+        self.easyconfigs = []
+        if easyconfigs is not None:
+            self.easyconfigs.extend(easyconfigs)
 
     def compose_ec_filenames(self):
         """Returns a list of all easyconfig names"""
         ec_filenames = []
+
+        # entries specified via 'software' top-level key
         for sw in self.software_list:
             full_ec_version = det_full_ec_version({
                 'toolchain': {'name': sw.toolchain_name, 'version': sw.toolchain_version},
@@ -80,6 +85,11 @@ def compose_ec_filenames(self):
             })
             ec_filename = '%s-%s.eb' % (sw.name, full_ec_version)
             ec_filenames.append(ec_filename)
+
+        # entries specified via 'easyconfigs' top-level key
+        for ec in self.easyconfigs:
+            ec_filenames.append(ec + '.eb')
+
         return ec_filenames
 
     # flags applicable to all sw (i.e. robot)
@@ -108,21 +118,53 @@ class EasyStackParser(object):
 
     @staticmethod
     def parse(filepath):
-        """Parses YAML file and assigns obtained values to SW config instances as well as general config instance"""
+        """
+        Parses YAML file and assigns obtained values to SW config instances as well as general config instance"""
         yaml_txt = read_file(filepath)
 
         try:
             easystack_raw = yaml.safe_load(yaml_txt)
         except yaml.YAMLError as err:
             raise EasyBuildError("Failed to parse %s: %s" % (filepath, err))
 
-        easystack = EasyStack()
+        easystack_data = None
+        top_keys = ('easyconfigs', 'software')
+        for key in top_keys:
+            if key in easystack_raw:
+                easystack_data = easystack_raw[key]
+                break
 
-        try:
-            software = easystack_raw["software"]
-        except KeyError:
-            wrong_structure_file = "Not a valid EasyStack YAML file: no 'software' key found"
-            raise EasyBuildError(wrong_structure_file)
+        if easystack_data is None:
+            msg = "Not a valid EasyStack YAML file: no 'easyconfigs' or 'software' top-level key found"
+            raise EasyBuildError(msg)
+        else:
+            parse_method_name = 'parse_by_' + key
+            parse_method = getattr(EasyStackParser, 'parse_by_%s' % key, None)
+            if parse_method is None:
+                raise EasyBuildError("Easystack parse method '%s' not found!", parse_method_name)
+
+        # assign general easystack attributes
+        easybuild_version = easystack_raw.get('easybuild_version', None)
+        robot = easystack_raw.get('robot', False)
+
+        return parse_method(filepath, easystack_data, easybuild_version=easybuild_version, robot=robot)
+
+    @staticmethod
+    def parse_by_easyconfigs(filepath, easyconfigs, easybuild_version=None, robot=False):
+        """
+        Parse easystack file with 'easyconfigs' as top-level key.
+        """
+        easystack = EasyStack(easyconfigs=easyconfigs)
+
+        return easystack
+
+    @staticmethod
+    def parse_by_software(filepath, software, easybuild_version=None, robot=False):
+        """
+        Parse easystack file with 'software' as top-level key.
+        """
+
+        easystack = EasyStack()
 
         # assign software-specific easystack attributes
         for name in software:
@@ -224,8 +266,8 @@ def parse(filepath):
                     easystack.software_list.append(sw)
 
             # assign general easystack attributes
-            easystack.easybuild_version = easystack_raw.get('easybuild_version', None)
-            easystack.robot = easystack_raw.get('robot', False)
+            easystack.easybuild_version = easybuild_version
+            easystack.robot = robot
 
         return easystack
 

test/framework/easystack.py

@@ -56,6 +56,36 @@ def tearDown(self):
         easybuild.tools.build_log.EXPERIMENTAL = self.orig_experimental
         super(EasyStackTest, self).tearDown()
 
+    def test_easystack_basic(self):
+        """Test for basic easystack file."""
+        topdir = os.path.dirname(os.path.abspath(__file__))
+        test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_basic.yaml')
+
+        ec_fns, opts = parse_easystack(test_easystack)
+        expected = [
+            'binutils-2.25-GCCcore-4.9.3.eb',
+            'binutils-2.26-GCCcore-4.9.3.eb',
+            'foss-2018a.eb',
+            'toy-0.0-gompi-2018a-test.eb',
+        ]
+        self.assertEqual(sorted(ec_fns), sorted(expected))
+        self.assertEqual(opts, {})
+
+    def test_easystack_easyconfigs(self):
+        """Test for easystack file using 'easyconfigs' key."""
+        topdir = os.path.dirname(os.path.abspath(__file__))
+        test_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_easyconfigs.yaml')
+
+        ec_fns, opts = parse_easystack(test_easystack)
+        expected = [
+            'binutils-2.25-GCCcore-4.9.3.eb',
+            'binutils-2.26-GCCcore-4.9.3.eb',
+            'foss-2018a.eb',
+            'toy-0.0-gompi-2018a-test.eb',
+        ]
+        self.assertEqual(sorted(ec_fns), sorted(expected))
+        self.assertEqual(opts, {})
+
     def test_parse_fail(self):
         """Test for clean error when easystack file fails to parse."""
         test_yml = os.path.join(self.test_prefix, 'test.yml')
@@ -120,15 +150,17 @@ def test_easystack_versions(self):
         versions = ('1.2.3', '1.2.30', '2021a', '1.2.3')
         for version in versions:
             write_file(test_easystack, tmpl_easystack_txt + ' ' + version)
-            ec_fns, _ = parse_easystack(test_easystack)
+            ec_fns, opts = parse_easystack(test_easystack)
             self.assertEqual(ec_fns, ['foo-%s.eb' % version])
+            self.assertEqual(opts, {})
 
         # multiple versions as a list
         test_easystack_txt = tmpl_easystack_txt + " [1.2.3, 3.2.1]"
         write_file(test_easystack, test_easystack_txt)
-        ec_fns, _ = parse_easystack(test_easystack)
+        ec_fns, opts = parse_easystack(test_easystack)
         expected = ['foo-1.2.3.eb', 'foo-3.2.1.eb']
         self.assertEqual(sorted(ec_fns), sorted(expected))
+        self.assertEqual(opts, {})
 
         # multiple versions listed with more info
         test_easystack_txt = '\n'.join([
@@ -139,9 +171,10 @@ def test_easystack_versions(self):
             "                 versionsuffix: -foo",
         ])
         write_file(test_easystack, test_easystack_txt)
-        ec_fns, _ = parse_easystack(test_easystack)
+        ec_fns, opts = parse_easystack(test_easystack)
         expected = ['foo-1.2.3.eb', 'foo-2021a.eb', 'foo-3.2.1-foo.eb']
         self.assertEqual(sorted(ec_fns), sorted(expected))
+        self.assertEqual(opts, {})
 
         # versions that get interpreted by YAML as float or int, single quotes required
         for version in ('1.2', '123', '3.50', '100', '2.44_01'):
@@ -152,8 +185,9 @@ def test_easystack_versions(self):
 
             # all is fine when wrapping the value in single quotes
             write_file(test_easystack, tmpl_easystack_txt + " '" + version + "'")
-            ec_fns, _ = parse_easystack(test_easystack)
+            ec_fns, opts = parse_easystack(test_easystack)
             self.assertEqual(ec_fns, ['foo-%s.eb' % version])
+            self.assertEqual(opts, {})
 
             # one rotten apple in the basket is enough
             test_easystack_txt = tmpl_easystack_txt + " [1.2.3, %s, 3.2.1]" % version
@@ -179,9 +213,10 @@ def test_easystack_versions(self):
                 "                 versionsuffix: -foo",
             ])
             write_file(test_easystack, test_easystack_txt)
-            ec_fns, _ = parse_easystack(test_easystack)
+            ec_fns, opts = parse_easystack(test_easystack)
             expected = ['foo-1.2.3.eb', 'foo-%s.eb' % version, 'foo-3.2.1-foo.eb']
             self.assertEqual(sorted(ec_fns), sorted(expected))
+            self.assertEqual(opts, {})
 
         # also check toolchain version that could be interpreted as a non-string value...
         test_easystack_txt = '\n'.join([
@@ -192,9 +227,10 @@ def test_easystack_versions(self):
             "        versions: [1.2.3, '2.3']",
         ])
         write_file(test_easystack, test_easystack_txt)
-        ec_fns, _ = parse_easystack(test_easystack)
+        ec_fns, opts = parse_easystack(test_easystack)
         expected = ['test-1.2.3-intel-2021.03.eb', 'test-2.3-intel-2021.03.eb']
         self.assertEqual(sorted(ec_fns), sorted(expected))
+        self.assertEqual(opts, {})
 
 
 def suite():

test/framework/easystacks/test_easystack_easyconfigs.yaml

@@ -0,0 +1,5 @@
+easyconfigs:
+    - binutils-2.25-GCCcore-4.9.3
+    - binutils-2.26-GCCcore-4.9.3
+    - foss-2018a
+    - toy-0.0-gompi-2018a-test