Skip to content

Commit ff3d33f

Browse files
deniskristakdeniskristakboegel
authored
add (experimental) support for specifying easyconfig files via "easystack" file (#3479)
Co-authored-by: deniskristak <[email protected]> Co-authored-by: Kenneth Hoste <[email protected]>
1 parent 14e939d commit ff3d33f

File tree

12 files changed

+345
-2
lines changed

12 files changed

+345
-2
lines changed

‎easybuild/framework/easystack.py‎

Lines changed: 234 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,234 @@
1+
# Copyright 2020-2020 Ghent University
2+
#
3+
# This file is part of EasyBuild,
4+
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
5+
# with support of Ghent University (http://ugent.be/hpc),
6+
# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
7+
# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
8+
# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
9+
#
10+
# https://github.com/easybuilders/easybuild
11+
#
12+
# EasyBuild is free software: you can redistribute it and/or modify
13+
# it under the terms of the GNU General Public License as published by
14+
# the Free Software Foundation v2.
15+
#
16+
# EasyBuild is distributed in the hope that it will be useful,
17+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
18+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19+
# GNU General Public License for more details.
20+
#
21+
# You should have received a copy of the GNU General Public License
22+
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
23+
#
24+
"""
25+
Support for easybuild-ing from multiple easyconfigs based on
26+
information obtained from provided file (easystack) with build specifications.
27+
28+
:author: Denis Kristak (Inuits)
29+
:author: Pavel Grochal (Inuits)
30+
"""
31+
32+
from easybuild.base import fancylogger
33+
from easybuild.tools.build_log import EasyBuildError
34+
from easybuild.tools.filetools import read_file
35+
from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
36+
from easybuild.tools.utilities import only_if_module_is_available
37+
try:
38+
import yaml
39+
except ImportError:
40+
pass
41+
_log = fancylogger.getLogger('easystack', fname=False)
42+
43+
44+
class EasyStack(object):
45+
"""One class instance per easystack. General options + list of all SoftwareSpecs instances"""
46+
47+
def __init__(self):
48+
self.easybuild_version = None
49+
self.robot = False
50+
self.software_list = []
51+
52+
def compose_ec_filenames(self):
53+
"""Returns a list of all easyconfig names"""
54+
ec_filenames = []
55+
for sw in self.software_list:
56+
full_ec_version = det_full_ec_version({
57+
'toolchain': {'name': sw.toolchain_name, 'version': sw.toolchain_version},
58+
'version': sw.version,
59+
'versionsuffix': sw.versionsuffix,
60+
})
61+
ec_filename = '%s-%s.eb' % (sw.name, full_ec_version)
62+
ec_filenames.append(ec_filename)
63+
return ec_filenames
64+
65+
# flags applicable to all sw (i.e. robot)
66+
def get_general_options(self):
67+
"""Returns general options (flags applicable to all sw (i.e. --robot))"""
68+
general_options = {}
69+
# TODO add support for general_options
70+
# general_options['robot'] = self.robot
71+
# general_options['easybuild_version'] = self.easybuild_version
72+
return general_options
73+
74+
75+
class SoftwareSpecs(object):
76+
"""Contains information about every software that should be installed"""
77+
78+
def __init__(self, name, version, versionsuffix, toolchain_version, toolchain_name):
79+
self.name = name
80+
self.version = version
81+
self.toolchain_version = toolchain_version
82+
self.toolchain_name = toolchain_name
83+
self.versionsuffix = versionsuffix
84+
85+
86+
class EasyStackParser(object):
87+
"""Parser for easystack files (in YAML syntax)."""
88+
89+
@only_if_module_is_available('yaml', pkgname='PyYAML')
90+
@staticmethod
91+
def parse(filepath):
92+
"""Parses YAML file and assigns obtained values to SW config instances as well as general config instance"""
93+
yaml_txt = read_file(filepath)
94+
easystack_raw = yaml.safe_load(yaml_txt)
95+
easystack = EasyStack()
96+
97+
try:
98+
software = easystack_raw["software"]
99+
except KeyError:
100+
wrong_structure_file = "Not a valid EasyStack YAML file: no 'software' key found"
101+
raise EasyBuildError(wrong_structure_file)
102+
103+
# assign software-specific easystack attributes
104+
for name in software:
105+
# ensure we have a string value (YAML parser returns type = dict
106+
# if levels under the current attribute are present)
107+
name = str(name)
108+
try:
109+
toolchains = software[name]['toolchains']
110+
except KeyError:
111+
raise EasyBuildError("Toolchains for software '%s' are not defined in %s", name, filepath)
112+
for toolchain in toolchains:
113+
toolchain = str(toolchain)
114+
toolchain_parts = toolchain.split('-', 1)
115+
if len(toolchain_parts) == 2:
116+
toolchain_name, toolchain_version = toolchain_parts
117+
elif len(toolchain_parts) == 1:
118+
toolchain_name, toolchain_version = toolchain, ''
119+
else:
120+
raise EasyBuildError("Incorrect toolchain specification for '%s' in %s, too many parts: %s",
121+
name, filepath, toolchain_parts)
122+
123+
try:
124+
# if version string containts asterisk or labels, raise error (asterisks not supported)
125+
versions = toolchains[toolchain]['versions']
126+
except TypeError as err:
127+
wrong_structure_err = "An error occurred when interpreting "
128+
wrong_structure_err += "the data for software %s: %s" % (name, err)
129+
raise EasyBuildError(wrong_structure_err)
130+
if '*' in str(versions):
131+
asterisk_err = "EasyStack specifications of '%s' in %s contain asterisk. "
132+
asterisk_err += "Wildcard feature is not supported yet."
133+
raise EasyBuildError(asterisk_err, name, filepath)
134+
135+
# yaml versions can be in different formats in yaml file
136+
# firstly, check if versions in yaml file are read as a dictionary.
137+
# Example of yaml structure:
138+
# ========================================================================
139+
# versions:
140+
# 2.25:
141+
# 2.23:
142+
# versionsuffix: '-R-4.0.0'
143+
# ========================================================================
144+
if isinstance(versions, dict):
145+
for version in versions:
146+
if versions[version] is not None:
147+
version_spec = versions[version]
148+
if 'versionsuffix' in version_spec:
149+
versionsuffix = str(version_spec['versionsuffix'])
150+
else:
151+
versionsuffix = ''
152+
if 'exclude-labels' in str(version_spec) or 'include-labels' in str(version_spec):
153+
lab_err = "EasyStack specifications of '%s' in %s "
154+
lab_err += "contain labels. Labels aren't supported yet."
155+
raise EasyBuildError(lab_err, name, filepath)
156+
else:
157+
versionsuffix = ''
158+
159+
specs = {
160+
'name': name,
161+
'toolchain_name': toolchain_name,
162+
'toolchain_version': toolchain_version,
163+
'version': version,
164+
'versionsuffix': versionsuffix,
165+
}
166+
sw = SoftwareSpecs(**specs)
167+
168+
# append newly created class instance to the list in instance of EasyStack class
169+
easystack.software_list.append(sw)
170+
continue
171+
172+
# is format read as a list of versions?
173+
# ========================================================================
174+
# versions:
175+
# [2.24, 2.51]
176+
# ========================================================================
177+
elif isinstance(versions, list):
178+
versions_list = versions
179+
180+
# format = multiple lines without ':' (read as a string)?
181+
# ========================================================================
182+
# versions:
183+
# 2.24
184+
# 2.51
185+
# ========================================================================
186+
elif isinstance(versions, str):
187+
versions_list = str(versions).split()
188+
189+
# format read as float (containing one version only)?
190+
# ========================================================================
191+
# versions:
192+
# 2.24
193+
# ========================================================================
194+
elif isinstance(versions, float):
195+
versions_list = [str(versions)]
196+
197+
# if no version is a dictionary, versionsuffix isn't specified
198+
versionsuffix = ''
199+
200+
for version in versions_list:
201+
sw = SoftwareSpecs(
202+
name=name, version=version, versionsuffix=versionsuffix,
203+
toolchain_name=toolchain_name, toolchain_version=toolchain_version)
204+
# append newly created class instance to the list in instance of EasyStack class
205+
easystack.software_list.append(sw)
206+
207+
# assign general easystack attributes
208+
easystack.easybuild_version = easystack_raw.get('easybuild_version', None)
209+
easystack.robot = easystack_raw.get('robot', False)
210+
211+
return easystack
212+
213+
214+
def parse_easystack(filepath):
215+
"""Parses through easystack file, returns what EC are to be installed together with their options."""
216+
log_msg = "Support for easybuild-ing from multiple easyconfigs based on "
217+
log_msg += "information obtained from provided file (easystack) with build specifications."
218+
_log.experimental(log_msg)
219+
_log.info("Building from easystack: '%s'" % filepath)
220+
221+
# class instance which contains all info about planned build
222+
easystack = EasyStackParser.parse(filepath)
223+
224+
easyconfig_names = easystack.compose_ec_filenames()
225+
226+
general_options = easystack.get_general_options()
227+
228+
_log.debug("EasyStack parsed. Proceeding to install these Easyconfigs: \n'%s'" % "',\n'".join(easyconfig_names))
229+
if len(general_options) != 0:
230+
_log.debug("General options for installation are: \n%s" % str(general_options))
231+
else:
232+
_log.debug("No general options were specified in easystack")
233+
234+
return easyconfig_names, general_options

‎easybuild/main.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747

4848
from easybuild.framework.easyblock import build_and_install_one, inject_checksums
4949
from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR
50+
from easybuild.framework.easystack import parse_easystack
5051
from easybuild.framework.easyconfig.easyconfig import clean_up_easyconfigs
5152
from easybuild.framework.easyconfig.easyconfig import fix_deprecated_easyconfigs, verify_easyconfig_filename
5253
from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check
@@ -224,6 +225,13 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
224225
last_log = find_last_log(logfile) or '(none)'
225226
print_msg(last_log, log=_log, prefix=False)
226227

228+
# if easystack is provided with the command, commands with arguments from it will be executed
229+
if options.easystack:
230+
# TODO add general_options (i.e. robot) to build options
231+
orig_paths, general_options = parse_easystack(options.easystack)
232+
if general_options:
233+
raise EasyBuildError("Specifying general configuration options in easystack file is not supported yet.")
234+
227235
# check whether packaging is supported when it's being used
228236
if options.package:
229237
check_pkg_support()

‎easybuild/tools/options.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -605,6 +605,8 @@ def informative_options(self):
605605
'show-full-config': ("Show current EasyBuild configuration (all settings)", None, 'store_true', False),
606606
'show-system-info': ("Show system information relevant to EasyBuild", None, 'store_true', False),
607607
'terse': ("Terse output (machine-readable)", None, 'store_true', False),
608+
'easystack': ("Path to easystack file in YAML format, specifying details of a software stack",
609+
None, 'store', None),
608610
})
609611

610612
self.log.debug("informative_options: descr %s opts %s" % (descr, opts))

‎easybuild/tools/utilities.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,7 @@ def wrap(orig):
173173
pass
174174

175175
if imported is None:
176-
raise ImportError("None of the specified modules %s is available" % ', '.join(modnames))
176+
raise ImportError("None of the specified modules (%s) is available" % ', '.join(modnames))
177177
else:
178178
return orig
179179

‎test/framework/easystacks/test_easystack_asterisk.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
software:
2+
binutils:
3+
toolchains:
4+
GCCcore-4.9.3:
5+
versions:
6+
"2.11.*"

‎test/framework/easystacks/test_easystack_basic.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
software:
2+
binutils:
3+
toolchains:
4+
GCCcore-4.9.3:
5+
versions:
6+
2.25:
7+
2.26:
8+
toy:
9+
toolchains:
10+
gompi-2018a:
11+
versions:
12+
0.0:
13+
versionsuffix: '-test'

‎test/framework/easystacks/test_easystack_labels.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
software:
2+
binutils:
3+
toolchains:
4+
GCCcore-4.9.3:
5+
versions:
6+
3.11:
7+
exclude-labels: arch:aarch64

‎test/framework/easystacks/test_easystack_wrong_structure.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
software:
2+
Bioconductor:
3+
toolchains:
4+
# foss-2020a:
5+
versions:
6+
3.11

‎test/framework/general.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ def bar():
8888
def bar2():
8989
pass
9090

91-
err_pat = "ImportError: None of the specified modules nosuchmodule, anothernosuchmodule is available"
91+
err_pat = r"ImportError: None of the specified modules \(nosuchmodule, anothernosuchmodule\) is available"
9292
self.assertErrorRegex(EasyBuildError, err_pat, bar2)
9393

9494
class Foo():

0 commit comments

Comments
 (0)