Merge pull request #3586 from boegel/cuda_templates_avail_easyconfig_templates

include %(mpi_cmd_prefix)s and %(cuda_*)s templates in output of --avail-easyconfig-templates

Author: verdurin

easybuild/framework/easyconfig/templates.py

@@ -46,7 +46,6 @@
 
 # derived from easyconfig, but not from ._config directly
 TEMPLATE_NAMES_EASYCONFIG = [
-    ('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"),
     ('module_name', "Module name"),
     ('nameletter', "First letter of software name"),
     ('toolchain_name', "Toolchain name"),
@@ -86,6 +85,18 @@
     ('Python', 'py'),
     ('R', 'r'),
 ]
+# template values which are only generated dynamically
+TEMPLATE_NAMES_DYNAMIC = [
+    ('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"),
+    ('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"),
+    ('cuda_compute_capabilities', "Comma-separated list of CUDA compute capabilities, as specified via "
+     "--cuda-compute-capabilities configuration option or via cuda_compute_capabilities easyconfig parameter"),
+    ('cuda_cc_space_sep', "Space-separated list of CUDA compute capabilities"),
+    ('cuda_cc_semicolon_sep', "Semicolon-separated list of CUDA compute capabilities"),
+    ('cuda_sm_comma_sep', "Comma-separated list of sm_* values that correspond with CUDA compute capabilities"),
+    ('cuda_sm_space_sep', "Space-separated list of sm_* values that correspond with CUDA compute capabilities"),
+]
+
 # constant templates that can be used in easyconfigs
 TEMPLATE_CONSTANTS = [
     # source url constants
@@ -300,6 +311,10 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None)
         except Exception:
             _log.warning("Failed to get .lower() for name %s value %s (type %s)", name, value, type(value))
 
+    # keep track of names of defined templates until now,
+    # so we can check whether names of additional dynamic template values are all known
+    common_template_names = set(template_values.keys())
+
     # step 5. add additional conditional templates
     if toolchain is not None and hasattr(toolchain, 'mpi_cmd_prefix'):
         try:
@@ -322,6 +337,14 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None)
         template_values['cuda_sm_comma_sep'] = ','.join(sm_values)
         template_values['cuda_sm_space_sep'] = ' '.join(sm_values)
 
+    unknown_names = []
+    for key in template_values:
+        dynamic_template_names = set(x for (x, _) in TEMPLATE_NAMES_DYNAMIC)
+        if not (key in common_template_names or key in dynamic_template_names):
+            unknown_names.append(key)
+    if unknown_names:
+        raise EasyBuildError("One or more template values found with unknown name: %s", ','.join(unknown_names))
+
     return template_values
 
 

easybuild/tools/docs.py

@@ -46,9 +46,9 @@
 from easybuild.framework.easyconfig.easyconfig import get_easyblock_class, process_easyconfig
 from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT
 from easybuild.framework.easyconfig.parser import EasyConfigParser
-from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_CONFIG, TEMPLATE_NAMES_EASYCONFIG
+from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_CONFIG, TEMPLATE_NAMES_DYNAMIC
+from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, TEMPLATE_NAMES_EASYCONFIG
 from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_LOWER, TEMPLATE_NAMES_LOWER_TEMPLATE
-from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, TEMPLATE_CONSTANTS
 from easybuild.framework.easyconfig.templates import TEMPLATE_SOFTWARE_VERSIONS, template_constant_dict
 from easybuild.framework.easyconfig.tools import avail_easyblocks
 from easybuild.framework.easyconfig.tweak import find_matching_easyconfigs
@@ -344,6 +344,12 @@ def avail_easyconfig_templates_txt():
     for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP:
         doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1]))
 
+    # some template values are only defined dynamically,
+    # see template_constant_dict function in easybuild.framework.easyconfigs.templates
+    doc.append('Template values which are defined dynamically')
+    for name in TEMPLATE_NAMES_DYNAMIC:
+        doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1]))
+
     doc.append('Template constants that can be used in easyconfigs')
     for cst in TEMPLATE_CONSTANTS:
         doc.append('%s%s: %s (%s)' % (INDENT_4SPACES, cst[0], cst[2], cst[1]))
@@ -395,6 +401,13 @@ def avail_easyconfig_templates_rst():
     ]
     doc.extend(rst_title_and_table(title, table_titles, table_values))
 
+    title = 'Template values which are defined dynamically'
+    table_values = [
+        ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_DYNAMIC],
+        [name[1] for name in TEMPLATE_NAMES_DYNAMIC],
+    ]
+    doc.extend(rst_title_and_table(title, table_titles, table_values))
+
     title = 'Template constants that can be used in easyconfigs'
     titles = ['Constant', 'Template value', 'Template name']
     table_values = [

test/framework/easyconfig.py

@@ -1128,6 +1128,7 @@ def test_templating_doc(self):
             easyconfig.templates.TEMPLATE_NAMES_CONFIG,
             easyconfig.templates.TEMPLATE_NAMES_LOWER,
             easyconfig.templates.TEMPLATE_NAMES_EASYBLOCK_RUN_STEP,
+            easyconfig.templates.TEMPLATE_NAMES_DYNAMIC,
             easyconfig.templates.TEMPLATE_CONSTANTS,
         ]
 

test/framework/options.py

@@ -509,6 +509,55 @@ def run_test(fmt=None):
         for fmt in [None, 'txt', 'rst']:
             run_test(fmt=fmt)
 
+    def test_avail_easyconfig_templates(self):
+        """Test listing available easyconfig file templates."""
+
+        def run_test(fmt=None):
+            """Helper function to test --avail-easyconfig-templates."""
+
+            args = ['--avail-easyconfig-templates']
+            if fmt is not None:
+                args.append('--output-format=%s' % fmt)
+
+            self.mock_stderr(True)
+            self.mock_stdout(True)
+            self.eb_main(args, verbose=True, raise_error=True)
+            stderr, stdout = self.get_stderr(), self.get_stdout()
+            self.mock_stderr(False)
+            self.mock_stdout(False)
+
+            self.assertFalse(stderr)
+
+            if fmt == 'rst':
+                pattern_lines = [
+                    r'^``%\(version_major\)s``\s+Major version\s*$',
+                    r'^``%\(cudaver\)s``\s+full version for CUDA\s*$',
+                    r'^``%\(pyshortver\)s``\s+short version for Python \(<major>.<minor>\)\s*$',
+                    r'^\* ``%\(name\)s``$',
+                    r'^``%\(namelower\)s``\s+lower case of value of name\s*$',
+                    r'^``%\(arch\)s``\s+System architecture \(e.g. x86_64, aarch64, ppc64le, ...\)\s*$',
+                    r'^``%\(cuda_cc_space_sep\)s``\s+Space-separated list of CUDA compute capabilities\s*$',
+                    r'^``SOURCE_TAR_GZ``\s+Source \.tar\.gz bundle\s+``%\(name\)s-%\(version\)s.tar.gz``\s*$',
+                ]
+            else:
+                pattern_lines = [
+                    r'^\s+%\(version_major\)s: Major version$',
+                    r'^\s+%\(cudaver\)s: full version for CUDA$',
+                    r'^\s+%\(pyshortver\)s: short version for Python \(<major>.<minor>\)$',
+                    r'^\s+%\(name\)s$',
+                    r'^\s+%\(namelower\)s: lower case of value of name$',
+                    r'^\s+%\(arch\)s: System architecture \(e.g. x86_64, aarch64, ppc64le, ...\)$',
+                    r'^\s+%\(cuda_cc_space_sep\)s: Space-separated list of CUDA compute capabilities$',
+                    r'^\s+SOURCE_TAR_GZ: Source \.tar\.gz bundle \(%\(name\)s-%\(version\)s.tar.gz\)$',
+                ]
+
+            for pattern_line in pattern_lines:
+                regex = re.compile(pattern_line, re.M)
+                self.assertTrue(regex.search(stdout), "Pattern '%s' should match in: %s" % (regex.pattern, stdout))
+
+        for fmt in [None, 'txt', 'rst']:
+            run_test(fmt=fmt)
+
     def test_avail_easyconfig_params(self):
         """Test listing available easyconfig parameters."""