Merge branch 'develop' into 5.0.x

Author: boegel

RELEASE_NOTES

@@ -4,6 +4,34 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.9.0 (30 December 2023)
+-------------------------
+
+feature release
+
+- various enhancements, including:
+  - allow tweaking of easyconfigs from different toolchains (#3669)
+  - add support for `--list-software --output-format=json` (#4152)
+  - add `--module-cache-suffix` configuration setting to allow multiple (Lmod) caches (#4403)
+- various bug fixes, including:
+  - deduplicate warnings & errors found in logs and add initial newline + tab in output (#4361)
+  - fix support for Environment Modules as modules tool to pass unit tests with v4.2+ (#4369)
+  - adapt module function check for Environment Modules v4+ (#4371)
+  - only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9 (#4375)
+  - use `-qopenmp` instead of `-fiopenmp` for OpenMP in Intel compilers (#4377)
+  - fix `LIBBLAS_MT` for FlexiBLAS, ensure `-lpthread` is included (#4379)
+  - relax major version match regex in `find_related_easyconfigs` using for `--review-pr` (#4385)
+  - eliminate duplicate multideps from generated module files (#4386)
+  - resolve templated values in extension names in `_make_extension_list` (#4392)
+  - use source toolchain version when passing only `--try-toolchain` (#4395)
+  - fix writing spider cache for Lmod >= 8.7.12 (#4402)
+  - fix `--inject-checksums` when extension specifies patch file in tuple format (#4405)
+  - fix `LooseVersion` when running with Python 2.7 (#4408)
+  - use more recent easyblocks PR in `test_github_merge_pr` (#4414)
+- other changes:
+  - extend test that checks build environment to recent `foss/2023a` toolchain (#4391)
+
+
 v4.8.2 (29 October 2023)
 ------------------------
 

easybuild/framework/easyblock.py

@@ -4651,7 +4651,7 @@ def inject_checksums(ecs, checksum_type):
     def make_list_lines(values, indent_level):
         """Make lines for list of values."""
         def to_str(s):
-            if isinstance(s, string_type):
+            if isinstance(s, str):
                 return "'%s'" % s
             else:
                 return str(s)

easybuild/tools/docs.py

@@ -38,6 +38,7 @@
 """
 import copy
 import inspect
+import json
 import os
 from collections import OrderedDict
 from easybuild.tools import LooseVersion
@@ -73,6 +74,7 @@
 DETAILED = 'detailed'
 SIMPLE = 'simple'
 
+FORMAT_JSON = 'json'
 FORMAT_MD = 'md'
 FORMAT_RST = 'rst'
 FORMAT_TXT = 'txt'
@@ -116,6 +118,11 @@ def avail_cfgfile_constants(go_cfg_constants, output_format=FORMAT_TXT):
     return generate_doc('avail_cfgfile_constants_%s' % output_format, [go_cfg_constants])
 
 
+def avail_cfgfile_constants_json(go_cfg_constants):
+    """Generate documentation on constants for configuration files in json format"""
+    raise NotImplementedError("JSON output format not supported for avail_cfgfile_constants_json")
+
+
 def avail_cfgfile_constants_txt(go_cfg_constants):
     """Generate documentation on constants for configuration files in txt format"""
     doc = [
@@ -185,6 +192,11 @@ def avail_easyconfig_constants(output_format=FORMAT_TXT):
     return generate_doc('avail_easyconfig_constants_%s' % output_format, [])
 
 
+def avail_easyconfig_constants_json():
+    """Generate easyconfig constant documentation in json format"""
+    raise NotImplementedError("JSON output format not supported for avail_easyconfig_constants_json")
+
+
 def avail_easyconfig_constants_txt():
     """Generate easyconfig constant documentation in txt format"""
     doc = ["Constants that can be used in easyconfigs"]
@@ -243,6 +255,11 @@ def avail_easyconfig_licenses(output_format=FORMAT_TXT):
     return generate_doc('avail_easyconfig_licenses_%s' % output_format, [])
 
 
+def avail_easyconfig_licenses_json():
+    """Generate easyconfig license documentation in json format"""
+    raise NotImplementedError("JSON output format not supported for avail_easyconfig_licenses_json")
+
+
 def avail_easyconfig_licenses_txt():
     """Generate easyconfig license documentation in txt format"""
     doc = ["License constants that can be used in easyconfigs"]
@@ -355,6 +372,13 @@ def avail_easyconfig_params_rst(title, grouped_params):
     return '\n'.join(doc)
 
 
+def avail_easyconfig_params_json():
+    """
+    Compose overview of available easyconfig parameters, in json format.
+    """
+    raise NotImplementedError("JSON output format not supported for avail_easyconfig_params_json")
+
+
 def avail_easyconfig_params_txt(title, grouped_params):
     """
     Compose overview of available easyconfig parameters, in plain text format.
@@ -427,6 +451,11 @@ def avail_easyconfig_templates(output_format=FORMAT_TXT):
     return generate_doc('avail_easyconfig_templates_%s' % output_format, [])
 
 
+def avail_easyconfig_templates_json():
+    """ Returns template documentation in json text format """
+    raise NotImplementedError("JSON output format not supported for avail_easyconfig_templates")
+
+
 def avail_easyconfig_templates_txt():
     """ Returns template documentation in plain text format """
     # This has to reflect the methods/steps used in easyconfig _generate_template_values
@@ -641,6 +670,8 @@ def avail_classes_tree(classes, class_names, locations, detailed, format_strings
 
 
 def list_easyblocks(list_easyblocks=SIMPLE, output_format=FORMAT_TXT):
+    if output_format == FORMAT_JSON:
+        raise NotImplementedError("JSON output format not supported for list_easyblocks")
     format_strings = {
         FORMAT_MD: {
             'det_root_templ': "- **%s** (%s%s)",
@@ -1025,6 +1056,38 @@ def list_software_txt(software, detailed=False):
     return '\n'.join(lines)
 
 
+def list_software_json(software, detailed=False):
+    """
+    Return overview of supported software in json
+
+    :param software: software information (strucuted like list_software does)
+    :param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info)
+    :return: multi-line string presenting requested info
+    """
+    lines = ['[']
+    for key in sorted(software, key=lambda x: x.lower()):
+        for entry in software[key]:
+            if detailed:
+                # deep copy here to avoid modifying the original dict
+                entry = copy.deepcopy(entry)
+                entry['description'] = ' '.join(entry['description'].split('\n')).strip()
+            else:
+                entry = {}
+            entry['name'] = key
+
+            lines.append(json.dumps(entry, indent=4, sort_keys=True, separators=(',', ': ')) + ",")
+            if not detailed:
+                break
+
+    # remove trailing comma on last line
+    if len(lines) > 1:
+        lines[-1] = lines[-1].rstrip(',')
+
+    lines.append(']')
+
+    return '\n'.join(lines)
+
+
 def list_toolchains(output_format=FORMAT_TXT):
     """Show list of known toolchains."""
     _, all_tcs = search_toolchain('')
@@ -1173,6 +1236,11 @@ def list_toolchains_txt(tcs):
     return '\n'.join(doc)
 
 
+def list_toolchains_json(tcs):
+    """ Returns overview of all toolchains in json format """
+    raise NotImplementedError("JSON output not implemented yet for --list-toolchains")
+
+
 def avail_toolchain_opts(name, output_format=FORMAT_TXT):
     """Show list of known options for given toolchain."""
     tc_class, _ = search_toolchain(name)
@@ -1226,6 +1294,11 @@ def avail_toolchain_opts_rst(name, tc_dict):
     return '\n'.join(doc)
 
 
+def avail_toolchain_opts_json(name, tc_dict):
+    """ Returns overview of toolchain options in jsonformat """
+    raise NotImplementedError("JSON output not implemented yet for --avail-toolchain-opts")
+
+
 def avail_toolchain_opts_txt(name, tc_dict):
     """ Returns overview of toolchain options in txt format """
     doc = ["Available options for %s toolchain:" % name]
@@ -1252,6 +1325,13 @@ def get_easyblock_classes(package_name):
     return easyblocks
 
 
+def gen_easyblocks_overview_json(package_name, path_to_examples, common_params=None, doc_functions=None):
+    """
+    Compose overview of all easyblocks in the given package in json format
+    """
+    raise NotImplementedError("JSON output not implemented yet for gen_easyblocks_overview")
+
+
 def gen_easyblocks_overview_md(package_name, path_to_examples, common_params=None, doc_functions=None):
     """
     Compose overview of all easyblocks in the given package in MarkDown format

easybuild/tools/options.py

@@ -79,7 +79,7 @@
 from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path
 from easybuild.tools.config import BuildOptions, ConfigurationVariables
 from easybuild.tools.configobj import ConfigObj, ConfigObjError
-from easybuild.tools.docs import FORMAT_MD, FORMAT_RST, FORMAT_TXT
+from easybuild.tools.docs import FORMAT_JSON, FORMAT_MD, FORMAT_RST, FORMAT_TXT
 from easybuild.tools.docs import avail_cfgfile_constants, avail_easyconfig_constants, avail_easyconfig_licenses
 from easybuild.tools.docs import avail_toolchain_opts, avail_easyconfig_params, avail_easyconfig_templates
 from easybuild.tools.docs import list_easyblocks, list_toolchains
@@ -466,7 +466,8 @@ def override_options(self):
             'mpi-tests': ("Run MPI tests (when relevant)", None, 'store_true', True),
             'optarch': ("Set architecture optimization, overriding native architecture optimizations",
                         None, 'store', None),
-            'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, [FORMAT_MD, FORMAT_RST, FORMAT_TXT]),
+            'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT,
+                              [FORMAT_JSON, FORMAT_MD, FORMAT_RST, FORMAT_TXT]),
             'output-style': ("Control output style; auto implies using Rich if available to produce rich output, "
                              "with fallback to basic colored output",
                              'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES),