Merge pull request #4534 from dagonzalezfo/4426_granular_exit_code

More granular exit codes

Author: boegel

easybuild/framework/easyblock.py

@@ -65,7 +65,7 @@
 from easybuild.framework.easyconfig.templates import ALTERNATIVE_EASYCONFIG_TEMPLATES, DEPRECATED_EASYCONFIG_TEMPLATES
 from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_DYNAMIC, template_constant_dict
 from easybuild.tools import LooseVersion
-from easybuild.tools.build_log import EasyBuildError, print_warning, print_msg
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_warning, print_msg
 from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG
 from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN
 from easybuild.tools.config import Singleton, build_option, get_module_naming_scheme
@@ -907,9 +907,12 @@ def validate_os_deps(self):
                 not_found.append(dep)
 
         if not_found:
-            raise EasyBuildError("One or more OS dependencies were not found: %s", not_found)
-        else:
-            self.log.info("OS dependencies ok: %s" % self['osdependencies'])
+            raise EasyBuildError(
+                "One or more OS dependencies were not found: %s", not_found,
+                exit_code=EasyBuildExit.MISSING_SYSTEM_DEPENDENCY
+            )
+
+        self.log.info("OS dependencies ok: %s" % self['osdependencies'])
 
         return True
 
@@ -1272,7 +1275,10 @@ def _validate(self, attr, values):  # private method
         if values is None:
             values = []
         if self[attr] and self[attr] not in values:
-            raise EasyBuildError("%s provided '%s' is not valid: %s", attr, self[attr], values)
+            raise EasyBuildError(
+                "%s provided '%s' is not valid: %s", attr, self[attr], values,
+                exit_code=EasyBuildExit.VALUE_ERROR
+            )
 
     def probe_external_module_metadata(self, mod_name, existing_metadata=None):
         """
@@ -1922,12 +1928,20 @@ def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error
                 error_re = re.compile(r"No module named '?.*/?%s'?" % modname)
                 _log.debug("error regexp for ImportError on '%s' easyblock: %s", modname, error_re.pattern)
                 if error_re.match(str(err)):
+                    # Missing easyblock type of error
                     if error_on_missing_easyblock:
-                        raise EasyBuildError("No software-specific easyblock '%s' found for %s", class_name, name)
-                elif error_on_failed_import:
-                    raise EasyBuildError("Failed to import %s easyblock: %s", class_name, err)
+                        raise EasyBuildError(
+                            "No software-specific easyblock '%s' found for %s", class_name, name,
+                            exit_code=EasyBuildExit.MISSING_EASYBLOCK
+                        ) from err
                 else:
-                    _log.debug("Failed to import easyblock for %s, but ignoring it: %s" % (class_name, err))
+                    # Broken import
+                    if error_on_failed_import:
+                        raise EasyBuildError(
+                            "Failed to import %s easyblock: %s", class_name, err,
+                            exit_code=EasyBuildExit.EASYBLOCK_ERROR
+                        ) from err
+                _log.debug("Failed to import easyblock for %s, but ignoring it: %s" % (class_name, err))
 
         if cls is not None:
             _log.info("Successfully obtained class '%s' for easyblock '%s' (software name '%s')",
@@ -1941,7 +1955,10 @@ def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error
         # simply reraise rather than wrapping it into another error
         raise err
     except Exception as err:
-        raise EasyBuildError("Failed to obtain class for %s easyblock (not available?): %s", easyblock, err)
+        raise EasyBuildError(
+            "Failed to obtain class for %s easyblock (not available?): %s", easyblock, err,
+            exit_code=EasyBuildExit.EASYBLOCK_ERROR
+        )
 
 
 def get_module_path(name, generic=None, decode=True):
@@ -2086,7 +2103,11 @@ def process_easyconfig(path, build_specs=None, validate=True, parse_only=False,
         try:
             ec = EasyConfig(spec, build_specs=build_specs, validate=validate, hidden=hidden)
         except EasyBuildError as err:
-            raise EasyBuildError("Failed to process easyconfig %s: %s", spec, err.msg)
+            try:
+                exit_code = err.exit_code
+            except AttributeError:
+                exit_code = EasyBuildExit.EASYCONFIG_ERROR
+            raise EasyBuildError("Failed to process easyconfig %s: %s", spec, err.msg, exit_code=exit_code)
 
         name = ec['name']
 

easybuild/framework/easyconfig/easyconfig.py

@@ -54,7 +54,7 @@
 from easybuild.framework.easyconfig.easyconfig import process_easyconfig
 from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check
 from easybuild.tools import LooseVersion
-from easybuild.tools.build_log import EasyBuildError, print_error, print_msg, print_warning
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_error, print_msg, print_warning
 from easybuild.tools.config import build_option
 from easybuild.tools.environment import restore_env
 from easybuild.tools.filetools import find_easyconfigs, get_cwd, is_patch_file, locate_files
@@ -409,7 +409,7 @@ def parse_easyconfigs(paths, validate=True):
         # keep track of whether any files were generated
         generated_ecs |= generated
         if not os.path.exists(path):
-            raise EasyBuildError("Can't find path %s", path)
+            raise EasyBuildError("Can't find path %s", path, exit_code=EasyBuildExit.MISSING_EASYCONFIG)
         try:
             ec_files = find_easyconfigs(path, ignore_dirs=build_option('ignore_dirs'))
             for ec_file in ec_files:

easybuild/framework/easyconfig/tools.py

@@ -40,7 +40,7 @@
 
 from easybuild.framework.easyconfig.easyconfig import resolve_template
 from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict
-from easybuild.tools.build_log import EasyBuildError, raise_nosupport
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, raise_nosupport
 from easybuild.tools.filetools import change_dir
 from easybuild.tools.run import run_shell_cmd
 
@@ -302,7 +302,7 @@ def sanity_check_step(self):
             cmd, stdin = resolve_exts_filter_template(exts_filter, self)
             cmd_res = run_shell_cmd(cmd, fail_on_error=False, stdin=stdin)
 
-            if cmd_res.exit_code:
+            if cmd_res.exit_code != EasyBuildExit.SUCCESS:
                 if stdin:
                     fail_msg = 'command "%s" (stdin: "%s") failed' % (cmd, stdin)
                 else:

easybuild/framework/extension.py

@@ -133,10 +133,10 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
 
         ec_res = {}
         try:
-            (ec_res['success'], app_log, err) = build_and_install_one(ec, init_env)
+            (ec_res['success'], app_log, err_msg, err_code) = build_and_install_one(ec, init_env)
             ec_res['log_file'] = app_log
             if not ec_res['success']:
-                ec_res['err'] = EasyBuildError(err)
+                ec_res['err'] = EasyBuildError(err_msg, exit_code=err_code)
         except Exception as err:
             # purposely catch all exceptions
             ec_res['success'] = False
@@ -174,7 +174,7 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
             if not isinstance(ec_res['err'], EasyBuildError):
                 raise ec_res['err']
             else:
-                raise EasyBuildError(test_msg)
+                raise EasyBuildError(test_msg, exit_code=err_code)
 
         res.append((ec, ec_res))
 
@@ -779,15 +779,15 @@ def main_with_hooks(args=None):
     try:
         init_session_state, eb_go, cfg_settings = prepare_main(args=args)
     except EasyBuildError as err:
-        print_error(err.msg)
+        print_error(err.msg, exit_code=err.exit_code)
 
     hooks = load_hooks(eb_go.options.hooks)
 
     try:
         main(args=args, prepared_cfg_data=(init_session_state, eb_go, cfg_settings))
     except EasyBuildError as err:
         run_hook(FAIL, hooks, args=[err])
-        print_error(err.msg, exit_on_error=True, exit_code=1)
+        print_error(err.msg, exit_on_error=True, exit_code=err.exit_code)
     except KeyboardInterrupt as err:
         run_hook(CANCEL, hooks, args=[err])
         print_error("Cancelled by user: %s" % err)

easybuild/main.py

@@ -40,12 +40,12 @@
 import tempfile
 from copy import copy
 from datetime import datetime
+from enum import IntEnum
 
 from easybuild.base import fancylogger
 from easybuild.base.exceptions import LoggedException
 from easybuild.tools.version import VERSION, this_is_easybuild
 
-
 # EasyBuild message prefix
 EB_MSG_PREFIX = "=="
 
@@ -71,6 +71,55 @@
 logging.addLevelName(DEVEL_LOG_LEVEL, 'DEVEL')
 
 
+class EasyBuildExit(IntEnum):
+    """
+    Table of exit codes
+    """
+    SUCCESS = 0
+    ERROR = 1
+    # core errors
+    OPTION_ERROR = 2
+    VALUE_ERROR = 3
+    MISSING_EASYCONFIG = 4
+    EASYCONFIG_ERROR = 5
+    MISSING_EASYBLOCK = 6
+    EASYBLOCK_ERROR = 7
+    MODULE_ERROR = 8
+    # step errors in order of execution
+    FAIL_FETCH_STEP = 10
+    FAIL_READY_STEP = 11
+    FAIL_SOURCE_STEP = 12
+    FAIL_PATCH_STEP = 13
+    FAIL_PREPARE_STEP = 14
+    FAIL_CONFIGURE_STEP = 15
+    FAIL_BUILD_STEP = 16
+    FAIL_TEST_STEP = 17
+    FAIL_INSTALL_STEP = 18
+    FAIL_EXTENSIONS_STEP = 19
+    FAIL_POST_ITER_STEP = 20
+    FAIL_POST_PROC_STEP = 21
+    FAIL_SANITY_CHECK_STEP = 22
+    FAIL_CLEANUP_STEP = 23
+    FAIL_MODULE_STEP = 24
+    FAIL_PERMISSIONS_STEP = 25
+    FAIL_PACKAGE_STEP = 26
+    FAIL_TEST_CASES_STEP = 27
+    # errors on missing things
+    MISSING_SOURCES = 30
+    MISSING_DEPENDENCY = 31
+    MISSING_SYSTEM_DEPENDENCY = 32
+    MISSING_EB_DEPENDENCY = 33
+    # errors on specific task failures
+    FAIL_SYSTEM_CHECK = 40
+    FAIL_DOWNLOAD = 41
+    FAIL_CHECKSUM = 42
+    FAIL_EXTRACT = 43
+    FAIL_PATCH_APPLY = 44
+    FAIL_SANITY_CHECK = 45
+    FAIL_MODULE_WRITE = 46
+    FAIL_GITHUB = 47
+
+
 class EasyBuildError(LoggedException):
     """
     EasyBuildError is thrown when EasyBuild runs into something horribly wrong.
@@ -80,12 +129,13 @@ class EasyBuildError(LoggedException):
     # always include location where error was raised from, even under 'python -O'
     INCLUDE_LOCATION = True
 
-    def __init__(self, msg, *args):
+    def __init__(self, msg, *args, exit_code=EasyBuildExit.ERROR, **kwargs):
         """Constructor: initialise EasyBuildError instance."""
         if args:
             msg = msg % args
-        LoggedException.__init__(self, msg)
+        LoggedException.__init__(self, msg, exit_code=exit_code, **kwargs)
         self.msg = msg
+        self.exit_code = exit_code
 
     def __str__(self):
         """Return string representation of this EasyBuildError instance."""

easybuild/tools/build_log.py

@@ -50,7 +50,7 @@
 from easybuild.base import fancylogger
 from easybuild.base.frozendict import FrozenDictKnownKeys
 from easybuild.base.wrapper import create_base_metaclass
-from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit
 
 try:
     import rich  # noqa
@@ -506,7 +506,10 @@ def get_items_check_required(self):
         """
         missing = [x for x in self.KNOWN_KEYS if x not in self]
         if len(missing) > 0:
-            raise EasyBuildError("Cannot determine value for configuration variables %s. Please specify it.", missing)
+            raise EasyBuildError(
+                "Cannot determine value for configuration variables %s. Please specify it.", ', '.join(missing),
+                exit_code=EasyBuildExit.OPTION_ERROR
+            )
 
         return self.items()
 
@@ -539,7 +542,10 @@ def init(options, config_options_dict):
         tmpdict['sourcepath'] = sourcepath.split(':')
         _log.debug("Converted source path ('%s') to a list of paths: %s" % (sourcepath, tmpdict['sourcepath']))
     elif not isinstance(sourcepath, (tuple, list)):
-        raise EasyBuildError("Value for sourcepath has invalid type (%s): %s", type(sourcepath), sourcepath)
+        raise EasyBuildError(
+            "Value for sourcepath has invalid type (%s): %s", type(sourcepath), sourcepath,
+            exit_code=EasyBuildExit.OPTION_ERROR
+        )
 
     # initialize configuration variables (any future calls to ConfigurationVariables() will yield the same instance
     variables = ConfigurationVariables(tmpdict, ignore_unknown_keys=True)
@@ -623,7 +629,7 @@ def build_option(key, **kwargs):
         error_msg = "Undefined build option: '%s'. " % key
         error_msg += "Make sure you have set up the EasyBuild configuration using set_up_configuration() "
         error_msg += "(from easybuild.tools.options) in case you're not using EasyBuild via the 'eb' CLI."
-        raise EasyBuildError(error_msg)
+        raise EasyBuildError(error_msg, exit_code=EasyBuildExit.OPTION_ERROR)
 
 
 def update_build_option(key, value):
@@ -688,7 +694,10 @@ def install_path(typ=None):
 
     known_types = ['modules', 'software']
     if typ not in known_types:
-        raise EasyBuildError("Unknown type specified in install_path(): %s (known: %s)", typ, ', '.join(known_types))
+        raise EasyBuildError(
+            "Unknown type specified in install_path(): %s (known: %s)", typ, ', '.join(known_types),
+            exit_code=EasyBuildExit.OPTION_ERROR
+        )
 
     variables = ConfigurationVariables()
 
@@ -780,7 +789,10 @@ def get_output_style():
             output_style = OUTPUT_STYLE_BASIC
 
     if output_style == OUTPUT_STYLE_RICH and not HAVE_RICH:
-        raise EasyBuildError("Can't use '%s' output style, Rich Python package is not available!", OUTPUT_STYLE_RICH)
+        raise EasyBuildError(
+            "Can't use '%s' output style, Rich Python package is not available!", OUTPUT_STYLE_RICH,
+            exit_code=EasyBuildExit.MISSING_EB_DEPENDENCY
+        )
 
     return output_style
 
@@ -805,8 +817,10 @@ def log_file_format(return_directory=False, ec=None, date=None, timestamp=None):
 
     logfile_format = ConfigurationVariables()['logfile_format']
     if not isinstance(logfile_format, tuple) or len(logfile_format) != 2:
-        raise EasyBuildError("Incorrect log file format specification, should be 2-tuple (<dir>, <filename>): %s",
-                             logfile_format)
+        raise EasyBuildError(
+            "Incorrect log file format specification, should be 2-tuple (<dir>, <filename>): %s", logfile_format,
+            exit_code=EasyBuildExit.OPTION_ERROR
+        )
 
     idx = int(not return_directory)
     res = ConfigurationVariables()['logfile_format'][idx] % {
@@ -913,7 +927,10 @@ def find_last_log(curlog):
         sorted_paths = [p for (_, p) in sorted(paths)]
 
     except OSError as err:
-        raise EasyBuildError("Failed to locate/select/order log files matching '%s': %s", glob_pattern, err)
+        raise EasyBuildError(
+            "Failed to locate/select/order log files matching '%s': %s", glob_pattern, err,
+            exit_code=EasyBuildExit.OPTION_ERROR
+        )
 
     try:
         # log of current session is typically listed last, should be taken into account

easybuild/tools/config.py

@@ -29,7 +29,7 @@
 import os
 import re
 
-from easybuild.tools.build_log import EasyBuildError, print_msg
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_msg
 from easybuild.tools.containers.singularity import SingularityContainer
 from easybuild.tools.config import CONT_IMAGE_FORMAT_EXT3, CONT_IMAGE_FORMAT_SANDBOX
 from easybuild.tools.config import CONT_IMAGE_FORMAT_SIF, CONT_IMAGE_FORMAT_SQUASHFS
@@ -49,7 +49,7 @@ def apptainer_version():
         """Get Apptainer version."""
         version_cmd = "apptainer --version"
         res = run_shell_cmd(version_cmd, hidden=True, in_dry_run=True)
-        if res.exit_code:
+        if res.exit_code != EasyBuildExit.SUCCESS:
             raise EasyBuildError(f"Error running '{version_cmd}': {res.output}")
 
         regex_res = re.search(r"\d+\.\d+(\.\d+)?", res.output.strip())

easybuild/tools/containers/apptainer.py

@@ -34,7 +34,7 @@
 import re
 
 from easybuild.tools import LooseVersion
-from easybuild.tools.build_log import EasyBuildError, print_msg
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_msg
 from easybuild.tools.config import CONT_IMAGE_FORMAT_EXT3, CONT_IMAGE_FORMAT_SANDBOX
 from easybuild.tools.config import CONT_IMAGE_FORMAT_SIF, CONT_IMAGE_FORMAT_SQUASHFS
 from easybuild.tools.config import build_option, container_path
@@ -163,7 +163,7 @@ def singularity_version():
         """Get Singularity version."""
         version_cmd = "singularity --version"
         res = run_shell_cmd(version_cmd, hidden=True, in_dry_run=True)
-        if res.exit_code:
+        if res.exit_code != EasyBuildExit.SUCCESS:
             raise EasyBuildError(f"Error running '{version_cmd}': {res.output}")
 
         regex_res = re.search(r"\d+\.\d+(\.\d+)?", res.output.strip())

easybuild/tools/containers/singularity.py

@@ -34,7 +34,7 @@
 from functools import reduce
 
 from easybuild.tools import LooseVersion
-from easybuild.tools.build_log import EasyBuildError, print_msg
+from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_msg
 from easybuild.tools.filetools import which
 from easybuild.tools.run import run_shell_cmd
 
@@ -77,7 +77,7 @@ def check_tool(tool_name, min_tool_version=None):
 
     version_cmd = f"{tool_name} --version"
     res = run_shell_cmd(version_cmd, hidden=True, in_dry_run=True)
-    if res.exit_code:
+    if res.exit_code != EasyBuildExit.SUCCESS:
         raise EasyBuildError(f"Error running '{version_cmd}' for tool {tool_name} with output: {res.output}")
 
     regex_res = re.search(r"\d+\.\d+(\.\d+)?", res.output.strip())