[feature] Copy build log and artifacts to a permanent location after failures

The files can be build in some selected build path (--buildpath), and
the logs of successful compilation are then concentrated to some other
location for permanent storage (--logfile-format). Logs of failed builds
remain in the build path location so that they can be inspected.

However, this setup is problematic when building software in HPC jobs. Quite
often in HPC systems the build path is set to some fast storage local to
the node, like NVME raid mounted on `/tmp` or `/dev/shm` (as suggested
in the documentation: https://docs.easybuild.io/configuration/#buildpath).
The node storage is often wiped out after the end of a job, so the log
files and the artifacts are no longer available after the termination of
the job.

This commit adds an option (--errorlogpath)to accumulate errors in some
more permanent location, so that the can be easily inspected after a
failed build.

Author: gkaf89

easybuild/framework/easyblock.py

@@ -73,13 +73,13 @@
 from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES
 from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES
 from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath
-from easybuild.tools.config import install_path, log_path, package_path, source_paths
+from easybuild.tools.config import install_path, log_path, package_path, source_paths, error_log_path
 from easybuild.tools.environment import restore_env, sanitize_env
 from easybuild.tools.filetools import CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256
 from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock
-from easybuild.tools.filetools import compute_checksum, convert_name, copy_file, create_lock, create_patch_info
+from easybuild.tools.filetools import convert_name, copy_file, copy_dir, create_lock, create_patch_info, is_readable
 from easybuild.tools.filetools import derive_alt_pypi_url, diff_files, dir_contains_files, download_file
-from easybuild.tools.filetools import encode_class_name, extract_file
+from easybuild.tools.filetools import encode_class_name, extract_file, compute_checksum
 from easybuild.tools.filetools import find_backup_name_candidate, get_source_tarball_from_git, is_alt_pypi_url
 from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
 from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file, symlink
@@ -4445,6 +4445,28 @@ def ensure_writable_log_dir(log_dir):
         # there may be multiple log files, or the file name may be different due to zipping
         logs = glob.glob('%s*' % application_log)
         print_msg("Results of the build can be found in the log file(s) %s" % ', '.join(logs), log=_log, silent=silent)
+        err_log_path = error_log_path(ec=ecdict['ec'])
+        if err_log_path and not(success):
+            print_msg("Build log and artifacts copied to permanent storage: %s" % err_log_path, log=_log, silent=silent)
+            for log_file in logs:
+                target_file = os.path.join(err_log_path, os.path.basename(log_file))
+                copy_file(log_file, target_file)
+
+            name = ecdict['ec'].name
+            version = ecdict['ec'].version
+
+            toolchain_dict = ecdict['ec'].toolchain.as_dict()
+            toolchain_components = [
+                toolchain_dict['name'],
+                toolchain_dict['version'],
+                toolchain_dict['versionsuffix'],
+            ]
+            toolchain_components = [s for s in toolchain_components if len(s) > 0]
+            toolchain = '-'.join(toolchain_components)
+
+            dest_build_path = os.path.join(err_log_path, name, version, toolchain)
+            if is_readable(app.builddir):
+                copy_dir(app.builddir, dest_build_path)
 
     del app
 

easybuild/tools/config.py

@@ -106,6 +106,7 @@
 DEFAULT_PATH_SUBDIRS = {
     'buildpath': 'build',
     'containerpath': 'containers',
+    'errorlogpath': 'error_log',
     'installpath': '',
     'packagepath': 'packages',
     'repositorypath': 'ebfiles_repo',
@@ -471,6 +472,7 @@ class ConfigurationVariables(BaseConfigurationVariables):
         'buildpath',
         'config',
         'containerpath',
+        'errorlogpath',
         'installpath',
         'installpath_modules',
         'installpath_software',
@@ -836,6 +838,25 @@ def log_path(ec=None):
     return log_file_format(return_directory=True, ec=ec, date=date, timestamp=timestamp)
 
 
+def error_log_path(ec=None):
+    """
+    Return the default error log path
+
+    This is a path where file from the build_log_path can be stored permanently
+    :param ec: dict-like value that provides values for %(name)s and %(version)s template values
+    """
+    error_log_path = ConfigurationVariables()['errorlogpath']
+
+    if ec is None:
+        ec = {}
+
+    name, version = ec.get('name', '%(name)s'), ec.get('version', '%(version)s')
+    date = time.strftime("%Y%m%d")
+    timestamp = time.strftime("%H%M%S")
+
+    return '/'.join([error_log_path, name + '-' + version, date + '-' + timestamp])
+
+
 def get_build_log_path():
     """
     Return (temporary) directory for build log

easybuild/tools/options.py

@@ -562,6 +562,8 @@ def config_options(self):
             'envvars-user-modules': ("List of environment variables that hold the base paths for which user-specific "
                                      "modules will be installed relative to", 'strlist', 'store',
                                      [DEFAULT_ENVVAR_USERS_MODULES]),
+            'errorlogpath': ("Location where logs and artifacts are copied in case of an error",
+                             None, 'store', mk_full_default_path('errorlogpath')),
             'external-modules-metadata': ("List of (glob patterns for) paths to files specifying metadata "
                                           "for external modules (INI format)", 'strlist', 'store', None),
             'hooks': ("Location of Python module with hook implementations", 'str', 'store', None),
@@ -1137,7 +1139,7 @@ def _postprocess_config(self):
         # - the <path> could also specify the location of a *remote* (Git( repository,
         #   which can be done in variety of formats (git@<url>:<org>/<repo>), https://<url>, etc.)
         #   (see also https://github.com/easybuilders/easybuild-framework/issues/3892);
-        path_opt_names = ['buildpath', 'containerpath', 'git_working_dirs_path', 'installpath',
+        path_opt_names = ['buildpath', 'containerpath', 'errorlogpath', 'git_working_dirs_path', 'installpath',
                           'installpath_modules', 'installpath_software', 'prefix', 'packagepath',
                           'robot_paths', 'sourcepath']
 
@@ -1147,8 +1149,8 @@ def _postprocess_config(self):
         if self.options.prefix is not None:
             # prefix applies to all paths, and repository has to be reinitialised to take new repositorypath in account
             # in the legacy-style configuration, repository is initialised in configuration file itself
-            path_opts = ['buildpath', 'containerpath', 'installpath', 'packagepath', 'repository', 'repositorypath',
-                         'sourcepath']
+            path_opts = ['buildpath', 'containerpath', 'errorlogpath', 'installpath', 'packagepath', 'repository',
+                         'repositorypath', 'sourcepath']
             for dest in path_opts:
                 if not self.options._action_taken.get(dest, False):
                     if dest == 'repository':