Merge branch 'develop' into modenvvar-expand-paths

Author: lexming

easybuild/framework/easyblock.py

@@ -92,6 +92,7 @@
     'checksums': [[], "Checksums for sources and patches", BUILD],
     'configopts': ['', 'Extra options passed to configure (default already has --prefix)', BUILD],
     'cuda_compute_capabilities': [[], "List of CUDA compute capabilities to build with (if supported)", BUILD],
+    'data_sources': [[], "List of source files for data", BUILD],
     'download_instructions': ['', "Specify steps to acquire necessary file, if obtaining it is difficult", BUILD],
     'easyblock': [None, "EasyBlock to use for building; if set to None, an easyblock is selected "
                         "based on the software name", BUILD],
@@ -132,7 +133,7 @@
     'skip_mod_files_sanity_check': [False, "Skip the check for .mod files in a GCCcore level install", BUILD],
     'skipsteps': [[], "Skip these steps", BUILD],
     'source_urls': [[], "List of URLs for source files", BUILD],
-    'sources': [[], "List of source files", BUILD],
+    'sources': [[], "List of source files for software", BUILD],
     'stop': [None, 'Keyword to halt the build process after a certain step.', BUILD],
     'testopts': ['', 'Extra options for test.', BUILD],
     'tests': [[], ("List of test-scripts to run after install. A test script should return a "

easybuild/framework/easyconfig/default.py

@@ -835,7 +835,7 @@ def count_files(self):
 
         # No need to resolve templates as we only need a count not the names
         with self.disable_templating():
-            cnt = len(self['sources']) + len(self['patches'])
+            cnt = sum(len(self[k]) for k in ['data_sources', 'sources', 'patches'])
             exts = self['exts_list']
 
         for ext in exts:

easybuild/framework/easyconfig/easyconfig.py

@@ -62,7 +62,7 @@
     ['name', 'version', 'versionprefix', 'versionsuffix'],
     ['homepage', 'description'],
     ['toolchain', 'toolchainopts'],
-    ['source_urls', 'sources', 'patches', 'checksums'],
+    ['source_urls', 'sources', 'data_sources', 'patches', 'checksums'],
     DEPENDENCY_PARAMETERS + ['multi_deps'],
     ['osdependencies'],
     ['preconfigopts', 'configopts'],

easybuild/framework/easyconfig/format/format.py

@@ -176,29 +176,22 @@ def sanity_check_step(self, exts_filter=None, custom_paths=None, custom_commands
             # make sure Extension sanity check step is run once, by using a single empty list of extra modules
             lists_of_extra_modules = [[]]
 
-        for extra_modules in lists_of_extra_modules:
-
-            fake_mod_data = None
-
-            # only load fake module + extra modules for stand-alone installations (not for extensions),
-            # since for extension the necessary modules should already be loaded at this point;
-            # take into account that module may already be loaded earlier in sanity check
-            if not (self.sanity_check_module_loaded or self.is_extension or self.dry_run):
-                # load fake module
-                fake_mod_data = self.load_fake_module(purge=True, extra_modules=extra_modules)
-
-                if extra_modules:
-                    info_msg = "Running extension sanity check with extra modules: %s" % ', '.join(extra_modules)
-                    self.log.info(info_msg)
-                    trace_msg(info_msg)
-
-            # perform extension sanity check
+        # only load fake module + extra modules for stand-alone installations (not for extensions),
+        # since for extension the necessary modules should already be loaded at this point;
+        # take into account that module may already be loaded earlier in sanity check
+        if not (self.sanity_check_module_loaded or self.is_extension or self.dry_run):
+            for extra_modules in lists_of_extra_modules:
+                with self.fake_module_environment(extra_modules=extra_modules):
+                    if extra_modules:
+                        info_msg = f"Running extension sanity check with extra modules: {', '.join(extra_modules)}"
+                        self.log.info(info_msg)
+                        trace_msg(info_msg)
+                    # perform sanity check for stand-alone extension
+                    (sanity_check_ok, fail_msg) = Extension.sanity_check_step(self)
+        else:
+            # perform single sanity check for extension
             (sanity_check_ok, fail_msg) = Extension.sanity_check_step(self)
 
-            if fake_mod_data:
-                # unload fake module and clean up
-                self.clean_up_fake_module(fake_mod_data)
-
         if custom_paths or custom_commands or not self.is_extension:
             super().sanity_check_step(custom_paths=custom_paths,
                                       custom_commands=custom_commands,

easybuild/framework/extensioneasyblock.py

@@ -111,7 +111,30 @@ def find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=
     return [(ec_file, generated)]
 
 
-def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
+def summary(ecs_with_res):
+    """
+    Compose summary of the build:
+    * [SUCCESS] for a successful build
+    * [FAILED] for a failed build
+    * [SKIPPED] for a build that didn’t run
+
+    :param ecs_with_res: list of tuples (ec, ec_res), ec is an EasyConfig object, and ec_res is a dict of the result
+    """
+    summary_fmt = "   * {} {}"
+    success_map = {
+        True: f'{"[SUCCESS]":<9}',
+        False: f'{"[FAILED]":<9}',
+        None: f'{"[SKIPPED]":<9}',
+    }
+    lines = ["Summary:"]
+    lines.extend([
+        summary_fmt.format(success_map[ec_res.get('success', False)], ec['full_mod_name'])
+        for ec, ec_res in ecs_with_res
+    ])
+    return '\n'.join(lines)
+
+
+def build_and_install_software(ecs, init_session_state, exit_on_failure=True, testing=False):
     """
     Build and install software for all provided parsed easyconfig files.
 
@@ -126,7 +149,7 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
 
     start_progress_bar(STATUS_BAR, size=len(ecs))
 
-    res = []
+    ecs_with_res = []
     ec_results = []
     failed_cnt = 0
 
@@ -171,14 +194,17 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
                 write_file(test_report_fp, test_report_txt['full'])
                 adjust_permissions(parent_dir, stat.S_IWUSR, add=False, recursive=False)
 
+        ecs_with_res.append((ec, ec_res))
+
         if not ec_res['success'] and exit_on_failure:
+            ecs_in_res = [res[0] for res in ecs_with_res]
+            ecs_without_res = [(ec, {'success': None}) for ec in ecs if ec not in ecs_in_res]
+            print_msg(summary(ecs_with_res + ecs_without_res), log=_log, silent=testing)
             error = ec_res['err']
             if isinstance(error, EasyBuildError):
                 error = EasyBuildError(test_msg, exit_code=error.exit_code)
             raise error
 
-        res.append((ec, ec_res))
-
         if failed_cnt:
             # if installations failed: indicate th
             status_label = ' (%s): ' % colorize('%s failed!' % failed_cnt, COLOR_RED)
@@ -192,7 +218,7 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True):
 
     stop_progress_bar(STATUS_BAR)
 
-    return res
+    return ecs_with_res
 
 
 def run_contrib_style_checks(ecs, check_contrib, check_style):
@@ -563,11 +589,11 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
 
         with rich_live_cm():
             run_hook(PRE_PREF + BUILD_AND_INSTALL_LOOP, hooks, args=[ordered_ecs])
-            ecs_with_res = build_and_install_software(ordered_ecs, init_session_state,
-                                                      exit_on_failure=exit_on_failure)
+            ecs_with_res = build_and_install_software(
+                ordered_ecs, init_session_state, exit_on_failure=exit_on_failure, testing=testing)
             run_hook(POST_PREF + BUILD_AND_INSTALL_LOOP, hooks, args=[ecs_with_res])
     else:
-        ecs_with_res = [(ec, {}) for ec in ordered_ecs]
+        ecs_with_res = [(ec, {'success': None}) for ec in ordered_ecs]
 
     correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)])
     overall_success = correct_builds_cnt == len(ordered_ecs)
@@ -585,6 +611,8 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
         print_msg(test_report_msg)
 
     print_msg(success_msg, log=_log, silent=testing)
+    if ecs_with_res:
+        print_msg(summary(ecs_with_res), log=_log, silent=testing)
 
     # cleanup and spec files
     for ec in easyconfigs:

easybuild/main.py

@@ -42,6 +42,7 @@ function log {
 
 # command name
 CMD=`basename $0`
+TOPDIR=`dirname $0`
 
 log "found CMD: $CMD | original command: %(orig_cmd)s | orig args: '$(echo \"$@\")'"
 

easybuild/scripts/rpath_wrapper_template.sh.in

@@ -71,6 +71,10 @@
 
 EMPTY_LIST = 'empty_list'
 
+DATA = 'data'
+MODULES = 'modules'
+SOFTWARE = 'software'
+
 PKG_TOOL_FPM = 'fpm'
 PKG_TYPE_RPM = 'rpm'
 
@@ -92,6 +96,8 @@
 DEFAULT_CONT_TYPE = CONT_TYPE_SINGULARITY
 
 DEFAULT_BRANCH = 'develop'
+DEFAULT_DOWNLOAD_INITIAL_WAIT_TIME = 10
+DEFAULT_DOWNLOAD_MAX_ATTEMPTS = 6
 DEFAULT_DOWNLOAD_TIMEOUT = 10
 DEFAULT_ENV_FOR_SHEBANG = '/usr/bin/env'
 DEFAULT_ENVVAR_USERS_MODULES = 'HOME'
@@ -112,8 +118,10 @@
     'packagepath': 'packages',
     'repositorypath': 'ebfiles_repo',
     'sourcepath': 'sources',
-    'subdir_modules': 'modules',
-    'subdir_software': 'software',
+    'sourcepath_data': 'sources',
+    'subdir_data': DATA,
+    'subdir_modules': MODULES,
+    'subdir_software': SOFTWARE,
 }
 DEFAULT_PKG_RELEASE = '1'
 DEFAULT_PKG_TOOL = PKG_TOOL_FPM
@@ -478,6 +486,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
     ('chem', "Chemistry, Computational Chemistry and Quantum Chemistry"),
     ('compiler', "Compilers"),
     ('data', "Data management & processing tools"),
+    ('dataset', "Datasets"),
     ('debugger', "Debuggers"),
     ('devel', "Development tools"),
     ('geo', "Earth Sciences"),
@@ -512,6 +521,7 @@ class ConfigurationVariables(BaseConfigurationVariables):
         'failed_install_build_dirs_path',
         'failed_install_logs_path',
         'installpath',
+        'installpath_data',
         'installpath_modules',
         'installpath_software',
         'job_backend',
@@ -526,6 +536,8 @@ class ConfigurationVariables(BaseConfigurationVariables):
         'repository',
         'repositorypath',
         'sourcepath',
+        'sourcepath_data',
+        'subdir_data',
         'subdir_modules',
         'subdir_software',
         'tmp_logdir',
@@ -569,16 +581,20 @@ def init(options, config_options_dict):
     """
     tmpdict = copy.deepcopy(config_options_dict)
 
-    # make sure source path is a list
-    sourcepath = tmpdict['sourcepath']
-    if isinstance(sourcepath, str):
-        tmpdict['sourcepath'] = sourcepath.split(':')
-        _log.debug("Converted source path ('%s') to a list of paths: %s" % (sourcepath, tmpdict['sourcepath']))
-    elif not isinstance(sourcepath, (tuple, list)):
-        raise EasyBuildError(
-            "Value for sourcepath has invalid type (%s): %s", type(sourcepath), sourcepath,
-            exit_code=EasyBuildExit.OPTION_ERROR
-        )
+    if tmpdict['sourcepath_data'] is None:
+        tmpdict['sourcepath_data'] = tmpdict['sourcepath'][:]
+
+    for srcpath in ['sourcepath', 'sourcepath_data']:
+        # make sure source path is a list
+        sourcepath = tmpdict[srcpath]
+        if isinstance(sourcepath, str):
+            tmpdict[srcpath] = sourcepath.split(':')
+            _log.debug("Converted source path ('%s') to a list of paths: %s" % (sourcepath, tmpdict[srcpath]))
+        elif not isinstance(sourcepath, (tuple, list)):
+            raise EasyBuildError(
+                "Value for %s has invalid type (%s): %s", srcpath, type(sourcepath), sourcepath,
+                exit_code=EasyBuildExit.OPTION_ERROR
+            )
 
     # initialize configuration variables (any future calls to ConfigurationVariables() will yield the same instance
     variables = ConfigurationVariables(tmpdict, ignore_unknown_keys=True)
@@ -704,11 +720,18 @@ def build_path():
 
 def source_paths():
     """
-    Return the list of source paths
+    Return the list of source paths for software
     """
     return ConfigurationVariables()['sourcepath']
 
 
+def source_paths_data():
+    """
+    Return the list of source paths for data
+    """
+    return ConfigurationVariables()['sourcepath_data']
+
+
 def source_path():
     """NO LONGER SUPPORTED: use source_paths instead"""
     _log.nosupport("source_path() is replaced by source_paths()", '2.0')
@@ -717,15 +740,16 @@ def source_path():
 def install_path(typ=None):
     """
     Returns the install path
-    - subdir 'software' for actual installation (default)
+    - subdir 'software' for actual software installation (default)
     - subdir 'modules' for environment modules (typ='mod')
+    - subdir 'data' for data installation (typ='data')
     """
     if typ is None:
-        typ = 'software'
+        typ = SOFTWARE
     elif typ == 'mod':
-        typ = 'modules'
+        typ = MODULES
 
-    known_types = ['modules', 'software']
+    known_types = [MODULES, SOFTWARE, DATA]
     if typ not in known_types:
         raise EasyBuildError(
             "Unknown type specified in install_path(): %s (known: %s)", typ, ', '.join(known_types),

easybuild/tools/config.py

@@ -69,6 +69,7 @@
 # import build_log must stay, to use of EasyBuildLog
 from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, CWD_NOTFOUND_ERROR
 from easybuild.tools.build_log import dry_run_msg, print_msg, print_warning
+from easybuild.tools.config import DEFAULT_DOWNLOAD_INITIAL_WAIT_TIME, DEFAULT_DOWNLOAD_MAX_ATTEMPTS
 from easybuild.tools.config import ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN, build_option, install_path
 from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar
 from easybuild.tools.hooks import load_source
@@ -777,8 +778,23 @@ def det_file_size(http_header):
     return res
 
 
-def download_file(filename, url, path, forced=False, trace=True):
-    """Download a file from the given URL, to the specified path."""
+def download_file(filename, url, path, forced=False, trace=True, max_attempts=None, initial_wait_time=None):
+    """
+    Download a file from the given URL, to the specified path.
+
+    :param filename: name of file to download
+    :param url: URL of file to download
+    :param path: path to download file to
+    :param forced: boolean to indicate whether force should be used to write the file
+    :param trace: boolean to indicate whether trace output should be printed
+    :param max_attempts: max. number of attempts to download file from specified URL
+    :param initial_wait_time: wait time (in seconds) after first attempt (doubled at each attempt)
+    """
+
+    if max_attempts is None:
+        max_attempts = DEFAULT_DOWNLOAD_MAX_ATTEMPTS
+    if initial_wait_time is None:
+        initial_wait_time = DEFAULT_DOWNLOAD_INITIAL_WAIT_TIME
 
     insecure = build_option('insecure_download')
 
@@ -802,7 +818,6 @@ def download_file(filename, url, path, forced=False, trace=True):
 
     # try downloading, three times max.
     downloaded = False
-    max_attempts = 3
     attempt_cnt = 0
 
     # use custom HTTP header
@@ -823,6 +838,9 @@ def download_file(filename, url, path, forced=False, trace=True):
     used_urllib = std_urllib
     switch_to_requests = False
 
+    wait = False
+    wait_time = initial_wait_time
+
     while not downloaded and attempt_cnt < max_attempts:
         attempt_cnt += 1
         try:
@@ -861,6 +879,9 @@ def download_file(filename, url, path, forced=False, trace=True):
                 status_code = err.code
             if status_code == 403 and attempt_cnt == 1:
                 switch_to_requests = True
+            elif status_code == 429:  # too many requests
+                _log.warning(f"Downloading of {url} failed with HTTP status code 429 (Too many requests)")
+                wait = True
             elif 400 <= status_code <= 499:
                 _log.warning("URL %s was not found (HTTP response code %s), not trying again" % (url, status_code))
                 break
@@ -887,6 +908,12 @@ def download_file(filename, url, path, forced=False, trace=True):
                 _log.info("Downloading using requests package instead of urllib2")
                 used_urllib = requests
 
+            if wait:
+                _log.info(f"Waiting for {wait_time} seconds before trying download of {url} again...")
+                time.sleep(wait_time)
+                # exponential backoff
+                wait_time *= 2
+
     if downloaded:
         _log.info("Successful download of file %s from url %s to path %s" % (filename, url, path))
         if trace: