Merge pull request #3820 from easybuilders/4.4.x

release EasyBuild v4.4.2

Author: boegel

.github/workflows/bootstrap_script.yml

@@ -107,7 +107,7 @@ jobs:
           EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
           EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
           EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
-          EB_BOOTSTRAP_EXPECTED="20210618.01 e5d477d717c6d3648ba2027ab735713ba5804fbf52f4b4adcca0bc1379b44618"
+          EB_BOOTSTRAP_EXPECTED="20210715.01 784dd29063d941be2d8b70e4c2eec12a9afe360f3ef8f753dcb518abf43ca7de"
           test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
 
           # test bootstrap script

RELEASE_NOTES

@@ -4,6 +4,42 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.4.2 (September 7th 2021)
+---------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - add per-extension timing in output produced by eb command (#3734)
+  - add definition for new toolchain nvpsmpic (NVHPC + ParaStationMPI + CUDA) (#3736)
+  - include list of missing libraries in warning about missing FFTW libraries in imkl toolchain component (#3776)
+  - check for recursive symlinks by default before copying a folder (#3784)
+  - add --filter-ecs configuration option to filter out easyconfigs from set of easyconfigs to install (#3796)
+  - check type of source_tmpl value for extensions, ensure it's a string value (not a list) (#3799)
+  - also define $BLAS_SHARED_LIBS & co in build environment (analogous to $BLAS_STATIC_LIBS) (#3800)
+  - report use of --ignore-test-failure in success message in output (#3806)
+  - add get_cuda_cc_template_value method to EasyConfig class (#3807)
+  - add support for fix_bash_shebang_for (#3808)
+  - pick up $MODULES_CMD to facilitate using Environment Modules 4.x as modules tool (#3816)
+  - use more sensible branch name for creating easyblocks PR with --new-pr (#3817)
+- various bug fixes, including:
+  - remove Python 2.6 from list of supported Python versions in setup.py (#3767)
+  - don't add directory that doesn't include any files to $PATH or $LD_LIBRARY_PATH (#3769)
+  - make logdir writable also when --stop/--fetch is used and --read-only-installdir is enabled (#3771)
+  - fix forgotten renaming of 'l' to 'char' __init__.py that is created for included Python modules (#3773)
+  - fix verify_imports by deleting all imported modules before re-importing them one by one (#3780)
+  - fix ignore_test_failure not set for Extension instances (#3782)
+  - update iompi toolchain to intel-compiler subtoolchain for oneAPI versions (>= iompi 2020.12) (#3785)
+  - don't parse patch files as easyconfigs when searching for where patch file is used (#3786)
+  - make sure git clone with a tag argument actually downloads a tag (#3795)
+  - fix CI by excluding GC3Pie 2.6.7 (which is broken with Python 2) and improve error reporting for option parsing (#3798)
+  - correctly resolve templates for patches in extensions when uploading to GitHub (#3805)
+  - add --easystack to ignored options when submitting job (#3813)
+- other changes:
+  - speed up tests by caching checked paths in set_tmpdir + less test cases for test_compiler_dependent_optarch (#3802)
+  - speed up set_parallel method in EasyBlock class (#3812)
+
+
 v4.4.1 (July 6th 2021)
 ----------------------
 

easybuild/framework/easyblock.py

@@ -93,7 +93,7 @@
 from easybuild.tools.py2vs3 import extract_method_name, string_type
 from easybuild.tools.repository.repository import init_repository
 from easybuild.tools.systemtools import check_linked_shared_libs, det_parallelism, get_shared_lib_ext, use_group
-from easybuild.tools.utilities import INDENT_4SPACES, get_class_for, quote_str
+from easybuild.tools.utilities import INDENT_4SPACES, get_class_for, nub, quote_str
 from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg
 from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION
 
@@ -258,8 +258,6 @@ def __init__(self, ec):
         if group_name is not None:
             self.group = use_group(group_name)
 
-        self.ignore_test_failure = build_option('ignore_test_failure')
-
         # generate build/install directories
         self.gen_builddir()
         self.gen_installdir()
@@ -595,7 +593,13 @@ def fetch_extension_sources(self, skip_checksums=False):
                         default_source_tmpl = resolve_template('%(name)s-%(version)s.tar.gz', template_values)
 
                         # if no sources are specified via 'sources', fall back to 'source_tmpl'
-                        src_fn = ext_options.get('source_tmpl', default_source_tmpl)
+                        src_fn = ext_options.get('source_tmpl')
+                        if src_fn is None:
+                            src_fn = default_source_tmpl
+                        elif not isinstance(src_fn, string_type):
+                            error_msg = "source_tmpl value must be a string! (found value of type '%s'): %s"
+                            raise EasyBuildError(error_msg, type(src_fn).__name__, src_fn)
+
                         src_path = self.obtain_file(src_fn, extension=True, urls=source_urls,
                                                     force_download=force_download)
                         if src_path:
@@ -1429,9 +1433,17 @@ def make_module_req(self):
             note += "for paths are skipped for the statements below due to dry run"
             lines.append(self.module_generator.comment(note))
 
-        # for these environment variables, the corresponding subdirectory must include at least one file
-        keys_requiring_files = set(('PATH', 'LD_LIBRARY_PATH', 'LIBRARY_PATH', 'CPATH',
-                                    'CMAKE_PREFIX_PATH', 'CMAKE_LIBRARY_PATH'))
+        # For these environment variables, the corresponding directory must include at least one file.
+        # The values determine if detection is done recursively, i.e. if it accepts directories where files
+        # are only in subdirectories.
+        keys_requiring_files = {
+            'PATH': False,
+            'LD_LIBRARY_PATH': False,
+            'LIBRARY_PATH': True,
+            'CPATH': True,
+            'CMAKE_PREFIX_PATH': True,
+            'CMAKE_LIBRARY_PATH': True,
+        }
 
         for key, reqs in sorted(requirements.items()):
             if isinstance(reqs, string_type):
@@ -1461,19 +1473,16 @@ def make_module_req(self):
                     if fixed_paths != paths:
                         self.log.info("Fixed symlink lib64 in paths for %s: %s -> %s", key, paths, fixed_paths)
                         paths = fixed_paths
-                # remove duplicate paths
-                # don't use 'set' here, since order in 'paths' is important!
-                uniq_paths = []
-                for path in paths:
-                    if path not in uniq_paths:
-                        uniq_paths.append(path)
-                paths = uniq_paths
+                # remove duplicate paths preserving order
+                paths = nub(paths)
                 if key in keys_requiring_files:
                     # only retain paths that contain at least one file
+                    recursive = keys_requiring_files[key]
                     retained_paths = [
-                        path for path in paths
-                        if os.path.isdir(os.path.join(self.installdir, path))
-                        and dir_contains_files(os.path.join(self.installdir, path))
+                        path
+                        for path, fullpath in ((path, os.path.join(self.installdir, path)) for path in paths)
+                        if os.path.isdir(fullpath)
+                        and dir_contains_files(fullpath, recursive=recursive)
                     ]
                     if retained_paths != paths:
                         self.log.info("Only retaining paths for %s that contain at least one file: %s -> %s",
@@ -1782,18 +1791,19 @@ def set_parallel(self):
         """Set 'parallel' easyconfig parameter to determine how many cores can/should be used for parallel builds."""
         # set level of parallelism for build
         par = build_option('parallel')
-        if self.cfg['parallel'] is not None:
-            if par is None:
-                par = self.cfg['parallel']
-                self.log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par)
-            else:
-                par = min(int(par), int(self.cfg['parallel']))
-                self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par)
-        else:
+        cfg_par = self.cfg['parallel']
+        if cfg_par is None:
             self.log.debug("Desired parallelism specified via 'parallel' build option: %s", par)
+        elif par is None:
+            par = cfg_par
+            self.log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par)
+        else:
+            par = min(int(par), int(cfg_par))
+            self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par)
 
-        self.cfg['parallel'] = det_parallelism(par=par, maxpar=self.cfg['maxparallel'])
-        self.log.info("Setting parallelism: %s" % self.cfg['parallel'])
+        par = det_parallelism(par, maxpar=self.cfg['maxparallel'])
+        self.log.info("Setting parallelism: %s" % par)
+        self.cfg['parallel'] = par
 
     def remove_module_file(self):
         """Remove module file (if it exists), and check for ghost installation directory (and deal with it)."""
@@ -1826,7 +1836,7 @@ def report_test_failure(self, msg_or_error):
 
         :param msg_or_error: failure description (string value or an EasyBuildError instance)
         """
-        if self.ignore_test_failure:
+        if build_option('ignore_test_failure'):
             print_warning("Test failure ignored: " + str(msg_or_error), log=self.log)
         else:
             exception = msg_or_error if isinstance(msg_or_error, EasyBuildError) else EasyBuildError(msg_or_error)
@@ -2409,6 +2419,7 @@ def extensions_step(self, fetch=False, install=True):
 
             tup = (ext.name, ext.version or '', idx + 1, exts_cnt)
             print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent)
+            start_time = datetime.now()
 
             if self.dry_run:
                 tup = (ext.name, ext.version, ext.__class__.__name__)
@@ -2429,11 +2440,19 @@ def extensions_step(self, fetch=False, install=True):
 
             # real work
             if install:
-                ext.prerun()
-                txt = ext.run()
-                if txt:
-                    self.module_extra_extensions += txt
-                ext.postrun()
+                try:
+                    ext.prerun()
+                    txt = ext.run()
+                    if txt:
+                        self.module_extra_extensions += txt
+                    ext.postrun()
+                finally:
+                    if not self.dry_run:
+                        ext_duration = datetime.now() - start_time
+                        if ext_duration.total_seconds() >= 1:
+                            print_msg("\t... (took %s)", time2str(ext_duration), log=self.log, silent=self.silent)
+                        elif self.logdebug or build_option('trace'):
+                            print_msg("\t... (took < 1 sec)", log=self.log, silent=self.silent)
 
         # cleanup (unload fake module, remove fake module dir)
         if fake_mod_data:
@@ -2466,7 +2485,7 @@ def package_step(self):
 
     def fix_shebang(self):
         """Fix shebang lines for specified files."""
-        for lang in ['perl', 'python']:
+        for lang in ['bash', 'perl', 'python']:
             shebang_regex = re.compile(r'^#![ ]*.*[/ ]%s.*' % lang)
             fix_shebang_for = self.cfg['fix_%s_shebang_for' % lang]
             if fix_shebang_for:
@@ -3685,15 +3704,21 @@ def build_and_install_one(ecdict, init_env):
 
             if os.path.exists(app.installdir) and build_option('read_only_installdir') and (
                     build_option('rebuild') or build_option('force')):
+                enabled_write_permissions = True
                 # re-enable write permissions so we can install additional modules
                 adjust_permissions(app.installdir, stat.S_IWUSR, add=True, recursive=True)
+            else:
+                enabled_write_permissions = False
 
         result = app.run_all_steps(run_test_cases=run_test_cases)
 
         if not dry_run:
             # also add any extension easyblocks used during the build for reproducibility
             if app.ext_instances:
                 copy_easyblocks_for_reprod(app.ext_instances, reprod_dir)
+            # If not already done remove the granted write permissions if we did so
+            if enabled_write_permissions and os.lstat(app.installdir)[stat.ST_MODE] & stat.S_IWUSR:
+                adjust_permissions(app.installdir, stat.S_IWUSR, add=False, recursive=True)
 
     except EasyBuildError as err:
         first_n = 300
@@ -3710,28 +3735,37 @@ def build_and_install_one(ecdict, init_env):
 
     # successful (non-dry-run) build
     if result and not dry_run:
+        def ensure_writable_log_dir(log_dir):
+            """Make sure we can write into the log dir"""
+            if build_option('read_only_installdir'):
+                # temporarily re-enable write permissions for copying log/easyconfig to install dir
+                if os.path.exists(log_dir):
+                    adjust_permissions(log_dir, stat.S_IWUSR, add=True, recursive=True)
+                else:
+                    parent_dir = os.path.dirname(log_dir)
+                    if os.path.exists(parent_dir):
+                        adjust_permissions(parent_dir, stat.S_IWUSR, add=True, recursive=False)
+                        mkdir(log_dir, parents=True)
+                        adjust_permissions(parent_dir, stat.S_IWUSR, add=False, recursive=False)
+                    else:
+                        mkdir(log_dir, parents=True)
+                        adjust_permissions(log_dir, stat.S_IWUSR, add=True, recursive=True)
 
         if app.cfg['stop']:
             ended = 'STOPPED'
             if app.builddir is not None:
                 new_log_dir = os.path.join(app.builddir, config.log_path(ec=app.cfg))
             else:
                 new_log_dir = os.path.dirname(app.logfile)
+            ensure_writable_log_dir(new_log_dir)
 
         # if we're only running the sanity check, we should not copy anything new to the installation directory
         elif build_option('sanity_check_only'):
             _log.info("Only running sanity check, so skipping build stats, easyconfigs archive, reprod files...")
 
         else:
             new_log_dir = os.path.join(app.installdir, config.log_path(ec=app.cfg))
-            if build_option('read_only_installdir'):
-                # temporarily re-enable write permissions for copying log/easyconfig to install dir
-                if os.path.exists(new_log_dir):
-                    adjust_permissions(new_log_dir, stat.S_IWUSR, add=True, recursive=True)
-                else:
-                    adjust_permissions(app.installdir, stat.S_IWUSR, add=True, recursive=False)
-                    mkdir(new_log_dir, parents=True)
-                    adjust_permissions(app.installdir, stat.S_IWUSR, add=False, recursive=False)
+            ensure_writable_log_dir(new_log_dir)
 
             # collect build stats
             _log.info("Collecting build stats...")

easybuild/framework/easyconfig/default.py

@@ -95,6 +95,8 @@
     'easybuild_version': [None, "EasyBuild-version this spec-file was written for", BUILD],
     'enhance_sanity_check': [False, "Indicate that additional sanity check commands & paths should enhance "
                              "the existin sanity check, not replace it", BUILD],
+    'fix_bash_shebang_for': [None, "List of files for which Bash shebang should be fixed "
+                                   "to '#!/usr/bin/env bash' (glob patterns supported)", BUILD],
     'fix_perl_shebang_for': [None, "List of files for which Perl shebang should be fixed "
                                    "to '#!/usr/bin/env perl' (glob patterns supported)", BUILD],
     'fix_python_shebang_for': [None, "List of files for which Python shebang should be fixed "

easybuild/framework/easyconfig/easyconfig.py

@@ -59,7 +59,7 @@
 from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT
 from easybuild.framework.easyconfig.parser import DEPRECATED_PARAMETERS, REPLACED_PARAMETERS
 from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig
-from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, template_constant_dict
+from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_DYNAMIC, template_constant_dict
 from easybuild.tools.build_log import EasyBuildError, print_warning, print_msg
 from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG
 from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN
@@ -559,6 +559,13 @@ def disable_templating(self):
         finally:
             self.enable_templating = old_enable_templating
 
+    def __str__(self):
+        """Return a string representation of this EasyConfig instance"""
+        if self.path:
+            return '%s EasyConfig @ %s' % (self.name, self.path)
+        else:
+            return 'Raw %s EasyConfig' % self.name
+
     def filename(self):
         """Determine correct filename for this easyconfig file."""
 
@@ -881,7 +888,7 @@ def validate_os_deps(self):
                 raise EasyBuildError("Non-tuple value type for OS dependency specification: %s (type %s)",
                                      dep, type(dep))
 
-            if not any([check_os_dependency(cand_dep) for cand_dep in dep]):
+            if not any(check_os_dependency(cand_dep) for cand_dep in dep):
                 not_found.append(dep)
 
         if not_found:
@@ -1803,6 +1810,25 @@ def asdict(self):
             res[key] = value
         return res
 
+    def get_cuda_cc_template_value(self, key):
+        """
+        Get template value based on --cuda-compute-capabilities EasyBuild configuration option
+        and cuda_compute_capabilities easyconfig parameter.
+        Returns user-friendly error message in case neither are defined,
+        or if an unknown key is used.
+        """
+        if key.startswith('cuda_') and any(x[0] == key for x in TEMPLATE_NAMES_DYNAMIC):
+            try:
+                return self.template_values[key]
+            except KeyError:
+                error_msg = "Template value '%s' is not defined!\n"
+                error_msg += "Make sure that either the --cuda-compute-capabilities EasyBuild configuration "
+                error_msg += "option is set, or that the cuda_compute_capabilities easyconfig parameter is defined."
+                raise EasyBuildError(error_msg, key)
+        else:
+            error_msg = "%s is not a template value based on --cuda-compute-capabilities/cuda_compute_capabilities"
+            raise EasyBuildError(error_msg, key)
+
 
 def det_installversion(version, toolchain_name, toolchain_version, prefix, suffix):
     """Deprecated 'det_installversion' function, to determine exact install version, based on supplied parameters."""

easybuild/framework/easyconfig/format/format.py

@@ -577,7 +577,7 @@ def get_version_toolchain(self, version=None, tcname=None, tcversion=None):
                 self.log.debug("No toolchain version specified, using default %s" % tcversion)
             else:
                 raise EasyBuildError("No toolchain version specified, no default found.")
-        elif any([tc.test(tcname, tcversion) for tc in tcs]):
+        elif any(tc.test(tcname, tcversion) for tc in tcs):
             self.log.debug("Toolchain '%s' version '%s' is supported in easyconfig" % (tcname, tcversion))
         else:
             raise EasyBuildError("Toolchain '%s' version '%s' not supported in easyconfig (only %s)",