Merge branch 'nompi' of github.com:SebastianAchilles/easybuild-framework into nompi

Author: SebastianAchilles

RELEASE_NOTES

@@ -4,6 +4,68 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.4.0 (June 2nd 2021)
+----------------------
+
+feature release
+
+- various enhancements, including:
+  - enhance apply_regex_substitutions to allow specifying action to take in case there are no matches (#3440)
+  - performance improvements for easyconfig parsing and eb startup (#3555)
+  - add support for downloading easyconfigs from multiple PRs with --from-pr (#3605, #3707, #3708)
+  - add support for prepending custom library paths in RPATH section via --rpath-override-dirs (#3650)
+  - allow amending easyconfig parameters which are not the default (#3651)
+  - update HierarchicalMNS for Intel OneAPI compilers (#3653)
+  - add support for --sanity-check-only (#3655)
+  - add support for running commands asynchronously via run_cmd + complete_cmd functions (#3656)
+  - add support for using oneAPI versions of 'intel' toolchain components (#3665)
+  - add toolchain definition for gofbf (foss with FlexiBLAS rather than OpenBLAS) (#3666)
+  - add support for using ARCH constant in easyconfig files (#3670)
+  - honor keyboard interrupt in 'eb' command (#3674)
+  - add toolchain definition for Fujitsu toolchain (#3677, #3704, #3712, #3713, #3714, #3717)
+  - extend sanity check step to check whether specific libraries are not linked into installed binaries/libraries (#3678)
+    - via banned-linked-shared-libs and required-linked-shared-libs EasyBuild configuration options
+    - via banned_linked_shared_libs and required_linked_shared_libs methods in toolchain support
+    - via banned_linked_shared_libs and required_linked_shared_libs methods in easyblock
+    - via banned_linked_shared_libs and required_linked_shared_libs easyconfig parameters
+  - add locate_solib function to locate Linux shared libraries (#3682)
+  - add system agnostic function to locate shared libraries (#3683)
+  - add update_build_option function to update specific build options after initializing the EasyBuild configuration (#3684)
+  - allow opting out of recursively unloaded of modules via recursive_module_unload easyconfig parameter (#3689)
+  - check for correct version values when parsing easystack file (#3693)
+  - run post-install commands specified for a specific extension (#3696)
+  - add support for --skip-extensions (#3702)
+  - include PR title in output produced by --merge-pr (#3706)
+- various bug fixes, including:
+  - avoid metadata greedy behaviour when probing for external module metadata (mostly relevant for integration with Cray Programming Environment) (#3559)
+  - catch problems early on if --github-user is not specified for --new-pr & co (#3644)
+  - re-enable write permissions when installing with read-only-installdir (#3649)
+  - also run sanity check for extensions when using --module-only (#3655)
+  - improve logging when failing to load class from exts_classmap (#3657)
+  - fix use of --module-only on existing installations without write permissions (#3659)
+  - correct help text for subdir-user-modules (#3660)
+  - avoid picking up easyblocks outside of sandbox in framework tests (#3680)
+  - use unload/load in ModuleGeneratorLua.swap_module, since 'swap' is not supported by Lmod (#3685)
+  - update HierarchicalMNS to also return 'Toolchain/<name>/<version>' as $MODULEPATH extension for cpe* Cray toolchains (#3686)
+  - make EasyConfigParser.get_config_dict return a copy rather than a reference (#3692)
+  - make sure that $TAPE is unset when using piped tar (#3698)
+  - fix extending message for changed files in new_pr_from_branch (#3699)
+  - enhance sched_getaffinity function to avoid early crash when counting available cores on systems with more than 1024 cores (#3701)
+  - correctly strip extension from filename in extract_cmd and back_up_file functions (#3705)
+- other changes:
+  - deprecate adding a non-existing path to $MODULEPATH (#3637)
+  - bump cryptography requirement from 3.2.1 to 3.3.2 (#3643, #3648)
+  - test bootstrap script in separate workflow, and limit test configurations a bit (#3646)
+  - update setup.py to indicate compatibility with Python 3.8 and 3.9 (#3647)
+  - replace log_error parameter of which() by on_error (#3661, #3664)
+  - don't skip sanity check for --module-only --rebuild (#3645)
+  - move disable_templating function into the EasyConfig class (#3668)
+  - pin GitPython version for Python<3.6, don't test bootstrap script against Python 3.8/3.9 (#3675)
+  - tweak foss toolchain definition to switch from OpenBLAS to FlexiBLAS in foss/2021a (#3679)
+  - suggest missing SSH key when not able to read from remote repository in --check-github (#3681)
+  - drop support for Python 2.6 (#3715)
+
+
 v4.3.4 (April 9th 2021)
 -----------------------
 

easybuild/framework/easyblock.py

@@ -81,7 +81,7 @@
 from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP
 from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP
 from easybuild.tools.hooks import PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP
-from easybuild.tools.hooks import load_hooks, run_hook
+from easybuild.tools.hooks import MODULE_WRITE, load_hooks, run_hook
 from easybuild.tools.run import run_cmd
 from easybuild.tools.jenkins import write_to_xml
 from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for
@@ -144,7 +144,7 @@ def __init__(self, ec):
         # keep track of original working directory, so we can go back there
         self.orig_workdir = os.getcwd()
 
-        # list of pre- and post-step hooks
+        # dict of all hooks (mapping of name to function)
         self.hooks = load_hooks(build_option('hooks'))
 
         # list of patch/source files, along with checksums
@@ -3165,6 +3165,10 @@ def make_module_step(self, fake=False):
         txt += self.make_module_extra()
         txt += self.make_module_footer()
 
+        hook_txt = run_hook(MODULE_WRITE, self.hooks, args=[self, mod_filepath, txt])
+        if hook_txt is not None:
+            txt = hook_txt
+
         if self.dry_run:
             # only report generating actual module file during dry run, don't mention temporary module files
             if not fake:
@@ -3528,7 +3532,16 @@ def run_all_steps(self, run_test_cases):
                     else:
                         print_msg("%s..." % descr, log=self.log, silent=self.silent)
                     self.current_step = step_name
-                    self.run_step(step_name, step_methods)
+                    start_time = datetime.now()
+                    try:
+                        self.run_step(step_name, step_methods)
+                    finally:
+                        if not self.dry_run:
+                            step_duration = datetime.now() - start_time
+                            if step_duration.total_seconds() >= 1:
+                                print_msg("... (took %s)", time2str(step_duration), log=self.log, silent=self.silent)
+                            elif self.logdebug or build_option('trace'):
+                                print_msg("... (took < 1 sec)", log=self.log, silent=self.silent)
 
         except StopException:
             pass

easybuild/framework/easyconfig/format/one.py

@@ -135,8 +135,14 @@ def get_config_dict(self):
         # we can't use copy.deepcopy() directly because in Python 2 copying the (irrelevant) __builtins__ key fails...
         cfg_copy = {}
         for key in cfg:
-            if key != '__builtins__':
-                cfg_copy[key] = copy.deepcopy(cfg[key])
+            # skip special variables like __builtins__, and imported modules (like 'os')
+            if key != '__builtins__' and "'module'" not in str(type(cfg[key])):
+                try:
+                    cfg_copy[key] = copy.deepcopy(cfg[key])
+                except Exception as err:
+                    raise EasyBuildError("Failed to copy '%s' easyconfig parameter: %s" % (key, err))
+            else:
+                self.log.debug("Not copying '%s' variable from parsed easyconfig", key)
 
         return cfg_copy
 

easybuild/framework/easyconfig/tools.py

@@ -334,7 +334,7 @@ def det_easyconfig_paths(orig_paths):
     :return: list of paths to easyconfig files
     """
     try:
-        from_prs = [int(pr_nr) for pr_nr in build_option('from_pr')]
+        from_prs = [int(x) for x in build_option('from_pr')]
     except ValueError:
         raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.")
 
@@ -623,6 +623,14 @@ def categorize_files_by_type(paths):
         # file must exist in order to check whether it's a patch file
         elif os.path.isfile(path) and is_patch_file(path):
             res['patch_files'].append(path)
+        elif path.endswith('.patch'):
+            if not os.path.exists(path):
+                raise EasyBuildError('File %s does not exist, did you mistype the path?', path)
+            elif not os.path.isfile(path):
+                raise EasyBuildError('File %s is expected to be a regular file, but is a folder instead', path)
+            else:
+                raise EasyBuildError('%s is not detected as a valid patch file. Please verify its contents!',
+                                     path)
         else:
             # anything else is considered to be an easyconfig file
             res['easyconfigs'].append(path)

easybuild/toolchains/compiler/fujitsu.py

@@ -30,7 +30,9 @@
 :author: Miguel Dias Costa (National University of Singapore)
 """
 import os
+import re
 
+import easybuild.tools.environment as env
 import easybuild.tools.systemtools as systemtools
 from easybuild.tools.toolchain.compiler import Compiler, DEFAULT_OPT_LEVEL
 
@@ -46,9 +48,8 @@ class FujitsuCompiler(Compiler):
     COMPILER_MODULE_NAME = [TC_CONSTANT_MODULE_NAME]
     COMPILER_FAMILY = TC_CONSTANT_FUJITSU
 
-    # make sure fcc is always called in clang compatibility mode
-    COMPILER_CC = 'fcc -Nclang'
-    COMPILER_CXX = 'FCC -Nclang'
+    COMPILER_CC = 'fcc'
+    COMPILER_CXX = 'FCC'
 
     COMPILER_F77 = 'frt'
     COMPILER_F90 = 'frt'
@@ -85,15 +86,31 @@ class FujitsuCompiler(Compiler):
         (systemtools.AARCH64, systemtools.ARM): '-mcpu=generic -mtune=generic',
     }
 
+    def prepare(self, *args, **kwargs):
+        super(FujitsuCompiler, self).prepare(*args, **kwargs)
+
+        # fcc doesn't accept e.g. -std=c++11 or -std=gnu++11, only -std=c11 or -std=gnu11
+        pattern = r'-std=(gnu|c)\+\+(\d+)'
+        if re.search(pattern, self.vars['CFLAGS']):
+            self.log.debug("Found '-std=(gnu|c)++' in CFLAGS, fcc doesn't accept '++' here, removing it")
+            self.vars['CFLAGS'] = re.sub(pattern, r'-std=\1\2', self.vars['CFLAGS'])
+            self._setenv_variables()
+
+        # make sure the fujitsu module libraries are found (and added to rpath by wrapper)
+        library_path = os.getenv('LIBRARY_PATH', '')
+        libdir = os.path.join(os.getenv(TC_CONSTANT_MODULE_VAR), 'lib64')
+        if libdir not in library_path:
+            self.log.debug("Adding %s to $LIBRARY_PATH" % libdir)
+            env.setvar('LIBRARY_PATH', os.pathsep.join([library_path, libdir]))
+
     def _set_compiler_vars(self):
         super(FujitsuCompiler, self)._set_compiler_vars()
 
         # enable clang compatibility mode
-        # moved to compiler constants to make sure it is always used
-        # self.variables.nappend('CFLAGS', ['Nclang'])
-        # self.variables.nappend('CXXFLAGS', ['Nclang'])
+        self.variables.nappend('CFLAGS', ['Nclang'])
+        self.variables.nappend('CXXFLAGS', ['Nclang'])
 
-        # make sure the fujitsu module libraries are found (and added to rpath by wrapper)
+        # also add fujitsu module library path to LDFLAGS
         libdir = os.path.join(os.getenv(TC_CONSTANT_MODULE_VAR), 'lib64')
         self.log.debug("Adding %s to $LDFLAGS" % libdir)
         self.variables.nappend('LDFLAGS', [libdir])

easybuild/toolchains/fft/fujitsufftw.py

@@ -33,5 +33,4 @@
 class FujitsuFFTW(Fftw):
     """Fujitsu FFTW FFT library"""
 
-    FFT_MODULE_NAME = ['FFTW3-sve']
     FFTW_API_VERSION = '3'

easybuild/toolchains/fujitsu.py

@@ -36,4 +36,3 @@ class Fujitsu(Ffmpi, FujitsuFFTW, FujitsuSSL):
     """Compiler toolchain for Fujitsu."""
     NAME = 'Fujitsu'
     SUBTOOLCHAIN = Ffmpi.NAME
-    COMPILER_MODULE_NAME = []

easybuild/tools/github.py

@@ -1232,7 +1232,7 @@ def reasons_for_closing(pr_data):
 
     robot_paths = build_option('robot_path')
 
-    pr_files = [path for path in fetch_easyconfigs_from_pr(pr_data['number']) if path.endswith('.eb')]
+    pr_files = [p for p in fetch_easyconfigs_from_pr(pr_data['number']) if p.endswith('.eb')]
 
     obsoleted = []
     uses_archived_tc = []
@@ -1491,17 +1491,17 @@ def add_pr_labels(pr, branch=GITHUB_DEVELOP_BRANCH):
 
     download_repo_path = download_repo(branch=branch, path=tmpdir)
 
-    pr_files = [path for path in fetch_easyconfigs_from_pr(pr) if path.endswith('.eb')]
+    pr_files = [p for p in fetch_easyconfigs_from_pr(pr) if p.endswith('.eb')]
 
     file_info = det_file_info(pr_files, download_repo_path)
 
     pr_target_account = build_option('pr_target_account')
     github_user = build_option('github_user')
     pr_data, _ = fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user)
-    pr_labels = [label['name'] for label in pr_data['labels']]
+    pr_labels = [x['name'] for x in pr_data['labels']]
 
     expected_labels = det_pr_labels(file_info, pr_target_repo)
-    missing_labels = [label for label in expected_labels if label not in pr_labels]
+    missing_labels = [x for x in expected_labels if x not in pr_labels]
 
     dry_run = build_option('dry_run') or build_option('extended_dry_run')
 

easybuild/tools/hooks.py

@@ -58,6 +58,7 @@
 
 START = 'start'
 PARSE = 'parse'
+MODULE_WRITE = 'module_write'
 END = 'end'
 
 PRE_PREF = 'pre_'
@@ -69,7 +70,7 @@
               INSTALL_STEP, EXTENSIONS_STEP, POSTPROC_STEP, SANITYCHECK_STEP, CLEANUP_STEP, MODULE_STEP,
               PERMISSIONS_STEP, PACKAGE_STEP, TESTCASES_STEP]
 
-HOOK_NAMES = [START, PARSE] + [p + s for s in STEP_NAMES for p in [PRE_PREF, POST_PREF]] + [END]
+HOOK_NAMES = [START, PARSE, MODULE_WRITE] + [p + s for s in STEP_NAMES for p in [PRE_PREF, POST_PREF]] + [END]
 KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES]
 
 
@@ -99,7 +100,7 @@ def load_hooks(hooks_path):
                         if attr.endswith(HOOK_SUFF):
                             hook = getattr(imported_hooks, attr)
                             if callable(hook):
-                                hooks.update({attr: hook})
+                                hooks[attr] = hook
                             else:
                                 _log.debug("Skipping non-callable attribute '%s' when loading hooks", attr)
                     _log.info("Found hooks: %s", sorted(hooks.keys()))
@@ -119,11 +120,8 @@ def load_hooks(hooks_path):
 
 
 def verify_hooks(hooks):
-    """Check whether list of obtained hooks only includes known hooks."""
-    unknown_hooks = []
-    for key in sorted(hooks):
-        if key not in KNOWN_HOOKS:
-            unknown_hooks.append(key)
+    """Check whether obtained hooks only includes known hooks."""
+    unknown_hooks = [key for key in sorted(hooks) if key not in KNOWN_HOOKS]
 
     if unknown_hooks:
         error_lines = ["Found one or more unknown hooks:"]
@@ -147,7 +145,7 @@ def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False):
     Find hook with specified label.
 
     :param label: name of hook
-    :param hooks: list of defined hooks
+    :param hooks: dict of defined hooks
     :param pre_step_hook: indicates whether hook to run is a pre-step hook
     :param post_step_hook: indicates whether hook to run is a post-step hook
     """
@@ -162,27 +160,26 @@ def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False):
 
     hook_name = hook_prefix + label + HOOK_SUFF
 
-    for key in hooks:
-        if key == hook_name:
-            _log.info("Found %s hook", hook_name)
-            res = hooks[key]
-            break
+    res = hooks.get(hook_name)
+    if res:
+        _log.info("Found %s hook", hook_name)
 
     return res
 
 
 def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, msg=None):
     """
-    Run hook with specified label.
+    Run hook with specified label and return result of calling the hook or None.
 
     :param label: name of hook
-    :param hooks: list of defined hooks
+    :param hooks: dict of defined hooks
     :param pre_step_hook: indicates whether hook to run is a pre-step hook
     :param post_step_hook: indicates whether hook to run is a post-step hook
     :param args: arguments to pass to hook function
     :param msg: custom message that is printed when hook is called
     """
     hook = find_hook(label, hooks, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook)
+    res = None
     if hook:
         if args is None:
             args = []
@@ -197,4 +194,5 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None,
         print_msg(msg)
 
         _log.info("Running '%s' hook function (arguments: %s)...", hook.__name__, args)
-        hook(*args)
+        res = hook(*args)
+    return res

easybuild/tools/options.py

@@ -1464,7 +1464,7 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False):
 
     # map list of strings --from-pr value to list of integers
     try:
-        from_prs = [int(pr_nr) for pr_nr in eb_go.options.from_pr]
+        from_prs = [int(x) for x in eb_go.options.from_pr]
     except ValueError:
         raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.")
 
@@ -1500,7 +1500,7 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False):
     # done here instead of in _postprocess_include because github integration requires build_options to be initialized
     if eb_go.options.include_easyblocks_from_pr:
         try:
-            easyblock_prs = [int(pr_nr) for pr_nr in eb_go.options.include_easyblocks_from_pr]
+            easyblock_prs = [int(x) for x in eb_go.options.include_easyblocks_from_pr]
         except ValueError:
             raise EasyBuildError("Argument to --include-easyblocks-from-pr must be a comma separated list of PR #s.")