Merge branch 'easybuilders:develop' into develop

Author: branfosj

RELEASE_NOTES

@@ -4,6 +4,68 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.4.0 (June 2nd 2021)
+----------------------
+
+feature release
+
+- various enhancements, including:
+  - enhance apply_regex_substitutions to allow specifying action to take in case there are no matches (#3440)
+  - performance improvements for easyconfig parsing and eb startup (#3555)
+  - add support for downloading easyconfigs from multiple PRs with --from-pr (#3605, #3707, #3708)
+  - add support for prepending custom library paths in RPATH section via --rpath-override-dirs (#3650)
+  - allow amending easyconfig parameters which are not the default (#3651)
+  - update HierarchicalMNS for Intel OneAPI compilers (#3653)
+  - add support for --sanity-check-only (#3655)
+  - add support for running commands asynchronously via run_cmd + complete_cmd functions (#3656)
+  - add support for using oneAPI versions of 'intel' toolchain components (#3665)
+  - add toolchain definition for gofbf (foss with FlexiBLAS rather than OpenBLAS) (#3666)
+  - add support for using ARCH constant in easyconfig files (#3670)
+  - honor keyboard interrupt in 'eb' command (#3674)
+  - add toolchain definition for Fujitsu toolchain (#3677, #3704, #3712, #3713, #3714, #3717)
+  - extend sanity check step to check whether specific libraries are not linked into installed binaries/libraries (#3678)
+    - via banned-linked-shared-libs and required-linked-shared-libs EasyBuild configuration options
+    - via banned_linked_shared_libs and required_linked_shared_libs methods in toolchain support
+    - via banned_linked_shared_libs and required_linked_shared_libs methods in easyblock
+    - via banned_linked_shared_libs and required_linked_shared_libs easyconfig parameters
+  - add locate_solib function to locate Linux shared libraries (#3682)
+  - add system agnostic function to locate shared libraries (#3683)
+  - add update_build_option function to update specific build options after initializing the EasyBuild configuration (#3684)
+  - allow opting out of recursively unloaded of modules via recursive_module_unload easyconfig parameter (#3689)
+  - check for correct version values when parsing easystack file (#3693)
+  - run post-install commands specified for a specific extension (#3696)
+  - add support for --skip-extensions (#3702)
+  - include PR title in output produced by --merge-pr (#3706)
+- various bug fixes, including:
+  - avoid metadata greedy behaviour when probing for external module metadata (mostly relevant for integration with Cray Programming Environment) (#3559)
+  - catch problems early on if --github-user is not specified for --new-pr & co (#3644)
+  - re-enable write permissions when installing with read-only-installdir (#3649)
+  - also run sanity check for extensions when using --module-only (#3655)
+  - improve logging when failing to load class from exts_classmap (#3657)
+  - fix use of --module-only on existing installations without write permissions (#3659)
+  - correct help text for subdir-user-modules (#3660)
+  - avoid picking up easyblocks outside of sandbox in framework tests (#3680)
+  - use unload/load in ModuleGeneratorLua.swap_module, since 'swap' is not supported by Lmod (#3685)
+  - update HierarchicalMNS to also return 'Toolchain/<name>/<version>' as $MODULEPATH extension for cpe* Cray toolchains (#3686)
+  - make EasyConfigParser.get_config_dict return a copy rather than a reference (#3692)
+  - make sure that $TAPE is unset when using piped tar (#3698)
+  - fix extending message for changed files in new_pr_from_branch (#3699)
+  - enhance sched_getaffinity function to avoid early crash when counting available cores on systems with more than 1024 cores (#3701)
+  - correctly strip extension from filename in extract_cmd and back_up_file functions (#3705)
+- other changes:
+  - deprecate adding a non-existing path to $MODULEPATH (#3637)
+  - bump cryptography requirement from 3.2.1 to 3.3.2 (#3643, #3648)
+  - test bootstrap script in separate workflow, and limit test configurations a bit (#3646)
+  - update setup.py to indicate compatibility with Python 3.8 and 3.9 (#3647)
+  - replace log_error parameter of which() by on_error (#3661, #3664)
+  - don't skip sanity check for --module-only --rebuild (#3645)
+  - move disable_templating function into the EasyConfig class (#3668)
+  - pin GitPython version for Python<3.6, don't test bootstrap script against Python 3.8/3.9 (#3675)
+  - tweak foss toolchain definition to switch from OpenBLAS to FlexiBLAS in foss/2021a (#3679)
+  - suggest missing SSH key when not able to read from remote repository in --check-github (#3681)
+  - drop support for Python 2.6 (#3715)
+
+
 v4.3.4 (April 9th 2021)
 -----------------------
 

easybuild/framework/easyblock.py

@@ -81,7 +81,7 @@
 from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP
 from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP
 from easybuild.tools.hooks import PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP
-from easybuild.tools.hooks import load_hooks, run_hook
+from easybuild.tools.hooks import MODULE_WRITE, load_hooks, run_hook
 from easybuild.tools.run import run_cmd
 from easybuild.tools.jenkins import write_to_xml
 from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for
@@ -144,7 +144,7 @@ def __init__(self, ec):
         # keep track of original working directory, so we can go back there
         self.orig_workdir = os.getcwd()
 
-        # list of pre- and post-step hooks
+        # dict of all hooks (mapping of name to function)
         self.hooks = load_hooks(build_option('hooks'))
 
         # list of patch/source files, along with checksums
@@ -3165,6 +3165,10 @@ def make_module_step(self, fake=False):
         txt += self.make_module_extra()
         txt += self.make_module_footer()
 
+        hook_txt = run_hook(MODULE_WRITE, self.hooks, args=[self, mod_filepath, txt])
+        if hook_txt is not None:
+            txt = hook_txt
+
         if self.dry_run:
             # only report generating actual module file during dry run, don't mention temporary module files
             if not fake:
@@ -3528,7 +3532,16 @@ def run_all_steps(self, run_test_cases):
                     else:
                         print_msg("%s..." % descr, log=self.log, silent=self.silent)
                     self.current_step = step_name
-                    self.run_step(step_name, step_methods)
+                    start_time = datetime.now()
+                    try:
+                        self.run_step(step_name, step_methods)
+                    finally:
+                        if not self.dry_run:
+                            step_duration = datetime.now() - start_time
+                            if step_duration.total_seconds() >= 1:
+                                print_msg("... (took %s)", time2str(step_duration), log=self.log, silent=self.silent)
+                            elif self.logdebug or build_option('trace'):
+                                print_msg("... (took < 1 sec)", log=self.log, silent=self.silent)
 
         except StopException:
             pass

easybuild/framework/easyconfig/format/one.py

@@ -135,8 +135,14 @@ def get_config_dict(self):
         # we can't use copy.deepcopy() directly because in Python 2 copying the (irrelevant) __builtins__ key fails...
         cfg_copy = {}
         for key in cfg:
-            if key != '__builtins__':
-                cfg_copy[key] = copy.deepcopy(cfg[key])
+            # skip special variables like __builtins__, and imported modules (like 'os')
+            if key != '__builtins__' and "'module'" not in str(type(cfg[key])):
+                try:
+                    cfg_copy[key] = copy.deepcopy(cfg[key])
+                except Exception as err:
+                    raise EasyBuildError("Failed to copy '%s' easyconfig parameter: %s" % (key, err))
+            else:
+                self.log.debug("Not copying '%s' variable from parsed easyconfig", key)
 
         return cfg_copy
 

easybuild/framework/easyconfig/tools.py

@@ -307,7 +307,7 @@ def get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None):
     return paths
 
 
-def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_pr=False):
+def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_prs=None):
     """Obtain alternative paths for easyconfig files."""
 
     # paths where tweaked easyconfigs will be placed, easyconfigs listed on the command line take priority and will be
@@ -318,12 +318,13 @@ def alt_easyconfig_paths(tmpdir, tweaked_ecs=False, from_pr=False):
         tweaked_ecs_paths = (os.path.join(tmpdir, 'tweaked_easyconfigs'),
                              os.path.join(tmpdir, 'tweaked_dep_easyconfigs'))
 
-    # path where files touched in PR will be downloaded to
-    pr_path = None
-    if from_pr:
-        pr_path = os.path.join(tmpdir, "files_pr%s" % '_'.join(str(pr) for pr in from_pr))
+    # paths where files touched in PRs will be downloaded to,
+    # which are picked up via 'pr_paths' build option in fetch_files_from_pr
+    pr_paths = None
+    if from_prs:
+        pr_paths = [os.path.join(tmpdir, 'files_pr%s' % pr) for pr in from_prs]
 
-    return tweaked_ecs_paths, pr_path
+    return tweaked_ecs_paths, pr_paths
 
 
 def det_easyconfig_paths(orig_paths):
@@ -333,7 +334,7 @@ def det_easyconfig_paths(orig_paths):
     :return: list of paths to easyconfig files
     """
     try:
-        from_pr_list = [int(pr_nr) for pr_nr in build_option('from_pr')]
+        from_prs = [int(x) for x in build_option('from_pr')]
     except ValueError:
         raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.")
 
@@ -342,9 +343,11 @@ def det_easyconfig_paths(orig_paths):
     # list of specified easyconfig files
     ec_files = orig_paths[:]
 
-    if from_pr_list is not None:
+    if from_prs:
         pr_files = []
-        for pr in from_pr_list:
+        for pr in from_prs:
+            # path to where easyconfig files should be downloaded is determined via 'pr_paths' build option,
+            # which corresponds to the list of PR paths returned by alt_easyconfig_paths
             pr_files.extend(fetch_easyconfigs_from_pr(pr))
 
         if ec_files:
@@ -620,6 +623,14 @@ def categorize_files_by_type(paths):
         # file must exist in order to check whether it's a patch file
         elif os.path.isfile(path) and is_patch_file(path):
             res['patch_files'].append(path)
+        elif path.endswith('.patch'):
+            if not os.path.exists(path):
+                raise EasyBuildError('File %s does not exist, did you mistype the path?', path)
+            elif not os.path.isfile(path):
+                raise EasyBuildError('File %s is expected to be a regular file, but is a folder instead', path)
+            else:
+                raise EasyBuildError('%s is not detected as a valid patch file. Please verify its contents!',
+                                     path)
         else:
             # anything else is considered to be an easyconfig file
             res['easyconfigs'].append(path)

easybuild/toolchains/compiler/fujitsu.py

@@ -30,7 +30,9 @@
 :author: Miguel Dias Costa (National University of Singapore)
 """
 import os
+import re
 
+import easybuild.tools.environment as env
 import easybuild.tools.systemtools as systemtools
 from easybuild.tools.toolchain.compiler import Compiler, DEFAULT_OPT_LEVEL
 
@@ -46,9 +48,8 @@ class FujitsuCompiler(Compiler):
     COMPILER_MODULE_NAME = [TC_CONSTANT_MODULE_NAME]
     COMPILER_FAMILY = TC_CONSTANT_FUJITSU
 
-    # make sure fcc is always called in clang compatibility mode
-    COMPILER_CC = 'fcc -Nclang'
-    COMPILER_CXX = 'FCC -Nclang'
+    COMPILER_CC = 'fcc'
+    COMPILER_CXX = 'FCC'
 
     COMPILER_F77 = 'frt'
     COMPILER_F90 = 'frt'
@@ -85,15 +86,31 @@ class FujitsuCompiler(Compiler):
         (systemtools.AARCH64, systemtools.ARM): '-mcpu=generic -mtune=generic',
     }
 
+    def prepare(self, *args, **kwargs):
+        super(FujitsuCompiler, self).prepare(*args, **kwargs)
+
+        # fcc doesn't accept e.g. -std=c++11 or -std=gnu++11, only -std=c11 or -std=gnu11
+        pattern = r'-std=(gnu|c)\+\+(\d+)'
+        if re.search(pattern, self.vars['CFLAGS']):
+            self.log.debug("Found '-std=(gnu|c)++' in CFLAGS, fcc doesn't accept '++' here, removing it")
+            self.vars['CFLAGS'] = re.sub(pattern, r'-std=\1\2', self.vars['CFLAGS'])
+            self._setenv_variables()
+
+        # make sure the fujitsu module libraries are found (and added to rpath by wrapper)
+        library_path = os.getenv('LIBRARY_PATH', '')
+        libdir = os.path.join(os.getenv(TC_CONSTANT_MODULE_VAR), 'lib64')
+        if libdir not in library_path:
+            self.log.debug("Adding %s to $LIBRARY_PATH" % libdir)
+            env.setvar('LIBRARY_PATH', os.pathsep.join([library_path, libdir]))
+
     def _set_compiler_vars(self):
         super(FujitsuCompiler, self)._set_compiler_vars()
 
         # enable clang compatibility mode
-        # moved to compiler constants to make sure it is always used
-        # self.variables.nappend('CFLAGS', ['Nclang'])
-        # self.variables.nappend('CXXFLAGS', ['Nclang'])
+        self.variables.nappend('CFLAGS', ['Nclang'])
+        self.variables.nappend('CXXFLAGS', ['Nclang'])
 
-        # make sure the fujitsu module libraries are found (and added to rpath by wrapper)
+        # also add fujitsu module library path to LDFLAGS
         libdir = os.path.join(os.getenv(TC_CONSTANT_MODULE_VAR), 'lib64')
         self.log.debug("Adding %s to $LDFLAGS" % libdir)
         self.variables.nappend('LDFLAGS', [libdir])

easybuild/toolchains/fft/fujitsufftw.py

@@ -33,5 +33,4 @@
 class FujitsuFFTW(Fftw):
     """Fujitsu FFTW FFT library"""
 
-    FFT_MODULE_NAME = ['FFTW3-sve']
     FFTW_API_VERSION = '3'

easybuild/toolchains/fujitsu.py

@@ -36,4 +36,3 @@ class Fujitsu(Ffmpi, FujitsuFFTW, FujitsuSSL):
     """Compiler toolchain for Fujitsu."""
     NAME = 'Fujitsu'
     SUBTOOLCHAIN = Ffmpi.NAME
-    COMPILER_MODULE_NAME = []

easybuild/tools/config.py

@@ -346,7 +346,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'build_specs',
         'command_line',
         'external_modules_metadata',
-        'pr_path',
+        'pr_paths',
         'robot_path',
         'valid_module_classes',
         'valid_stops',

easybuild/tools/github.py

@@ -447,7 +447,15 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi
 
     if path is None:
         if github_repo == GITHUB_EASYCONFIGS_REPO:
-            path = build_option('pr_path')
+            pr_paths = build_option('pr_paths')
+            if pr_paths:
+                # figure out directory for this specific PR (see also alt_easyconfig_paths)
+                cands = [p for p in pr_paths if p.endswith('files_pr%s' % pr)]
+                if len(cands) == 1:
+                    path = cands[0]
+                else:
+                    raise EasyBuildError("Failed to isolate path for PR #%s from list of PR paths: %s", pr, pr_paths)
+
         elif github_repo == GITHUB_EASYBLOCKS_REPO:
             path = os.path.join(tempfile.gettempdir(), 'ebs_pr%s' % pr)
         else:
@@ -1224,7 +1232,7 @@ def reasons_for_closing(pr_data):
 
     robot_paths = build_option('robot_path')
 
-    pr_files = [path for path in fetch_easyconfigs_from_pr(pr_data['number']) if path.endswith('.eb')]
+    pr_files = [p for p in fetch_easyconfigs_from_pr(pr_data['number']) if p.endswith('.eb')]
 
     obsoleted = []
     uses_archived_tc = []
@@ -1381,6 +1389,7 @@ def merge_pr(pr):
 
     msg = "\n%s/%s PR #%s was submitted by %s, " % (pr_target_account, pr_target_repo, pr, pr_data['user']['login'])
     msg += "you are using GitHub account '%s'\n" % github_user
+    msg += "\nPR title: %s\n\n" % pr_data['title']
     print_msg(msg, prefix=False)
     if pr_data['user']['login'] == github_user:
         raise EasyBuildError("Please do not merge your own PRs!")
@@ -1482,17 +1491,17 @@ def add_pr_labels(pr, branch=GITHUB_DEVELOP_BRANCH):
 
     download_repo_path = download_repo(branch=branch, path=tmpdir)
 
-    pr_files = [path for path in fetch_easyconfigs_from_pr(pr) if path.endswith('.eb')]
+    pr_files = [p for p in fetch_easyconfigs_from_pr(pr) if p.endswith('.eb')]
 
     file_info = det_file_info(pr_files, download_repo_path)
 
     pr_target_account = build_option('pr_target_account')
     github_user = build_option('github_user')
     pr_data, _ = fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user)
-    pr_labels = [label['name'] for label in pr_data['labels']]
+    pr_labels = [x['name'] for x in pr_data['labels']]
 
     expected_labels = det_pr_labels(file_info, pr_target_repo)
-    missing_labels = [label for label in expected_labels if label not in pr_labels]
+    missing_labels = [x for x in expected_labels if x not in pr_labels]
 
     dry_run = build_option('dry_run') or build_option('extended_dry_run')