Merge pull request #4042 from easybuilders/4.6.x

release EasyBuild v4.6.0

Author: SebastianAchilles

.github/workflows/bootstrap_script.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
         lmod7: Lmod-7.8.22
-        lmod8: Lmod-8.4.27
+        lmod8: Lmod-8.7.6
         modulesTcl: modules-tcl-1.147
         modules3: modules-3.2.10
         modules4: modules-4.1.4

.github/workflows/container_tests.yml

@@ -39,7 +39,7 @@ jobs:
           cd $HOME
           export INSTALL_DEP=$GITHUB_WORKSPACE/easybuild/scripts/install_eb_dep.sh
           # install Lmod
-          source $INSTALL_DEP Lmod-8.4.27 $HOME
+          source $INSTALL_DEP Lmod-8.7.6 $HOME
           # changes in environment are not passed to other steps, so need to create files...
           echo $MOD_INIT > mod_init
           echo $PATH > path

.github/workflows/eb_command.yml

@@ -38,7 +38,7 @@ jobs:
           cd $HOME
           export INSTALL_DEP=$GITHUB_WORKSPACE/easybuild/scripts/install_eb_dep.sh
           # install Lmod
-          source $INSTALL_DEP Lmod-8.4.26 $HOME
+          source $INSTALL_DEP Lmod-8.7.6 $HOME
           # changes in environment are not passed to other steps, so need to create files...
           echo $MOD_INIT > mod_init
           echo $PATH > path

.github/workflows/unit_tests.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
         lmod7: Lmod-7.8.22
-        lmod8: Lmod-8.4.27
+        lmod8: Lmod-8.7.6
         modulesTcl: modules-tcl-1.147
         modules3: modules-3.2.10
         modules4: modules-4.1.4

RELEASE_NOTES

@@ -4,6 +4,28 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.6.0 (July 8th 2022)
+----------------------
+
+feature release
+
+- various enhancements, including:
+  - allow searching for sources/patches in alternative location by specifying 'alt_location' in source/patch spec (#3994)
+  - show URLs used for download attempts in trace output (#4026)
+  - add support for setting environment variables via 'pushenv' with modextravars (#4030)
+  - add support for OneAPI compilers using toolchain option 'oneapi' (#4031, #4032, #4039)
+  - make check_linked_shared_libs more robust by taking into account that 'ldd' may fail (#4033)
+  - fall back to sequential extension install if parallel install is not implemented (#4034)
+  - add support for using template values in name/version of extensions (#4036)
+- various bug fixes, including:
+  - make sure that ARCH constant has 'aarch64' (rather than 'arm64') as value on macOS ARM (#4018)
+  - tweak 'eb' wrapper script to correctly handle errors when python command being considered fails to run (#4019)
+  - tweak is_patch_for function to make it more robust (#4028)
+- other changes:
+  - update Lmod used to run tests to version 8.7.6 (#4027, #4030)
+  - tweak apply_patch to not create .orig files (by default) when applying patch files (#4038)
+
+
 v4.5.5 (June 8th 2022)
 ----------------------
 

easybuild/framework/easyblock.py

@@ -373,7 +373,8 @@ def fetch_source(self, source, checksum=None, extension=False, download_instruct
         :param checksum: checksum corresponding to source
         :param extension: flag if being called from collect_exts_file_info()
         """
-        filename, download_filename, extract_cmd, source_urls, git_config = None, None, None, None, None
+        filename, download_filename, extract_cmd = None, None, None
+        source_urls, git_config, alt_location = None, None, None
 
         if source is None:
             raise EasyBuildError("fetch_source called with empty 'source' argument")
@@ -387,6 +388,7 @@ def fetch_source(self, source, checksum=None, extension=False, download_instruct
             download_filename = source.pop('download_filename', None)
             source_urls = source.pop('source_urls', None)
             git_config = source.pop('git_config', None)
+            alt_location = source.pop('alt_location', None)
             if source:
                 raise EasyBuildError("Found one or more unexpected keys in 'sources' specification: %s", source)
 
@@ -401,7 +403,7 @@ def fetch_source(self, source, checksum=None, extension=False, download_instruct
         force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES]
         path = self.obtain_file(filename, extension=extension, download_filename=download_filename,
                                 force_download=force_download, urls=source_urls, git_config=git_config,
-                                download_instructions=download_instructions)
+                                download_instructions=download_instructions, alt_location=alt_location)
         if path is None:
             raise EasyBuildError('No file found for source %s', filename)
 
@@ -468,7 +470,9 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None):
             patch_info['postinstall'] = patch_spec in post_install_patches
 
             force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES]
-            path = self.obtain_file(patch_info['name'], extension=extension, force_download=force_download)
+            alt_location = patch_info.pop('alt_location', None)
+            path = self.obtain_file(patch_info['name'], extension=extension, force_download=force_download,
+                                    alt_location=alt_location)
             if path:
                 self.log.debug('File %s found for patch %s', path, patch_spec)
                 patch_info['path'] = path
@@ -517,11 +521,13 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
         for ext in exts_list:
             if isinstance(ext, (list, tuple)) and ext:
                 # expected format: (name, version, options (dict))
-                ext_name = ext[0]
+                # name and version can use templates, resolved via parent EC
+
+                ext_name = resolve_template(ext[0], self.cfg.template_values)
                 if len(ext) == 1:
                     exts_sources.append({'name': ext_name})
                 else:
-                    ext_version = ext[1]
+                    ext_version = resolve_template(ext[1], self.cfg.template_values)
 
                     # make sure we grab *raw* dict of default options for extension,
                     # since it may use template values like %(name)s & %(version)s
@@ -687,7 +693,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
         return exts_sources
 
     def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False,
-                    git_config=None, download_instructions=None):
+                    git_config=None, download_instructions=None, alt_location=None):
         """
         Locate the file with the given name
         - searches in different subdirectories of source path
@@ -698,11 +704,18 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
         :param download_filename: filename with which the file should be downloaded, and then renamed to <filename>
         :param force_download: always try to download file, even if it's already available in source path
         :param git_config: dictionary to define how to download a git repository
+        :param download_instructions: instructions to manually add source (used for complex cases)
+        :param alt_location: alternative location to use instead of self.name
         """
         srcpaths = source_paths()
 
         update_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, label=filename)
 
+        if alt_location is None:
+            location = self.name
+        else:
+            location = alt_location
+
         # should we download or just try and find it?
         if re.match(r"^(https?|ftp)://", filename):
             # URL detected, so let's try and download it
@@ -711,7 +724,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
             filename = url.split('/')[-1]
 
             # figure out where to download the file to
-            filepath = os.path.join(srcpaths[0], letter_dir_for(self.name), self.name)
+            filepath = os.path.join(srcpaths[0], letter_dir_for(location), location)
             if extension:
                 filepath = os.path.join(filepath, "extensions")
             self.log.info("Creating path %s to download file to" % filepath)
@@ -750,8 +763,8 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
 
             for path in ebpath + common_filepaths + srcpaths:
                 # create list of candidate filepaths
-                namepath = os.path.join(path, self.name)
-                letterpath = os.path.join(path, letter_dir_for(self.name), self.name)
+                namepath = os.path.join(path, location)
+                letterpath = os.path.join(path, letter_dir_for(location), location)
 
                 # most likely paths
                 candidate_filepaths = [
@@ -790,8 +803,8 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
 
                     break  # no need to try other source paths
 
-            name_letter = self.name.lower()[0]
-            targetdir = os.path.join(srcpaths[0], name_letter, self.name)
+            name_letter = location.lower()[0]
+            targetdir = os.path.join(srcpaths[0], name_letter, location)
 
             if foundfile:
                 if self.dry_run:
@@ -808,7 +821,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                 source_urls.extend(self.cfg['source_urls'])
 
                 # add https://sources.easybuild.io as fallback source URL
-                source_urls.append(EASYBUILD_SOURCES_URL + '/' + os.path.join(name_letter, self.name))
+                source_urls.append(EASYBUILD_SOURCES_URL + '/' + os.path.join(name_letter, location))
 
                 mkdir(targetdir, parents=True)
 
@@ -1754,7 +1767,13 @@ def install_extensions(self, install=True):
 
         if build_option('parallel_extensions_install'):
             self.log.experimental("installing extensions in parallel")
-            self.install_extensions_parallel(install=install)
+            try:
+                self.install_extensions_parallel(install=install)
+            except NotImplementedError:
+                # If parallel extension install is not supported for this type of extension then install sequentially
+                msg = "Parallel extensions install not supported for %s - using sequential install" % self.name
+                self.log.experimental(msg)
+                self.install_extensions_sequential(install=install)
         else:
             self.install_extensions_sequential(install=install)
 

easybuild/framework/easyconfig/constants.py

@@ -34,17 +34,35 @@
 import platform
 
 from easybuild.base import fancylogger
-from easybuild.tools.systemtools import get_os_name, get_os_type, get_os_version
+from easybuild.tools.build_log import print_warning
+from easybuild.tools.systemtools import KNOWN_ARCH_CONSTANTS, get_os_name, get_os_type, get_os_version
 
 
 _log = fancylogger.getLogger('easyconfig.constants', fname=False)
 
 
 EXTERNAL_MODULE_MARKER = 'EXTERNAL_MODULE'
 
+
+def _get_arch_constant():
+    """
+    Get value for ARCH constant.
+    """
+    arch = platform.uname()[4]
+
+    # macOS on Arm produces 'arm64' rather than 'aarch64'
+    if arch == 'arm64':
+        arch = 'aarch64'
+
+    if arch not in KNOWN_ARCH_CONSTANTS:
+        print_warning("Using unknown value for ARCH constant: %s", arch)
+
+    return arch
+
+
 # constants that can be used in easyconfig
 EASYCONFIG_CONSTANTS = {
-    'ARCH': (platform.uname()[4], "CPU architecture of current system (aarch64, x86_64, ppc64le, ...)"),
+    'ARCH': (_get_arch_constant(), "CPU architecture of current system (aarch64, x86_64, ppc64le, ...)"),
     'EXTERNAL_MODULE': (EXTERNAL_MODULE_MARKER, "External module marker"),
     'HOME': (os.path.expanduser('~'), "Home directory ($HOME)"),
     'OS_TYPE': (get_os_type(), "System type (e.g. 'Linux' or 'Darwin')"),

easybuild/toolchains/compiler/intel_compilers.py

@@ -29,6 +29,8 @@
 """
 import os
 
+from distutils.version import LooseVersion
+
 from easybuild.toolchains.compiler.inteliccifort import IntelIccIfort
 from easybuild.tools.toolchain.compiler import Compiler
 
@@ -39,6 +41,8 @@ class IntelCompilers(IntelIccIfort):
     """
 
     COMPILER_MODULE_NAME = ['intel-compilers']
+    COMPILER_UNIQUE_OPTS = dict(IntelIccIfort.COMPILER_UNIQUE_OPTS,
+                                oneapi=(False, "Use oneAPI compilers icx/icpx/ifx instead of classic compilers"))
 
     def _set_compiler_vars(self):
         """Intel compilers-specific adjustments after setting compiler variables."""
@@ -59,5 +63,22 @@ def _set_compiler_vars(self):
     def set_variables(self):
         """Set the variables."""
 
+        if self.options.get('oneapi', False):
+            self.COMPILER_CXX = 'icpx'
+            self.COMPILER_CC = 'icx'
+            self.COMPILER_F77 = 'ifx'
+            self.COMPILER_F90 = 'ifx'
+            self.COMPILER_FC = 'ifx'
+            # fp-model source is not supported by icx but is equivalent to precise
+            self.options.options_map['defaultprec'] = ['fp-speculation=safe', 'fp-model precise']
+            if LooseVersion(self.get_software_version(self.COMPILER_MODULE_NAME)[0]) >= LooseVersion('2022'):
+                self.options.options_map['defaultprec'].insert(0, 'ftz')
+            # icx doesn't like -fp-model fast=1; fp-model fast is equivalent
+            self.options.options_map['loose'] = ['fp-model fast']
+            # fp-model fast=2 gives "warning: overriding '-ffp-model=fast=2' option with '-ffp-model=fast'"
+            self.options.options_map['veryloose'] = ['fp-model fast']
+            # recommended in porting guide
+            self.options.options_map['openmp'] = ['fiopenmp']
+
         # skip IntelIccIfort.set_variables (no longer relevant for recent versions)
         Compiler.set_variables(self)

easybuild/tools/config.py

@@ -250,6 +250,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'add_dummy_to_minimal_toolchains',
         'add_system_to_minimal_toolchains',
         'allow_modules_tool_mismatch',
+        'backup_patched_files',
         'consider_archived_easyconfigs',
         'container_build_image',
         'debug',

easybuild/tools/filetools.py

@@ -65,7 +65,7 @@
 from easybuild.tools.config import build_option, install_path
 from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar
 from easybuild.tools.py2vs3 import HTMLParser, std_urllib, string_type
-from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars
+from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars, trace_msg
 
 try:
     import requests
@@ -856,9 +856,11 @@ def download_file(filename, url, path, forced=False):
 
     if downloaded:
         _log.info("Successful download of file %s from url %s to path %s" % (filename, url, path))
+        trace_msg("download succeeded: %s" % url)
         return path
     else:
         _log.warning("Download of %s to %s failed, done trying" % (url, path))
+        trace_msg("download failed: %s" % url)
         return None
 
 
@@ -1469,6 +1471,9 @@ def create_patch_info(patch_spec):
     """
     Create info dictionary from specified patch spec.
     """
+    # Valid keys that can be used in a patch spec dict
+    valid_keys = ['name', 'copy', 'level', 'sourcepath', 'alt_location']
+
     if isinstance(patch_spec, (list, tuple)):
         if not len(patch_spec) == 2:
             error_msg = "Unknown patch specification '%s', only 2-element lists/tuples are supported!"
@@ -1495,19 +1500,44 @@ def create_patch_info(patch_spec):
                                  str(patch_spec))
 
     elif isinstance(patch_spec, string_type):
-        allowed_patch_exts = ['.patch' + x for x in ('',) + ZIPPED_PATCH_EXTS]
-        if not any(patch_spec.endswith(x) for x in allowed_patch_exts):
-            msg = "Use of patch file with filename that doesn't end with correct extension: %s " % patch_spec
-            msg += "(should be any of: %s)" % (', '.join(allowed_patch_exts))
-            _log.deprecated(msg, '5.0')
+        validate_patch_spec(patch_spec)
         patch_info = {'name': patch_spec}
+    elif isinstance(patch_spec, dict):
+        patch_info = {}
+        for key in patch_spec.keys():
+            if key in valid_keys:
+                patch_info[key] = patch_spec[key]
+            else:
+                raise EasyBuildError("Wrong patch spec '%s', use of unknown key %s in dict (valid keys are %s)",
+                                     str(patch_spec), key, valid_keys)
+
+        # Dict must contain at least the patchfile name
+        if 'name' not in patch_info.keys():
+            raise EasyBuildError("Wrong patch spec '%s', when using a dict 'name' entry must be supplied",
+                                 str(patch_spec))
+        if 'copy' not in patch_info.keys():
+            validate_patch_spec(patch_info['name'])
+        else:
+            if 'sourcepath' in patch_info.keys() or 'level' in patch_info.keys():
+                raise EasyBuildError("Wrong patch spec '%s', you can't use 'sourcepath' or 'level' with 'copy' (since "
+                                     "this implies you want to copy a file to the 'copy' location)",
+                                     str(patch_spec))
     else:
-        error_msg = "Wrong patch spec, should be string of 2-tuple with patch name + argument: %s"
-        raise EasyBuildError(error_msg, patch_spec)
+        error_msg = "Wrong patch spec, should be string, 2-tuple with patch name + argument, or a dict " \
+                    "(with possible keys %s): %s" % (valid_keys, patch_spec)
+        raise EasyBuildError(error_msg)
 
     return patch_info
 
 
+def validate_patch_spec(patch_spec):
+    allowed_patch_exts = ['.patch' + x for x in ('',) + ZIPPED_PATCH_EXTS]
+    if not any(patch_spec.endswith(x) for x in allowed_patch_exts):
+        msg = "Use of patch file with filename that doesn't end with correct extension: %s " % patch_spec
+        msg += "(should be any of: %s)" % (', '.join(allowed_patch_exts))
+        _log.deprecated(msg, '5.0')
+
+
 def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git_am=False, use_git=False):
     """
     Apply a patch to source code in directory dest
@@ -1592,7 +1622,8 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git_am=Fa
         else:
             _log.debug("Using specified patch level %d for patch %s" % (level, patch_file))
 
-        patch_cmd = "patch -b -p%s -i %s" % (level, abs_patch_file)
+        backup_option = '-b ' if build_option('backup_patched_files') else ''
+        patch_cmd = "patch " + backup_option + "-p%s -i %s" % (level, abs_patch_file)
 
     out, ec = run.run_cmd(patch_cmd, simple=False, path=abs_dest, log_ok=False, trace=False)