Merge branch 'easystack_easyconfigs' of https://github.com/boegel/easybuild-framework into easystack_easyconfigs

Author: casparvl

.github/workflows/bootstrap_script.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
         lmod7: Lmod-7.8.22
-        lmod8: Lmod-8.4.27
+        lmod8: Lmod-8.7.6
         modulesTcl: modules-tcl-1.147
         modules3: modules-3.2.10
         modules4: modules-4.1.4

.github/workflows/container_tests.yml

@@ -39,7 +39,7 @@ jobs:
           cd $HOME
           export INSTALL_DEP=$GITHUB_WORKSPACE/easybuild/scripts/install_eb_dep.sh
           # install Lmod
-          source $INSTALL_DEP Lmod-8.4.27 $HOME
+          source $INSTALL_DEP Lmod-8.7.6 $HOME
           # changes in environment are not passed to other steps, so need to create files...
           echo $MOD_INIT > mod_init
           echo $PATH > path

.github/workflows/eb_command.yml

@@ -38,7 +38,7 @@ jobs:
           cd $HOME
           export INSTALL_DEP=$GITHUB_WORKSPACE/easybuild/scripts/install_eb_dep.sh
           # install Lmod
-          source $INSTALL_DEP Lmod-8.4.26 $HOME
+          source $INSTALL_DEP Lmod-8.7.6 $HOME
           # changes in environment are not passed to other steps, so need to create files...
           echo $MOD_INIT > mod_init
           echo $PATH > path

.github/workflows/unit_tests.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
         lmod7: Lmod-7.8.22
-        lmod8: Lmod-8.4.27
+        lmod8: Lmod-8.7.6
         modulesTcl: modules-tcl-1.147
         modules3: modules-3.2.10
         modules4: modules-4.1.4

RELEASE_NOTES

@@ -4,6 +4,49 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.6.0 (July 8th 2022)
+----------------------
+
+feature release
+
+- various enhancements, including:
+  - allow searching for sources/patches in alternative location by specifying 'alt_location' in source/patch spec (#3994)
+  - show URLs used for download attempts in trace output (#4026)
+  - add support for setting environment variables via 'pushenv' with modextravars (#4030)
+  - add support for OneAPI compilers using toolchain option 'oneapi' (#4031, #4032, #4039)
+  - make check_linked_shared_libs more robust by taking into account that 'ldd' may fail (#4033)
+  - fall back to sequential extension install if parallel install is not implemented (#4034)
+  - add support for using template values in name/version of extensions (#4036)
+- various bug fixes, including:
+  - make sure that ARCH constant has 'aarch64' (rather than 'arm64') as value on macOS ARM (#4018)
+  - tweak 'eb' wrapper script to correctly handle errors when python command being considered fails to run (#4019)
+  - tweak is_patch_for function to make it more robust (#4028)
+- other changes:
+  - update Lmod used to run tests to version 8.7.6 (#4027, #4030)
+  - tweak apply_patch to not create .orig files (by default) when applying patch files (#4038)
+
+
+v4.5.5 (June 8th 2022)
+----------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - add toolchain definitions for nvompi (NVHPC + OpenMPI) (#3969), nvpsmpi (NVHPC + ParaStationMPI) (#3970), gfbf (GCC, FlexiBLAS, FFTW) (#4006)
+  - add support for FFTW.MPI toolchain component ($FFT*DIR variables) (#4012)
+  - add support for customizing EasyBuild command used in jobs via --job-eb-cmd (#4016)
+- various bug fixes, including:
+  - fix copying of easyconfig file with --copy-ec without --rebuild if module is already installed (#3993)
+  - ignore deprecation warnings regarding cryptography in Python 3.6 + Blowfish with Python 3.10 in test suite output (#3999)
+  - fix typo in debug log message in easyblock.py (#4000)
+  - fix printing of list of attempted download URLs when url-encoded characters are used in URL (#4005)
+  - set $FFT(W)_LIB_DIR to imkl-FFTW's lib path in build environment if usempi toolchain option is enabled (#4011)
+  - correctly identify Apple Silicon M1 as Arm 64-bit by also considering arm64 next to aarch64 (#4014)
+  - fix 'eb --show-system-info' on Apple M1 system (#4015)
+- other changes:
+  - change 'eb' command to import easybuild.framework to check if EasyBuild framework is available (#3995, #3998)
+
+
 v4.5.4 (March 31st 2022)
 ------------------------
 

easybuild/base/fancylogger.py

@@ -380,9 +380,9 @@ def streamLog(self, levelno, data):
 
         def write_and_flush_stream(hdlr, data=None):
             """Write to stream and flush the handler"""
-            if (not hasattr(hdlr, 'stream')) or hdlr.stream is None:
+            if getattr(hdlr, 'stream', None) is None:
                 # no stream or not initialised.
-                raise("write_and_flush_stream failed. No active stream attribute.")
+                raise ValueError("write_and_flush_stream failed. No active stream attribute.")
             if data is not None:
                 hdlr.stream.write(data)
                 hdlr.flush()

easybuild/framework/easyblock.py

@@ -373,7 +373,8 @@ def fetch_source(self, source, checksum=None, extension=False, download_instruct
         :param checksum: checksum corresponding to source
         :param extension: flag if being called from collect_exts_file_info()
         """
-        filename, download_filename, extract_cmd, source_urls, git_config = None, None, None, None, None
+        filename, download_filename, extract_cmd = None, None, None
+        source_urls, git_config, alt_location = None, None, None
 
         if source is None:
             raise EasyBuildError("fetch_source called with empty 'source' argument")
@@ -387,6 +388,7 @@ def fetch_source(self, source, checksum=None, extension=False, download_instruct
             download_filename = source.pop('download_filename', None)
             source_urls = source.pop('source_urls', None)
             git_config = source.pop('git_config', None)
+            alt_location = source.pop('alt_location', None)
             if source:
                 raise EasyBuildError("Found one or more unexpected keys in 'sources' specification: %s", source)
 
@@ -401,7 +403,7 @@ def fetch_source(self, source, checksum=None, extension=False, download_instruct
         force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES]
         path = self.obtain_file(filename, extension=extension, download_filename=download_filename,
                                 force_download=force_download, urls=source_urls, git_config=git_config,
-                                download_instructions=download_instructions)
+                                download_instructions=download_instructions, alt_location=alt_location)
         if path is None:
             raise EasyBuildError('No file found for source %s', filename)
 
@@ -468,7 +470,9 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None):
             patch_info['postinstall'] = patch_spec in post_install_patches
 
             force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES]
-            path = self.obtain_file(patch_info['name'], extension=extension, force_download=force_download)
+            alt_location = patch_info.pop('alt_location', None)
+            path = self.obtain_file(patch_info['name'], extension=extension, force_download=force_download,
+                                    alt_location=alt_location)
             if path:
                 self.log.debug('File %s found for patch %s', path, patch_spec)
                 patch_info['path'] = path
@@ -517,11 +521,13 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
         for ext in exts_list:
             if isinstance(ext, (list, tuple)) and ext:
                 # expected format: (name, version, options (dict))
-                ext_name = ext[0]
+                # name and version can use templates, resolved via parent EC
+
+                ext_name = resolve_template(ext[0], self.cfg.template_values)
                 if len(ext) == 1:
                     exts_sources.append({'name': ext_name})
                 else:
-                    ext_version = ext[1]
+                    ext_version = resolve_template(ext[1], self.cfg.template_values)
 
                     # make sure we grab *raw* dict of default options for extension,
                     # since it may use template values like %(name)s & %(version)s
@@ -687,7 +693,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
         return exts_sources
 
     def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False,
-                    git_config=None, download_instructions=None):
+                    git_config=None, download_instructions=None, alt_location=None):
         """
         Locate the file with the given name
         - searches in different subdirectories of source path
@@ -698,11 +704,18 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
         :param download_filename: filename with which the file should be downloaded, and then renamed to <filename>
         :param force_download: always try to download file, even if it's already available in source path
         :param git_config: dictionary to define how to download a git repository
+        :param download_instructions: instructions to manually add source (used for complex cases)
+        :param alt_location: alternative location to use instead of self.name
         """
         srcpaths = source_paths()
 
         update_progress_bar(PROGRESS_BAR_DOWNLOAD_ALL, label=filename)
 
+        if alt_location is None:
+            location = self.name
+        else:
+            location = alt_location
+
         # should we download or just try and find it?
         if re.match(r"^(https?|ftp)://", filename):
             # URL detected, so let's try and download it
@@ -711,7 +724,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
             filename = url.split('/')[-1]
 
             # figure out where to download the file to
-            filepath = os.path.join(srcpaths[0], letter_dir_for(self.name), self.name)
+            filepath = os.path.join(srcpaths[0], letter_dir_for(location), location)
             if extension:
                 filepath = os.path.join(filepath, "extensions")
             self.log.info("Creating path %s to download file to" % filepath)
@@ -750,8 +763,8 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
 
             for path in ebpath + common_filepaths + srcpaths:
                 # create list of candidate filepaths
-                namepath = os.path.join(path, self.name)
-                letterpath = os.path.join(path, letter_dir_for(self.name), self.name)
+                namepath = os.path.join(path, location)
+                letterpath = os.path.join(path, letter_dir_for(location), location)
 
                 # most likely paths
                 candidate_filepaths = [
@@ -790,8 +803,8 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
 
                     break  # no need to try other source paths
 
-            name_letter = self.name.lower()[0]
-            targetdir = os.path.join(srcpaths[0], name_letter, self.name)
+            name_letter = location.lower()[0]
+            targetdir = os.path.join(srcpaths[0], name_letter, location)
 
             if foundfile:
                 if self.dry_run:
@@ -808,7 +821,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                 source_urls.extend(self.cfg['source_urls'])
 
                 # add https://sources.easybuild.io as fallback source URL
-                source_urls.append(EASYBUILD_SOURCES_URL + '/' + os.path.join(name_letter, self.name))
+                source_urls.append(EASYBUILD_SOURCES_URL + '/' + os.path.join(name_letter, location))
 
                 mkdir(targetdir, parents=True)
 
@@ -960,7 +973,7 @@ def gen_builddir(self):
         if not self.cfg.get('cleanupoldbuild', False):
             uniq_builddir = builddir
             suff = 0
-            while(os.path.isdir(uniq_builddir)):
+            while os.path.isdir(uniq_builddir):
                 uniq_builddir = "%s.%d" % (builddir, suff)
                 suff += 1
             builddir = uniq_builddir
@@ -1754,7 +1767,13 @@ def install_extensions(self, install=True):
 
         if build_option('parallel_extensions_install'):
             self.log.experimental("installing extensions in parallel")
-            self.install_extensions_parallel(install=install)
+            try:
+                self.install_extensions_parallel(install=install)
+            except NotImplementedError:
+                # If parallel extension install is not supported for this type of extension then install sequentially
+                msg = "Parallel extensions install not supported for %s - using sequential install" % self.name
+                self.log.experimental(msg)
+                self.install_extensions_sequential(install=install)
         else:
             self.install_extensions_sequential(install=install)
 

easybuild/framework/easyconfig/constants.py

@@ -34,17 +34,35 @@
 import platform
 
 from easybuild.base import fancylogger
-from easybuild.tools.systemtools import get_os_name, get_os_type, get_os_version
+from easybuild.tools.build_log import print_warning
+from easybuild.tools.systemtools import KNOWN_ARCH_CONSTANTS, get_os_name, get_os_type, get_os_version
 
 
 _log = fancylogger.getLogger('easyconfig.constants', fname=False)
 
 
 EXTERNAL_MODULE_MARKER = 'EXTERNAL_MODULE'
 
+
+def _get_arch_constant():
+    """
+    Get value for ARCH constant.
+    """
+    arch = platform.uname()[4]
+
+    # macOS on Arm produces 'arm64' rather than 'aarch64'
+    if arch == 'arm64':
+        arch = 'aarch64'
+
+    if arch not in KNOWN_ARCH_CONSTANTS:
+        print_warning("Using unknown value for ARCH constant: %s", arch)
+
+    return arch
+
+
 # constants that can be used in easyconfig
 EASYCONFIG_CONSTANTS = {
-    'ARCH': (platform.uname()[4], "CPU architecture of current system (aarch64, x86_64, ppc64le, ...)"),
+    'ARCH': (_get_arch_constant(), "CPU architecture of current system (aarch64, x86_64, ppc64le, ...)"),
     'EXTERNAL_MODULE': (EXTERNAL_MODULE_MARKER, "External module marker"),
     'HOME': (os.path.expanduser('~'), "Home directory ($HOME)"),
     'OS_TYPE': (get_os_type(), "System type (e.g. 'Linux' or 'Darwin')"),

easybuild/framework/easyconfig/format/version.py

@@ -117,7 +117,7 @@ def __bool__(self):
 
     def is_valid(self):
         """Check if this is a valid VersionOperator. Suffix can be anything."""
-        return not(self.version is None or self.operator is None)
+        return not (self.version is None or self.operator is None)
 
     def set(self, versop_str):
         """

easybuild/main.py

@@ -55,7 +55,6 @@
 from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for
 from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, run_contrib_checks, skip_available
 from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak
-from easybuild.tools.build_log import print_warning
 from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option
 from easybuild.tools.containers.common import containerize
 from easybuild.tools.docs import list_software