Merge pull request #4023 from easybuilders/4.5.x

release EasyBuild v4.5.5

Author: SebastianAchilles

.github/workflows/eb_command.yml

@@ -73,15 +73,15 @@ jobs:
           pymajver=$(python -c 'import sys; print(sys.version_info[0])')
           pymajminver=$(python -c 'import sys; print(".".join(str(x) for x in sys.version_info[:2]))')
           # check patterns in verbose output
-          for pattern in "^>> Considering .python.\.\.\." "^>> .python. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> 'python' is able to import 'easybuild.main', so retaining it" "^>> Selected Python command: python \(.*/bin/python\)" "^This is EasyBuild 4\.[0-9.]\+"; do
+          for pattern in "^>> Considering .python.\.\.\." "^>> .python. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> 'python' is able to import 'easybuild.framework', so retaining it" "^>> Selected Python command: python \(.*/bin/python\)" "^This is EasyBuild 4\.[0-9.]\+"; do
               echo "Looking for pattern \"${pattern}\" in eb_version.out..."
               grep "$pattern" eb_version.out
           done
           # also check when specifying Python command via $EB_PYTHON
           for eb_python in "python${pymajver}" "python${pymajminver}"; do
               export EB_PYTHON="${eb_python}"
               eb --version | tee eb_version.out 2>&1
-              for pattern in "^>> Considering .${eb_python}.\.\.\." "^>> .${eb_python}. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> '${eb_python}' is able to import 'easybuild.main', so retaining it" "^>> Selected Python command: ${eb_python} \(.*/bin/${eb_python}\)" "^This is EasyBuild 4\.[0-9.]\+"; do
+              for pattern in "^>> Considering .${eb_python}.\.\.\." "^>> .${eb_python}. version: ${pymajminver}\.[0-9]\+, which matches Python ${pymajver} version requirement" "^>> '${eb_python}' is able to import 'easybuild.framework', so retaining it" "^>> Selected Python command: ${eb_python} \(.*/bin/${eb_python}\)" "^This is EasyBuild 4\.[0-9.]\+"; do
                   echo "Looking for pattern \"${pattern}\" in eb_version.out..."
                   grep "$pattern" eb_version.out
               done

.github/workflows/unit_tests.yml

@@ -200,7 +200,7 @@ jobs:
           # run test suite
           python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
           # try and make sure output of running tests is clean (no printed messages/warnings)
-          IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.5|from cryptography.*default_backend|CryptographyDeprecationWarning: Python 2|from cryptography.utils import int_from_bytes"
+          IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.*default_backend|CryptographyDeprecationWarning: Python 2|from cryptography.utils import int_from_bytes|Blowfish"
           # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches)
           PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
           test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite\n${PRINTED_MSG}" && exit 1)

RELEASE_NOTES

@@ -4,6 +4,27 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.5.5 (June 8th 2022)
+----------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - add toolchain definitions for nvompi (NVHPC + OpenMPI) (#3969), nvpsmpi (NVHPC + ParaStationMPI) (#3970), gfbf (GCC, FlexiBLAS, FFTW) (#4006)
+  - add support for FFTW.MPI toolchain component ($FFT*DIR variables) (#4012)
+  - add support for customizing EasyBuild command used in jobs via --job-eb-cmd (#4016)
+- various bug fixes, including:
+  - fix copying of easyconfig file with --copy-ec without --rebuild if module is already installed (#3993)
+  - ignore deprecation warnings regarding cryptography in Python 3.6 + Blowfish with Python 3.10 in test suite output (#3999)
+  - fix typo in debug log message in easyblock.py (#4000)
+  - fix printing of list of attempted download URLs when url-encoded characters are used in URL (#4005)
+  - set $FFT(W)_LIB_DIR to imkl-FFTW's lib path in build environment if usempi toolchain option is enabled (#4011)
+  - correctly identify Apple Silicon M1 as Arm 64-bit by also considering arm64 next to aarch64 (#4014)
+  - fix 'eb --show-system-info' on Apple M1 system (#4015)
+- other changes:
+  - change 'eb' command to import easybuild.framework to check if EasyBuild framework is available (#3995, #3998)
+
+
 v4.5.4 (March 31st 2022)
 ------------------------
 

easybuild/framework/easyblock.py

@@ -883,8 +883,10 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                         error_msg += "please follow the download instructions above, and make the file available "
                         error_msg += "in the active source path (%s)" % ':'.join(source_paths())
                     else:
+                        # flatten list to string with '%' characters escaped (literal '%' desired in 'sprintf')
+                        failedpaths_msg = ', '.join(failedpaths).replace('%', '%%')
                         error_msg += "and downloading it didn't work either... "
-                        error_msg += "Paths attempted (in order): %s " % ', '.join(failedpaths)
+                        error_msg += "Paths attempted (in order): %s " % failedpaths_msg
 
                     raise EasyBuildError(error_msg, filename)
 
@@ -3286,7 +3288,7 @@ def _sanity_check_step_dry_run(self, custom_paths=None, custom_commands=None, **
         if self.toolchain.use_rpath:
             self.sanity_check_rpath()
         else:
-            self.log.debug("Skiping RPATH sanity check")
+            self.log.debug("Skipping RPATH sanity check")
 
     def _sanity_check_step_extensions(self):
         """Sanity check on extensions (if any)."""
@@ -3452,7 +3454,7 @@ def xs2str(xs):
                 self.log.warning("RPATH sanity check failed!")
                 self.sanity_check_fail_msgs.extend(rpath_fails)
         else:
-            self.log.debug("Skiping RPATH sanity check")
+            self.log.debug("Skipping RPATH sanity check")
 
         # pass or fail
         if self.sanity_check_fail_msgs:

easybuild/main.py

@@ -447,7 +447,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     forced = options.force or options.rebuild
     dry_run_mode = options.dry_run or options.dry_run_short or options.missing_modules
 
-    keep_available_modules = forced or dry_run_mode or options.extended_dry_run or pr_options
+    keep_available_modules = forced or dry_run_mode or options.extended_dry_run or pr_options or options.copy_ec
     keep_available_modules = keep_available_modules or options.inject_checksums or options.sanity_check_only
 
     # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping

easybuild/toolchains/fft/fftw.py

@@ -33,6 +33,7 @@
 
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.toolchain.fft import Fft
+from easybuild.tools.modules import get_software_root
 
 
 class Fftw(Fft):
@@ -56,6 +57,11 @@ def _set_fftw_variables(self):
         fftw_libs = ["fftw%s" % suffix]
         if self.options.get('usempi', False):
             fftw_libs.insert(0, "fftw%s_mpi" % suffix)
+            fftwmpiroot = get_software_root('FFTW.MPI')
+            if fftwmpiroot:
+                # get libfft%_mpi via the FFTW.MPI module
+                self.FFT_MODULE_NAME = ['FFTW.MPI']
+
         fftw_libs_mt = ["fftw%s" % suffix]
         if self.options.get('openmp', False):
             fftw_libs_mt.insert(0, "fftw%s_omp" % suffix)

easybuild/toolchains/fft/intelfftw.py

@@ -49,7 +49,10 @@ def _set_fftw_variables(self):
         if not hasattr(self, 'BLAS_LIB_DIR'):
             raise EasyBuildError("_set_fftw_variables: IntelFFT based on IntelMKL (no BLAS_LIB_DIR found)")
 
+        imklroot = get_software_root(self.FFT_MODULE_NAME[0])
         imklver = get_software_version(self.FFT_MODULE_NAME[0])
+        self.FFT_LIB_DIR = self.BLAS_LIB_DIR
+        self.FFT_INCLUDE_DIR = [os.path.join(d, 'fftw') for d in self.BLAS_INCLUDE_DIR]
 
         picsuff = ''
         if self.options.get('pic', None):
@@ -69,54 +72,47 @@ def _set_fftw_variables(self):
             raise EasyBuildError(error_msg)
 
         interface_lib = "fftw3xc%s%s" % (compsuff, picsuff)
-        fftw_libs = [interface_lib]
-        cluster_interface_lib = None
+        fft_lib_dirs = [os.path.join(imklroot, d) for d in self.FFT_LIB_DIR]
+
+        def fftw_lib_exists(libname):
+            """Helper function to check whether FFTW library with specified name exists."""
+            return any(os.path.exists(os.path.join(d, "lib%s.a" % libname)) for d in fft_lib_dirs)
+
+        # interface libs can be optional:
+        # MKL >= 10.2 include fftw3xc and fftw3xf interfaces in LIBBLAS=libmkl_gf/libmkl_intel
+        # See https://software.intel.com/en-us/articles/intel-mkl-main-libraries-contain-fftw3-interfaces
+        # The cluster interface libs (libfftw3x_cdft*) can be omitted if the toolchain does not provide MPI-FFTW
+        # interfaces.
+        fftw_libs = []
+        if fftw_lib_exists(interface_lib) or LooseVersion(imklver) < LooseVersion("10.2"):
+            fftw_libs = [interface_lib]
+
         if self.options.get('usempi', False):
             # add cluster interface for recent imkl versions
-            if LooseVersion(imklver) >= LooseVersion('10.3'):
+            # only get cluster_interface_lib from seperate module imkl-FFTW, rest via libmkl_gf/libmkl_intel
+            imklfftwroot = get_software_root('imkl-FFTW')
+            if LooseVersion(imklver) >= LooseVersion('10.3') and (fftw_libs or imklfftwroot):
                 suff = picsuff
                 if LooseVersion(imklver) >= LooseVersion('11.0.2'):
                     suff = bitsuff + suff
                 cluster_interface_lib = 'fftw3x_cdft%s' % suff
                 fftw_libs.append(cluster_interface_lib)
             fftw_libs.append("mkl_cdft_core")  # add cluster dft
             fftw_libs.extend(self.variables['LIBBLACS'].flatten())  # add BLACS; use flatten because ListOfList
+            if imklfftwroot:
+                fft_lib_dirs += [os.path.join(imklfftwroot, 'lib')]
+                self.FFT_LIB_DIR = [os.path.join(imklfftwroot, 'lib')]
 
         fftw_mt_libs = fftw_libs + [x % self.BLAS_LIB_MAP for x in self.BLAS_LIB_MT]
 
         self.log.debug('fftw_libs %s' % fftw_libs.__repr__())
         fftw_libs.extend(self.variables['LIBBLAS'].flatten())  # add BLAS libs (contains dft)
         self.log.debug('fftw_libs %s' % fftw_libs.__repr__())
 
-        self.FFT_LIB_DIR = self.BLAS_LIB_DIR
-        self.FFT_INCLUDE_DIR = [os.path.join(d, 'fftw') for d in self.BLAS_INCLUDE_DIR]
-
         # building the FFTW interfaces is optional,
         # so make sure libraries are there before FFT_LIB is set
-        imklroot = get_software_root(self.FFT_MODULE_NAME[0])
-        fft_lib_dirs = [os.path.join(imklroot, d) for d in self.FFT_LIB_DIR]
-        imklfftwroot = get_software_root('imkl-FFTW')
-        if imklfftwroot:
-            # only get cluster_interface_lib from seperate module imkl-FFTW, rest via libmkl_gf/libmkl_intel
-            fft_lib_dirs += [os.path.join(imklfftwroot, 'lib')]
-            fftw_libs.remove(interface_lib)
-            fftw_mt_libs.remove(interface_lib)
-
-        def fftw_lib_exists(libname):
-            """Helper function to check whether FFTW library with specified name exists."""
-            return any(os.path.exists(os.path.join(d, "lib%s.a" % libname)) for d in fft_lib_dirs)
-
-        if not fftw_lib_exists(interface_lib) and LooseVersion(imklver) >= LooseVersion("10.2"):
-            # interface libs can be optional:
-            # MKL >= 10.2 include fftw3xc and fftw3xf interfaces in LIBBLAS=libmkl_gf/libmkl_intel
-            # See https://software.intel.com/en-us/articles/intel-mkl-main-libraries-contain-fftw3-interfaces
-            # The cluster interface libs (libfftw3x_cdft*) can be omitted if the toolchain does not provide MPI-FFTW
-            # interfaces.
-            fftw_libs = [lib for lib in fftw_libs if lib not in [interface_lib, cluster_interface_lib]]
-            fftw_mt_libs = [lib for lib in fftw_mt_libs if lib not in [interface_lib, cluster_interface_lib]]
-
         # filter out libraries from list of FFTW libraries to check for if they are not provided by Intel MKL
-        check_fftw_libs = [lib for lib in fftw_libs if lib not in ['dl', 'gfortran']]
+        check_fftw_libs = [lib for lib in fftw_libs + fftw_mt_libs if lib not in ['dl', 'gfortran']]
 
         missing_fftw_libs = [lib for lib in check_fftw_libs if not fftw_lib_exists(lib)]
         if missing_fftw_libs:
@@ -128,5 +124,4 @@ def fftw_lib_exists(libname):
                 raise EasyBuildError(msg)
         else:
             self.FFT_LIB = fftw_libs
-
-        self.FFT_LIB_MT = fftw_mt_libs
+            self.FFT_LIB_MT = fftw_mt_libs

easybuild/toolchains/gfbf.py

@@ -0,0 +1,41 @@
+##
+# Copyright 2021-2022 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+"""
+EasyBuild support for gfbf compiler toolchain (includes GCC, FlexiBLAS and FFTW)
+
+:author: Kenneth Hoste (Ghent University)
+:author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada)
+"""
+
+from easybuild.toolchains.gcc import GccToolchain
+from easybuild.toolchains.fft.fftw import Fftw
+from easybuild.toolchains.linalg.flexiblas import FlexiBLAS
+
+
+class Gfbf(GccToolchain, FlexiBLAS, Fftw):
+    """Compiler toolchain with GCC, FlexiBLAS and FFTW."""
+    NAME = 'gfbf'
+    SUBTOOLCHAIN = GccToolchain.NAME
+    OPTIONAL = True

easybuild/toolchains/nvompi.py

@@ -0,0 +1,38 @@
+##
+# Copyright 2013-2021 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+"""
+EasyBuild support for nvompi compiler toolchain (NVHPC + Open MPI).
+
+:author: Robert Mijakovic <[email protected]> (LuxProvide)
+"""
+
+from easybuild.toolchains.nvhpc import NVHPCToolchain
+from easybuild.toolchains.mpi.openmpi import OpenMPI
+
+
+class Nvompi(NVHPCToolchain, OpenMPI):
+    """Compiler toolchain with NVHPC and Open MPI."""
+    NAME = 'nvompi'
+    SUBTOOLCHAIN = NVHPCToolchain.NAME

easybuild/toolchains/nvpsmpi.py

@@ -0,0 +1,40 @@
+##
+# Copyright 2016-2021 Ghent University
+# Copyright 2016-2021 Forschungszentrum Juelich
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+"""
+EasyBuild support for nvsmpi compiler toolchain (includes NVHPC and ParaStationMPI).
+
+:author: Robert Mijakovic <[email protected]> (LuxProvide)
+"""
+
+from easybuild.toolchains.nvhpc import NVHPCToolchain
+from easybuild.toolchains.mpi.psmpi import Psmpi
+
+
+# Order matters!
+class NVpsmpi(NVHPCToolchain, Psmpi):
+    """Compiler toolchain with NVHPC and ParaStationMPI."""
+    NAME = 'nvpsmpi'
+    SUBTOOLCHAIN = NVHPCToolchain.NAME