Merge branch 'develop' into 5.0.x

Author: boegel

easybuild/toolchains/ifbf.py

@@ -0,0 +1,44 @@
+##
+# Copyright 2012-2023 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+"""
+EasyBuild support for ifbf toolchain (includes Intel compilers, FlexiBLAS, and FFTW).
+
+Authors:
+
+* Sebastian Achilles (Juelich Supercomputing Centre)
+"""
+
+from easybuild.toolchains.intel_compilers import IntelCompilersToolchain
+from easybuild.toolchains.fft.fftw import Fftw
+from easybuild.toolchains.linalg.flexiblas import FlexiBLAS
+
+
+class Ifbf(IntelCompilersToolchain, FlexiBLAS, Fftw):
+    """
+    Compiler toolchain with Intel compilers, FlexiBLAS, and FFTW
+    """
+    NAME = 'ifbf'
+    SUBTOOLCHAIN = IntelCompilersToolchain.NAME
+    OPTIONAL = True

easybuild/toolchains/iofbf.py

@@ -0,0 +1,47 @@
+##
+# Copyright 2012-2023 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+"""
+EasyBuild support for iofbf toolchain (includes Intel compilers, OpenMPI,
+FlexiBLAS, LAPACK, ScaLAPACK and FFTW).
+
+Authors:
+
+* Sebastian Achilles (Juelich Supercomputing Centre)
+"""
+
+from easybuild.toolchains.iompi import Iompi
+from easybuild.toolchains.ifbf import Ifbf
+from easybuild.toolchains.fft.fftw import Fftw
+from easybuild.toolchains.linalg.flexiblas import FlexiBLAS
+from easybuild.toolchains.linalg.scalapack import ScaLAPACK
+
+
+class Iofbf(Iompi, FlexiBLAS, ScaLAPACK, Fftw):
+    """
+    Compiler toolchain with Intel compilers (icc/ifort), OpenMPI,
+    FlexiBLAS, LAPACK, ScaLAPACK and FFTW.
+    """
+    NAME = 'iofbf'
+    SUBTOOLCHAIN = [Iompi.NAME, Ifbf.NAME]

easybuild/tools/config.py

@@ -300,6 +300,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'sequential',
         'set_default_module',
         'set_gid_bit',
+        'silence_hook_trigger',
         'skip_extensions',
         'skip_test_cases',
         'skip_test_step',

easybuild/tools/filetools.py

@@ -2591,6 +2591,8 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
     recursive = git_config.pop('recursive', False)
     clone_into = git_config.pop('clone_into', False)
     keep_git_dir = git_config.pop('keep_git_dir', False)
+    extra_config_params = git_config.pop('extra_config_params', None)
+    recurse_submodules = git_config.pop('recurse_submodules', None)
 
     # input validation of git_config dict
     if git_config:
@@ -2616,7 +2618,11 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
     targetpath = os.path.join(targetdir, filename)
 
     # compose 'git clone' command, and run it
-    clone_cmd = ['git', 'clone']
+    if extra_config_params:
+        git_cmd = 'git ' + ' '.join(['-c %s' % param for param in extra_config_params])
+    else:
+        git_cmd = 'git'
+    clone_cmd = [git_cmd, 'clone']
 
     if not keep_git_dir and not commit:
         # Speed up cloning by only fetching the most recent commit, not the whole history
@@ -2627,6 +2633,8 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
         clone_cmd.extend(['--branch', tag])
         if recursive:
             clone_cmd.append('--recursive')
+        if recurse_submodules:
+            clone_cmd.extend(["--recurse-submodules='%s'" % pat for pat in recurse_submodules])
     else:
         # checkout is done separately below for specific commits
         clone_cmd.append('--no-checkout')
@@ -2646,17 +2654,22 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
 
     # if a specific commit is asked for, check it out
     if commit:
-        checkout_cmd = ['git', 'checkout', commit]
-        if recursive:
-            checkout_cmd.extend(['&&', 'git', 'submodule', 'update', '--init', '--recursive'])
+        checkout_cmd = [git_cmd, 'checkout', commit]
+
+        if recursive or recurse_submodules:
+            checkout_cmd.extend(['&&', git_cmd, 'submodule', 'update', '--init'])
+            if recursive:
+                checkout_cmd.append('--recursive')
+            if recurse_submodules:
+                checkout_cmd.extend(["--recurse-submodules='%s'" % pat for pat in recurse_submodules])
 
         work_dir = os.path.join(tmpdir, repo_name) if repo_name else tmpdir
         run_shell_cmd(' '.join(checkout_cmd), work_dir=work_dir, hidden=True, verbose_dry_run=True)
 
     elif not build_option('extended_dry_run'):
         # If we wanted to get a tag make sure we actually got a tag and not a branch with the same name
         # This doesn't make sense in dry-run mode as we don't have anything to check
-        cmd = "git describe --exact-match --tags HEAD"
+        cmd = f"{git_cmd} describe --exact-match --tags HEAD"
         work_dir = os.path.join(tmpdir, repo_name) if repo_name else tmpdir
         res = run_shell_cmd(cmd, fail_on_error=False, work_dir=work_dir, hidden=True, verbose_dry_run=True)
 
@@ -2671,13 +2684,17 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
                 # make the repo unshallow first;
                 # this is equivalent with 'git fetch -unshallow' in Git 1.8.3+
                 # (first fetch seems to do nothing, unclear why)
-                cmds.append("git fetch --depth=2147483647 && git fetch --depth=2147483647")
+                cmds.append(f"{git_cmd} fetch --depth=2147483647 && git fetch --depth=2147483647")
 
-            cmds.append(f"git checkout refs/tags/{tag}")
+            cmds.append(f"{git_cmd} checkout refs/tags/{tag}")
             # Clean all untracked files, e.g. from left-over submodules
-            cmds.append("git clean --force -d -x")
+            cmds.append(f"{git_cmd} clean --force -d -x")
             if recursive:
-                cmds.append("git submodule update --init --recursive")
+                cmds.append(f"{git_cmd} submodule update --init --recursive")
+            elif recurse_submodules:
+                cmds.append(f"{git_cmd} submodule update --init ")
+                cmds[-1] += ' '.join(["--recurse-submodules='%s'" % pat for pat in recurse_submodules])
+
             for cmd in cmds:
                 run_shell_cmd(cmd, work_dir=work_dir, hidden=True, verbose_dry_run=True)
 

easybuild/tools/github.py

@@ -2251,15 +2251,18 @@ def install_github_token(github_user, silent=False):
 def validate_github_token(token, github_user):
     """
     Check GitHub token:
-    * see if it conforms expectations (only [a-f]+[0-9] characters, length of 40)
-    * see if it can be used for authenticated access
+   * see if it conforms expectations (classic GitHub token with only [0-9a-f] characters
+     and length of 40 starting with 'ghp_', or fine-grained GitHub token with only
+     alphanumeric ([a-zA-Z0-9]) characters + '_' and length of 93 starting with 'github_pat_'),
+    * see if it can be used for authenticated access.
     """
     # cfr. https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/
     token_regex = re.compile('^ghp_[a-zA-Z0-9]{36}$')
     token_regex_old_format = re.compile('^[0-9a-f]{40}$')
+    # https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/about-authentication-to-github#githubs-token-formats
+    token_regex_fine_grained = re.compile('github_pat_[a-zA-Z0-9_]{82}')
 
-    # token should be 40 characters long, and only contain characters in [0-9a-f]
-    sanity_check = bool(token_regex.match(token))
+    sanity_check = bool(token_regex.match(token)) or bool(token_regex_fine_grained.match(token))
     if sanity_check:
         _log.info("Sanity check on token passed")
     else:

easybuild/tools/hooks.py

@@ -241,7 +241,7 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None,
 
         if msg is None:
             msg = "Running %s hook..." % label
-        if build_option('debug'):
+        if build_option('debug') and not build_option('silence_hook_trigger'):
             print_msg(msg)
 
         _log.info("Running '%s' hook function (args: %s, keyword args: %s)...", hook.__name__, args, kwargs)

easybuild/tools/module_generator.py

@@ -621,7 +621,8 @@ def _generate_extensions_list(self):
         """
         Generate a list of all extensions in name/version format
         """
-        return self.app.make_extension_string(name_version_sep='/', ext_sep=',').split(',')
+        exts_str = self.app.make_extension_string(name_version_sep='/', ext_sep=',')
+        return exts_str.split(',') if exts_str else []
 
     def _generate_help_text(self):
         """

easybuild/tools/options.py

@@ -495,6 +495,8 @@ def override_options(self):
             'silence-deprecation-warnings': (
                 "Silence specified deprecation warnings out of (%s)" % ', '.join(all_deprecations),
                 'strlist', 'extend', []),
+            'silence-hook-trigger': ("Supress printing of debug message every time a hook is triggered",
+                                     None, 'store_true', False),
             'skip-extensions': ("Skip installation of extensions", None, 'store_true', False),
             'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'),
             'skip-test-step': ("Skip running the test step (e.g. unit tests)", None, 'store_true', False),

test/framework/filetools.py

@@ -2815,6 +2815,32 @@ def run_check():
         ]) % git_repo
         run_check()
 
+        git_config['recurse_submodules'] = ['!vcflib', '!sdsl-lite']
+        expected = '\n'.join([
+            '  running command "git clone --depth 1 --branch tag_for_tests --recursive'
+            + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"',
+            r"  \(in .*/tmp.*\)",
+            r'  running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
+            r"  \(in .*/tmp.*\)",
+        ]) % git_repo
+        run_check()
+
+        git_config['extra_config_params'] = [
+            'submodule."fastahack".active=false',
+            'submodule."sha1".active=false',
+        ]
+        expected = '\n'.join([
+            '  running command "git -c submodule."fastahack".active=false -c submodule."sha1".active=false'
+            + ' clone --depth 1 --branch tag_for_tests --recursive'
+            + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"',
+            r"  \(in .*/tmp.*\)",
+            r'  running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
+            r"  \(in .*/tmp.*\)",
+        ]) % git_repo
+        run_check()
+        del git_config['recurse_submodules']
+        del git_config['extra_config_params']
+
         git_config['keep_git_dir'] = True
         expected = '\n'.join([
             r'  running command "git clone --branch tag_for_tests --recursive %(git_repo)s"',
@@ -2837,7 +2863,20 @@ def run_check():
         ]) % git_repo
         run_check()
 
+        git_config['recurse_submodules'] = ['!vcflib', '!sdsl-lite']
+        expected = '\n'.join([
+            r'  running command "git clone --no-checkout %(git_repo)s"',
+            r"  \(in .*/tmp.*\)",
+            '  running command "git checkout 8456f86 && git submodule update --init --recursive'
+            + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\'"',
+            r"  \(in /.*/testrepository\)",
+            r'  running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
+            r"  \(in .*/tmp.*\)",
+        ]) % git_repo
+        run_check()
+
         del git_config['recursive']
+        del git_config['recurse_submodules']
         expected = '\n'.join([
             r'  running command "git clone --no-checkout %(git_repo)s"',
             r"  \(in /.*\)",

test/framework/github.py

@@ -655,6 +655,11 @@ def test_validate_github_token(self):
         if token_old_format:
             self.assertTrue(gh.validate_github_token(token_old_format, GITHUB_TEST_ACCOUNT))
 
+        # if a fine-grained token is available, test with that too
+        finegrained_token = os.getenv('TEST_GITHUB_TOKEN_FINEGRAINED')
+        if finegrained_token:
+            self.assertTrue(gh.validate_github_token(finegrained_token, GITHUB_TEST_ACCOUNT))
+
     def test_github_find_easybuild_easyconfig(self):
         """Test for find_easybuild_easyconfig function"""
         if self.skip_github_tests: