Merge branch 'develop' into 5.0.x

Author: boegel

RELEASE_NOTES

@@ -4,6 +4,34 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.8.1 (11 September 2023)
+--------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - add end-to-end test for running EasyBuild in different Linux distros using containers (#3968)
+  - suggest default title in `--review-pr` (#4287)
+  - add `build_and_install_loop` hooks to run before and after the install loop for individual easyconfigs (#4304)
+  - implement support for `cancel_hook` and `fail_hook` (#4315, #4325)
+  - add postiter hook to the list of steps so the corresponding hook can be used (#4316)
+  - add `run_shell_cmd` hook (#4323)
+  - add `build_info_msg` easyconfig parameter to print message during installation of an easyconfig (#4324)
+  - add `--silence-hook-trigger` configuration option to supress printing of debug message every time a hook is triggered (#4329)
+  - add support for using fine grained Github tokens (#4332)
+  - add definitions for ifbf and iofbf toolchain (#4337)
+  - add support for submodule filtering and specifying extra Git configuration in `git_config` (#4338, #4339)
+- various bug fixes, including:
+  - improve error when checksum dict has no entry for a file (#4150)
+  - avoid error being logged when `checksums.json` is not found (#4261)
+  - don't fail in `mkdir` if path gets created while processing it (#4300, #4328)
+  - ignore request for external module (meta)data when no modules tool is active (#4308)
+  - use sys.executable to obtain path to `python` command in tests, rather than assuming that `python` command is available in `$PATH` (#4309)
+  - fix `test_add_and_remove_module_path` by replacing string comparison of paths by checking whether they point to the same path (since symlinks may cause trouble) (#4312)
+  - enhance `Toolchain.get_flag` to handle lists (#4319)
+  - only add extensions in module file if there are extensions (#4331)
+
+
 v4.8.0 (7 July 2023)
 --------------------
 

easybuild/framework/easyblock.py

@@ -394,13 +394,13 @@ def get_checksums_from_json(self, always_read=False):
         :param always_read: always read the checksums.json file, even if it has been read before
         """
         if always_read or self.json_checksums is None:
-            try:
-                path = self.obtain_file("checksums.json", no_download=True)
+            path = self.obtain_file("checksums.json", no_download=True, warning_only=True)
+            if path is not None:
                 self.log.info("Loading checksums from file %s", path)
                 json_txt = read_file(path)
                 self.json_checksums = json.loads(json_txt)
-            # if the file can't be found, return an empty dict
-            except EasyBuildError:
+            else:
+                # if the file can't be found, return an empty dict
                 self.json_checksums = {}
 
         return self.json_checksums
@@ -727,7 +727,8 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
         return exts_sources
 
     def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False,
-                    git_config=None, no_download=False, download_instructions=None, alt_location=None):
+                    git_config=None, no_download=False, download_instructions=None, alt_location=None,
+                    warning_only=False):
         """
         Locate the file with the given name
         - searches in different subdirectories of source path
@@ -780,7 +781,13 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                     return fullpath
 
             except IOError as err:
-                raise EasyBuildError("Downloading file %s from url %s to %s failed: %s", filename, url, fullpath, err)
+                if not warning_only:
+                    raise EasyBuildError("Downloading file %s "
+                                         "from url %s to %s failed: %s", filename, url, fullpath, err)
+                else:
+                    self.log.warning("Downloading file %s "
+                                     "from url %s to %s failed: %s", filename, url, fullpath, err)
+                    return None
 
         else:
             # try and find file in various locations
@@ -857,8 +864,13 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                     self.dry_run_msg("  * %s (MISSING)", filename)
                     return filename
                 else:
-                    raise EasyBuildError("Couldn't find file %s anywhere, and downloading it is disabled... "
+                    if not warning_only:
+                        raise EasyBuildError("Couldn't find file %s anywhere, and downloading it is disabled... "
+                                             "Paths attempted (in order): %s ", filename, ', '.join(failedpaths))
+                    else:
+                        self.log.warning("Couldn't find file %s anywhere, and downloading it is disabled... "
                                          "Paths attempted (in order): %s ", filename, ', '.join(failedpaths))
+                        return None
             elif git_config:
                 return get_source_tarball_from_git(filename, targetdir, git_config)
             else:
@@ -950,7 +962,11 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                         error_msg += "and downloading it didn't work either... "
                         error_msg += "Paths attempted (in order): %s " % failedpaths_msg
 
-                    raise EasyBuildError(error_msg, filename)
+                    if not warning_only:
+                        raise EasyBuildError(error_msg, filename)
+                    else:
+                        self.log.warning(error_msg, filename)
+                        return None
 
     #
     # GETTER/SETTER UTILITY FUNCTIONS

easybuild/tools/filetools.py

@@ -1252,14 +1252,11 @@ def verify_checksum(path, checksums):
 
     for checksum in checksums:
         if isinstance(checksum, dict):
-            if filename in checksum:
+            try:
                 # Set this to a string-type checksum
                 checksum = checksum[filename]
-            elif build_option('enforce_checksums'):
-                raise EasyBuildError("Missing checksum for %s", filename)
-            else:
-                # Set to None and allow to fail elsewhere
-                checksum = None
+            except KeyError:
+                raise EasyBuildError("Missing checksum for %s in %s", filename, checksum)
 
         if isinstance(checksum, str):
             # if no checksum type is specified, it is assumed to be MD5 (32 characters) or SHA256 (64 characters)
@@ -1289,7 +1286,8 @@ def verify_checksum(path, checksums):
                 # no matching checksums
                 return False
         else:
-            raise EasyBuildError("Invalid checksum spec '%s', should be a string (MD5) or 2-tuple (type, value).",
+            raise EasyBuildError("Invalid checksum spec '%s': should be a string (MD5 or SHA256), "
+                                 "2-tuple (type, value), or tuple of alternative checksum specs.",
                                  checksum)
 
         actual_checksum = compute_checksum(path, typ)

easybuild/tools/options.py

@@ -495,7 +495,7 @@ def override_options(self):
             'silence-deprecation-warnings': (
                 "Silence specified deprecation warnings out of (%s)" % ', '.join(all_deprecations),
                 'strlist', 'extend', []),
-            'silence-hook-trigger': ("Supress printing of debug message every time a hook is triggered",
+            'silence-hook-trigger': ("Suppress printing of debug message every time a hook is triggered",
                                      None, 'store_true', False),
             'skip-extensions': ("Skip installation of extensions", None, 'store_true', False),
             'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'),

test/framework/easyblock.py

@@ -39,6 +39,7 @@
 from unittest import TextTestRunner
 
 import easybuild.tools.systemtools as st
+from easybuild.base import fancylogger
 from easybuild.framework.easyblock import EasyBlock, get_easyblock_instance
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.framework.easyconfig.easyconfig import EasyConfig
@@ -2456,6 +2457,29 @@ def test_checksum_step(self):
         eb.fetch_sources()
         eb.checksum_step()
 
+        with self.mocked_stdout_stderr() as (stdout, stderr):
+
+            # using checksum-less test easyconfig in location that does not provide checksums.json
+            test_ec = os.path.join(self.test_prefix, 'test-no-checksums.eb')
+            copy_file(toy_ec, test_ec)
+            write_file(test_ec, 'checksums = []', append=True)
+            ec = process_easyconfig(test_ec)[0]
+
+            # enable logging to screen, so we can check whether error is logged when checksums.json is not found
+            fancylogger.logToScreen(enable=True, stdout=True)
+
+            eb = get_easyblock_instance(ec)
+            eb.fetch_sources()
+            eb.checksum_step()
+
+            fancylogger.logToScreen(enable=False, stdout=True)
+            stdout = self.get_stdout()
+
+            # make sure there's no error logged for not finding checksums.json,
+            # see also https://github.com/easybuilders/easybuild-framework/issues/4301
+            regex = re.compile("ERROR .*Couldn't find file checksums.json anywhere", re.M)
+            self.assertFalse(regex.search(stdout), "Pattern '%s' should not be found in log" % regex.pattern)
+
         # fiddle with checksum to check whether faulty checksum is catched
         copy_file(toy_ec, self.test_prefix)
         toy_ec = os.path.join(self.test_prefix, os.path.basename(toy_ec))

test/framework/filetools.py

@@ -351,6 +351,14 @@ def test_checksums(self):
         alt_checksums = ('7167b64b1ca062b9674ffef46f9325db7167b64b1ca062b9674ffef46f9325db', broken_checksums['sha256'])
         self.assertFalse(ft.verify_checksum(fp, alt_checksums))
 
+        # Check dictionary
+        alt_checksums = (known_checksums['sha256'],)
+        self.assertTrue(ft.verify_checksum(fp, {os.path.basename(fp): known_checksums['sha256']}))
+        faulty_dict = {'wrong-name': known_checksums['sha256']}
+        self.assertErrorRegex(EasyBuildError,
+                              "Missing checksum for " + os.path.basename(fp) + " in .*wrong-name.*",
+                              ft.verify_checksum, fp, faulty_dict)
+
         # check whether missing checksums are enforced
         build_options = {
             'enforce_checksums': True,
@@ -365,6 +373,8 @@ def test_checksums(self):
         for checksum in [known_checksums[x] for x in ('md5', 'sha256')]:
             dict_checksum = {os.path.basename(fp): checksum, 'foo': 'baa'}
             self.assertTrue(ft.verify_checksum(fp, dict_checksum))
+            del dict_checksum[os.path.basename(fp)]
+            self.assertErrorRegex(EasyBuildError, "Missing checksum for", ft.verify_checksum, fp, dict_checksum)
 
     def test_common_path_prefix(self):
         """Test get common path prefix for a list of paths."""