Merge pull request #4724 from boegel/5.0.x

sync with develop (20241218) + version bump to `5.0.0beta1`

Author: lexming

easybuild/framework/easyblock.py

@@ -3187,21 +3187,24 @@ def post_processing_step(self):
             )
         return self.post_install_step()
 
+    def _dispatch_sanity_check_step(self, *args, **kwargs):
+        """Decide whether to run the dry-run or the real version of the sanity-check step"""
+        if self.dry_run:
+            self._sanity_check_step_dry_run(*args, **kwargs)
+        else:
+            self._sanity_check_step(*args, **kwargs)
+
     def sanity_check_step(self, *args, **kwargs):
         """
         Do a sanity check on the installation
         - if *any* of the files/subdirectories in the installation directory listed
           in sanity_check_paths are non-existent (or empty), the sanity check fails
         """
-        if self.dry_run:
-            self._sanity_check_step_dry_run(*args, **kwargs)
-
         # handling of extensions that were installed for multiple dependency versions is done in ExtensionEasyBlock
-        elif self.cfg['multi_deps'] and not self.is_extension:
+        if self.cfg['multi_deps'] and not self.is_extension:
             self._sanity_check_step_multi_deps(*args, **kwargs)
-
         else:
-            self._sanity_check_step(*args, **kwargs)
+            self._dispatch_sanity_check_step(*args, **kwargs)
 
     def _sanity_check_step_multi_deps(self, *args, **kwargs):
         """Perform sanity check for installations that iterate over a list a versions for particular dependencies."""
@@ -3233,7 +3236,7 @@ def _sanity_check_step_multi_deps(self, *args, **kwargs):
             self.log.info(info_msg)
 
             kwargs['extra_modules'] = extra_modules
-            self._sanity_check_step(*args, **kwargs)
+            self._dispatch_sanity_check_step(*args, **kwargs)
 
         # restore list of lists of build dependencies & stop iterating again
         self.cfg['builddependencies'] = builddeps
@@ -3509,7 +3512,7 @@ def _sanity_check_step_common(self, custom_paths, custom_commands):
             # if no sanity_check_paths are specified in easyconfig,
             # we fall back to the ones provided by the easyblock via custom_paths
             if custom_paths:
-                paths = custom_paths
+                paths = self.cfg.resolve_template(custom_paths)
                 self.log.info("Using customized sanity check paths: %s", paths)
             # if custom_paths is empty, we fall back to a generic set of paths:
             # non-empty bin/ + /lib or /lib64 directories
@@ -3523,14 +3526,13 @@ def _sanity_check_step_common(self, custom_paths, custom_commands):
             # if enhance_sanity_check is enabled *and* sanity_check_paths are specified in the easyconfig,
             # those paths are used to enhance the paths provided by the easyblock
             if enhance_sanity_check and ec_paths:
-                for key in ec_paths:
-                    val = ec_paths[key]
+                for key, val in ec_paths.items():
                     if isinstance(val, list):
                         paths[key] = paths.get(key, []) + val
                     else:
-                        error_pattern = "Incorrect value type in sanity_check_paths, should be a list: "
-                        error_pattern += "%s (type: %s)" % (val, type(val))
-                        raise EasyBuildError(error_pattern)
+                        error_msg = "Incorrect value type in sanity_check_paths, should be a list: "
+                        error_msg += "%s (type: %s)" % (val, type(val))
+                        raise EasyBuildError(error_msg)
                 self.log.info("Enhanced sanity check paths after taking into account easyconfig file: %s", paths)
 
         sorted_keys = sorted(paths.keys())
@@ -3560,7 +3562,7 @@ def _sanity_check_step_common(self, custom_paths, custom_commands):
             self.log.info("Using (only) sanity check commands specified by easyconfig file: %s", commands)
         else:
             if custom_commands:
-                commands = custom_commands
+                commands = self.cfg.resolve_template(custom_commands)
                 self.log.info("Using customised sanity check commands: %s", commands)
             else:
                 commands = []

easybuild/framework/easyconfig/easyconfig.py

@@ -1769,6 +1769,12 @@ def _generate_template_values(self, ignore=None):
             if self.template_values[key] is None:
                 del self.template_values[key]
 
+    def resolve_template(self, value):
+        """Resolve all templates in the given value using this easyconfig"""
+        if not self.template_values:
+            self.generate_template_values()
+        return resolve_template(value, self.template_values)
+
     @handle_deprecated_or_replaced_easyconfig_parameters
     def __contains__(self, key):
         """Check whether easyconfig parameter is defined"""
@@ -1784,9 +1790,7 @@ def __getitem__(self, key):
             raise EasyBuildError("Use of unknown easyconfig parameter '%s' when getting parameter value", key)
 
         if self.enable_templating:
-            if self.template_values is None or len(self.template_values) == 0:
-                self.generate_template_values()
-            value = resolve_template(value, self.template_values)
+            value = self.resolve_template(value)
 
         return value
 

easybuild/tools/include.py

@@ -198,10 +198,8 @@ def include_easyblocks(tmpdir, paths):
 
     # hard inject location to included (generic) easyblocks into Python search path
     # only prepending to sys.path is not enough due to 'pkgutil.extend_path' in easybuild/easyblocks/__init__.py
-    new_path = os.path.join(easyblocks_path, 'easybuild', 'easyblocks')
-    easybuild.easyblocks.__path__.insert(0, new_path)
-    new_path = os.path.join(new_path, 'generic')
-    easybuild.easyblocks.generic.__path__.insert(0, new_path)
+    easybuild.easyblocks.__path__.insert(0, easyblocks_dir)
+    easybuild.easyblocks.generic.__path__.insert(0, os.path.join(easyblocks_dir, 'generic'))
 
     # sanity check: verify that included easyblocks can be imported (from expected location)
     for subdir, ebs in [('', included_ebs), ('generic', included_generic_ebs)]:

easybuild/tools/version.py

@@ -45,7 +45,7 @@
 # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
 #   UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
 # This causes problems further up the dependency chain...
-VERSION = LooseVersion('5.0.0.dev0')
+VERSION = LooseVersion('5.0.0beta1')
 UNKNOWN = 'UNKNOWN'
 UNKNOWN_EASYBLOCKS_VERSION = '0.0.UNKNOWN.EASYBLOCKS'
 

test/framework/easyconfig.py

@@ -1279,6 +1279,29 @@ def test_template_constant_import(self):
         self.assertEqual(GNU_SOURCE, TEMPLATE_CONSTANTS['GNU_SOURCE'][0])
         self.assertEqual(SHLIB_EXT, get_shared_lib_ext())
 
+    def test_ec_method_resolve_template(self):
+        """Test the `resolve_template` method of easyconfig instances."""
+        # don't use any escaping insanity here, since it is templated itself
+        self.contents = textwrap.dedent("""
+            easyblock = "ConfigureMake"
+            name = "PI"
+            version = "3.14"
+            homepage = "http://example.com"
+            description = "test easyconfig %(name)s version %(version_major)s"
+            toolchain = SYSTEM
+        """)
+        self.prep()
+        ec = EasyConfig(self.eb_file, validate=False)
+
+        # We can resolve anything with values from the EC
+        self.assertEqual(ec.resolve_template('%(namelower)s %(version_major)s begins with %(nameletterlower)s'),
+                         'pi 3 begins with p')
+
+        # `resolve_template` does basically the same resolving any value on acccess
+        description = ec.get('description', resolve=False)
+        self.assertIn('%', description, 'Description needs a template for the next test')
+        self.assertEqual(ec.resolve_template(description), ec['description'])
+
     def test_templating_cuda_toolchain(self):
         """Test templates via toolchain component, like setting %(cudaver)s with fosscuda toolchain."""
 

test/framework/easyconfigs/test_ecs/p/Python/Python-2.7.15.eb

@@ -0,0 +1,23 @@
+easyblock = 'ConfigureMake'
+
+name = 'Python'
+version = '2.7.15'
+
+homepage = 'http://python.org/'
+description = """Python is a programming language that lets you work more quickly and integrate your systems
+more effectively."""
+
+toolchain = SYSTEM
+
+source_urls = ['http://www.python.org/ftp/%(namelower)s/%(version)s/']
+sources = [SOURCE_TGZ]
+
+# This just serves to have a Python as a dependency to test e.g. the Python version templates
+# So all dependencies and extensions are removed
+dependencies = []
+
+osdependencies = []
+
+exts_list = []
+
+moduleclass = 'lang'

test/framework/easyconfigs/test_ecs/p/Python/Python-3.7.2.eb

@@ -0,0 +1,23 @@
+easyblock = 'ConfigureMake'
+
+name = 'Python'
+version = '3.7.2'
+
+homepage = 'http://python.org/'
+description = """Python is a programming language that lets you work more quickly and integrate your systems
+more effectively."""
+
+toolchain = SYSTEM
+
+source_urls = ['http://www.python.org/ftp/%(namelower)s/%(version)s/']
+sources = [SOURCE_TGZ]
+
+# This just serves to have a Python as a dependency to test e.g. the Python version templates
+# So all dependencies and extensions are removed
+dependencies = []
+
+osdependencies = []
+
+exts_list = []
+
+moduleclass = 'lang'

test/framework/filetools.py

@@ -2545,7 +2545,7 @@ def test_index_functions(self):
         # test with specified path with and without trailing '/'s
         for path in [test_ecs, test_ecs + '/', test_ecs + '//']:
             index = ft.create_index(path)
-            self.assertEqual(len(index), 92)
+            self.assertEqual(len(index), 94)
 
             expected = [
                 os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'),

test/framework/sandbox/sources/p/Python/Python-3.7.2.tgz

@@ -2673,6 +2673,99 @@ def test_toy_build_enhanced_sanity_check(self):
 
         del sys.modules['easybuild.easyblocks.toy']
 
+    def test_toy_build_enhanced_sanity_check_templated_multi_dep(self):
+        """Test enhancing of sanity check by easyblocks with templates and in the presence of multi_deps."""
+
+        # if toy easyblock was imported, get rid of corresponding entry in sys.modules,
+        # to avoid that it messes up the use of --include-easyblocks=toy.py below...
+        if 'easybuild.easyblocks.toy' in sys.modules:
+            del sys.modules['easybuild.easyblocks.toy']
+
+        test_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)))
+        toy_ec = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
+        toy_ec_txt = read_file(toy_ec)
+
+        test_ec = os.path.join(self.test_prefix, 'test.eb')
+
+        # get rid of custom sanity check paths in test easyconfig
+        regex = re.compile(r'^sanity_check_paths\s*=\s*{[^}]+}', re.M)
+        test_ec_txt = regex.sub('', toy_ec_txt)
+        self.assertNotIn('sanity_check_', test_ec_txt)
+
+        test_ec_txt += "\nmulti_deps = {'Python': ['3.7.2', '2.7.15']}"
+        write_file(test_ec, test_ec_txt)
+
+        # create custom easyblock for toy that has a custom sanity_check_step
+        toy_easyblock = os.path.join(test_dir, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py')
+
+        toy_easyblock_txt = read_file(toy_easyblock)
+
+        toy_custom_sanity_check_step = textwrap.dedent("""
+            # Add to class to indent
+                def sanity_check_step(self):
+                    paths = {
+                        'files': ['bin/python%(pyshortver)s'],
+                        'dirs': ['lib/py-%(pyshortver)s'],
+                    }
+                    cmds = ['python%(pyshortver)s']
+                    return super(EB_toy, self).sanity_check_step(custom_paths=paths, custom_commands=cmds)
+        """)
+        test_toy_easyblock = os.path.join(self.test_prefix, 'toy.py')
+        write_file(test_toy_easyblock, toy_easyblock_txt + toy_custom_sanity_check_step)
+
+        eb_args = [
+            '--extended-dry-run',
+            '--include-easyblocks=%s' % test_toy_easyblock,
+        ]
+
+        # by default, sanity check commands & paths specified by easyblock are used
+        with self.mocked_stdout_stderr():
+            self._test_toy_build(ec_file=test_ec, extra_args=eb_args, verify=False, testing=False, raise_error=True)
+            stdout = self.get_stdout()
+            # Cut output to start of the toy-ec, after the Python installations
+            stdout = stdout[stdout.index(test_ec):]
+
+        pattern_template = textwrap.dedent(r"""
+            Sanity check paths - file.*
+            \s*\* bin/python{pyshortver}
+            Sanity check paths - \(non-empty\) directory.*
+            \s*\* lib/py-{pyshortver}
+            Sanity check commands
+            \s*\* python{pyshortver}
+        """)
+        for pyshortver in ('2.7', '3.7'):
+            regex = re.compile(pattern_template.format(pyshortver=pyshortver), re.M)
+            self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
+
+        # Enhance sanity check by extra paths to check for, the ones from the easyblock should be kept
+        test_ec_txt += textwrap.dedent("""
+            enhance_sanity_check = True
+            sanity_check_paths = {
+                'files': ['bin/pip%(pyshortver)s'],
+                'dirs': ['bin'],
+            }
+        """)
+        write_file(test_ec, test_ec_txt)
+        with self.mocked_stdout_stderr():
+            self._test_toy_build(ec_file=test_ec, extra_args=eb_args, verify=False, testing=False, raise_error=True)
+            stdout = self.get_stdout()
+            # Cut output to start of the toy-ec, after the Python installations
+            stdout = stdout[stdout.index(test_ec):]
+
+        pattern_template = textwrap.dedent(r"""
+            Sanity check paths - file.*
+            \s*\* bin/pip{pyshortver}
+            \s*\* bin/python{pyshortver}
+            Sanity check paths - \(non-empty\) directory.*
+            \s*\* bin
+            \s*\* lib/py-{pyshortver}
+            Sanity check commands
+            \s*\* python{pyshortver}
+        """)
+        for pyshortver in ('2.7', '3.7'):
+            regex = re.compile(pattern_template.format(pyshortver=pyshortver), re.M)
+            self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
+
     def test_toy_dumped_easyconfig(self):
         """ Test dumping of file in eb_filerepo in both .eb format """
         filename = 'toy-0.0'