Merge branch 'develop' into make_rpath_readelf_check_optional

Author: boegel

.github/workflows/bootstrap_script.yml

@@ -1436,7 +1436,7 @@ def make_module_extra_extensions(self):
 
         # set environment variable that specifies list of extensions
         # We need only name and version, so don't resolve templates
-        exts_list = ','.join(['-'.join(ext[:2]) for ext in self.cfg.get_ref('exts_list')])
+        exts_list = self.make_extension_string(ext_sep=',', sort=False)
         env_var_name = convert_name(self.name, upper=True)
         lines.append(self.module_generator.set_environment('EBEXTSLIST%s' % env_var_name, exts_list))
 
@@ -1717,6 +1717,27 @@ def load_dependency_modules(self):
     # EXTENSIONS UTILITY FUNCTIONS
     #
 
+    def _make_extension_list(self):
+        """
+        Return a list of extension names and their versions included in this installation
+
+        Each entry should be a (name, version) tuple or just (name, ) if no version exists
+        """
+        # We need only name and version, so don't resolve templates
+        # Each extension in exts_list is either a string or a list/tuple with name, version as first entries
+        return [(ext, ) if isinstance(ext, string_type) else ext[:2] for ext in self.cfg.get_ref('exts_list')]
+
+    def make_extension_string(self, name_version_sep='-', ext_sep=', ', sort=True):
+        """
+        Generate a string with a list of extensions.
+
+        The name and version are separated by name_version_sep and each extension is separated by ext_sep
+        """
+        exts_list = (name_version_sep.join(ext) for ext in self._make_extension_list())
+        if sort:
+            exts_list = sorted(exts_list, key=str.lower)
+        return ext_sep.join(exts_list)
+
     def prepare_for_extensions(self):
         """
         Also do this before (eg to set the template)

easybuild/framework/easyblock.py

@@ -612,28 +612,17 @@ def use(self, paths, prefix=None, guarded=False, user_modpath=None):
 
     def _generate_extension_list(self):
         """
-        Generate a string with a comma-separated list of extensions.
-        """
-        # We need only name and version, so don't resolve templates
-        exts_list = self.app.cfg.get_ref('exts_list')
-        extensions = ', '.join(sorted(['-'.join(ext[:2]) for ext in exts_list], key=str.lower))
+        Generate a string with a list of extensions.
 
-        return extensions
+        The name and version are separated by name_version_sep and each extension is separated by ext_sep
+        """
+        return self.app.make_extension_string()
 
     def _generate_extensions_list(self):
         """
         Generate a list of all extensions in name/version format
         """
-        exts_list = self.app.cfg['exts_list']
-        # the format is extension_name/extension_version
-        exts_ver_list = []
-        for ext in exts_list:
-            if isinstance(ext, tuple):
-                exts_ver_list.append('%s/%s' % (ext[0], ext[1]))
-            elif isinstance(ext, string_type):
-                exts_ver_list.append(ext)
-
-        return sorted(exts_ver_list, key=str.lower)
+        return self.app.make_extension_string(name_version_sep='/', ext_sep=',').split(',')
 
     def _generate_help_text(self):
         """

easybuild/tools/module_generator.py

@@ -623,19 +623,19 @@ def parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp):
     """
     if strictness == IGNORE:
         check_ec = False
-        use_regexp = False
+        fail_on_error_match = False
     elif strictness == WARN:
         check_ec = True
-        use_regexp = False
+        fail_on_error_match = False
     elif strictness == ERROR:
         check_ec = True
-        use_regexp = True
+        fail_on_error_match = True
     else:
         raise EasyBuildError("invalid strictness setting: %s", strictness)
 
     # allow for overriding the regexp setting
     if not regexp:
-        use_regexp = False
+        fail_on_error_match = False
 
     if ec and (log_all or log_ok):
         # We don't want to error if the user doesn't care
@@ -650,14 +650,16 @@ def parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp):
             _log.debug('cmd "%s" exited with exit code %s and output:\n%s' % (cmd, ec, stdouterr))
 
     # parse the stdout/stderr for errors when strictness dictates this or when regexp is passed in
-    if use_regexp or regexp:
-        res = parse_log_for_error(stdouterr, regexp, msg="Command used: %s" % cmd)
-        if len(res) > 0:
-            message = "Found %s errors in command output (output: %s)" % (len(res), "\n\t".join([r[0] for r in res]))
-            if use_regexp:
-                raise EasyBuildError(message)
+    if fail_on_error_match or regexp:
+        res = parse_log_for_error(stdouterr, regexp, stdout=False)
+        if res:
+            errors = "\n\t" + "\n\t".join([r[0] for r in res])
+            error_str = "error" if len(res) == 1 else "errors"
+            if fail_on_error_match:
+                raise EasyBuildError("Found %s %s in output of %s:%s", len(res), error_str, cmd, errors)
             else:
-                _log.warning(message)
+                _log.warning("Found %s potential %s (some may be harmless) in output of %s:%s",
+                             len(res), error_str, cmd, errors)
 
     if simple:
         if ec:

easybuild/tools/run.py

@@ -1033,7 +1033,7 @@ def test_init_extensions(self):
         toy_ec_txt = read_file(toy_ec_file)
 
         test_ec = os.path.join(self.test_prefix, 'test.eb')
-        test_ec_txt = toy_ec_txt.replace("('barbar', '0.0', {", "('barbar', '0.0', {'easyblock': 'DummyExtension',")
+        test_ec_txt = toy_ec_txt.replace("('barbar', '1.2', {", "('barbar', '1.2', {'easyblock': 'DummyExtension',")
         write_file(test_ec, test_ec_txt)
         ec = process_easyconfig(test_ec)[0]
         eb = get_easyblock_instance(ec)
@@ -1837,7 +1837,7 @@ def test_collect_exts_file_info(self):
         self.assertIsInstance(exts_file_info, list)
         self.assertEqual(len(exts_file_info), 4)
 
-        self.assertEqual(exts_file_info[0], {'name': 'ls'})
+        self.assertEqual(exts_file_info[0], {'name': 'ulimit'})
 
         self.assertEqual(exts_file_info[1]['name'], 'bar')
         self.assertEqual(exts_file_info[1]['src'], os.path.join(toy_ext_sources, 'bar-0.0.tar.gz'))
@@ -1849,7 +1849,7 @@ def test_collect_exts_file_info(self):
         self.assertEqual(exts_file_info[1]['patches'][1]['path'], os.path.join(toy_ext_sources, bar_patch2))
 
         self.assertEqual(exts_file_info[2]['name'], 'barbar')
-        self.assertEqual(exts_file_info[2]['src'], os.path.join(toy_ext_sources, 'barbar-0.0.tar.gz'))
+        self.assertEqual(exts_file_info[2]['src'], os.path.join(toy_ext_sources, 'barbar-1.2.tar.gz'))
         self.assertNotIn('patches', exts_file_info[2])
 
         self.assertEqual(exts_file_info[3]['name'], 'toy')
@@ -1862,7 +1862,7 @@ def test_collect_exts_file_info(self):
         self.assertIsInstance(exts_file_info, list)
         self.assertEqual(len(exts_file_info), 4)
 
-        self.assertEqual(exts_file_info[0], {'name': 'ls'})
+        self.assertEqual(exts_file_info[0], {'name': 'ulimit'})
 
         self.assertEqual(exts_file_info[1]['name'], 'bar')
         self.assertNotIn('src', exts_file_info[1])

test/framework/easyblock.py

@@ -27,10 +27,10 @@ exts_default_options = {
 }
 
 exts_list = [
-    'ls',  # extension that is part of "standard library"
+    'ulimit',  # extension that is part of "standard library"
     ('bar', '0.0', {
         'buildopts': " && gcc bar.c -o anotherbar",
-        'checksums': ['f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7'],  # checksum for 
+        'checksums': ['f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7'],  # checksum for
         # custom extension filter to verify use of stdin value being passed to filter command
         'exts_filter': ("cat | grep '^bar$'", '%(name)s'),
         'patches': [
@@ -43,7 +43,7 @@ exts_list = [
         # cfr. https://github.com/easybuilders/easybuild-framework/pull/3034
         'keepsymlinks': True,
     }),
-    ('barbar', '0.0', {
+    ('barbar', '1.2', {
         'start_dir': 'src',
     }),
     (name, version, {

test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb

@@ -749,7 +749,7 @@ def test_module_extensions(self):
 
         patterns = [
             r'^if convertToCanonical\(LmodVersion\(\)\) >= convertToCanonical\("8\.2\.8"\) then\n' +
-            r'\s*extensions\("bar/0.0,barbar/0.0,ls,toy/0.0"\)\nend$',
+            r'\s*extensions\("bar/0.0,barbar/1.2,toy/0.0,ulimit"\)\nend$',
         ]
 
         for pattern in patterns:

test/framework/module_generator.py

@@ -5747,7 +5747,7 @@ def test_inject_checksums(self):
             r"^==  \* bar-0\.0\.tar\.gz: %s$" % bar_tar_gz_sha256,
             r"^==  \* %s: %s$" % (bar_patch, bar_patch_sha256),
             r"^==  \* %s: %s$" % (bar_patch_bis, bar_patch_bis_sha256),
-            r"^==  \* barbar-0\.0\.tar\.gz: d5bd9908cdefbe2d29c6f8d5b45b2aaed9fd904b5e6397418bb5094fbdb3d838$",
+            r"^==  \* barbar-1\.2\.tar\.gz: d5bd9908cdefbe2d29c6f8d5b45b2aaed9fd904b5e6397418bb5094fbdb3d838$",
         ]
         for pattern in patterns:
             regex = re.compile(pattern, re.M)
@@ -5803,7 +5803,7 @@ def test_inject_checksums(self):
         self.assertEqual(ec['checksums'], [{'toy-0.0.tar.gz': toy_source_sha256},
                                            {'toy-0.0_fix-silly-typo-in-printf-statement.patch': toy_patch_sha256}])
         self.assertEqual(ec['exts_default_options'], {'source_urls': ['http://example.com/%(name)s']})
-        self.assertEqual(ec['exts_list'][0], 'ls')
+        self.assertEqual(ec['exts_list'][0], 'ulimit')
         self.assertEqual(ec['exts_list'][1], ('bar', '0.0', {
             'buildopts': " && gcc bar.c -o anotherbar",
             'checksums': [
@@ -5817,7 +5817,7 @@ def test_inject_checksums(self):
             'unknowneasyconfigparameterthatshouldbeignored': 'foo',
             'keepsymlinks': True,
         }))
-        self.assertEqual(ec['exts_list'][2], ('barbar', '0.0', {
+        self.assertEqual(ec['exts_list'][2], ('barbar', '1.2', {
             'checksums': ['d5bd9908cdefbe2d29c6f8d5b45b2aaed9fd904b5e6397418bb5094fbdb3d838'],
             'start_dir': 'src',
         }))

test/framework/options.py

@@ -1088,7 +1088,7 @@ def test_toy_advanced(self):
         toy_mod_txt = read_file(toy_module)
 
         patterns = [
-            '^setenv.*EBEXTSLISTTOY.*bar-0.0,barbar-0.0',
+            '^setenv.*EBEXTSLISTTOY.*ulimit,bar-0.0,barbar-1.2',
             # set by ToyExtension easyblock used to install extensions
             '^setenv.*TOY_EXT_BAR.*bar',
             '^setenv.*TOY_EXT_BARBAR.*barbar',
@@ -2197,7 +2197,7 @@ def test_reproducibility_ext_easyblocks(self):
         ec1 = os.path.join(self.test_prefix, 'toy1.eb')
         ec1_txt = '\n'.join([
             toy_ec_txt,
-            "exts_list = [('barbar', '0.0', {'start_dir': 'src'})]",
+            "exts_list = [('barbar', '1.2', {'start_dir': 'src'})]",
             "",
         ])
         write_file(ec1, ec1_txt)
@@ -2998,7 +2998,7 @@ def test_toy_multi_deps(self):
         test_ec = os.path.join(self.test_prefix, 'test.eb')
 
         # also inject (minimal) list of extensions to test iterative installation of extensions
-        test_ec_txt += "\nexts_list = [('barbar', '0.0', {'start_dir': 'src'})]"
+        test_ec_txt += "\nexts_list = [('barbar', '1.2', {'start_dir': 'src'})]"
 
         test_ec_txt += "\nmulti_deps = {'GCC': ['4.6.3', '7.3.0-2.30']}"
         write_file(test_ec, test_ec_txt)