Add make_extension_string and _make_extension_list to EasyBlock

This allows custom EasyBlocks to override thise and e.g. add more
extensions which should be mentioned in the module.
This also unifies the handling of getting and converting those into a
string which fixes a bug with e.g. R ECs where extensions may only be a
plain string.

Author: Flamefire

easybuild/framework/easyblock.py

@@ -1429,7 +1429,7 @@ def make_module_extra_extensions(self):
 
         # set environment variable that specifies list of extensions
         # We need only name and version, so don't resolve templates
-        exts_list = ','.join(['-'.join(ext[:2]) for ext in self.cfg.get_ref('exts_list')])
+        exts_list = self.make_extension_string(ext_sep=',', sort=False)
         env_var_name = convert_name(self.name, upper=True)
         lines.append(self.module_generator.set_environment('EBEXTSLIST%s' % env_var_name, exts_list))
 
@@ -1710,6 +1710,27 @@ def load_dependency_modules(self):
     # EXTENSIONS UTILITY FUNCTIONS
     #
 
+    def _make_extension_list(self):
+        """
+        Return a list of extension names and their versions included in this installation
+
+        Each entry should be a (name, version) tuple or just (name, ) if no version exists
+        """
+        # We need only name and version, so don't resolve templates
+        # Each extension in exts_list is either a string or a list/tuple with name, version as first entries
+        return [(ext, ) if isinstance(ext, string_type) else ext[:2] for ext in self.cfg.get_ref('exts_list')]
+
+    def make_extension_string(self, name_version_sep='-', ext_sep=', ', sort=True):
+        """
+        Generate a string with a list of extensions.
+
+        The name and version are separated by name_version_sep and each extension is separated by ext_sep
+        """
+        exts_list = (name_version_sep.join(ext) for ext in self._make_extension_list())
+        if sort:
+            exts_list = sorted(exts_list, key=str.lower)
+        return ext_sep.join(exts_list)
+
     def prepare_for_extensions(self):
         """
         Also do this before (eg to set the template)

easybuild/tools/module_generator.py

@@ -606,28 +606,17 @@ def use(self, paths, prefix=None, guarded=False, user_modpath=None):
 
     def _generate_extension_list(self):
         """
-        Generate a string with a comma-separated list of extensions.
-        """
-        # We need only name and version, so don't resolve templates
-        exts_list = self.app.cfg.get_ref('exts_list')
-        extensions = ', '.join(sorted(['-'.join(ext[:2]) for ext in exts_list], key=str.lower))
+        Generate a string with a list of extensions.
 
-        return extensions
+        The name and version are separated by name_version_sep and each extension is separated by ext_sep
+        """
+        return self.app.make_extension_string()
 
     def _generate_extensions_list(self):
         """
         Generate a list of all extensions in name/version format
         """
-        exts_list = self.app.cfg['exts_list']
-        # the format is extension_name/extension_version
-        exts_ver_list = []
-        for ext in exts_list:
-            if isinstance(ext, tuple):
-                exts_ver_list.append('%s/%s' % (ext[0], ext[1]))
-            elif isinstance(ext, string_type):
-                exts_ver_list.append(ext)
-
-        return sorted(exts_ver_list, key=str.lower)
+        return self.app.make_extension_string(name_version_sep='/', ext_sep=',').split(',')
 
     def _generate_help_text(self):
         """

test/framework/easyblock.py

@@ -1033,7 +1033,7 @@ def test_init_extensions(self):
         toy_ec_txt = read_file(toy_ec_file)
 
         test_ec = os.path.join(self.test_prefix, 'test.eb')
-        test_ec_txt = toy_ec_txt.replace("('barbar', '0.0', {", "('barbar', '0.0', {'easyblock': 'DummyExtension',")
+        test_ec_txt = toy_ec_txt.replace("('barbar', '1.2', {", "('barbar', '1.2', {'easyblock': 'DummyExtension',")
         write_file(test_ec, test_ec_txt)
         ec = process_easyconfig(test_ec)[0]
         eb = get_easyblock_instance(ec)
@@ -1837,7 +1837,7 @@ def test_collect_exts_file_info(self):
         self.assertTrue(isinstance(exts_file_info, list))
         self.assertEqual(len(exts_file_info), 4)
 
-        self.assertEqual(exts_file_info[0], {'name': 'ls'})
+        self.assertEqual(exts_file_info[0], {'name': 'ulimit'})
 
         self.assertEqual(exts_file_info[1]['name'], 'bar')
         self.assertEqual(exts_file_info[1]['src'], os.path.join(toy_ext_sources, 'bar-0.0.tar.gz'))
@@ -1849,7 +1849,7 @@ def test_collect_exts_file_info(self):
         self.assertEqual(exts_file_info[1]['patches'][1]['path'], os.path.join(toy_ext_sources, bar_patch2))
 
         self.assertEqual(exts_file_info[2]['name'], 'barbar')
-        self.assertEqual(exts_file_info[2]['src'], os.path.join(toy_ext_sources, 'barbar-0.0.tar.gz'))
+        self.assertEqual(exts_file_info[2]['src'], os.path.join(toy_ext_sources, 'barbar-1.2.tar.gz'))
         self.assertNotIn('patches', exts_file_info[2])
 
         self.assertEqual(exts_file_info[3]['name'], 'toy')
@@ -1862,7 +1862,7 @@ def test_collect_exts_file_info(self):
         self.assertTrue(isinstance(exts_file_info, list))
         self.assertEqual(len(exts_file_info), 4)
 
-        self.assertEqual(exts_file_info[0], {'name': 'ls'})
+        self.assertEqual(exts_file_info[0], {'name': 'ulimit'})
 
         self.assertEqual(exts_file_info[1]['name'], 'bar')
         self.assertNotIn('src', exts_file_info[1])

test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb

@@ -27,10 +27,10 @@ exts_default_options = {
 }
 
 exts_list = [
-    'ls',  # extension that is part of "standard library"
+    'ulimit',  # extension that is part of "standard library"
     ('bar', '0.0', {
         'buildopts': " && gcc bar.c -o anotherbar",
-        'checksums': ['f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7'],  # checksum for 
+        'checksums': ['f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7'],  # checksum for
         # custom extension filter to verify use of stdin value being passed to filter command
         'exts_filter': ("cat | grep '^bar$'", '%(name)s'),
         'patches': [
@@ -43,7 +43,7 @@ exts_list = [
         # cfr. https://github.com/easybuilders/easybuild-framework/pull/3034
         'keepsymlinks': True,
     }),
-    ('barbar', '0.0', {
+    ('barbar', '1.2', {
         'start_dir': 'src',
     }),
     (name, version, {

test/framework/module_generator.py

@@ -749,7 +749,7 @@ def test_module_extensions(self):
 
         patterns = [
             r'^if convertToCanonical\(LmodVersion\(\)\) >= convertToCanonical\("8\.2\.8"\) then\n' +
-            r'\s*extensions\("bar/0.0,barbar/0.0,ls,toy/0.0"\)\nend$',
+            r'\s*extensions\("bar/0.0,barbar/1.2,toy/0.0,ulimit"\)\nend$',
         ]
 
         for pattern in patterns:

test/framework/options.py

@@ -5688,7 +5688,7 @@ def test_inject_checksums(self):
             r"^==  \* bar-0\.0\.tar\.gz: %s$" % bar_tar_gz_sha256,
             r"^==  \* %s: %s$" % (bar_patch, bar_patch_sha256),
             r"^==  \* %s: %s$" % (bar_patch_bis, bar_patch_bis_sha256),
-            r"^==  \* barbar-0\.0\.tar\.gz: d5bd9908cdefbe2d29c6f8d5b45b2aaed9fd904b5e6397418bb5094fbdb3d838$",
+            r"^==  \* barbar-1\.2\.tar\.gz: d5bd9908cdefbe2d29c6f8d5b45b2aaed9fd904b5e6397418bb5094fbdb3d838$",
         ]
         for pattern in patterns:
             regex = re.compile(pattern, re.M)
@@ -5744,7 +5744,7 @@ def test_inject_checksums(self):
         self.assertEqual(ec['checksums'], [{'toy-0.0.tar.gz': toy_source_sha256},
                                            {'toy-0.0_fix-silly-typo-in-printf-statement.patch': toy_patch_sha256}])
         self.assertEqual(ec['exts_default_options'], {'source_urls': ['http://example.com/%(name)s']})
-        self.assertEqual(ec['exts_list'][0], 'ls')
+        self.assertEqual(ec['exts_list'][0], 'ulimit')
         self.assertEqual(ec['exts_list'][1], ('bar', '0.0', {
             'buildopts': " && gcc bar.c -o anotherbar",
             'checksums': [
@@ -5758,7 +5758,7 @@ def test_inject_checksums(self):
             'unknowneasyconfigparameterthatshouldbeignored': 'foo',
             'keepsymlinks': True,
         }))
-        self.assertEqual(ec['exts_list'][2], ('barbar', '0.0', {
+        self.assertEqual(ec['exts_list'][2], ('barbar', '1.2', {
             'checksums': ['d5bd9908cdefbe2d29c6f8d5b45b2aaed9fd904b5e6397418bb5094fbdb3d838'],
             'start_dir': 'src',
         }))

test/framework/sandbox/sources/toy/extensions/barbar-1.2.tar.gz

@@ -1088,7 +1088,7 @@ def test_toy_advanced(self):
         toy_mod_txt = read_file(toy_module)
 
         patterns = [
-            '^setenv.*EBEXTSLISTTOY.*bar-0.0,barbar-0.0',
+            '^setenv.*EBEXTSLISTTOY.*ulimit,bar-0.0,barbar-1.2',
             # set by ToyExtension easyblock used to install extensions
             '^setenv.*TOY_EXT_BAR.*bar',
             '^setenv.*TOY_EXT_BARBAR.*barbar',
@@ -2198,7 +2198,7 @@ def test_reproducibility_ext_easyblocks(self):
         ec1 = os.path.join(self.test_prefix, 'toy1.eb')
         ec1_txt = '\n'.join([
             toy_ec_txt,
-            "exts_list = [('barbar', '0.0', {'start_dir': 'src'})]",
+            "exts_list = [('barbar', '1.2', {'start_dir': 'src'})]",
             "",
         ])
         write_file(ec1, ec1_txt)
@@ -2968,7 +2968,7 @@ def test_toy_multi_deps(self):
         test_ec = os.path.join(self.test_prefix, 'test.eb')
 
         # also inject (minimal) list of extensions to test iterative installation of extensions
-        test_ec_txt += "\nexts_list = [('barbar', '0.0', {'start_dir': 'src'})]"
+        test_ec_txt += "\nexts_list = [('barbar', '1.2', {'start_dir': 'src'})]"
 
         test_ec_txt += "\nmulti_deps = {'GCC': ['4.6.3', '7.3.0-2.30']}"
         write_file(test_ec, test_ec_txt)