Merge pull request #4687 from Micket/modulesep

Allow use of custom delimiter for paths in module generator

Author: lexming

easybuild/framework/easyblock.py

@@ -1469,22 +1469,27 @@ def make_module_extra(self, altroot=None, altversion=None):
         for (key, value) in self.cfg['modextravars'].items():
             lines.append(self.module_generator.set_environment(key, value))
 
-        for (key, value) in self.cfg['modextrapaths'].items():
-            if isinstance(value, str):
-                value = [value]
-            elif not isinstance(value, (tuple, list)):
-                raise EasyBuildError("modextrapaths dict value %s (type: %s) is not a list or tuple",
-                                     value, type(value))
-            lines.append(self.module_generator.prepend_paths(key, value, allow_abs=self.cfg['allow_prepend_abs_path']))
-
-        for (key, value) in self.cfg['modextrapaths_append'].items():
-            if isinstance(value, str):
-                value = [value]
-            elif not isinstance(value, (tuple, list)):
-                raise EasyBuildError("modextrapaths_append dict value %s (type: %s) is not a list or tuple",
-                                     value, type(value))
-            lines.append(self.module_generator.append_paths(key, value, allow_abs=self.cfg['allow_append_abs_path']))
+        for extrapaths_type, prepend in [('modextrapaths', True), ('modextrapaths_append', False)]:
+            allow_abs = self.cfg['allow_prepend_abs_path'] if prepend else self.cfg['allow_append_abs_path']
 
+            for (key, value) in self.cfg[extrapaths_type].items():
+                if not isinstance(value, (tuple, list, dict, str)):
+                    raise EasyBuildError(
+                        f"{extrapaths_type} dict value '{value}' (type {type(value)}) is not a 'list, dict or str'"
+                    )
+
+                try:
+                    paths = value['paths']
+                    delim = value['delimiter']
+                except KeyError:
+                    raise EasyBuildError(f'{extrapaths_type} dict "{value}" lacks "paths" or "delimiter" items')
+                except TypeError:
+                    paths = value
+                    delim = ':'
+
+                lines.append(
+                    self.module_generator.update_paths(key, paths, prepend=prepend, delim=delim, allow_abs=allow_abs)
+                )
         # add lines to update $PYTHONPATH or $EBPYTHONPREFIXES
         lines.extend(self.make_module_pythonpath())
 

easybuild/tools/module_generator.py

@@ -235,33 +235,37 @@ def _filter_paths(self, key, paths):
                 filtered_paths = None
         return filtered_paths
 
-    def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
+    def append_paths(self, key, paths, allow_abs=False, expand_relpaths=True, delim=':'):
         """
         Generate append-path statements for the given list of paths.
 
         :param key: environment variable to append paths to
         :param paths: list of paths to append
         :param allow_abs: allow providing of absolute paths
         :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
+        :param delim: delimiter used between paths
         """
         paths = self._filter_paths(key, paths)
         if paths is None:
             return ''
-        return self.update_paths(key, paths, prepend=False, allow_abs=allow_abs, expand_relpaths=expand_relpaths)
+        return self.update_paths(key, paths, prepend=False, allow_abs=allow_abs, expand_relpaths=expand_relpaths,
+                                 delim=delim)
 
-    def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True):
+    def prepend_paths(self, key, paths, allow_abs=False, expand_relpaths=True, delim=':'):
         """
         Generate prepend-path statements for the given list of paths.
 
         :param key: environment variable to append paths to
         :param paths: list of paths to append
         :param allow_abs: allow providing of absolute paths
         :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
+        :param delim: delimiter used between paths
         """
         paths = self._filter_paths(key, paths)
         if paths is None:
             return ''
-        return self.update_paths(key, paths, prepend=True, allow_abs=allow_abs, expand_relpaths=expand_relpaths)
+        return self.update_paths(key, paths, prepend=True, allow_abs=allow_abs, expand_relpaths=expand_relpaths,
+                                 delim=delim)
 
     def _modulerc_check_module_version(self, module_version):
         """
@@ -552,15 +556,16 @@ def unload_module(self, mod_name):
         """
         raise NotImplementedError
 
-    def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True):
+    def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True, delim=':'):
         """
         Generate prepend-path or append-path statements for the given list of paths.
 
         :param key: environment variable to prepend/append paths to
-        :param paths: list of paths to prepend
+        :param paths: list of paths to prepend/append
         :param prepend: whether to prepend (True) or append (False) paths
         :param allow_abs: allow providing of absolute paths
         :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
+        :param delim: delimiter used between paths
         """
         raise NotImplementedError
 
@@ -970,15 +975,16 @@ def msg_on_unload(self, msg):
         print_cmd = "puts stderr %s" % quote_str(msg, tcl=True)
         return '\n'.join(['', self.conditional_statement("module-info mode unload", print_cmd, indent=False)])
 
-    def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True):
+    def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True, delim=':'):
         """
         Generate prepend-path or append-path statements for the given list of paths.
 
         :param key: environment variable to prepend/append paths to
-        :param paths: list of paths to prepend
+        :param paths: list of paths to prepend/append
         :param prepend: whether to prepend (True) or append (False) paths
         :param allow_abs: allow providing of absolute paths
         :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
+        :param delim: delimiter used between paths
         """
         if prepend:
             update_type = 'prepend'
@@ -1010,7 +1016,8 @@ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpath
             else:
                 abspaths.append(path)
 
-        statements = ['%s-path\t%s\t\t%s\n' % (update_type, key, p) for p in abspaths]
+        delim_opt = '' if delim == ':' else f' -d "{delim}"'
+        statements = [f'{update_type}-path{delim_opt}\t{key}\t\t{p}\n' for p in abspaths]
         return ''.join(statements)
 
     def set_alias(self, key, value):
@@ -1161,6 +1168,7 @@ class ModuleGeneratorLua(ModuleGenerator):
 
     PATH_JOIN_TEMPLATE = 'pathJoin(root, "%s")'
     UPDATE_PATH_TEMPLATE = '%s_path("%s", %s)'
+    UPDATE_PATH_TEMPLATE_DELIM = '%s_path("%s", %s, "%s")'
 
     START_STR = '[==['
     END_STR = ']==]'
@@ -1422,7 +1430,7 @@ def modulerc(self, module_version=None, filepath=None, modulerc_txt=None):
         return super(ModuleGeneratorLua, self).modulerc(module_version=module_version, filepath=filepath,
                                                         modulerc_txt=modulerc_txt)
 
-    def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True):
+    def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpaths=True, delim=':'):
         """
         Generate prepend_path or append_path statements for the given list of paths
 
@@ -1431,6 +1439,7 @@ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpath
         :param prepend: whether to prepend (True) or append (False) paths
         :param allow_abs: allow providing of absolute paths
         :param expand_relpaths: expand relative paths into absolute paths (by prefixing install dir)
+        :param delim: delimiter used between paths
         """
         if prepend:
             update_type = 'prepend'
@@ -1463,7 +1472,10 @@ def update_paths(self, key, paths, prepend=True, allow_abs=False, expand_relpath
                 else:
                     abspaths.append('root')
 
-        statements = [self.UPDATE_PATH_TEMPLATE % (update_type, key, p) for p in abspaths]
+        if delim != ':':
+            statements = [self.UPDATE_PATH_TEMPLATE_DELIM % (update_type, key, p, delim) for p in abspaths]
+        else:
+            statements = [self.UPDATE_PATH_TEMPLATE % (update_type, key, p) for p in abspaths]
         statements.append('')
         return '\n'.join(statements)
 

test/framework/easyblock.py

@@ -1259,6 +1259,7 @@ def test_make_module_step(self):
         modextrapaths = {
             'PATH': ('xbin', 'pibin'),
             'CPATH': 'pi/include',
+            'TCLLIBPATH': {'paths': 'pi', 'delimiter': ' '},
         }
         modextrapaths_append = {'APPEND_PATH': 'append_path'}
         self.contents = '\n'.join([
@@ -1333,28 +1334,47 @@ def test_make_module_step(self):
             self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
 
         for (key, vals) in modextrapaths.items():
-            if isinstance(vals, str):
-                vals = [vals]
-            for val in vals:
-                if get_module_syntax() == 'Tcl':
-                    regex = re.compile(r'^prepend-path\s+%s\s+\$root/%s$' % (key, val), re.M)
-                elif get_module_syntax() == 'Lua':
-                    regex = re.compile(r'^prepend_path\("%s", pathJoin\(root, "%s"\)\)$' % (key, val), re.M)
-                else:
-                    self.fail("Unknown module syntax: %s" % get_module_syntax())
-                self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
-                # Check for duplicates
-                num_prepends = len(regex.findall(txt))
-                self.assertEqual(num_prepends, 1, "Expected exactly 1 %s command in %s" % (regex.pattern, txt))
+            if isinstance(vals, dict):
+                delim = vals['delimiter']
+                paths = vals['paths']
+                if isinstance(paths, str):
+                    paths = [paths]
+
+                for val in paths:
+                    if get_module_syntax() == 'Tcl':
+                        regex = re.compile(fr'^prepend-path\s+-d\s+"{delim}"\s+{key}\s+\$root/{val}$', re.M)
+                    elif get_module_syntax() == 'Lua':
+                        regex = re.compile(fr'^prepend_path\("{key}", pathJoin\(root, "{val}"\), "{delim}"\)$', re.M)
+                    else:
+                        self.fail("Unknown module syntax: %s" % get_module_syntax())
+                    self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
+                    # Check for duplicates
+                    num_prepends = len(regex.findall(txt))
+                    self.assertEqual(num_prepends, 1, "Expected exactly 1 %s command in %s" % (regex.pattern, txt))
+            else:
+                if isinstance(vals, str):
+                    vals = [vals]
+
+                for val in vals:
+                    if get_module_syntax() == 'Tcl':
+                        regex = re.compile(fr'^prepend-path\s+{key}\s+\$root/{val}$', re.M)
+                    elif get_module_syntax() == 'Lua':
+                        regex = re.compile(fr'^prepend_path\("{key}", pathJoin\(root, "{val}"\)\)$', re.M)
+                    else:
+                        self.fail("Unknown module syntax: %s" % get_module_syntax())
+                    self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
+                    # Check for duplicates
+                    num_prepends = len(regex.findall(txt))
+                    self.assertEqual(num_prepends, 1, "Expected exactly 1 %s command in %s" % (regex.pattern, txt))
 
         for (key, vals) in modextrapaths_append.items():
             if isinstance(vals, str):
                 vals = [vals]
             for val in vals:
                 if get_module_syntax() == 'Tcl':
-                    regex = re.compile(r'^append-path\s+%s\s+\$root/%s$' % (key, val), re.M)
+                    regex = re.compile(r'^append-path\s+(-d ".")?%s\s+\$root/%s$' % (key, val), re.M)
                 elif get_module_syntax() == 'Lua':
-                    regex = re.compile(r'^append_path\("%s", pathJoin\(root, "%s"\)\)$' % (key, val), re.M)
+                    regex = re.compile(r'^append_path\("%s", pathJoin\(root, "%s"\)(, ".")?\)$' % (key, val), re.M)
                 else:
                     self.fail("Unknown module syntax: %s" % get_module_syntax())
                 self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))