Add module-write hook

This hook will be called anytime a module is written, so even for the
fake modules used for sanity checks etc.
It gets passed the easyblock as the first argument and the filename and
content to be written as the next 2 and may return a new text to be used
for the module file. If nothing or None is returned the original text
will be used.

Author: Flamefire

easybuild/framework/easyblock.py

@@ -81,7 +81,7 @@
 from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP
 from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP
 from easybuild.tools.hooks import PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP
-from easybuild.tools.hooks import load_hooks, run_hook
+from easybuild.tools.hooks import MODULE_WRITE, load_hooks, run_hook
 from easybuild.tools.run import run_cmd
 from easybuild.tools.jenkins import write_to_xml
 from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for
@@ -3165,6 +3165,10 @@ def make_module_step(self, fake=False):
         txt += self.make_module_extra()
         txt += self.make_module_footer()
 
+        hook_txt = run_hook(MODULE_WRITE, self.hooks, args=[self, mod_filepath, txt])
+        if hook_txt is not None:
+            txt = hook_txt
+
         if self.dry_run:
             # only report generating actual module file during dry run, don't mention temporary module files
             if not fake:

easybuild/tools/hooks.py

@@ -58,6 +58,7 @@
 
 START = 'start'
 PARSE = 'parse'
+MODULE_WRITE = 'module_write'
 END = 'end'
 
 PRE_PREF = 'pre_'
@@ -69,7 +70,7 @@
               INSTALL_STEP, EXTENSIONS_STEP, POSTPROC_STEP, SANITYCHECK_STEP, CLEANUP_STEP, MODULE_STEP,
               PERMISSIONS_STEP, PACKAGE_STEP, TESTCASES_STEP]
 
-HOOK_NAMES = [START, PARSE] + [p + s for s in STEP_NAMES for p in [PRE_PREF, POST_PREF]] + [END]
+HOOK_NAMES = [START, PARSE, MODULE_WRITE] + [p + s for s in STEP_NAMES for p in [PRE_PREF, POST_PREF]] + [END]
 KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES]
 
 
@@ -99,7 +100,7 @@ def load_hooks(hooks_path):
                         if attr.endswith(HOOK_SUFF):
                             hook = getattr(imported_hooks, attr)
                             if callable(hook):
-                                hooks.update({attr: hook})
+                                hooks[attr] = hook
                             else:
                                 _log.debug("Skipping non-callable attribute '%s' when loading hooks", attr)
                     _log.info("Found hooks: %s", sorted(hooks.keys()))
@@ -120,10 +121,7 @@ def load_hooks(hooks_path):
 
 def verify_hooks(hooks):
     """Check whether list of obtained hooks only includes known hooks."""
-    unknown_hooks = []
-    for key in sorted(hooks):
-        if key not in KNOWN_HOOKS:
-            unknown_hooks.append(key)
+    unknown_hooks = [key for key in sorted(hooks) if key not in KNOWN_HOOKS]
 
     if unknown_hooks:
         error_lines = ["Found one or more unknown hooks:"]
@@ -147,7 +145,7 @@ def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False):
     Find hook with specified label.
 
     :param label: name of hook
-    :param hooks: list of defined hooks
+    :param hooks: dict of defined hooks
     :param pre_step_hook: indicates whether hook to run is a pre-step hook
     :param post_step_hook: indicates whether hook to run is a post-step hook
     """
@@ -162,18 +160,16 @@ def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False):
 
     hook_name = hook_prefix + label + HOOK_SUFF
 
-    for key in hooks:
-        if key == hook_name:
-            _log.info("Found %s hook", hook_name)
-            res = hooks[key]
-            break
+    res = hooks.get(hook_name)
+    if res:
+        _log.info("Found %s hook", hook_name)
 
     return res
 
 
 def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, msg=None):
     """
-    Run hook with specified label.
+    Run hook with specified label and return result of hook or None.
 
     :param label: name of hook
     :param hooks: list of defined hooks
@@ -183,6 +179,7 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None,
     :param msg: custom message that is printed when hook is called
     """
     hook = find_hook(label, hooks, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook)
+    res = None
     if hook:
         if args is None:
             args = []
@@ -197,4 +194,5 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None,
         print_msg(msg)
 
         _log.info("Running '%s' hook function (arguments: %s)...", hook.__name__, args)
-        hook(*args)
+        res = hook(*args)
+    return res

test/framework/toy_build.py

@@ -39,6 +39,7 @@
 import stat
 import sys
 import tempfile
+import textwrap
 from distutils.version import LooseVersion
 from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered
 from test.framework.package import mock_fpm
@@ -2638,34 +2639,38 @@ def test_toy_build_trace(self):
     def test_toy_build_hooks(self):
         """Test use of --hooks."""
         hooks_file = os.path.join(self.test_prefix, 'my_hooks.py')
-        hooks_file_txt = '\n'.join([
-            "import os",
-            '',
-            "def start_hook():",
-            "   print('start hook triggered')",
-            '',
-            "def parse_hook(ec):",
-            "   print('%s %s' % (ec.name, ec.version))",
+        hooks_file_txt = textwrap.dedent("""
+            import os
+
+            def start_hook():
+               print('start hook triggered')
+
+            def parse_hook(ec):
+               print('%s %s' % (ec.name, ec.version))
             # print sources value to check that raw untemplated strings are exposed in parse_hook
-            "   print(ec['sources'])",
+               print(ec['sources'])
             # try appending to postinstallcmd to see whether the modification is actually picked up
             # (required templating to be disabled before parse_hook is called)
-            "   ec['postinstallcmds'].append('echo toy')",
-            "   print(ec['postinstallcmds'][-1])",
-            '',
-            "def pre_configure_hook(self):",
-            "    print('pre-configure: toy.source: %s' % os.path.exists('toy.source'))",
-            '',
-            "def post_configure_hook(self):",
-            "    print('post-configure: toy.source: %s' % os.path.exists('toy.source'))",
-            '',
-            "def post_install_hook(self):",
-            "    print('in post-install hook for %s v%s' % (self.name, self.version))",
-            "    print(', '.join(sorted(os.listdir(self.installdir))))",
-            '',
-            "def end_hook():",
-            "   print('end hook triggered, all done!')",
-        ])
+               ec['postinstallcmds'].append('echo toy')
+               print(ec['postinstallcmds'][-1])
+
+            def pre_configure_hook(self):
+                print('pre-configure: toy.source: %s' % os.path.exists('toy.source'))
+
+            def post_configure_hook(self):
+                print('post-configure: toy.source: %s' % os.path.exists('toy.source'))
+
+            def post_install_hook(self):
+                print('in post-install hook for %s v%s' % (self.name, self.version))
+                print(', '.join(sorted(os.listdir(self.installdir))))
+
+            def module_write_hook(self, module_path, module_txt):
+                print('in module-write hook hook for %s' % os.path.basename(module_path))
+                return module_txt.replace('Toy C program, 100% toy.', 'Not a toy anymore')
+
+            def end_hook():
+               print('end hook triggered, all done!')
+        """)
         write_file(hooks_file, hooks_file_txt)
 
         self.mock_stderr(True)
@@ -2676,26 +2681,44 @@ def test_toy_build_hooks(self):
         self.mock_stderr(False)
         self.mock_stdout(False)
 
+        test_mod_path = os.path.join(self.test_installpath, 'modules', 'all')
+        toy_mod_file = os.path.join(test_mod_path, 'toy', '0.0')
+        if get_module_syntax() == 'Lua':
+            toy_mod_file += '.lua'
+
         self.assertEqual(stderr, '')
-        expected_output = '\n'.join([
-            "== Running start hook...",
-            "start hook triggered",
-            "== Running parse hook for toy-0.0.eb...",
-            "toy 0.0",
-            "['%(name)s-%(version)s.tar.gz']",
-            "echo toy",
-            "== Running pre-configure hook...",
-            "pre-configure: toy.source: True",
-            "== Running post-configure hook...",
-            "post-configure: toy.source: False",
-            "== Running post-install hook...",
-            "in post-install hook for toy v0.0",
-            "bin, lib",
-            "== Running end hook...",
-            "end hook triggered, all done!",
-        ])
+        # There are 4 modules written:
+        # Sanitycheck for extensions and main easyblock (1 each), main and devel module
+        expected_output = textwrap.dedent("""
+            == Running start hook...
+            start hook triggered
+            == Running parse hook for toy-0.0.eb...
+            toy 0.0
+            ['%(name)s-%(version)s.tar.gz']
+            echo toy
+            == Running pre-configure hook...
+            pre-configure: toy.source: True
+            == Running post-configure hook...
+            post-configure: toy.source: False
+            == Running post-install hook...
+            in post-install hook for toy v0.0
+            bin, lib
+            == Running module_write hook...
+            in module-write hook hook for {mod_name}
+            == Running module_write hook...
+            in module-write hook hook for {mod_name}
+            == Running module_write hook...
+            in module-write hook hook for {mod_name}
+            == Running module_write hook...
+            in module-write hook hook for {mod_name}
+            == Running end hook...
+            end hook triggered, all done!
+        """).strip().format(mod_name=os.path.basename(toy_mod_file))
         self.assertEqual(stdout.strip(), expected_output)
 
+        toy_mod = read_file(toy_mod_file)
+        self.assertIn('Not a toy anymore', toy_mod)
+
     def test_toy_multi_deps(self):
         """Test installation of toy easyconfig that uses multi_deps."""
         test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')