Merge pull request #3728 from Flamefire/module-write-hook

Add module-write hook

Author: smoors

easybuild/framework/easyblock.py

@@ -81,7 +81,7 @@
 from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP
 from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP
 from easybuild.tools.hooks import PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP
-from easybuild.tools.hooks import load_hooks, run_hook
+from easybuild.tools.hooks import MODULE_WRITE, load_hooks, run_hook
 from easybuild.tools.run import run_cmd
 from easybuild.tools.jenkins import write_to_xml
 from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for
@@ -144,7 +144,7 @@ def __init__(self, ec):
         # keep track of original working directory, so we can go back there
         self.orig_workdir = os.getcwd()
 
-        # list of pre- and post-step hooks
+        # dict of all hooks (mapping of name to function)
         self.hooks = load_hooks(build_option('hooks'))
 
         # list of patch/source files, along with checksums
@@ -3165,6 +3165,10 @@ def make_module_step(self, fake=False):
         txt += self.make_module_extra()
         txt += self.make_module_footer()
 
+        hook_txt = run_hook(MODULE_WRITE, self.hooks, args=[self, mod_filepath, txt])
+        if hook_txt is not None:
+            txt = hook_txt
+
         if self.dry_run:
             # only report generating actual module file during dry run, don't mention temporary module files
             if not fake:

easybuild/tools/hooks.py

@@ -58,6 +58,7 @@
 
 START = 'start'
 PARSE = 'parse'
+MODULE_WRITE = 'module_write'
 END = 'end'
 
 PRE_PREF = 'pre_'
@@ -69,7 +70,7 @@
               INSTALL_STEP, EXTENSIONS_STEP, POSTPROC_STEP, SANITYCHECK_STEP, CLEANUP_STEP, MODULE_STEP,
               PERMISSIONS_STEP, PACKAGE_STEP, TESTCASES_STEP]
 
-HOOK_NAMES = [START, PARSE] + [p + s for s in STEP_NAMES for p in [PRE_PREF, POST_PREF]] + [END]
+HOOK_NAMES = [START, PARSE, MODULE_WRITE] + [p + s for s in STEP_NAMES for p in [PRE_PREF, POST_PREF]] + [END]
 KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES]
 
 
@@ -99,7 +100,7 @@ def load_hooks(hooks_path):
                         if attr.endswith(HOOK_SUFF):
                             hook = getattr(imported_hooks, attr)
                             if callable(hook):
-                                hooks.update({attr: hook})
+                                hooks[attr] = hook
                             else:
                                 _log.debug("Skipping non-callable attribute '%s' when loading hooks", attr)
                     _log.info("Found hooks: %s", sorted(hooks.keys()))
@@ -119,11 +120,8 @@ def load_hooks(hooks_path):
 
 
 def verify_hooks(hooks):
-    """Check whether list of obtained hooks only includes known hooks."""
-    unknown_hooks = []
-    for key in sorted(hooks):
-        if key not in KNOWN_HOOKS:
-            unknown_hooks.append(key)
+    """Check whether obtained hooks only includes known hooks."""
+    unknown_hooks = [key for key in sorted(hooks) if key not in KNOWN_HOOKS]
 
     if unknown_hooks:
         error_lines = ["Found one or more unknown hooks:"]
@@ -147,7 +145,7 @@ def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False):
     Find hook with specified label.
 
     :param label: name of hook
-    :param hooks: list of defined hooks
+    :param hooks: dict of defined hooks
     :param pre_step_hook: indicates whether hook to run is a pre-step hook
     :param post_step_hook: indicates whether hook to run is a post-step hook
     """
@@ -162,27 +160,26 @@ def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False):
 
     hook_name = hook_prefix + label + HOOK_SUFF
 
-    for key in hooks:
-        if key == hook_name:
-            _log.info("Found %s hook", hook_name)
-            res = hooks[key]
-            break
+    res = hooks.get(hook_name)
+    if res:
+        _log.info("Found %s hook", hook_name)
 
     return res
 
 
 def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, msg=None):
     """
-    Run hook with specified label.
+    Run hook with specified label and return result of calling the hook or None.
 
     :param label: name of hook
-    :param hooks: list of defined hooks
+    :param hooks: dict of defined hooks
     :param pre_step_hook: indicates whether hook to run is a pre-step hook
     :param post_step_hook: indicates whether hook to run is a post-step hook
     :param args: arguments to pass to hook function
     :param msg: custom message that is printed when hook is called
     """
     hook = find_hook(label, hooks, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook)
+    res = None
     if hook:
         if args is None:
             args = []
@@ -197,4 +194,5 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None,
         print_msg(msg)
 
         _log.info("Running '%s' hook function (arguments: %s)...", hook.__name__, args)
-        hook(*args)
+        res = hook(*args)
+    return res

test/framework/toy_build.py

@@ -39,6 +39,7 @@
 import stat
 import sys
 import tempfile
+import textwrap
 from distutils.version import LooseVersion
 from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered
 from test.framework.package import mock_fpm
@@ -2638,34 +2639,38 @@ def test_toy_build_trace(self):
     def test_toy_build_hooks(self):
         """Test use of --hooks."""
         hooks_file = os.path.join(self.test_prefix, 'my_hooks.py')
-        hooks_file_txt = '\n'.join([
-            "import os",
-            '',
-            "def start_hook():",
-            "   print('start hook triggered')",
-            '',
-            "def parse_hook(ec):",
-            "   print('%s %s' % (ec.name, ec.version))",
+        hooks_file_txt = textwrap.dedent("""
+            import os
+
+            def start_hook():
+               print('start hook triggered')
+
+            def parse_hook(ec):
+               print('%s %s' % (ec.name, ec.version))
             # print sources value to check that raw untemplated strings are exposed in parse_hook
-            "   print(ec['sources'])",
+               print(ec['sources'])
             # try appending to postinstallcmd to see whether the modification is actually picked up
             # (required templating to be disabled before parse_hook is called)
-            "   ec['postinstallcmds'].append('echo toy')",
-            "   print(ec['postinstallcmds'][-1])",
-            '',
-            "def pre_configure_hook(self):",
-            "    print('pre-configure: toy.source: %s' % os.path.exists('toy.source'))",
-            '',
-            "def post_configure_hook(self):",
-            "    print('post-configure: toy.source: %s' % os.path.exists('toy.source'))",
-            '',
-            "def post_install_hook(self):",
-            "    print('in post-install hook for %s v%s' % (self.name, self.version))",
-            "    print(', '.join(sorted(os.listdir(self.installdir))))",
-            '',
-            "def end_hook():",
-            "   print('end hook triggered, all done!')",
-        ])
+               ec['postinstallcmds'].append('echo toy')
+               print(ec['postinstallcmds'][-1])
+
+            def pre_configure_hook(self):
+                print('pre-configure: toy.source: %s' % os.path.exists('toy.source'))
+
+            def post_configure_hook(self):
+                print('post-configure: toy.source: %s' % os.path.exists('toy.source'))
+
+            def post_install_hook(self):
+                print('in post-install hook for %s v%s' % (self.name, self.version))
+                print(', '.join(sorted(os.listdir(self.installdir))))
+
+            def module_write_hook(self, module_path, module_txt):
+                print('in module-write hook hook for %s' % os.path.basename(module_path))
+                return module_txt.replace('Toy C program, 100% toy.', 'Not a toy anymore')
+
+            def end_hook():
+               print('end hook triggered, all done!')
+        """)
         write_file(hooks_file, hooks_file_txt)
 
         self.mock_stderr(True)
@@ -2676,26 +2681,44 @@ def test_toy_build_hooks(self):
         self.mock_stderr(False)
         self.mock_stdout(False)
 
+        test_mod_path = os.path.join(self.test_installpath, 'modules', 'all')
+        toy_mod_file = os.path.join(test_mod_path, 'toy', '0.0')
+        if get_module_syntax() == 'Lua':
+            toy_mod_file += '.lua'
+
         self.assertEqual(stderr, '')
-        expected_output = '\n'.join([
-            "== Running start hook...",
-            "start hook triggered",
-            "== Running parse hook for toy-0.0.eb...",
-            "toy 0.0",
-            "['%(name)s-%(version)s.tar.gz']",
-            "echo toy",
-            "== Running pre-configure hook...",
-            "pre-configure: toy.source: True",
-            "== Running post-configure hook...",
-            "post-configure: toy.source: False",
-            "== Running post-install hook...",
-            "in post-install hook for toy v0.0",
-            "bin, lib",
-            "== Running end hook...",
-            "end hook triggered, all done!",
-        ])
+        # There are 4 modules written:
+        # Sanitycheck for extensions and main easyblock (1 each), main and devel module
+        expected_output = textwrap.dedent("""
+            == Running start hook...
+            start hook triggered
+            == Running parse hook for toy-0.0.eb...
+            toy 0.0
+            ['%(name)s-%(version)s.tar.gz']
+            echo toy
+            == Running pre-configure hook...
+            pre-configure: toy.source: True
+            == Running post-configure hook...
+            post-configure: toy.source: False
+            == Running post-install hook...
+            in post-install hook for toy v0.0
+            bin, lib
+            == Running module_write hook...
+            in module-write hook hook for {mod_name}
+            == Running module_write hook...
+            in module-write hook hook for {mod_name}
+            == Running module_write hook...
+            in module-write hook hook for {mod_name}
+            == Running module_write hook...
+            in module-write hook hook for {mod_name}
+            == Running end hook...
+            end hook triggered, all done!
+        """).strip().format(mod_name=os.path.basename(toy_mod_file))
         self.assertEqual(stdout.strip(), expected_output)
 
+        toy_mod = read_file(toy_mod_file)
+        self.assertIn('Not a toy anymore', toy_mod)
+
     def test_toy_multi_deps(self):
         """Test installation of toy easyconfig that uses multi_deps."""
         test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')