Merge pull request #4923 from boegel/easyblock_hook

add support for pre/post `easyblock` hook

Author: ocaisa

easybuild/framework/easyblock.py

@@ -98,9 +98,9 @@
 from easybuild.tools.filetools import is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
 from easybuild.tools.filetools import remove_file, remove_lock, symlink, verify_checksum, weld_paths, write_file
 from easybuild.tools.hooks import (
-    BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, EXTRACT_STEP, FETCH_STEP, INSTALL_STEP, MODULE_STEP,
-    MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP, PREPARE_STEP, READY_STEP,
-    SANITYCHECK_STEP, SINGLE_EXTENSION, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook,
+    BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EASYBLOCK, EXTENSIONS_STEP, EXTRACT_STEP, FETCH_STEP, INSTALL_STEP,
+    MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP, PREPARE_STEP,
+    READY_STEP, SANITYCHECK_STEP, SINGLE_EXTENSION, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook,
 )
 from easybuild.tools.run import RunShellCmdError, raise_run_shell_cmd_error, run_shell_cmd
 from easybuild.tools.jenkins import write_to_xml
@@ -5001,6 +5001,8 @@ def build_and_install_one(ecdict, init_env):
         _log.debug("Skip set to %s" % skip)
         app.cfg['skip'] = skip
 
+    hooks = load_hooks(build_option('hooks'))
+
     # build easyconfig
     error_msg = '(no error)'
     exit_code = None
@@ -5022,6 +5024,8 @@ def build_and_install_one(ecdict, init_env):
             else:
                 enabled_write_permissions = False
 
+        run_hook(EASYBLOCK, hooks, pre_step_hook=True, args=[app])
+
         result = app.run_all_steps(run_test_cases=run_test_cases)
 
         if not dry_run:
@@ -5127,6 +5131,8 @@ def ensure_writable_log_dir(log_dir):
             except EasyBuildError as err:
                 _log.warning("Unable to commit easyconfig to repository: %s", err)
 
+        run_hook(EASYBLOCK, hooks, post_step_hook=True, args=[app])
+
         # cleanup logs
         app.close_log()
 

easybuild/tools/hooks.py

@@ -62,6 +62,7 @@
 START = 'start'
 PARSE = 'parse'
 BUILD_AND_INSTALL_LOOP = 'build_and_install_loop'
+EASYBLOCK = 'easyblock'
 SINGLE_EXTENSION = 'single_extension'
 MODULE_WRITE = 'module_write'
 END = 'end'
@@ -86,6 +87,7 @@
     START,
     PARSE,
     PRE_PREF + BUILD_AND_INSTALL_LOOP,
+    PRE_PREF + EASYBLOCK,
 ] + [p + x for x in STEP_NAMES[:STEP_NAMES.index(EXTENSIONS_STEP)]
      for p in [PRE_PREF, POST_PREF]] + [
     # pre-extensions hook is triggered before starting installation of extensions,
@@ -105,6 +107,7 @@
     POST_PREF + MODULE_STEP,
 ] + [p + x for x in STEP_NAMES[STEP_NAMES.index(MODULE_STEP)+1:]
      for p in [PRE_PREF, POST_PREF]] + [
+    POST_PREF + EASYBLOCK,
     POST_PREF + BUILD_AND_INSTALL_LOOP,
     END,
     CANCEL,

test/framework/hooks.py

@@ -57,6 +57,9 @@ def setUp(self):
             'def pre_build_and_install_loop_hook(ecs):',
             '    print("About to start looping for %d easyconfigs!" % len(ecs))',
             '',
+            'def pre_easyblock_hook(self):',
+            '    print(f"Starting installation of {self.name} {self.version}")',
+            '',
             'def foo():',
             '    print("running foo helper method")',
             '',
@@ -78,6 +81,9 @@ def setUp(self):
             '        if cmd == "make install":',
             '            return "sudo " + cmd',
             '',
+            'def post_easyblock_hook(self):',
+            '    print(f"Done with installation of {self.name} {self.version}")',
+            '',
             'def fail_hook(err):',
             '    print("EasyBuild FAIL: %s" % err)',
             '',
@@ -101,13 +107,15 @@ def test_load_hooks(self):
 
         hooks = load_hooks(self.test_hooks_pymod)
 
-        self.assertEqual(len(hooks), 9)
+        self.assertEqual(len(hooks), 11)
         expected = [
             'crash_hook',
             'fail_hook',
             'parse_hook',
             'post_configure_hook',
+            'post_easyblock_hook',
             'pre_build_and_install_loop_hook',
+            'pre_easyblock_hook',
             'pre_install_hook',
             'pre_run_shell_cmd_hook',
             'pre_single_extension_hook',
@@ -197,12 +205,18 @@ def test_run_hook(self):
 
         init_config(build_options={'debug': True})
 
+        class FakeEasyBlock():
+            def __init__(self, *args, **kwargs):
+                self.name = 'fake'
+                self.version = '1.2.3'
+
         def run_hooks():
             self.mock_stdout(True)
             self.mock_stderr(True)
             run_hook('start', hooks)
             run_hook('parse', hooks, args=['<EasyConfig instance>'], msg="Running parse hook for example.eb...")
             run_hook('build_and_install_loop', hooks, args=[['ec1', 'ec2']], pre_step_hook=True)
+            run_hook('easyblock', hooks, args=[FakeEasyBlock()], pre_step_hook=True)
             run_hook('configure', hooks, pre_step_hook=True, args=[None])
             run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["configure.sh"], kwargs={'interactive': True})
             run_hook('configure', hooks, post_step_hook=True, args=[None])
@@ -220,6 +234,7 @@ def run_hooks():
             run_hook('extensions', hooks, post_step_hook=True, args=[None])
             run_hook('fail', hooks, args=[EasyBuildError('oops')])
             run_hook('crash', hooks, args=[RuntimeError('boom!')])
+            run_hook('easyblock', hooks, args=[FakeEasyBlock()], post_step_hook=True)
             stdout = self.get_stdout()
             stderr = self.get_stderr()
             self.mock_stdout(False)
@@ -236,6 +251,8 @@ def run_hooks():
             "Parse hook with argument <EasyConfig instance>",
             "== Running pre-build_and_install_loop hook...",
             "About to start looping for 2 easyconfigs!",
+            "== Running pre-easyblock hook...",
+            "Starting installation of fake 1.2.3",
             "== Running pre-run_shell_cmd hook...",
             "this is run before running interactive command 'configure.sh'",
             "== Running post-configure hook...",
@@ -257,6 +274,8 @@ def run_hooks():
             "EasyBuild FAIL: 'oops'",
             "== Running crash hook...",
             "EasyBuild CRASHED, oh no! => boom!",
+            "== Running post-easyblock hook...",
+            "Done with installation of fake 1.2.3",
         ]
         expected_stdout = '\n'.join(expected_stdout_lines)
 

test/framework/options.py

@@ -797,6 +797,7 @@ def test_avail_hooks(self):
             "	start_hook",
             "	parse_hook",
             "	pre_build_and_install_loop_hook",
+            "	pre_easyblock_hook",
             "	pre_fetch_hook",
             "	post_fetch_hook",
             "	pre_ready_hook",
@@ -836,6 +837,7 @@ def test_avail_hooks(self):
             "	post_package_hook",
             "	pre_testcases_hook",
             "	post_testcases_hook",
+            "	post_easyblock_hook",
             "	post_build_and_install_loop_hook",
             "	end_hook",
             "	cancel_hook",

test/framework/toy_build.py

@@ -3654,13 +3654,20 @@ def start_hook():
 
             def parse_hook(ec):
                print('%s %s' % (ec.name, ec.version))
-            # print sources value to check that raw untemplated strings are exposed in parse_hook
+               # print sources value to check that raw untemplated strings are exposed in parse_hook
                print(ec['sources'])
-            # try appending to postinstallcmd to see whether the modification is actually picked up
-            # (required templating to be disabled before parse_hook is called)
+               # try appending to postinstallcmd to see whether the modification is actually picked up
+               # (required templating to be disabled before parse_hook is called)
                ec['postinstallcmds'].append('echo toy')
                print(ec['postinstallcmds'][-1])
 
+            def pre_build_and_install_loop_hook(ecs):
+                mod_names = ' '.join(ec['full_mod_name'] for ec in ecs)
+                print(f"installing {len(ecs)} easyconfigs: {mod_names}")
+
+            def pre_easyblock_hook(self):
+                print(f'starting installation of {self.name} {self.version}')
+
             def pre_configure_hook(self):
                 print('pre-configure: toy.source: %s' % os.path.exists('toy.source'))
 
@@ -3704,6 +3711,13 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
                     copy_file('toy', 'copy_of_toy')
                     change_dir(cwd)
                     print("'%s' command failed (exit code %s), but I fixed it!" % (cmd, exit_code))
+
+            def post_easyblock_hook(self):
+                print(f'done with installation of {self.name} {self.version}')
+
+            def post_build_and_install_loop_hook(ecs):
+                mod_names = ' '.join(ec[0]['full_mod_name'] for ec in ecs)
+                print(f"done with installing {len(ecs)} easyconfigs: {mod_names}")
         """)
         write_file(hooks_file, hooks_file_txt)
 
@@ -3741,6 +3755,8 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
             toy 0.0
             ['%(name)s-%(version)s.tar.gz']
             echo toy
+            installing 1 easyconfigs: toy/0.0
+            starting installation of toy 0.0
             pre-configure: toy.source: True
             post-configure: toy.source: False
             pre_run_shell_cmd_hook triggered for ' gcc toy.c -o toy '
@@ -3764,6 +3780,8 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
             in module-write hook hook for {mod_name}
             in module-write hook hook for {mod_name}
             in module-write hook hook for {mod_name}
+            done with installation of toy 0.0
+            done with installing 1 easyconfigs: toy/0.0
             end hook triggered, all done!
         """).strip().format(mod_name=os.path.basename(toy_mod_file))
         self.assertEqual(stdout.strip(), expected_output)