Merge pull request #3375 from ocaisa/track_all_easyblocks_used

Also put easyblocks used by extensions in `reprod` directory

Author: akesandgren

easybuild/framework/easyblock.py

@@ -3278,11 +3278,19 @@ def build_and_install_one(ecdict, init_env):
     start_time = time.time()
     try:
         run_test_cases = not build_option('skip_test_cases') and app.cfg['tests']
+
         if not dry_run:
             # create our reproducability files before carrying out the easyblock steps
             reprod_dir_root = os.path.dirname(app.logfile)
             reprod_dir = reproduce_build(app, reprod_dir_root)
+
         result = app.run_all_steps(run_test_cases=run_test_cases)
+
+        if not dry_run:
+            # also add any extension easyblocks used during the build for reproducability
+            if app.ext_instances:
+                copy_easyblocks_for_reprod(app.ext_instances, reprod_dir)
+
     except EasyBuildError as err:
         first_n = 300
         errormsg = "build failed (first %d chars): %s" % (first_n, err.msg[:first_n])
@@ -3416,6 +3424,24 @@ def build_and_install_one(ecdict, init_env):
     return (success, application_log, errormsg)
 
 
+def copy_easyblocks_for_reprod(easyblock_instances, reprod_dir):
+    reprod_easyblock_dir = os.path.join(reprod_dir, 'easyblocks')
+    easyblock_paths = set()
+    for easyblock_instance in easyblock_instances:
+        for easyblock_class in inspect.getmro(type(easyblock_instance)):
+            easyblock_path = inspect.getsourcefile(easyblock_class)
+            # if we reach EasyBlock or ExtensionEasyBlock class, we are done
+            # (ExtensionEasyblock is hardcoded to avoid a cyclical import)
+            if easyblock_class.__name__ in [EasyBlock.__name__, 'ExtensionEasyBlock']:
+                break
+            else:
+                easyblock_paths.add(easyblock_path)
+    for easyblock_path in easyblock_paths:
+        easyblock_basedir, easyblock_filename = os.path.split(easyblock_path)
+        copy_file(easyblock_path, os.path.join(reprod_easyblock_dir, easyblock_filename))
+        _log.info("Dumped easyblock %s required for reproduction to %s", easyblock_filename, reprod_easyblock_dir)
+
+
 def reproduce_build(app, reprod_dir_root):
     """
     Create reproducability files (processed easyconfig and easyblocks used) from class instance
@@ -3437,18 +3463,8 @@ def reproduce_build(app, reprod_dir_root):
     except NotImplementedError as err:
         _log.warning("Unable to dump easyconfig instance to %s: %s", reprod_spec, err)
 
-    # also archive the relevant easyblocks
-    reprod_easyblock_dir = os.path.join(reprod_dir, 'easyblocks')
-    for easyblock_class in inspect.getmro(type(app)):
-        easyblock_path = inspect.getsourcefile(easyblock_class)
-        easyblock_basedir, easyblock_filename = os.path.split(easyblock_path)
-        # if we reach EasyBlock or ExtensionEasyBlock class, we are done
-        # (ExtensionEasyblock is hardcoded to avoid a cyclical import)
-        if easyblock_class.__name__ in [EasyBlock.__name__, 'ExtensionEasyBlock']:
-            break
-        else:
-            copy_file(easyblock_path, os.path.join(reprod_easyblock_dir, easyblock_filename))
-            _log.info("Dumped easyblock %s required for reproduction to %s", easyblock_filename, reprod_easyblock_dir)
+    # also archive all the relevant easyblocks (including any used by extensions)
+    copy_easyblocks_for_reprod([app], reprod_dir)
 
     # if there is a hook file we should also archive it
     hooks_path = build_option('hooks')

test/framework/toy_build.py

@@ -1839,6 +1839,44 @@ def test_reproducability(self):
         reprod_hooks = os.path.join(reprod_dir, 'hooks', hooks_filename)
         self.assertTrue(os.path.exists(reprod_hooks))
 
+    def test_reproducability_ext_easyblocks(self):
+        """Test toy build produces expected reproducability files also when extensions are used"""
+
+        topdir = os.path.dirname(os.path.abspath(__file__))
+        toy_ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
+        toy_ec_txt = read_file(toy_ec_file)
+
+        ec1 = os.path.join(self.test_prefix, 'toy1.eb')
+        ec1_txt = '\n'.join([
+            toy_ec_txt,
+            "exts_list = [('barbar', '0.0')]",
+            "",
+        ])
+        write_file(ec1, ec1_txt)
+
+        self.test_toy_build(ec_file=ec1, verify=False, extra_args=['--minimal-toolchains', '--easyblock=EB_toytoy'])
+
+        # Check whether easyconfig is dumped to reprod/ subdir
+        reprod_dir = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'easybuild', 'reprod')
+        reprod_ec = os.path.join(reprod_dir, 'toy-0.0.eb')
+
+        self.assertTrue(os.path.exists(reprod_ec))
+
+        # Check for child easyblock existence
+        child_easyblock = os.path.join(reprod_dir, 'easyblocks', 'toytoy.py')
+        self.assertTrue(os.path.exists(child_easyblock))
+        # Check for parent easyblock existence
+        parent_easyblock = os.path.join(reprod_dir, 'easyblocks', 'toy.py')
+        self.assertTrue(os.path.exists(parent_easyblock))
+        # Check for extension easyblock existence
+        ext_easyblock = os.path.join(reprod_dir, 'easyblocks', 'toy_extension.py')
+        self.assertTrue(os.path.exists(ext_easyblock))
+
+        # Make sure framework easyblock modules are not included
+        for framework_easyblock in ['easyblock.py', 'extensioneasyblock.py']:
+            path = os.path.join(reprod_dir, 'easyblocks', framework_easyblock)
+            self.assertFalse(os.path.exists(path))
+
     def test_toy_toy(self):
         """Test building two easyconfigs in a single go, with one depending on the other."""
         topdir = os.path.dirname(os.path.abspath(__file__))