Merge pull request #2544 from vkarak/feat/improve-test-load-perf

[feat] Optimise the test case generation

Author: Vasileios Karakasis

reframe/frontend/executors/__init__.py

@@ -36,11 +36,11 @@ class TestCase:
 
     def __init__(self, check, partition, environ):
         self._check_orig = check
-        self._check = copy.deepcopy(check)
-        self._partition = copy.deepcopy(partition)
-        self._environ = copy.deepcopy(environ)
-        self._check._case = weakref.ref(self)
+        self._check = check
+        self._partition = partition
+        self._environ = environ
         self._deps = []
+        self._is_ready = False
 
         # Incoming dependencies
         self.in_degree = 0
@@ -72,6 +72,15 @@ def __repr__(self):
         e = self.environ.name if self.environ else None
         return f'({c!r}, {p!r}, {e!r})'
 
+    def prepare(self):
+        '''Prepare test case for sending down the test pipeline'''
+        if self._is_ready:
+            return
+
+        self._check = copy.deepcopy(self._check)
+        self._check._case = weakref.ref(self)
+        self._is_ready = True
+
     @property
     def check(self):
         return self._check
@@ -97,8 +106,14 @@ def clone(self):
         return TestCase(self._check_orig, self._partition, self._environ)
 
 
-def generate_testcases(checks):
-    '''Generate concrete test cases from checks.'''
+def generate_testcases(checks, prepare=False):
+    '''Generate concrete test cases from checks.
+
+    If `prepare` is true then each of the cases will also be prepared for
+    being sent to the test pipeline. Note that setting this to true may slow down
+    the test case generation.
+
+    '''
 
     rt = runtime.runtime()
     cases = []
@@ -107,7 +122,11 @@ def generate_testcases(checks):
                                                c.valid_prog_environs)
         for part, environs in valid_comb.items():
             for env in environs:
-                cases.append(TestCase(c, part, env))
+                case = TestCase(c, part, env)
+                if prepare:
+                    case.prepare()
+
+                cases.append(case)
 
     return cases
 
@@ -295,6 +314,7 @@ def __exit__(this, exc_type, exc_value, traceback):
             raise TaskExit from e
 
     def setup(self, *args, **kwargs):
+        self.testcase.prepare()
         self._safe_call(self.check.setup, *args, **kwargs)
         self._notify_listeners('on_task_setup')
 

reframe/schemas/config.json

@@ -514,7 +514,7 @@
         "environments/features": [],
         "environments/target_systems": ["*"],
         "general/dump_pipeline_progress": false,
-        "general/pipeline_timeout": null,
+        "general/pipeline_timeout": 3,
         "general/check_search_path": ["${RFM_INSTALL_PREFIX}/checks/"],
         "general/check_search_recursive": false,
         "general/clean_stagedir": true,

unittests/test_dependencies.py

@@ -375,7 +375,11 @@ def assert_deps(self):
 
 def test_build_deps(loader, default_exec_ctx):
     checks = loader.load_all(force=True)
-    cases = executors.generate_testcases(checks)
+
+    # We need to prepare the test cases as if we were about to run them,
+    # because we want to test `getdep()` as well, which normally gets resolved
+    # during the `setup` phase of the pipeline
+    cases = executors.generate_testcases(checks, prepare=True)
 
     # Test calling getdep() before having built the graph
     t = find_check('Test1_fully', checks)

unittests/test_pipeline.py

@@ -1210,7 +1210,7 @@ def sety(self, T0):
         def setz(self, T0):
             self.z = T0().x + 2
 
-    cases = executors.generate_testcases([T0(), T1()])
+    cases = executors.generate_testcases([T0(), T1()], prepare=True)
     deps, _ = dependencies.build_deps(cases)
     for c in dependencies.toposort(deps):
         _run(*c)