Merge pull request #2948 from vkarak/enhancement/warn-unset-fixt-vars

[enhancement] Warn on unset variables in fixtures when using the `-S` option

Author: vkarak

reframe/core/decorators.py

@@ -56,14 +56,15 @@ class TestRegistry:
     and the constructor arguments for the different instantiations of the
     test are stored as the dictionary value as a list of (args, kwargs)
     tuples.
-
-    For backward compatibility reasons, the registry also contains a set of
-    tests to be skipped. The machinery related to this should be dropped with
-    the ``required_version`` decorator.
     '''
 
     def __init__(self):
-        self._tests = dict()
+        self._tests = {}
+        self._unset_vars = {}
+
+    @property
+    def unset_vars(self):
+        return self._unset_vars
 
     @classmethod
     def create(cls, test, *args, **kwargs):
@@ -85,6 +86,10 @@ def instantiate_all(self, reset_sysenv=0, external_vars=None):
         :param reset_sysenv: Reset valid_systems and valid_prog_environs after
             instantiating the tests. Bit 0 resets the valid_systems, bit 1
             resets the valid_prog_environs.
+
+        :param external_vars: Test variables to set in the instantiated
+            fixtures.
+
         '''
 
         # We first instantiate the leaf tests and then walk up their
@@ -118,6 +123,7 @@ def instantiate_all(self, reset_sysenv=0, external_vars=None):
         # candidate tests; the leaf tests are consumed at the end of the
         # traversal and all instantiated tests (including fixtures) are stored
         # in `final_tests`.
+        unset_vars = {}
         final_tests = []
         fixture_registry = FixtureRegistry()
         while leaf_tests:
@@ -132,7 +138,9 @@ def instantiate_all(self, reset_sysenv=0, external_vars=None):
             # Instantiate the new fixtures and update the registry
             new_fixtures = tmp_registry.difference(fixture_registry)
             if external_vars:
-                _setvars(new_fixtures.uninst_tests(), external_vars)
+                self._unset_vars.update(
+                    _setvars(new_fixtures.uninst_tests(), external_vars)
+                )
 
             leaf_tests = new_fixtures.instantiate_all()
             fixture_registry.update(new_fixtures)

reframe/frontend/cli.py

@@ -1069,15 +1069,6 @@ def print_infoline(param, value):
             f'{len(testcases)} remaining'
         )
 
-        # Warn on any unset test variables for the final set of selected tests
-        for clsname in {type(tc.check).__name__ for tc in testcases}:
-            varlist = ', '.join(f'{v!r}' for v in loader.unset_vars(clsname))
-            if varlist:
-                printer.warning(
-                    f'test {clsname!r}: '
-                    f'the following variables were not set: {varlist}'
-                )
-
         # Filter in failed cases
         if options.failed:
             if options.restore_session is None:
@@ -1195,6 +1186,16 @@ def _sort_testcases(testcases):
         )
         printer.verbose(f'Final number of test cases: {len(testcases)}')
 
+        # Warn on any unset test variables for the final set of selected tests
+        # including any dependencies
+        for clsname in {type(tc.check).__name__ for tc in testcases}:
+            varlist = ', '.join(f'{v!r}' for v in loader.unset_vars(clsname))
+            if varlist:
+                printer.warning(
+                    f'test {clsname!r}: '
+                    f'the following variables were not set: {varlist}'
+                )
+
         # Disable hooks
         for tc in testcases:
             for h in options.hooks:

reframe/frontend/loader.py

@@ -155,6 +155,7 @@ def load_from_module(self, module):
         if registry:
             candidate_tests = registry.instantiate_all(reset_sysenv,
                                                        self._external_vars)
+            self._unset_vars.update(registry.unset_vars)
         else:
             candidate_tests = []
 

tutorials/cscs-webinar-2022/config/mysettings.py

@@ -12,7 +12,7 @@
         {
             'name': 'tresa',
             'descr': 'My laptop',
-            'hostnames': ['tresa\.local'],
+            'hostnames': ['tresa\.local', 'tresa'],
             'partitions': [
                 {
                     'name': 'default',