Merge pull request #3010 from vkarak/feat/filter-by-expr

[feat] Filter tests using arbitrary expressions

Author: vkarak

docs/manpage.rst

@@ -68,6 +68,21 @@ This happens recursively so that if test ``T1`` depends on ``T2`` and ``T2`` dep
    The value of this attribute is not required to be non-zero for GPU tests.
    Tests may or may not make use of it.
 
+   .. deprecated:: 4.4
+
+      Please use ``-E 'not num_gpus_per_node'`` instead.
+
+.. option:: -E, --filter-expr=EXPR
+
+   Select only tests that satisfy the given expression.
+
+   The expression ``EXPR`` can be any valid Python expression on the test variables or parameters.
+   For example, ``-E num_tasks > 10`` will select all tests, whose :attr:`~reframe.core.pipeline.RegressionTest.num_tasks` exceeds ``10``.
+   You may use any test variable in expression, even user-defined.
+   Multiple variables can also be included such as ``-E num_tasks >= my_param``, where ``my_param`` is user-defined parameter.
+
+   .. versionadded:: 4.4
+
 .. option:: --failed
 
    Select only the failed test cases for a previous run.
@@ -77,13 +92,18 @@ This happens recursively so that if test ``T1`` depends on ``T2`` and ``T2`` dep
 
    .. versionadded:: 3.4
 
+
 .. option:: --gpu-only
 
    Select tests that can run on GPUs.
 
    These are all tests with :attr:`num_gpus_per_node` greater than zero.
    This option and :option:`--cpu-only` are mutually exclusive.
 
+   .. deprecated:: 4.4
+
+      Please use ``-E num_gpus_per_node`` instead.
+
 .. option:: --maintainer=MAINTAINER
 
    Filter tests by maintainer.
@@ -101,6 +121,7 @@ This happens recursively so that if test ``T1`` depends on ``T2`` and ``T2`` dep
       The ``MAINTAINER`` pattern is matched anywhere in the maintainer's name and not at its beginning.
       If you want to match at the beginning of the name, you should prepend ``^``.
 
+
 .. option:: -n, --name=NAME
 
    Filter tests by name.

reframe/frontend/cli.py

@@ -356,6 +356,10 @@ def main():
         metavar='PATTERN', default=[],
         help='Exclude checks whose name matches PATTERN'
     )
+    select_options.add_argument(
+        '-E', '--filter-expr', action='store', metavar='EXPR',
+        help='Select checks that satisfy the expression EXPR'
+    )
 
     # Action options
     action_options.add_argument(
@@ -1048,6 +1052,16 @@ def print_infoline(param, value):
             f'Filtering test cases(s) by tags: {len(testcases)} remaining'
         )
 
+        if options.filter_expr:
+            testcases = filter(filters.validates(options.filter_expr),
+                               testcases)
+
+            testcases = list(testcases)
+            printer.verbose(
+                f'Filtering test cases(s) by {options.filter_expr}: '
+                f'{len(testcases)} remaining'
+            )
+
         # Filter test cases by maintainers
         for maint in options.maintainers:
             testcases = filter(filters.have_maintainer(maint), testcases)
@@ -1059,8 +1073,12 @@ def print_infoline(param, value):
             sys.exit(1)
 
         if options.gpu_only:
+            printer.warning('the `--gpu-only` option is deprecated; '
+                            'please use `-E num_gpus_per_node` instead')
             testcases = filter(filters.have_gpu_only(), testcases)
         elif options.cpu_only:
+            printer.warning('the `--cpu-only` option is deprecated; '
+                            'please use `-E "not num_gpus_per_node"` instead')
             testcases = filter(filters.have_cpu_only(), testcases)
 
         testcases = list(testcases)

reframe/frontend/filters.py

@@ -109,16 +109,18 @@ def _fn(case):
 
 
 def have_gpu_only():
-    def _fn(case):
-        # NOTE: This takes into account num_gpus_per_node being None
-        return case.check.num_gpus_per_node
-
-    return _fn
+    return validates('num_gpus_per_node')
 
 
 def have_cpu_only():
+    return validates('not num_gpus_per_node')
+
+
+def validates(expr):
     def _fn(case):
-        # NOTE: This takes into account num_gpus_per_node being None
-        return not case.check.num_gpus_per_node
+        try:
+            return eval(expr, None, case.check.__dict__)
+        except Exception as err:
+            raise ReframeError(f'invalid expression `{expr}`') from err
 
     return _fn

unittests/test_cli.py

@@ -698,6 +698,42 @@ def test_filtering_exclude_hash(run_reframe):
     assert returncode == 0
 
 
+def test_filtering_cpu_only(run_reframe):
+    returncode, stdout, stderr = run_reframe(
+        checkpath=['unittests/resources/checks/hellocheck.py'],
+        action='list',
+        more_options=['--cpu-only']
+    )
+    assert 'Traceback' not in stdout
+    assert 'Traceback' not in stderr
+    assert 'Found 2 check(s)' in stdout
+    assert returncode == 0
+
+
+def test_filtering_gpu_only(run_reframe):
+    returncode, stdout, stderr = run_reframe(
+        checkpath=['unittests/resources/checks/hellocheck.py'],
+        action='list',
+        more_options=['--gpu-only']
+    )
+    assert 'Traceback' not in stdout
+    assert 'Traceback' not in stderr
+    assert 'Found 0 check(s)' in stdout
+    assert returncode == 0
+
+
+def test_filtering_by_expr(run_reframe):
+    returncode, stdout, stderr = run_reframe(
+        checkpath=['unittests/resources/checks/hellocheck.py'],
+        action='list',
+        more_options=['-E num_tasks==1']
+    )
+    assert 'Traceback' not in stdout
+    assert 'Traceback' not in stderr
+    assert 'Found 2 check(s)' in stdout
+    assert returncode == 0
+
+
 def test_show_config_all(run_reframe):
     # Just make sure that this option does not make the frontend crash
     returncode, stdout, stderr = run_reframe(

unittests/test_filters.py

@@ -11,6 +11,7 @@
 import reframe.frontend.filters as filters
 import reframe.utility.sanity as sn
 import unittests.utility as test_util
+from reframe.core.exceptions import ReframeError
 
 
 def count_checks(filter_fn, checks):
@@ -140,3 +141,30 @@ def test_invalid_regex(sample_cases):
 
     with pytest.raises(errors.ReframeError):
         count_checks(filters.have_tag('*foo'), sample_cases).evaluate()
+
+
+def test_validates_expr(sample_cases, sample_param_cases):
+    validates = filters.validates
+    assert count_checks(validates('"a" in tags'), sample_cases) == 2
+    assert count_checks(validates('num_gpus_per_node == 1'), sample_cases) == 2
+    assert count_checks(validates('p > 5'), sample_param_cases) == 5
+    assert count_checks(validates('p > 5 or p < 1'), sample_param_cases) == 6
+    assert count_checks(validates('num_tasks in tags'), sample_cases) == 0
+
+
+def test_validates_expr_invalid(sample_cases):
+    validates = filters.validates
+
+    # undefined variables
+    with pytest.raises(ReframeError):
+        assert count_checks(validates('foo == 3'), sample_cases)
+
+    # invalid syntax
+    with pytest.raises(ReframeError):
+        assert count_checks(validates('num_tasks = 2'), sample_cases)
+
+    with pytest.raises(ReframeError):
+        assert count_checks(validates('import os'), sample_cases)
+
+    with pytest.raises(ReframeError):
+        assert count_checks(validates('"foo" i tags'), sample_cases)