Merge pull request #3310 from vkarak/bugfix/warn-on-filter-expr-errors

[enhancement] Warn instead of failing when a filter expression is invalid for test

Author: vkarak

reframe/frontend/executors/__init__.py

@@ -124,6 +124,10 @@ def __repr__(self):
         e = self.environ.name if self.environ else None
         return f'({c!r}, {p!r}, {e!r})'
 
+    def __str__(self):
+        check, partition, environ = self
+        return f'{check.name} @{partition.fullname}+{environ.name}'
+
     def prepare(self):
         '''Prepare test case for sending down the test pipeline'''
         if self._is_ready:

reframe/frontend/filters.py

@@ -6,6 +6,7 @@
 import re
 
 from reframe.core.exceptions import ReframeError
+from reframe.core.logging import getlogger
 
 
 def re_compile(patt):
@@ -118,6 +119,8 @@ def _fn(case):
         try:
             return eval(expr, None, case.check.__dict__)
         except Exception as err:
-            raise ReframeError(f'invalid expression `{expr}`') from err
+            getlogger().warning(f'error while evaluating expression `{expr}` '
+                                f'for test case `{case}`: {err}')
+            return False
 
     return _fn

unittests/test_filters.py

@@ -4,23 +4,25 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 import pytest
+from collections import namedtuple
 
 import reframe as rfm
 import reframe.core.exceptions as errors
 import reframe.frontend.executors as executors
 import reframe.frontend.filters as filters
 import reframe.utility.sanity as sn
 import unittests.utility as test_util
-from reframe.core.exceptions import ReframeError
 
 
 def count_checks(filter_fn, checks):
     return sn.count(filter(filter_fn, checks))
 
 
 def make_case(*args, **kwargs):
+    _P = namedtuple('_Partition', ['fullname'])
+    _E = namedtuple('_Environment', ['name'])
     test = test_util.make_check(*args, **kwargs)
-    return executors.TestCase(test, None, None)
+    return executors.TestCase(test, _P('generic:default'), _E('builtin'))
 
 
 @pytest.fixture
@@ -156,15 +158,13 @@ def test_validates_expr_invalid(sample_cases):
     validates = filters.validates
 
     # undefined variables
-    with pytest.raises(ReframeError):
-        assert count_checks(validates('foo == 3'), sample_cases)
+    assert count_checks(validates('foo == 3'), sample_cases) == 0
 
-    # invalid syntax
-    with pytest.raises(ReframeError):
-        assert count_checks(validates('num_tasks = 2'), sample_cases)
+    # assignments
+    assert count_checks(validates('num_tasks = 2'), sample_cases) == 0
 
-    with pytest.raises(ReframeError):
-        assert count_checks(validates('import os'), sample_cases)
+    # imports
+    assert count_checks(validates('import os'), sample_cases) == 0
 
-    with pytest.raises(ReframeError):
-        assert count_checks(validates('"foo" i tags'), sample_cases)
+    # invalid syntax
+    assert count_checks(validates('"foo" i tags'), sample_cases) == 0