Merge pull request #2533 from ekouts/bugfix/id_filtering

[bugfix] Improve test selection by their variant id

Author: Vasileios Karakasis

reframe/frontend/filters.py

@@ -49,7 +49,10 @@ def have_any_name(names):
     regex_matches = []
     for n in names:
         if has_compact_names and '@' in n:
-            exact_matches.append(n.replace('@', '_'))
+            test, _, variant = n.rpartition('@')
+            if variant.isdigit():
+                exact_matches.append((test, int(variant)))
+
         else:
             regex_matches.append(n)
 
@@ -61,7 +64,8 @@ def have_any_name(names):
     def _fn(case):
         # Check if we have an exact match
         for m in exact_matches:
-            if m == case.check.unique_name:
+            cls_name = type(case.check).__name__
+            if (cls_name, case.check.variant_num) == m:
                 return True
 
         display_name = case.check.display_name.replace(' ', '')

reframe/frontend/runreport.py

@@ -18,7 +18,7 @@
 # The schema data version
 # Major version bumps are expected to break the validation of previous schemas
 
-DATA_VERSION = '2.0'
+DATA_VERSION = '2.1'
 _SCHEMA = os.path.join(rfm.INSTALL_PREFIX, 'reframe/schemas/runreport.json')
 
 

reframe/frontend/statistics.py

@@ -110,6 +110,7 @@ def json(self, force=False):
                     'environment': None,
                     'fail_phase': None,
                     'fail_reason': None,
+                    'fixture': check.is_fixture(),
                     'jobid': None,
                     'job_stderr': None,
                     'job_stdout': None,
@@ -234,7 +235,7 @@ def print_failure_report(self, printer, rerun_info=True):
                          f"{r['dependencies_actual']}")
             printer.info(f"  * Maintainers: {r['maintainers']}")
             printer.info(f"  * Failing phase: {r['fail_phase']}")
-            if rerun_info:
+            if rerun_info and not r['fixture']:
                 if rt.runtime().get_option('general/0/compact_test_names'):
                     cls = r['display_name'].split(' ')[0]
                     variant = r['unique_name'].replace(cls, '')

reframe/schemas/runreport.json

@@ -40,6 +40,7 @@
                 "fail_reason": {"type": ["string", "null"]},
                 "fail_severe": {"type": "boolean"},
                 "filename": {"type": "string"},
+                "fixture": {"type": "boolean"},
                 "jobid": {"type": ["string", "null"]},
                 "job_stderr": {"type": ["string", "null"]},
                 "job_stdout": {"type": ["string", "null"]},

unittests/test_filters.py

@@ -52,7 +52,7 @@ def use_compact_names(make_exec_ctx_g):
 @pytest.fixture
 def sample_param_cases(use_compact_names):
     class _X(rfm.RegressionTest):
-        p = parameter([1, 1, 3])
+        p = parameter([1] + list(range(11)))
         valid_systems = ['*']
         valid_prog_environs = ['*']
 
@@ -66,7 +66,7 @@ def sample_param_cases_compat():
     # `general/compact_test_names=False`
 
     class _X(rfm.RegressionTest):
-        p = parameter([1, 1, 3])
+        p = parameter([1] + list(range(11)))
         valid_systems = ['*']
         valid_prog_environs = ['*']
 
@@ -87,17 +87,22 @@ def test_have_any_name(sample_cases):
 
 
 def test_have_any_name_param_test(sample_param_cases):
-    assert 2 == count_checks(filters.have_any_name(['.*%p=1']),
+    # The regex will match "_X%p=1" as well as "_X%p=10"
+    assert 3 == count_checks(filters.have_any_name(['.*%p=1']),
+                             sample_param_cases)
+    assert 2 == count_checks(filters.have_any_name(['.*%p=1$']),
                              sample_param_cases)
     assert 1 == count_checks(filters.have_any_name(['_X%p=3']),
                              sample_param_cases)
     assert 1 == count_checks(filters.have_any_name(['_X@2']),
                              sample_param_cases)
-    assert 0 == count_checks(filters.have_any_name(['_X@3']),
+    assert 1 == count_checks(filters.have_any_name(['_X@002']),
+                             sample_param_cases)
+    assert 0 == count_checks(filters.have_any_name(['_X@12']),
                              sample_param_cases)
     assert 2 == count_checks(filters.have_any_name(['_X@0', '_X@1']),
                              sample_param_cases)
-    assert 3 == count_checks(filters.have_any_name(['_X@0', '_X.*']),
+    assert 12 == count_checks(filters.have_any_name(['_X@0', '_X.*']),
                              sample_param_cases)
 
 
@@ -108,11 +113,14 @@ def test_have_any_name_param_test_compat(sample_param_cases_compat):
                              sample_param_cases_compat)
     assert 0 == count_checks(filters.have_any_name(['_X@2']),
                              sample_param_cases_compat)
-    assert 2 == count_checks(filters.have_any_name(['_X_1']),
+    # The regex will match "_X_1" as well as "_X_10"
+    assert 3 == count_checks(filters.have_any_name(['_X_1']),
+                             sample_param_cases_compat)
+    assert 2 == count_checks(filters.have_any_name(['_X_1$']),
                              sample_param_cases_compat)
     assert 0 == count_checks(filters.have_any_name(['_X@0', '_X@1']),
                              sample_param_cases_compat)
-    assert 3 == count_checks(filters.have_any_name(['_X@0', '_X.*']),
+    assert 12 == count_checks(filters.have_any_name(['_X@0', '_X.*']),
                              sample_param_cases_compat)