Merge pull request #664 from ChristopherBignamini/regular_expressions

[feat] Add support for selecting tests using regular expressions

Author: vkarak

reframe/core/pipeline.py

@@ -6,7 +6,6 @@
            'RunOnlyRegressionTest', 'CompileOnlyRegressionTest']
 
 
-import fnmatch
 import inspect
 import itertools
 import os
@@ -67,12 +66,18 @@ class RegressionTest:
 
     #: List of programming environments supported by this test.
     #:
+    #: If ``*`` is in the list then all programming environments are supported
+    #: by this test.
+    #:
     #: :type: :class:`List[str]`
     #: :default: ``[]``
     #:
     #: .. note::
     #:     .. versionchanged:: 2.12
     #:        Programming environments can now be specified using wildcards.
+    #:
+    #:     .. versionchanged:: 2.17
+    #:        Support for wildcards is dropped.
     valid_prog_environs = fields.TypedField('valid_prog_environs',
                                             typ.List[str])
 
@@ -787,11 +792,10 @@ def supports_system(self, partition_name):
         return partition_name in self.valid_systems
 
     def supports_environ(self, env_name):
-        for env in self.valid_prog_environs:
-            if fnmatch.fnmatch(env_name, env):
-                return True
+        if '*' in self.valid_prog_environs:
+            return True
 
-        return False
+        return env_name in self.valid_prog_environs
 
     def is_local(self):
         """Check if the test will execute locally.

reframe/frontend/check_filters.py

@@ -1,34 +1,42 @@
+import re
+
 import reframe.core.runtime as rt
 import reframe.utility.sanity as util
 
 
-def have_name(names):
+def have_name(patt):
+    regex = re.compile(patt)
+
     def _fn(c):
-        return c.name in names
+        return regex.match(c.name)
 
     return _fn
 
 
-def have_not_name(names):
+def have_not_name(patt):
     def _fn(c):
-        return not have_name(names)(c)
+        return not have_name(patt)(c)
 
     return _fn
 
 
-def have_tag(tags):
+def have_tag(patt):
+    regex = re.compile(patt)
+
     def _fn(c):
-        return (set(tags)).issubset(c.tags)
+        return any(regex.match(p) for p in c.tags)
 
     return _fn
 
 
-def have_prgenv(prgenv):
+def have_prgenv(patt):
+    regex = re.compile(patt)
+
     def _fn(c):
-        if prgenv:
-            return util.allx(c.supports_environ(e) for e in prgenv)
+        if '*' in c.valid_prog_environs:
+            return True
         else:
-            return bool(c.valid_prog_environs)
+            return any(regex.match(p) for p in c.valid_prog_environs)
 
     return _fn
 

reframe/frontend/cli.py

@@ -1,6 +1,6 @@
-import os
 import inspect
 import json
+import os
 import socket
 import sys
 import traceback
@@ -111,7 +111,7 @@ def main():
         '-x', '--exclude', action='append', dest='exclude_names',
         metavar='NAME', default=[], help='Exclude checks with NAME')
     select_options.add_argument(
-        '-p', '--prgenv', action='append', default=[],
+        '-p', '--prgenv', action='append', default=[r'.*'],
         help='Select tests for PRGENV programming environment only')
     select_options.add_argument(
         '--gpu-only', action='store_true',
@@ -414,20 +414,25 @@ def main():
             raise ReframeError from e
 
         # Filter checks by name
-        checks_matched = filter(filters.have_not_name(options.exclude_names),
-                                checks_found)
+        checks_matched = checks_found
+        if options.exclude_names:
+            for name in options.exclude_names:
+                checks_matched = filter(filters.have_not_name(name),
+                                        checks_matched)
 
         if options.names:
-            checks_matched = filter(filters.have_name(options.names),
+            checks_matched = filter(filters.have_name('|'.join(options.names)),
                                     checks_matched)
 
         # Filter checks by tags
-        checks_matched = filter(filters.have_tag(options.tags), checks_matched)
+        for tag in options.tags:
+            checks_matched = filter(filters.have_tag(tag), checks_matched)
 
         # Filter checks by prgenv
         if not options.skip_prgenv_check:
-            checks_matched = filter(filters.have_prgenv(options.prgenv),
-                                    checks_matched)
+            for prgenv in options.prgenv:
+                checks_matched = filter(filters.have_prgenv(prgenv),
+                                        checks_matched)
 
         # Filter checks by system
         if not options.skip_system_check:

unittests/test_check_filters.py

@@ -37,51 +37,56 @@ def setUp(self):
                 'num_gpus_per_node': 1})
         ]
 
+    def count_checks(self, filter_fn):
+        return sn.count(filter(filter_fn, self.checks))
+
     def test_have_name(self):
-        self.assertEqual(1, sn.count(filter(filters.have_name('check1'),
-                                            self.checks)))
-        self.assertEqual(1, sn.count(filter(filters.have_name('check2'),
-                                            self.checks)))
-        self.assertEqual(1, sn.count(filter(filters.have_name('check3'),
-                                            self.checks)))
-        self.assertEqual(0, sn.count(filter(filters.have_name('check4'),
-                                            self.checks)))
+        self.assertEqual(1, self.count_checks(filters.have_name('check1')))
+        self.assertEqual(3, self.count_checks(filters.have_name('check')))
+        self.assertEqual(2, self.count_checks(filters.have_name(r'\S*1|\S*3')))
+        self.assertEqual(0, self.count_checks(filters.have_name('Check')))
+        self.assertEqual(3, self.count_checks(filters.have_name('(?i)Check')))
+        self.assertEqual(
+            2, self.count_checks(filters.have_name('check1|(?i)CHECK2'))
+        )
 
     def test_have_not_name(self):
-        self.assertEqual(2, sn.count(filter(filters.have_not_name('check1'),
-                                            self.checks)))
+        self.assertEqual(2, self.count_checks(filters.have_not_name('check1')))
+        self.assertEqual(
+            1, self.count_checks(filters.have_not_name('check1|check3'))
+        )
+        self.assertEqual(
+            0, self.count_checks(filters.have_not_name('check1|check2|check3'))
+        )
+        self.assertEqual(3, self.count_checks(filters.have_not_name('Check1')))
+        self.assertEqual(
+            2, self.count_checks(filters.have_not_name('(?i)Check1'))
+        )
 
     def test_have_tags(self):
-        self.assertEqual(1, sn.count(filter(filters.have_tag(['a', 'c']),
-                                            self.checks)))
-        self.assertEqual(0, sn.count(filter(filters.have_tag(['p', 'q']),
-                                            self.checks)))
-        self.assertEqual(2, sn.count(filter(filters.have_tag(['z']),
-                                            self.checks)))
+        self.assertEqual(2, self.count_checks(filters.have_tag('a|c')))
+        self.assertEqual(0, self.count_checks(filters.have_tag('p|q')))
+        self.assertEqual(2, self.count_checks(filters.have_tag('z')))
 
     def test_have_prgenv(self):
-        self.assertEqual(1, sn.count(filter(
-            filters.have_prgenv(['env1', 'env2']), self.checks)))
-        self.assertEqual(2, sn.count(filter(filters.have_prgenv(['env3']),
-                                            self.checks)))
-        self.assertEqual(1, sn.count(filter(filters.have_prgenv(['env4']),
-                                            self.checks)))
-        self.assertEqual(0, sn.count(filter(
-            filters.have_prgenv(['env1', 'env3']), self.checks)))
+        self.assertEqual(
+            1, self.count_checks(filters.have_prgenv('env1|env2'))
+        )
+        self.assertEqual(2, self.count_checks(filters.have_prgenv('env3')))
+        self.assertEqual(1, self.count_checks(filters.have_prgenv('env4')))
+        self.assertEqual(
+            3, self.count_checks(filters.have_prgenv('env1|env3'))
+        )
 
     @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'testsys')
     def test_partition(self):
         p = rt.runtime().system.partition('gpu')
-        self.assertEqual(2, sn.count(filter(filters.have_partition([p]),
-                                            self.checks)))
+        self.assertEqual(2, self.count_checks(filters.have_partition([p])))
         p = rt.runtime().system.partition('login')
-        self.assertEqual(0, sn.count(filter(filters.have_partition([p]),
-                                            self.checks)))
+        self.assertEqual(0, self.count_checks(filters.have_partition([p])))
 
     def test_have_gpu_only(self):
-        self.assertEqual(2, sn.count(filter(filters.have_gpu_only(),
-                                            self.checks)))
+        self.assertEqual(2, self.count_checks(filters.have_gpu_only()))
 
     def test_have_cpu_only(self):
-        self.assertEqual(1, sn.count(filter(filters.have_cpu_only(),
-                                            self.checks)))
+        self.assertEqual(1, self.count_checks(filters.have_cpu_only()))

unittests/test_pipeline.py

@@ -253,12 +253,6 @@ def test_supports_environ(self):
         self.assertTrue(test.supports_environ('foo-env'))
         self.assertTrue(test.supports_environ('*'))
 
-        test.valid_prog_environs = ['PrgEnv-foo-*']
-        self.assertTrue(test.supports_environ('PrgEnv-foo-version1'))
-        self.assertTrue(test.supports_environ('PrgEnv-foo-version2'))
-        self.assertFalse(test.supports_environ('PrgEnv-boo-version1'))
-        self.assertFalse(test.supports_environ('Prgenv-foo-version1'))
-
     def test_sourcesdir_none(self):
         test = RegressionTest('hellocheck', 'unittests/resources/checks')
         test.sourcesdir = None