Merge branch 'master' into intel_vtune

Author: vkarak

docs/running.rst

@@ -365,10 +365,29 @@ For example the following will select all the checks that can run with both ``Pr
 
 If you are going to run a set of tests selected by programming environment, they will run only for the selected programming environment(s).
 
+The ``-p`` option accepts also the Python `regular expression syntax <https://docs.python.org/3.6/library/re.html#regular-expression-syntax>`__.
+In fact, the argument to ``-p`` option is treated as a regular expression always. This means that the ``-p PrgEnv-gnu`` will match also tests that support a ``PrgEnv-gnuXX`` environment.
+If you would like to stricly select tests that support ``PrgEnv-gnu`` only and not ``PrgEnv-gnuXX``, you should write ``-p PrgEnv-gnu$``.
+As described above multiple ``-p`` options are AND-ed.
+Combining that with regular expressions can be quite powerful.
+For example, the following will select all tests that support programming environment ``foo`` and either ``PrgEnv-gnu`` or ``PrgEnv-pgi``:
+
+.. code-block:: bash
+
+   ./bin/reframe -p foo -p 'PrgEnv-(gnu|pgi)' -l
+
+
+.. note::
+   .. versionadded:: 2.17
+
+   The ``-p`` option recognizes regular expressions as arguments.
+
+
 Selecting tests by tags
 ^^^^^^^^^^^^^^^^^^^^^^^
 
-As we have seen in the `"ReFrame tutorial" <tutorial.html>`__, every regression test may be associated with a set of tags. Using the ``-t`` or ``--tag`` option you can select the regression tests associated with a specific tag.
+As we have seen in the `"ReFrame tutorial" <tutorial.html>`__, every regression test may be associated with a set of tags.
+Using the ``-t`` or ``--tag`` option you can select the regression tests associated with a specific tag.
 For example the following will list all the tests that have a ``maintenance`` tag and can run on the current system:
 
 .. code-block:: bash
@@ -378,6 +397,11 @@ For example the following will list all the tests that have a ``maintenance`` ta
 Similarly to the ``-p`` option, you can chain multiple ``-t`` options together, in which case a regression test will be selected if it is associated with all the tags specified in the command line.
 The list of tags associated with a check can be viewed in the listing output when specifying the ``-l`` option.
 
+.. note::
+   .. versionadded:: 2.17
+
+   The ``-t`` option recognizes regular expressions as arguments.
+
 Selecting tests by name
 ^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -439,6 +463,18 @@ Similarly, you can exclude this test by passing the ``-x Example7Test`` option:
   Found 12 check(s).
 
 
+Both ``-n`` and ``-x`` options can be chained, in which case either the tests that have any of the specified names are selected or excluded from running.
+They may also accept regular expressions as arguments.
+
+.. note::
+   .. versionadded:: 2.17
+
+      The ``-n`` and ``-x`` options recognize regular expressions as arguments.
+      Chaining these options, e.g., ``-n A -n B``, is equivalent to a regular expression that applies OR to the individual arguments, i.e., equivalent to ``-n 'A|B'``.
+
+
+
+
 Controlling the Execution of Regression Tests
 ---------------------------------------------
 

reframe/core/pipeline.py

@@ -6,7 +6,6 @@
            'RunOnlyRegressionTest', 'CompileOnlyRegressionTest']
 
 
-import fnmatch
 import inspect
 import itertools
 import os
@@ -67,12 +66,18 @@ class RegressionTest:
 
     #: List of programming environments supported by this test.
     #:
+    #: If ``*`` is in the list then all programming environments are supported
+    #: by this test.
+    #:
     #: :type: :class:`List[str]`
     #: :default: ``[]``
     #:
     #: .. note::
     #:     .. versionchanged:: 2.12
     #:        Programming environments can now be specified using wildcards.
+    #:
+    #:     .. versionchanged:: 2.17
+    #:        Support for wildcards is dropped.
     valid_prog_environs = fields.TypedField('valid_prog_environs',
                                             typ.List[str])
 
@@ -787,11 +792,10 @@ def supports_system(self, partition_name):
         return partition_name in self.valid_systems
 
     def supports_environ(self, env_name):
-        for env in self.valid_prog_environs:
-            if fnmatch.fnmatch(env_name, env):
-                return True
+        if '*' in self.valid_prog_environs:
+            return True
 
-        return False
+        return env_name in self.valid_prog_environs
 
     def is_local(self):
         """Check if the test will execute locally.

reframe/frontend/check_filters.py

@@ -1,34 +1,42 @@
+import re
+
 import reframe.core.runtime as rt
 import reframe.utility.sanity as util
 
 
-def have_name(names):
+def have_name(patt):
+    regex = re.compile(patt)
+
     def _fn(c):
-        return c.name in names
+        return regex.match(c.name)
 
     return _fn
 
 
-def have_not_name(names):
+def have_not_name(patt):
     def _fn(c):
-        return not have_name(names)(c)
+        return not have_name(patt)(c)
 
     return _fn
 
 
-def have_tag(tags):
+def have_tag(patt):
+    regex = re.compile(patt)
+
     def _fn(c):
-        return (set(tags)).issubset(c.tags)
+        return any(regex.match(p) for p in c.tags)
 
     return _fn
 
 
-def have_prgenv(prgenv):
+def have_prgenv(patt):
+    regex = re.compile(patt)
+
     def _fn(c):
-        if prgenv:
-            return util.allx(c.supports_environ(e) for e in prgenv)
+        if '*' in c.valid_prog_environs:
+            return True
         else:
-            return bool(c.valid_prog_environs)
+            return any(regex.match(p) for p in c.valid_prog_environs)
 
     return _fn
 

reframe/frontend/cli.py

@@ -1,6 +1,7 @@
-import os
 import inspect
 import json
+import os
+import re
 import socket
 import sys
 import traceback
@@ -111,7 +112,7 @@ def main():
         '-x', '--exclude', action='append', dest='exclude_names',
         metavar='NAME', default=[], help='Exclude checks with NAME')
     select_options.add_argument(
-        '-p', '--prgenv', action='append', default=[],
+        '-p', '--prgenv', action='append', default=[r'.*'],
         help='Select tests for PRGENV programming environment only')
     select_options.add_argument(
         '--gpu-only', action='store_true',
@@ -414,20 +415,25 @@ def main():
             raise ReframeError from e
 
         # Filter checks by name
-        checks_matched = filter(filters.have_not_name(options.exclude_names),
-                                checks_found)
+        checks_matched = checks_found
+        if options.exclude_names:
+            for name in options.exclude_names:
+                checks_matched = filter(filters.have_not_name(name),
+                                        checks_matched)
 
         if options.names:
-            checks_matched = filter(filters.have_name(options.names),
+            checks_matched = filter(filters.have_name('|'.join(options.names)),
                                     checks_matched)
 
         # Filter checks by tags
-        checks_matched = filter(filters.have_tag(options.tags), checks_matched)
+        for tag in options.tags:
+            checks_matched = filter(filters.have_tag(tag), checks_matched)
 
         # Filter checks by prgenv
         if not options.skip_prgenv_check:
-            checks_matched = filter(filters.have_prgenv(options.prgenv),
-                                    checks_matched)
+            for prgenv in options.prgenv:
+                checks_matched = filter(filters.have_prgenv(prgenv),
+                                        checks_matched)
 
         # Filter checks by system
         if not options.skip_system_check:
@@ -447,6 +453,12 @@ def main():
 
         checks_matched = [c for c in checks_matched]
 
+        # Determine the programming environments to run with
+        run_environs = {e.name
+                        for env_patt in options.prgenv
+                        for p in rt.system.partitions
+                        for e in p.environs if re.match(env_patt, e.name)}
+
         # Act on checks
 
         # Unload regression's module and load user-specified modules
@@ -491,7 +503,7 @@ def main():
             exec_policy.skip_environ_check = options.skip_prgenv_check
             exec_policy.skip_sanity_check = options.skip_sanity_check
             exec_policy.skip_performance_check = options.skip_performance_check
-            exec_policy.only_environs = options.prgenv
+            exec_policy.only_environs = run_environs
             exec_policy.keep_stage_files = options.keep_stage_files
             try:
                 errmsg = "invalid option for --flex-alloc-tasks: '{0}'"

unittests/test_check_filters.py

@@ -37,51 +37,56 @@ def setUp(self):
                 'num_gpus_per_node': 1})
         ]
 
+    def count_checks(self, filter_fn):
+        return sn.count(filter(filter_fn, self.checks))
+
     def test_have_name(self):
-        self.assertEqual(1, sn.count(filter(filters.have_name('check1'),
-                                            self.checks)))
-        self.assertEqual(1, sn.count(filter(filters.have_name('check2'),
-                                            self.checks)))
-        self.assertEqual(1, sn.count(filter(filters.have_name('check3'),
-                                            self.checks)))
-        self.assertEqual(0, sn.count(filter(filters.have_name('check4'),
-                                            self.checks)))
+        self.assertEqual(1, self.count_checks(filters.have_name('check1')))
+        self.assertEqual(3, self.count_checks(filters.have_name('check')))
+        self.assertEqual(2, self.count_checks(filters.have_name(r'\S*1|\S*3')))
+        self.assertEqual(0, self.count_checks(filters.have_name('Check')))
+        self.assertEqual(3, self.count_checks(filters.have_name('(?i)Check')))
+        self.assertEqual(
+            2, self.count_checks(filters.have_name('check1|(?i)CHECK2'))
+        )
 
     def test_have_not_name(self):
-        self.assertEqual(2, sn.count(filter(filters.have_not_name('check1'),
-                                            self.checks)))
+        self.assertEqual(2, self.count_checks(filters.have_not_name('check1')))
+        self.assertEqual(
+            1, self.count_checks(filters.have_not_name('check1|check3'))
+        )
+        self.assertEqual(
+            0, self.count_checks(filters.have_not_name('check1|check2|check3'))
+        )
+        self.assertEqual(3, self.count_checks(filters.have_not_name('Check1')))
+        self.assertEqual(
+            2, self.count_checks(filters.have_not_name('(?i)Check1'))
+        )
 
     def test_have_tags(self):
-        self.assertEqual(1, sn.count(filter(filters.have_tag(['a', 'c']),
-                                            self.checks)))
-        self.assertEqual(0, sn.count(filter(filters.have_tag(['p', 'q']),
-                                            self.checks)))
-        self.assertEqual(2, sn.count(filter(filters.have_tag(['z']),
-                                            self.checks)))
+        self.assertEqual(2, self.count_checks(filters.have_tag('a|c')))
+        self.assertEqual(0, self.count_checks(filters.have_tag('p|q')))
+        self.assertEqual(2, self.count_checks(filters.have_tag('z')))
 
     def test_have_prgenv(self):
-        self.assertEqual(1, sn.count(filter(
-            filters.have_prgenv(['env1', 'env2']), self.checks)))
-        self.assertEqual(2, sn.count(filter(filters.have_prgenv(['env3']),
-                                            self.checks)))
-        self.assertEqual(1, sn.count(filter(filters.have_prgenv(['env4']),
-                                            self.checks)))
-        self.assertEqual(0, sn.count(filter(
-            filters.have_prgenv(['env1', 'env3']), self.checks)))
+        self.assertEqual(
+            1, self.count_checks(filters.have_prgenv('env1|env2'))
+        )
+        self.assertEqual(2, self.count_checks(filters.have_prgenv('env3')))
+        self.assertEqual(1, self.count_checks(filters.have_prgenv('env4')))
+        self.assertEqual(
+            3, self.count_checks(filters.have_prgenv('env1|env3'))
+        )
 
     @rt.switch_runtime(fixtures.TEST_SITE_CONFIG, 'testsys')
     def test_partition(self):
         p = rt.runtime().system.partition('gpu')
-        self.assertEqual(2, sn.count(filter(filters.have_partition([p]),
-                                            self.checks)))
+        self.assertEqual(2, self.count_checks(filters.have_partition([p])))
         p = rt.runtime().system.partition('login')
-        self.assertEqual(0, sn.count(filter(filters.have_partition([p]),
-                                            self.checks)))
+        self.assertEqual(0, self.count_checks(filters.have_partition([p])))
 
     def test_have_gpu_only(self):
-        self.assertEqual(2, sn.count(filter(filters.have_gpu_only(),
-                                            self.checks)))
+        self.assertEqual(2, self.count_checks(filters.have_gpu_only()))
 
     def test_have_cpu_only(self):
-        self.assertEqual(1, sn.count(filter(filters.have_cpu_only(),
-                                            self.checks)))
+        self.assertEqual(1, self.count_checks(filters.have_cpu_only()))

unittests/test_cli.py

@@ -148,12 +148,16 @@ def test_check_submit_success(self):
         self.local = False
         self.system = partition.fullname
 
-        # pick up the programming environment of the partition
-        self.environs = [partition.environs[0].name]
+        # Pick up the programming environment of the partition
+        # Prepend ^ and append $ so as to much exactly the given name
+        self.environs = ['^' + partition.environs[0].name + '$']
 
         returncode, stdout, _ = self._run_reframe()
         self.assertNotIn('FAILED', stdout)
         self.assertIn('PASSED', stdout)
+
+        # Assert that we have run only one test case
+        self.assertIn('Ran 1 test case(s)', stdout)
         self.assertEqual(0, returncode)
 
     def test_check_failure(self):

unittests/test_pipeline.py

@@ -253,12 +253,6 @@ def test_supports_environ(self):
         self.assertTrue(test.supports_environ('foo-env'))
         self.assertTrue(test.supports_environ('*'))
 
-        test.valid_prog_environs = ['PrgEnv-foo-*']
-        self.assertTrue(test.supports_environ('PrgEnv-foo-version1'))
-        self.assertTrue(test.supports_environ('PrgEnv-foo-version2'))
-        self.assertFalse(test.supports_environ('PrgEnv-boo-version1'))
-        self.assertFalse(test.supports_environ('Prgenv-foo-version1'))
-
     def test_sourcesdir_none(self):
         test = RegressionTest('hellocheck', 'unittests/resources/checks')
         test.sourcesdir = None