options: add new --run-pattern

Add the --run-pattern|-S option to filter tests based on regexp. For
example, the following command will filter all tests which contain
"sendfile" or "madvise" in their name (including arguments) inside the
syscalls testing suite:

    kirk -S "sendfile|madvise" -r syscalls

Reviewed-by: Petr Vorel <pvorel@suse.cz>
Acked-by: Cyril Hrubis <chrubis@suse.cz>
Signed-off-by: Andrea Cervesato <andrea.cervesato@suse.com>

Author: acerv

libkirk/main.py

@@ -296,6 +296,14 @@ def _start_session(
     # start event loop
     exit_code = RC_OK
 
+    # read tests regex filter
+    run_pattern = args.run_pattern
+    if run_pattern:
+        try:
+            re.compile(run_pattern)
+        except re.error:
+            parser.error(f"'{run_pattern}' is not a valid regular expression")
+
     async def session_run() -> None:
         """
         Run session then stop events handler.
@@ -304,6 +312,7 @@ async def session_run() -> None:
             await session.run(
                 command=args.run_command,
                 suites=args.run_suite,
+                pattern=run_pattern,
                 report_path=args.json_report,
                 restore=restore_dir,
             )
@@ -414,6 +423,10 @@ def run(cmd_args: list = None) -> None:
         "-r",
         nargs="*",
         help="List of suites to run")
+    parser.add_argument(
+        "--run-pattern",
+        "-S",
+        help="Run all tests matching the regex pattern")
     parser.add_argument(
         "--run-command",
         "-c",
@@ -465,6 +478,9 @@ def run(cmd_args: list = None) -> None:
     if args.json_report and os.path.exists(args.json_report):
         parser.error(f"JSON report file already exists: {args.json_report}")
 
+    if args.run_pattern and not args.run_suite:
+        parser.error("--run-pattern must be used with --run-suite")
+
     if not args.run_suite and not args.run_command:
         parser.error("--run-suite/--run-command are required")
 

libkirk/session.py

@@ -6,6 +6,7 @@
 .. moduleauthor:: Andrea Cervesato <andrea.cervesato@suse.com>
 """
 import os
+import re
 import logging
 import asyncio
 import libkirk
@@ -186,16 +187,16 @@ async def _stop_sut(self) -> None:
         await libkirk.events.fire("sut_stop", self._sut.name)
         await self._sut.stop(iobuffer=RedirectSUTStdout(self._sut, False))
 
-    async def _read_suites(self, suites: list, restore: str) -> list:
+    async def _get_suites_objects(self, names: list) -> list:
         """
-        Read suites and return a list of Suite objects.
+        Return suites objects by giving their names.
         """
         coros = []
-        for suite in suites:
+        for suite in names:
             coros.append(self._framework.find_suite(self._sut, suite))
 
         if not coros:
-            raise KirkException(f"Can't find suites: {suites}")
+            raise KirkException(f"Can't find suites: {names}")
 
         suites_obj = await asyncio.gather(*coros, return_exceptions=True)
         for suite in suites_obj:
@@ -205,6 +206,31 @@ async def _read_suites(self, suites: list, restore: str) -> list:
             if not suite:
                 raise KirkException("Can't find suite objects")
 
+        return suites_obj
+
+    async def _read_suites(
+            self,
+            suites: list,
+            pattern: str,
+            restore: str) -> list:
+        """
+        Read suites and return a list of Suite objects.
+        """
+        suites_obj = await self._get_suites_objects(suites)
+
+        if pattern:
+            matcher = re.compile(pattern)
+
+            for suite_obj in suites_obj:
+                toremove = []
+
+                for test in suite_obj.tests:
+                    if not matcher.search(test.name):
+                        toremove.append(test)
+
+                for item in toremove:
+                    suite_obj.tests.remove(item)
+
         restored = self._read_restored_session(restore)
         if restored:
             await libkirk.events.fire("session_restore", restore)
@@ -224,6 +250,13 @@ async def _read_suites(self, suites: list, restore: str) -> list:
 
                 toremove.clear()
 
+        num_tests = 0
+        for suite_obj in suites_obj:
+            num_tests += len(suite_obj.tests)
+
+        if num_tests == 0:
+            raise KirkException("No tests selected")
+
         return suites_obj
 
     async def _exec_command(self, command: str) -> None:
@@ -286,10 +319,13 @@ async def stop(self) -> None:
             await libkirk.events.fire("session_stopped")
             self._stop = False
 
+    # consider changing the arguments handling if new ones must be added
+    # pylint: disable=too-many-positional-arguments
     async def run(
             self,
             command: str = None,
             suites: list = None,
+            pattern: str = None,
             report_path: str = None,
             restore: str = None) -> None:
         """
@@ -298,6 +334,8 @@ async def run(
         :type command: str
         :param suites: list of suites to execute
         :type suites: list
+        :param pattern: string pattern to match tests
+        :types pattern: str
         :param report_path: JSON report path
         :type report_path: str
         :param restore: temporary directory generated by a previous session
@@ -318,7 +356,8 @@ async def run(
                     await self._exec_command(command)
 
                 if suites:
-                    suites_obj = await self._read_suites(suites, restore)
+                    suites_obj = await self._read_suites(
+                        suites, pattern, restore)
                     await self._scheduler.schedule(suites_obj)
             except KirkException as err:
                 if not self._stop:

libkirk/tests/test_session.py

@@ -44,6 +44,20 @@ async def test_run(self, session):
         """
         await session.run(suites=["suite01", "suite02"])
 
+    async def test_run_pattern(self, tmpdir, session):
+        """
+        Test run method when executing tests filtered out with a pattern.
+        """
+        report = str(tmpdir / "report.json")
+        await session.run(
+            suites=["suite01", "suite02"],
+            pattern="test01|test02",
+            report_path=report)
+
+        with open(report, "r", encoding="utf-8") as report_file:
+            report_data = json.loads(report_file.read())
+            assert len(report_data["results"]) == 4
+
     async def test_run_report(self, tmpdir, session):
         """
         Test run method when executing suites, generating a report.