options: add --runtime option

Iterate testing suites until they reach a specific runtime value,
given by the --runtime option. If the given runtime is smaller
than the testing suite execution time, session will gracefully stop.

Reviewed-by: Petr Vorel <pvorel@suse.cz>
Signed-off-by: Andrea Cervesato <andrea.cervesato@suse.com>

Author: acerv

libkirk/main.py

@@ -363,6 +363,7 @@ async def session_run() -> None:
                 suite_iterate=args.suite_iterate,
                 skip_tests=skip_tests,
                 randomize=args.randomize,
+                runtime=args.runtime,
             )
         except asyncio.CancelledError:
             await session.stop()
@@ -501,6 +502,12 @@ def run(cmd_args: list = None) -> None:
         "-x",
         action="store_true",
         help="Force parallelization execution of all tests")
+    parser.add_argument(
+        "--runtime",
+        "-u",
+        type=_time_config,
+        default="0",
+        help="Set for how long we want to run the session in seconds")
 
     # session arguments
     parser.add_argument(

libkirk/session.py

@@ -68,6 +68,7 @@ def __init__(self, **kwargs) -> None:
         self._stop = False
         self._exec_lock = asyncio.Lock()
         self._run_lock = asyncio.Lock()
+        self._results = []
 
         if not self._tmpdir:
             raise ValueError("tmpdir is empty")
@@ -358,6 +359,48 @@ async def stop(self) -> None:
             await libkirk.events.fire("session_stopped")
             self._stop = False
 
+    async def _schedule_once(self, suites_obj: list) -> None:
+        """
+        Schedule tests only once.
+        """
+        await self._scheduler.schedule(suites_obj)
+        self._results.extend(self._scheduler.results)
+
+    async def _schedule_infinite(self, suites_obj: list) -> None:
+        """
+        Schedule all testing suites infinite times.
+        """
+        suites_list = []
+        suites_list.extend(suites_obj)
+
+        count = 1
+        while not self._stop:
+            await self._schedule_once(suites_obj)
+            if self._scheduler.stopped:
+                break
+
+            count += 1
+
+            suites_list.clear()
+            for suite in copy.deepcopy(suites_obj):
+                suite.name = f"{suite.name}[{count}]"
+                suites_list.append(suite)
+
+    async def _run_scheduler(self, suites_obj: list, runtime: int) -> None:
+        """
+        Run the scheduler for specific amount of time given by `runtime`.
+        """
+        if runtime <= 0:
+            await self._schedule_once(suites_obj)
+            return
+
+        try:
+            await asyncio.wait_for(
+                self._schedule_infinite(suites_obj),
+                runtime)
+        except asyncio.TimeoutError:
+            await self._scheduler.stop()
+
     async def run(self, **kwargs: dict) -> None:
         """
         Run a new session and store results inside a JSON file.
@@ -377,16 +420,9 @@ async def run(self, **kwargs: dict) -> None:
         :type suite_iterate: int
         :param randomize: randomize all tests if True
         :type randomize: bool
+        :param runtime: for how long we want to run the session
+        :type runtime: int
         """
-        command = kwargs.get("command", None)
-        suites = kwargs.get("suites", None)
-        pattern = kwargs.get("pattern", None)
-        skip_tests = kwargs.get("skip_tests", None)
-        report_path = kwargs.get("report_path", None)
-        restore = kwargs.get("restore", None)
-        suite_iterate = kwargs.get("suite_iterate", 1)
-        randomize = kwargs.get("randomize", False)
-
         async with self._run_lock:
             await libkirk.events.fire("session_started", self._tmpdir.abspath)
 
@@ -398,44 +434,53 @@ async def run(self, **kwargs: dict) -> None:
             try:
                 await self._start_sut()
 
+                command = kwargs.get("command", None)
                 if command:
                     await self._exec_command(command)
 
+                suites = kwargs.get("suites", None)
                 if suites:
                     suites_obj = await self._read_suites(
-                        suites, pattern, skip_tests, restore)
+                        suites,
+                        kwargs.get("pattern", None),
+                        kwargs.get("skip_tests", None),
+                        kwargs.get("restore", False))
 
                     suites_obj = self._apply_iterate(
-                        suites_obj, suite_iterate)
+                        suites_obj,
+                        kwargs.get("suite_iterate", 1))
 
-                    if randomize:
+                    if kwargs.get("randomize", False):
                         for suite in suites_obj:
                             random.shuffle(suite.tests)
 
-                    await self._scheduler.schedule(suites_obj)
+                    await self._run_scheduler(
+                        suites_obj,
+                        kwargs.get("runtime", 0))
             except KirkException as err:
                 if not self._stop:
                     self._logger.exception(err)
                     await libkirk.events.fire("session_error", str(err))
                     raise err
             finally:
                 try:
-                    if self._scheduler.results:
+                    if self._results:
                         exporter = JSONExporter()
 
                         tasks = []
                         tasks.append(
                             exporter.save_file(
-                                self._scheduler.results,
+                                self._results,
                                 os.path.join(
                                     self._tmpdir.abspath,
                                     "results.json")
                             ))
 
+                        report_path = kwargs.get("report_path", None)
                         if report_path:
                             tasks.append(
                                 exporter.save_file(
-                                    self._scheduler.results,
+                                    self._results,
                                     report_path
                                 ))
 
@@ -449,4 +494,5 @@ async def run(self, **kwargs: dict) -> None:
                     await libkirk.events.fire("session_error", str(err))
                     raise err
                 finally:
+                    self._results.clear()
                     await self._inner_stop()

libkirk/tests/test_main.py

@@ -4,7 +4,6 @@
 import os
 import sys
 import pwd
-import time
 import json
 import pytest
 import libkirk.main
@@ -22,7 +21,7 @@ def setup(self, dummy_framework):
         """
         libkirk.main.LOADED_FRAMEWORK.append(dummy_framework)
 
-    def read_report(self, temp, tests_num) -> dict:
+    def read_report(self, temp) -> dict:
         """
         Check if report file contains the given number of tests.
         """
@@ -32,11 +31,9 @@ def read_report(self, temp, tests_num) -> dict:
 
         # read report and check if all suite's tests have been executed
         report_d = None
-        with open(report, 'r') as report_f:
+        with open(report, 'r', encoding="utf-8") as report_f:
             report_d = json.loads(report_f.read())
 
-        assert len(report_d["results"]) == tests_num
-
         return report_d
 
     def test_wrong_options(self):
@@ -100,7 +97,8 @@ def test_run_suite(self, tmpdir):
 
         assert excinfo.value.code == libkirk.main.RC_OK
 
-        self.read_report(temp, 2)
+        report = self.read_report(temp)
+        assert len(report["results"]) == 2
 
     def test_run_suite_timeout(self, tmpdir):
         """
@@ -119,8 +117,10 @@ def test_run_suite_timeout(self, tmpdir):
 
         assert excinfo.value.code == libkirk.main.RC_OK
 
-        report_d = self.read_report(temp, 2)
-        for param in report_d["results"]:
+        report = self.read_report(temp)
+        assert len(report["results"]) == 2
+
+        for param in report["results"]:
             assert param["test"]["passed"] == 0
             assert param["test"]["failed"] == 0
             assert param["test"]["broken"] == 0
@@ -186,7 +186,8 @@ def test_restore_suite(self, tmpdir):
 
         assert excinfo.value.code == libkirk.main.RC_OK
 
-        self.read_report(temp, 2)
+        report = self.read_report(temp)
+        assert len(report["results"]) == 2
 
         # restore session
         name = pwd.getpwuid(os.getuid()).pw_name
@@ -202,7 +203,8 @@ def test_restore_suite(self, tmpdir):
 
         assert excinfo.value.code == libkirk.main.RC_OK
 
-        self.read_report(temp, 1)
+        report = self.read_report(temp)
+        assert len(report["results"]) == 1
 
     def test_json_report(self, tmpdir):
         """
@@ -223,9 +225,11 @@ def test_json_report(self, tmpdir):
         assert excinfo.value.code == libkirk.main.RC_OK
         assert os.path.isfile(report)
 
-        report_a = self.read_report(temp, 2)
+        report_a = self.read_report(temp)
+        assert len(report_a["results"]) == 2
+
         report_b = None
-        with open(report, 'r') as report_f:
+        with open(report, 'r', encoding="utf-8") as report_f:
             report_b = json.loads(report_f.read())
 
         assert report_a == report_b
@@ -247,7 +251,8 @@ def test_skip_tests(self, tmpdir):
 
         assert excinfo.value.code == libkirk.main.RC_OK
 
-        self.read_report(temp, 1)
+        report = self.read_report(temp)
+        assert len(report["results"]) == 1
 
     def test_skip_file(self, tmpdir):
         """
@@ -269,7 +274,8 @@ def test_skip_file(self, tmpdir):
 
         assert excinfo.value.code == libkirk.main.RC_OK
 
-        self.read_report(temp, 1)
+        report = self.read_report(temp)
+        assert len(report["results"]) == 1
 
     def test_skip_tests_and_file(self, tmpdir):
         """
@@ -292,7 +298,8 @@ def test_skip_tests_and_file(self, tmpdir):
 
         assert excinfo.value.code == libkirk.main.RC_OK
 
-        self.read_report(temp, 1)
+        report = self.read_report(temp)
+        assert len(report["results"]) == 1
 
     def test_workers(self, tmpdir):
         """
@@ -312,7 +319,9 @@ def test_workers(self, tmpdir):
             libkirk.main.run(cmd_args=cmd_args)
 
         assert excinfo.value.code == libkirk.main.RC_OK
-        self.read_report(temp, 2)
+
+        report = self.read_report(temp)
+        assert len(report["results"]) == 2
 
     def test_sut_help(self):
         """
@@ -360,8 +369,9 @@ def test_env(self, tmpdir):
 
         assert excinfo.value.code == libkirk.main.RC_OK
 
-        report_d = self.read_report(temp, 1)
-        assert report_d["results"][0]["test"]["log"] == "ciao"
+        report = self.read_report(temp)
+        assert len(report["results"]) == 1
+        assert report["results"][0]["test"]["log"] == "ciao"
 
     def test_suite_iterate(self, tmpdir):
         """
@@ -380,7 +390,8 @@ def test_suite_iterate(self, tmpdir):
 
         assert excinfo.value.code == libkirk.main.RC_OK
 
-        self.read_report(temp, 8)
+        report = self.read_report(temp)
+        assert len(report["results"]) == 8
 
     def test_randomize(self, tmpdir):
         """
@@ -402,10 +413,31 @@ def test_randomize(self, tmpdir):
 
         assert excinfo.value.code == libkirk.main.RC_OK
 
-        report_d = self.read_report(temp, 2 * num_of_suites)
+        report = self.read_report(temp)
+        assert len(report["results"]) == 2 * num_of_suites
 
         tests_names = []
-        for test in report_d["results"]:
+        for test in report["results"]:
             tests_names.append(test["test_fqn"])
 
         assert ["test01", "test02"] * num_of_suites != tests_names
+
+    def test_runtime(self, tmpdir):
+        """
+        Test --runtime option.
+        """
+        temp = tmpdir.mkdir("temp")
+        cmd_args = [
+            "--tmp-dir", str(temp),
+            "--framework", "dummy",
+            "--run-suite", "suite01",
+            "--runtime", "1",
+        ]
+
+        with pytest.raises(SystemExit) as excinfo:
+            libkirk.main.run(cmd_args=cmd_args)
+
+        assert excinfo.value.code == libkirk.main.RC_OK
+
+        report = self.read_report(temp)
+        assert len(report["results"]) > 2

libkirk/tests/test_session.py

@@ -163,3 +163,17 @@ async def test_run_randomize(self, tmpdir, session):
             tests_names.append(test["test_fqn"])
 
         assert ["test01", "test02"] * num_of_suites != tests_names
+
+    async def test_run_runtime(self, tmpdir, session):
+        """
+        Test run method when executing suites for a certain amount of time.
+        """
+        report = str(tmpdir / "report.json")
+        await session.run(
+            suites=["suite01"],
+            runtime=1,
+            report_path=report)
+
+        with open(report, "r", encoding="utf-8") as report_file:
+            report_data = json.loads(report_file.read())
+            assert len(report_data["results"]) > 2