Guards against timeout configuration error (#1232)

* Guards against timeout configuration error

* Relaxing timeout condition by allowing equal

* Fixed tests and used new argument based timing configurations of measurement durations [skip ci]

* (fix): placeholder value removed

Author: ArneTR

lib/scenario_runner.py

@@ -163,9 +163,31 @@ def __init__(self,
         self.__volume_sizes = {}
         self.__warnings = []
 
+        self._check_all_durations()
+
         # we currently do not use this variable
         # self.__filename = self._original_filename # this can be changed later if working directory changes
 
+
+    def _check_all_durations(self):
+        if self._measurement_total_duration is None: # exit early if no max timeout specififed
+            return
+
+        durations = {
+            "measurement_flow_process_duration": self._measurement_flow_process_duration,
+            'pre_test_sleep': self._measurement_pre_test_sleep,
+            'post_test_sleep': self._measurement_post_test_sleep,
+            'idle_duration': self._measurement_idle_duration,
+            'baseline_duration': self._measurement_baseline_duration,
+            'phase_transition_time': self._measurement_phase_transition_time,
+            'wait_time_dependencies': self._measurement_wait_time_dependencies,
+        }
+
+        for key, value in durations.items():
+            if value is not None and value > self._measurement_total_duration:
+                raise ValueError(f"Cannot run flows due to configuration error. Measurement_total_duration must be >= {key}, otherwise the flow will run into a timeout in every case. Values are: {key}: {value} and measurement_total_duration: {self._measurement_total_duration}")
+
+
     def custom_sleep(self, sleep_time):
         if not self._dev_no_sleeps:
             print(TerminalColors.HEADER, '\nSleeping for : ', sleep_time, TerminalColors.ENDC)

tests/test_config_opts.py

@@ -11,23 +11,11 @@
 from tests import test_functions as Tests
 from lib.scenario_runner import ScenarioRunner
 
-#pylint: disable=unused-argument # unused arguement off for now - because there are no running tests in this file
-@pytest.fixture(name="reset_config")
-def reset_config_fixture():
-    config = GlobalConfig().config
-    idle_start_time = config['measurement']['idle-time-start']
-    idle_time_end = config['measurement']['idle-time-end']
-    flow_process_runtime = config['measurement']['flow-process-runtime']
-    yield
-    config['measurement']['idle-time-start'] = idle_start_time
-    config['measurement']['idle-time-end'] = idle_time_end
-    config['measurement']['flow-process-runtime'] = flow_process_runtime
-
 def test_global_timeout():
 
     measurement_total_duration = 1
 
-    runner = ScenarioRunner(uri=GMT_DIR, uri_type='folder', filename='tests/data/usage_scenarios/basic_stress.yml', skip_system_checks=True, dev_cache_build=False, dev_no_sleeps=True, dev_no_metrics=True, dev_no_phase_stats=True, measurement_total_duration=1)
+    runner = ScenarioRunner(uri=GMT_DIR, uri_type='folder', filename='tests/data/usage_scenarios/basic_stress.yml', skip_system_checks=True, dev_cache_build=False, dev_no_sleeps=True, dev_no_metrics=True, dev_no_phase_stats=True, measurement_total_duration=1, measurement_pre_test_sleep=1, measurement_baseline_duration=1, measurement_idle_duration=1, measurement_post_test_sleep=1, measurement_wait_time_dependencies=1)
 
     out = io.StringIO()
     err = io.StringIO()
@@ -47,6 +35,13 @@ def test_global_timeout():
         Tests.assertion_info('Timeout was not raised', str(out.getvalue()))
 
 
+def test_invalid_combination_measurement_flow_process_duration():
+
+    with pytest.raises(ValueError) as err:
+        ScenarioRunner(uri=GMT_DIR, uri_type='folder', filename='tests/data/usage_scenarios/basic_stress.yml', skip_system_checks=True, dev_cache_build=False, dev_no_sleeps=True, dev_no_metrics=True, dev_no_phase_stats=True, measurement_total_duration=10, measurement_flow_process_duration=20)
+
+    assert str(err.value) == 'Cannot run flows due to configuration error. Measurement_total_duration must be >= measurement_flow_process_duration, otherwise the flow will run into a timeout in every case. Values are: measurement_flow_process_duration: 20 and measurement_total_duration: 10'
+
 def test_provider_disabling_not_active_by_default():
 
 
@@ -136,65 +131,3 @@ def test_phase_padding_active():
     assert notes[-7][0] - notes[-8][0] == runner._phase_padding_ms*FROM_MS_TO_US
 
     assert notes[-6][1] == 'Ending phase [RUNTIME] [UNPADDED]'
-
-
-
-# Rethink how to do this test entirely
-def wip_test_idle_start_time(reset_config):
-    GlobalConfig().config['measurement']['idle-time-start'] = 2
-    runner = ScenarioRunner(uri=GMT_DIR, uri_type='folder', filename='tests/data/usage_scenarios/basic_stress.yml', skip_system_checks=True, dev_no_metrics=True, dev_no_phase_stats=True, dev_no_sleeps=True, dev_cache_build=True)
-    run_id = runner.run()
-    query = """
-            SELECT
-                time, note
-            FROM
-                notes
-            WHERE
-                run_id = %s
-            ORDER BY
-                time
-            """
-
-    notes = DB().fetch_all(query, (run_id,))
-
-    timestamp_preidle = [note for note in notes if "Booting" in note[1]][0][0]
-    timestamp_start = [note for note in notes if note[1] == 'Start of measurement'][0][0]
-
-    #assert that the difference between the two timestamps is roughly 2 seconds
-    diff = (timestamp_start - timestamp_preidle)/1000000
-    assert 1.9 <= diff <= 2.1, \
-        Tests.assertion_info('2s apart', f"timestamp difference of notes: {diff}s")
-
-# Rethink how to do this test entirely
-def wip_test_idle_end_time(reset_config):
-    GlobalConfig().config['measurement']['idle-time-end'] = 2
-    runner = ScenarioRunner(uri=GMT_DIR, uri_type='folder', filename='tests/data/usage_scenarios/basic_stress.yml', skip_system_checks=True, dev_no_metrics=True, dev_no_phase_stats=True, dev_no_sleeps=True, dev_cache_build=True)
-    run_id = runner.run()
-    query = """
-            SELECT
-                time, note
-            FROM
-                notes
-            WHERE
-                run_id = %s
-            ORDER BY
-                time
-            """
-
-    notes = DB().fetch_all(query, (run_id,))
-    timestamp_postidle = [note for note in notes if note[1] == 'End of post-measurement idle'][0][0]
-    timestamp_end = [note for note in notes if note[1] == 'End of measurement'][0][0]
-
-    #assert that the difference between the two timestamps is roughly 2 seconds
-    diff = (timestamp_postidle - timestamp_end)/1000000
-    assert 1.9 <= diff <= 2.1, \
-        Tests.assertion_info('2s apart', f"timestamp difference of notes: {diff}s")
-
-def wip_test_process_runtime_exceeded(reset_config):
-    GlobalConfig().config['measurement']['flow-process-runtime'] = .1
-    runner = ScenarioRunner(uri=GMT_DIR, uri_type='folder', filename='tests/data/usage_scenarios/basic_stress.yml', skip_system_checks=True, dev_no_metrics=True, dev_no_phase_stats=True, dev_no_sleeps=True, dev_cache_build=True)
-    with pytest.raises(RuntimeError) as err:
-        runner.run()
-    expected_exception = 'Process exceeded runtime of 0.1s: stress-ng -c 1 -t 1 -q'
-    assert expected_exception in str(err.value), \
-        Tests.assertion_info(expected_exception, str(err.value))