From f5a763ae9284e71887483b1377ba7a4a56bad1ed Mon Sep 17 00:00:00 2001 From: Freddie Akeroyd Date: Sun, 20 Apr 2025 01:49:09 +0100 Subject: [PATCH 1/5] Add check for archiving value at begin and end --- test_genie_python_dae.py | 72 ++++++++++++++++++++++++++++++++-------- 1 file changed, 59 insertions(+), 13 deletions(-) diff --git a/test_genie_python_dae.py b/test_genie_python_dae.py index 2106df7..1085eb5 100644 --- a/test_genie_python_dae.py +++ b/test_genie_python_dae.py @@ -8,7 +8,7 @@ from datetime import timedelta from threading import Thread from time import sleep -from typing import Any, Callable +from typing import Callable import h5py from parameterized import parameterized @@ -36,7 +36,7 @@ def nexus_file_with_retry( - instrument: str, run_number: int, test_func: Callable[[h5py.File], None] + instrument: str, run_number: int, test_func: Callable[[h5py.File, int], None] ) -> None: # isisicp writes files asynchronously, so need to retry file read # in case file not completed and still locked @@ -46,7 +46,8 @@ def nexus_file_with_retry( for i in range(num_of_tries): try: with h5py.File(nexus_file, "r") as f: - test_func(f) + test_func(f, run_number) + break except IOError: if i == num_of_tries - 1: print("{} not found, giving up".format(nexus_file)) @@ -153,6 +154,52 @@ def test_GIVEN_running_instrument_WHEN_pars_changed_THEN_pars_saved_in_file(self else: self.assertEqual(0, saved_beamstop) + def test_GIVEN_running_instrument_WHEN_block_logging_but_not_changing_THEN_block_value_saved_in_file( + self, + ): + load_config_if_not_already_loaded("rcptt_simple") + self.fail_if_not_in_setup() + set_genie_python_raises_exceptions(True) + test_block_name = "FLOAT_BLOCK" + r_cnt_start = g.get_pv("DAE:_RESTART_ARCHIVER_CNT", is_local=True) + fr_cnt_start = g.get_pv("DAE:_FR_ARCHIVER_CNT", is_local=True) + ncheck = 10 + # block is not changing but check we get at least one value logged + # test quick begin/end and some with delays + # there is a 5 second flush time in archiver -> mysql hence + # trying shorter and longer delays + delays = [(10, 10), (2, 10), (10, 2), (2, 2), (2, 0), (0, 2), (0, 0)] + for delay in delays: + print(f"Testing (pre, post) delay {delay}") + for _ in range(ncheck): + sleep(delay[0]) + g.begin() + sleep(delay[1]) + run_number = g.get_runnumber() + g.end() + g.waitfor_runstate("SETUP", maxwaitsecs=self.TIMEOUT) + nexus_path = r"/raw_data_1/selog/{}/value_log".format(test_block_name) + + def test_function(f: h5py.File, run_number: int) -> None: + # if no values are logged this will fail with a + # "Unable to synchronously open object (component not found)" exception + values = [val for val in f[nexus_path + r"/value"][:]] + print( + f"Found {len(values)} value(s) for block {test_block_name} run {run_number}" + ) + self.assertTrue( + len(values) > 0, + f"Not enough values logged to file with delay {delay}", + ) + + nexus_file_with_retry(g.adv.get_instrument(), run_number, test_function) + # check archiver has restarted expected number of times and no force restarts + # ARBLOCK restarted on begin and end hence factor 2 + r_cnt = g.get_pv("DAE:_RESTART_ARCHIVER_CNT", is_local=True) + fr_cnt = g.get_pv("DAE:_FR_ARCHIVER_CNT", is_local=True) + self.assertEqual(r_cnt, r_cnt_start + 2 * ncheck * len(delays)) + self.assertEqual(fr_cnt, fr_cnt_start) + def test_GIVEN_running_instrument_WHEN_block_logging_THEN_block_saved_in_file(self): load_config_if_not_already_loaded("rcptt_simple") self.fail_if_not_in_setup() @@ -195,7 +242,7 @@ def test_GIVEN_running_instrument_WHEN_block_logging_THEN_block_saved_in_file(se nexus_path = r"/raw_data_1/selog/{}/value_log".format(test_block_name) - def test_function(f: Any) -> None: + def test_function(f: h5py.File, run_number: int) -> None: value_valid = f[nexus_path + r"/value_valid"][:] is_valid = [sample == 1 for sample in value_valid] values = [int(val) for val in f[nexus_path + r"/value"][:]] @@ -270,7 +317,7 @@ def _assert_title_correct(self, test_title, expected_title): g.waitfor_runstate("SETUP", maxwaitsecs=self.TIMEOUT) - def test_func(f): + def test_func(f: h5py.File, run_number: int) -> None: saved_title = f["/raw_data_1/title"][0].decode() self.assertEqual(expected_title, saved_title) @@ -567,7 +614,10 @@ def _adjust_icp_begin_delay(self, delay_seconds): begindelay_property = "isisicp.begindelay" config_line = "{} = {}\r\n".format(begindelay_property, delay_seconds) - if g.get_runstate() != "SETUP": + g.waitfor_runstate("PROCESSING", maxwaitsecs=30, onexit=True) + runstate = g.get_runstate() + if runstate != "SETUP": + print(f"Aborting run as currently {runstate}") g.abort() # make sure not left in a funny state from e.g. previous aborted test with g._genie_api.dae.temporarily_kill_icp(): @@ -813,9 +863,7 @@ def test_GIVEN_change_title_called_WHEN_valid_argument_THEN_get_title_correct_im g.change_title(title) self.assertEqual(g.get_title(), title) - def test_GIVEN_dae_setup_WHEN_read_x_and_xe_THEN_centre_and_edges_correct( - self - ) -> None: + def test_GIVEN_dae_setup_WHEN_read_x_and_xe_THEN_centre_and_edges_correct(self) -> None: set_genie_python_raises_exceptions(True) # check x and xe are correct length ntc = g.get_number_timechannels() @@ -826,12 +874,10 @@ def test_GIVEN_dae_setup_WHEN_read_x_and_xe_THEN_centre_and_edges_correct( # check x is a bin centre of xe edges x = g.get_pv("DAE:SPEC:1:1:X", is_local=True) xe = g.get_pv("DAE:SPEC:1:1:XE", is_local=True) - self.assertAlmostEqual(x[0], (xe[0] + xe[1]) / 2.0, delta=.001) + self.assertAlmostEqual(x[0], (xe[0] + xe[1]) / 2.0, delta=0.001) set_genie_python_raises_exceptions(False) - def test_GIVEN_dae_setup_WHEN_paused_THEN_period_values_correct( - self - ) -> None: + def test_GIVEN_dae_setup_WHEN_paused_THEN_period_values_correct(self) -> None: set_genie_python_raises_exceptions(True) sleep_time = 5 num_periods = 2 From c97c961eca34e880192f7bfd0529dba10635689e Mon Sep 17 00:00:00 2001 From: Freddie Akeroyd Date: Sun, 20 Apr 2025 01:56:35 +0100 Subject: [PATCH 2/5] Extra comments --- test_genie_python_dae.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test_genie_python_dae.py b/test_genie_python_dae.py index 1085eb5..d7af0c7 100644 --- a/test_genie_python_dae.py +++ b/test_genie_python_dae.py @@ -161,7 +161,9 @@ def test_GIVEN_running_instrument_WHEN_block_logging_but_not_changing_THEN_block self.fail_if_not_in_setup() set_genie_python_raises_exceptions(True) test_block_name = "FLOAT_BLOCK" + # number of planned restarts due to a begin/end r_cnt_start = g.get_pv("DAE:_RESTART_ARCHIVER_CNT", is_local=True) + # number of forced restarts due to background check fr_cnt_start = g.get_pv("DAE:_FR_ARCHIVER_CNT", is_local=True) ncheck = 10 # block is not changing but check we get at least one value logged @@ -193,7 +195,7 @@ def test_function(f: h5py.File, run_number: int) -> None: ) nexus_file_with_retry(g.adv.get_instrument(), run_number, test_function) - # check archiver has restarted expected number of times and no force restarts + # check archiver has restarted expected number of times and no forced restarts # ARBLOCK restarted on begin and end hence factor 2 r_cnt = g.get_pv("DAE:_RESTART_ARCHIVER_CNT", is_local=True) fr_cnt = g.get_pv("DAE:_FR_ARCHIVER_CNT", is_local=True) From ffa340e6a8406363274b542ad3991c2e7d1ff137 Mon Sep 17 00:00:00 2001 From: Freddie Akeroyd Date: Sun, 20 Apr 2025 23:55:17 +0100 Subject: [PATCH 3/5] pyright --- test_genie_python_dae.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test_genie_python_dae.py b/test_genie_python_dae.py index d7af0c7..3abd04e 100644 --- a/test_genie_python_dae.py +++ b/test_genie_python_dae.py @@ -8,7 +8,7 @@ from datetime import timedelta from threading import Thread from time import sleep -from typing import Callable +from typing import Any, Callable import h5py from parameterized import parameterized @@ -182,7 +182,7 @@ def test_GIVEN_running_instrument_WHEN_block_logging_but_not_changing_THEN_block g.waitfor_runstate("SETUP", maxwaitsecs=self.TIMEOUT) nexus_path = r"/raw_data_1/selog/{}/value_log".format(test_block_name) - def test_function(f: h5py.File, run_number: int) -> None: + def test_function(f: Any, run_number: int) -> None: # if no values are logged this will fail with a # "Unable to synchronously open object (component not found)" exception values = [val for val in f[nexus_path + r"/value"][:]] @@ -191,7 +191,7 @@ def test_function(f: h5py.File, run_number: int) -> None: ) self.assertTrue( len(values) > 0, - f"Not enough values logged to file with delay {delay}", + "Not enough values logged to file", ) nexus_file_with_retry(g.adv.get_instrument(), run_number, test_function) @@ -319,7 +319,7 @@ def _assert_title_correct(self, test_title, expected_title): g.waitfor_runstate("SETUP", maxwaitsecs=self.TIMEOUT) - def test_func(f: h5py.File, run_number: int) -> None: + def test_func(f: Any, run_number: int) -> None: saved_title = f["/raw_data_1/title"][0].decode() self.assertEqual(expected_title, saved_title) From dfc51b84ac636d69cd19a507a47b35e25983c177 Mon Sep 17 00:00:00 2001 From: Freddie Akeroyd Date: Mon, 21 Apr 2025 02:10:52 +0100 Subject: [PATCH 4/5] Call direct rather than via thread Test fails on timedelta part if use thread - not sure why, seems second call to get_time_since_begin() in thread happens 15 seconds after first call! --- test_genie_python_dae.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test_genie_python_dae.py b/test_genie_python_dae.py index 3abd04e..df1b176 100644 --- a/test_genie_python_dae.py +++ b/test_genie_python_dae.py @@ -772,12 +772,13 @@ def get_time_thread(return_value): return_value.append(g.get_time_since_begin(True)) time_taken = [] - thread = Thread(target=get_time_thread, args=(time_taken,)) - thread.start() + #thread = Thread(target=get_time_thread, args=(time_taken,)) + #thread.start() g.begin() + get_time_thread(time_taken) - thread.join() + #thread.join() # Taking the fluctuation of actual runtime into account and tolerating up to 1 sec difference self.assertAlmostEqual( From bc1558d0c36b2425bda06f8fb959aa4fe03d6c7f Mon Sep 17 00:00:00 2001 From: Freddie Akeroyd Date: Mon, 21 Apr 2025 02:16:36 +0100 Subject: [PATCH 5/5] ruff --- test_genie_python_dae.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test_genie_python_dae.py b/test_genie_python_dae.py index df1b176..ebe5e38 100644 --- a/test_genie_python_dae.py +++ b/test_genie_python_dae.py @@ -6,7 +6,8 @@ import unittest from contextlib import contextmanager from datetime import timedelta -from threading import Thread + +# from threading import Thread from time import sleep from typing import Any, Callable @@ -772,13 +773,13 @@ def get_time_thread(return_value): return_value.append(g.get_time_since_begin(True)) time_taken = [] - #thread = Thread(target=get_time_thread, args=(time_taken,)) - #thread.start() + # thread = Thread(target=get_time_thread, args=(time_taken,)) + # thread.start() g.begin() get_time_thread(time_taken) - #thread.join() + # thread.join() # Taking the fluctuation of actual runtime into account and tolerating up to 1 sec difference self.assertAlmostEqual(