diff --git a/src/ibex_bluesky_core/callbacks/_utils.py b/src/ibex_bluesky_core/callbacks/_utils.py new file mode 100644 index 00000000..95eb9e55 --- /dev/null +++ b/src/ibex_bluesky_core/callbacks/_utils.py @@ -0,0 +1,20 @@ +from pathlib import Path +from platform import node + +INSTRUMENT = node() +DEFAULT_PATH = Path("//isis.cclrc.ac.uk/inst$") / INSTRUMENT / "user" / "TEST" / "scans" + +# Common document metadata +UID = "uid" +TIME = "time" +DATA = "data" +RB = "rb_number" +START_TIME = "start_time" +NAME = "name" +SEQ_NUM = "seq_num" +DATA_KEYS = "data_keys" +DESCRIPTOR = "descriptor" +UNITS = "units" +PRECISION = "precision" +MOTORS = "motors" +UNKNOWN_RB = "Unknown RB" diff --git a/src/ibex_bluesky_core/callbacks/file_logger.py b/src/ibex_bluesky_core/callbacks/file_logger.py index cdd05d4f..67d9770c 100644 --- a/src/ibex_bluesky_core/callbacks/file_logger.py +++ b/src/ibex_bluesky_core/callbacks/file_logger.py @@ -2,9 +2,9 @@ import csv import logging +import os from datetime import datetime from pathlib import Path -from platform import node from typing import Optional from zoneinfo import ZoneInfo @@ -14,21 +14,25 @@ from event_model.documents.run_start import RunStart from event_model.documents.run_stop import RunStop -logger = logging.getLogger(__name__) +from ibex_bluesky_core.callbacks._utils import ( + DATA, + DATA_KEYS, + DEFAULT_PATH, + DESCRIPTOR, + INSTRUMENT, + MOTORS, + NAME, + PRECISION, + RB, + SEQ_NUM, + START_TIME, + TIME, + UID, + UNITS, + UNKNOWN_RB, +) -TIME = "time" -START_TIME = "start_time" -NAME = "name" -SEQ_NUM = "seq_num" -DATA_KEYS = "data_keys" -DATA = "data" -DESCRIPTOR = "descriptor" -UNITS = "units" -UID = "uid" -RB = "rb_number" -PRECISION = "precision" -INSTRUMENT = node() -DEFAULT_PATH = Path("//isis/inst$") / INSTRUMENT / "user" / "TEST" / "scans" +logger = logging.getLogger(__name__) class HumanReadableFileCallback(CallbackBase): @@ -56,16 +60,21 @@ def start(self, doc: RunStart) -> None: self.current_start_document = doc[UID] datetime_obj = datetime.fromtimestamp(doc[TIME]) - title_format_datetime = datetime_obj.astimezone(ZoneInfo("Europe/London")).strftime( + title_format_datetime = datetime_obj.astimezone(ZoneInfo("UTC")).strftime( "%Y-%m-%d_%H-%M-%S" ) - axes = "_".join(self.fields) - rb_num = doc.get("rb_number", "Unknown RB") + rb_num = doc.get(RB, UNKNOWN_RB) + + # motors is a tuple, we need to convert to a list to join the two below + motors = list(doc.get(MOTORS, [])) + self.filename = ( - self.output_dir / f"{rb_num}" / f"{INSTRUMENT}_{axes}_{title_format_datetime}Z.txt" + self.output_dir + / f"{rb_num}" + / f"{INSTRUMENT}{'_' + '_'.join(motors) if motors else ''}_{title_format_datetime}Z.txt" ) - if rb_num == "Unknown RB": - logger.warning('No RB number found, saving to "Unknown RB"') + if rb_num == UNKNOWN_RB: + logger.warning('No RB number found, saving to "%s"', UNKNOWN_RB) assert self.filename is not None logger.info("starting new file %s", self.filename) @@ -74,14 +83,14 @@ def start(self, doc: RunStart) -> None: ] header_data = {k: v for k, v in doc.items() if k not in exclude_list} - formatted_time = datetime_obj.astimezone(ZoneInfo("Europe/London")).strftime( - "%Y-%m-%d %H:%M:%S" - ) + formatted_time = datetime_obj.astimezone(ZoneInfo("UTC")).strftime("%Y-%m-%d %H:%M:%S") header_data[START_TIME] = formatted_time - with open(self.filename, "a", newline="", encoding="utf-8") as outfile: - for key, value in header_data.items(): - outfile.write(f"{key}: {value}\n") + # make sure the parent directory exists, create it if not + os.makedirs(self.filename.parent, exist_ok=True) + + with open(self.filename, "a", newline="\n", encoding="utf-8") as outfile: + outfile.writelines([f"{key}: {value}\n" for key, value in header_data.items()]) logger.debug("successfully wrote header in %s", self.filename) diff --git a/src/ibex_bluesky_core/callbacks/fitting/livefit_logger.py b/src/ibex_bluesky_core/callbacks/fitting/livefit_logger.py index 4da5a272..a0e0d756 100644 --- a/src/ibex_bluesky_core/callbacks/fitting/livefit_logger.py +++ b/src/ibex_bluesky_core/callbacks/fitting/livefit_logger.py @@ -1,11 +1,10 @@ -"""Creates a readable .csv file of Bluesky fitting metrics.""" +"""Creates a readable text file, with comma separated values of Bluesky fitting metrics.""" import csv import logging import os from datetime import datetime from pathlib import Path -from platform import node from typing import Optional from zoneinfo import ZoneInfo @@ -15,14 +14,17 @@ from event_model.documents.run_start import RunStart from event_model.documents.run_stop import RunStop +from ibex_bluesky_core.callbacks._utils import ( + DATA, + DEFAULT_PATH, + INSTRUMENT, + RB, + TIME, + UID, + UNKNOWN_RB, +) from ibex_bluesky_core.callbacks.fitting import LiveFit -UID = "uid" -TIME = "time" -DATA = "data" -RB = "rb_number" -INSTRUMENT = node() -DEFAULT_PATH = Path("//isis/inst$") / INSTRUMENT / "user" / "TEST" / "scans" logger = logging.getLogger(__name__) @@ -72,15 +74,15 @@ def start(self, doc: RunStart) -> None: """ datetime_obj = datetime.fromtimestamp(doc[TIME]) - title_format_datetime = datetime_obj.astimezone(ZoneInfo("Europe/London")).strftime( + title_format_datetime = datetime_obj.astimezone(ZoneInfo("UTC")).strftime( "%Y-%m-%d_%H-%M-%S" ) self.output_dir.mkdir(parents=True, exist_ok=True) self.current_start_document = doc[UID] - file = f"{INSTRUMENT}_{self.x}_{self.y}_{title_format_datetime}Z{self.postfix}.csv" - rb_num = doc.get("rb_number", "Unknown RB") - if rb_num == "Unknown RB": - logger.warning('No RB number found, will save to "Unknown RB"') + file = f"{INSTRUMENT}_{self.x}_{self.y}_{title_format_datetime}Z{self.postfix}.txt" + rb_num = doc.get(RB, UNKNOWN_RB) + if rb_num == UNKNOWN_RB: + logger.warning('No RB number found, will save to "%s"', UNKNOWN_RB) self.filename = self.output_dir / f"{rb_num}" / file def event(self, doc: Event) -> Event: @@ -126,13 +128,15 @@ def stop(self, doc: RunStop) -> None: self.stats = self.livefit.result.fit_report().split("\n") + # make sure the parent directory exists, create it if not + os.makedirs(self.filename.parent, exist_ok=True) + # Writing to csv file with open(self.filename, "w", newline="", encoding="utf-8") as csvfile: # Writing the data self.csvwriter = csv.writer(csvfile) - for row in self.stats: - csvfile.write(row + os.linesep) + csvfile.writelines([row + os.linesep for row in self.stats]) csvfile.write(os.linesep) # Space out file csvfile.write(os.linesep) diff --git a/tests/callbacks/fitting/test_fit_logging_callback.py b/tests/callbacks/fitting/test_fit_logging_callback.py index 9a2af19e..ad44a0d8 100644 --- a/tests/callbacks/fitting/test_fit_logging_callback.py +++ b/tests/callbacks/fitting/test_fit_logging_callback.py @@ -28,21 +28,25 @@ def test_after_fitting_callback_writes_to_file_successfully_no_y_uncertainty( lf = LiveFit(Linear.fit(), y="invariant", x="motor", update_every=50) lfl = LiveFitLogger(lf, y="invariant", x="motor", postfix=postfix, output_dir=filepath) - with patch("ibex_bluesky_core.callbacks.fitting.livefit_logger.open", m): + with ( + patch("ibex_bluesky_core.callbacks.fitting.livefit_logger.open", m), + patch("ibex_bluesky_core.callbacks.fitting.livefit_logger.os.makedirs"), + ): with patch("time.time", MagicMock(return_value=time)): RE(scan([invariant], mot, -1, 1, 3), [lf, lfl], rb_number="0") assert m.call_args_list[0].args == ( - filepath / "0" / f"{node()}_motor_invariant_2024-10-04_14-43-43Z{postfix}.csv", + filepath / "0" / f"{node()}_motor_invariant_2024-10-04_13-43-43Z{postfix}.txt", "w", ) # type: ignore handle = m() + rows_writelines = next(i.args[0] for i in handle.writelines.call_args_list) rows = [i.args[0] for i in handle.write.call_args_list] # Check that it starts writing to the file in the expected way - assert f" Model({Linear.__name__} [{Linear.equation}])" + os.linesep in rows + assert f" Model({Linear.__name__} [{Linear.equation}])" + os.linesep in rows_writelines assert "x,y,modelled y\r\n" in rows @@ -58,12 +62,15 @@ def test_fitting_callback_handles_no_rb_number_save( lf = LiveFit(Linear.fit(), y="invariant", x="motor", update_every=50) lfl = LiveFitLogger(lf, y="invariant", x="motor", postfix=postfix, output_dir=filepath) - with patch("ibex_bluesky_core.callbacks.fitting.livefit_logger.open", m): + with ( + patch("ibex_bluesky_core.callbacks.fitting.livefit_logger.open", m), + patch("ibex_bluesky_core.callbacks.fitting.livefit_logger.os.makedirs"), + ): with patch("time.time", MagicMock(return_value=time)): RE(scan([invariant], mot, -1, 1, 3), [lf, lfl]) assert m.call_args_list[0].args == ( - filepath / "Unknown RB" / f"{node()}_motor_invariant_2024-10-04_14-43-43Z{postfix}.csv", + filepath / "Unknown RB" / f"{node()}_motor_invariant_2024-10-04_13-43-43Z{postfix}.txt", "w", ) # type: ignore @@ -84,21 +91,25 @@ def test_after_fitting_callback_writes_to_file_successfully_with_y_uncertainty( lf, y="invariant", x="motor", postfix=postfix, output_dir=filepath, yerr="uncertainty" ) - with patch("ibex_bluesky_core.callbacks.fitting.livefit_logger.open", m): + with ( + patch("ibex_bluesky_core.callbacks.fitting.livefit_logger.open", m), + patch("ibex_bluesky_core.callbacks.fitting.livefit_logger.os.makedirs"), + ): with patch("time.time", MagicMock(return_value=time)): RE(scan([invariant, uncertainty], mot, -1, 1, 3), [lf, lfl], rb_number="0") assert m.call_args_list[0].args == ( - filepath / "0" / f"{node()}_motor_invariant_2024-10-04_14-43-43Z{postfix}.csv", + filepath / "0" / f"{node()}_motor_invariant_2024-10-04_13-43-43Z{postfix}.txt", "w", ) # type: ignore handle = m() rows = [i.args[0] for i in handle.write.call_args_list] + rows_writelines = next(i.args[0] for i in handle.writelines.call_args_list) # Check that it starts writing to the file in the expected way - assert f" Model({Linear.__name__} [{Linear.equation}])" + os.linesep in rows + assert f" Model({Linear.__name__} [{Linear.equation}])" + os.linesep in rows_writelines assert "x,y,y uncertainty,modelled y\r\n" in rows @@ -117,7 +128,10 @@ def test_file_not_written_if_no_fitting_result(RE: run_engine.RunEngine): lf = LiveFit(method, y="invariant", x="motor") lfl = LiveFitLogger(lf, y="invariant", x="motor", postfix=postfix, output_dir=filepath) - with patch("ibex_bluesky_core.callbacks.fitting.livefit_logger.open", m): + with ( + patch("ibex_bluesky_core.callbacks.fitting.livefit_logger.open", m), + patch("ibex_bluesky_core.callbacks.fitting.livefit_logger.os.makedirs"), + ): RE(scan([invariant], mot, -1, 1, 3), [lf, lfl], rb_number="0") assert not m.called diff --git a/tests/callbacks/test_write_log_callback.py b/tests/callbacks/test_write_log_callback.py index bb539c9a..a8bc99a6 100644 --- a/tests/callbacks/test_write_log_callback.py +++ b/tests/callbacks/test_write_log_callback.py @@ -22,34 +22,67 @@ def test_header_data_all_available_on_start(cb): time = 1728049423.5860472 uid = "test123" scan_id = 1234 - run_start = RunStart(time=time, uid=uid, scan_id=scan_id, rb_number="0") - with patch("ibex_bluesky_core.callbacks.file_logger.open", mock_open()) as mock_file: + run_start = RunStart( + time=time, uid=uid, scan_id=scan_id, rb_number="0", detectors=["dae"], motors=("block",) + ) + with ( + patch("ibex_bluesky_core.callbacks.file_logger.open", mock_open()) as mock_file, + patch("ibex_bluesky_core.callbacks.file_logger.os.makedirs"), + ): cb.start(run_start) result = ( save_path / f"{run_start.get('rb_number', None)}" - / f"{node()}_block_dae_2024-10-04_14-43-43Z.txt" + / f"{node()}_block_2024-10-04_13-43-43Z.txt" ) - mock_file.assert_called_with(result, "a", newline="", encoding="utf-8") + mock_file.assert_called_with(result, "a", newline="\n", encoding="utf-8") + writelines_call_args = mock_file().writelines.call_args[0][0] # time should have been renamed to start_time and converted to human readable - mock_file().write.assert_any_call("start_time: 2024-10-04 14:43:43\n") - mock_file().write.assert_any_call(f"uid: {uid}\n") + assert "start_time: 2024-10-04 13:43:43\n" in writelines_call_args + assert f"uid: {uid}\n" in writelines_call_args def test_no_rb_number_folder(cb): time = 1728049423.5860472 uid = "test123" scan_id = 1234 - run_start = RunStart(time=time, uid=uid, scan_id=scan_id) - with patch("ibex_bluesky_core.callbacks.file_logger.open", mock_open()) as mock_file: + run_start = RunStart(time=time, uid=uid, scan_id=scan_id, detectors=["dae"], motors=("block",)) + + with ( + patch("ibex_bluesky_core.callbacks.file_logger.open", mock_open()) as mock_file, + patch("ibex_bluesky_core.callbacks.file_logger.os.makedirs") as mock_mkdir, + ): + cb.start(run_start) + result = save_path / "Unknown RB" / f"{node()}_block_2024-10-04_13-43-43Z.txt" + assert mock_mkdir.called + + mock_file.assert_called_with(result, "a", newline="\n", encoding="utf-8") + # time should have been renamed to start_time and converted to human readable + writelines_call_args = mock_file().writelines.call_args[0][0] + assert "start_time: 2024-10-04 13:43:43\n" in writelines_call_args + assert f"uid: {uid}\n" in writelines_call_args + + +def test_no_motors_doesnt_append_to_filename(cb): + time = 1728049423.5860472 + uid = "test123" + scan_id = 1234 + run_start = RunStart(time=time, uid=uid, scan_id=scan_id, detectors=["dae"]) + + with ( + patch("ibex_bluesky_core.callbacks.file_logger.open", mock_open()) as mock_file, + patch("ibex_bluesky_core.callbacks.file_logger.os.makedirs") as mock_mkdir, + ): cb.start(run_start) - result = save_path / "Unknown RB" / f"{node()}_block_dae_2024-10-04_14-43-43Z.txt" + result = save_path / "Unknown RB" / f"{node()}_2024-10-04_13-43-43Z.txt" + assert mock_mkdir.called - mock_file.assert_called_with(result, "a", newline="", encoding="utf-8") + mock_file.assert_called_with(result, "a", newline="\n", encoding="utf-8") # time should have been renamed to start_time and converted to human readable - mock_file().write.assert_any_call("start_time: 2024-10-04 14:43:43\n") - mock_file().write.assert_any_call(f"uid: {uid}\n") + writelines_call_args = mock_file().writelines.call_args[0][0] + assert "start_time: 2024-10-04 13:43:43\n" in writelines_call_args + assert f"uid: {uid}\n" in writelines_call_args def test_descriptor_data_does_nothing_if_doc_not_called_primary(cb):