fix exception if output file (rbnum) dir doesn't exist

src/ibex_bluesky_core/callbacks/file_logger.py

@@ -2,6 +2,7 @@
 
 import csv
 import logging
+import os
 from datetime import datetime
 from pathlib import Path
 from platform import node
@@ -28,7 +29,7 @@
 RB = "rb_number"
 PRECISION = "precision"
 INSTRUMENT = node()
-DEFAULT_PATH = Path("//isis/inst$") / INSTRUMENT / "user" / "TEST" / "scans"
+DEFAULT_PATH = Path("//isis.cclrc.ac.uk/inst$") / INSTRUMENT / "user" / "TEST" / "scans"
 
 
 class HumanReadableFileCallback(CallbackBase):
@@ -59,10 +60,15 @@ def start(self, doc: RunStart) -> None:
         title_format_datetime = datetime_obj.astimezone(ZoneInfo("Europe/London")).strftime(
             "%Y-%m-%d_%H-%M-%S"
         )
-        axes = "_".join(self.fields)
         rb_num = doc.get("rb_number", "Unknown RB")
+
+        # motors is a tuple, we need to convert to a list to join the two below
+        motors = list(doc.get("motors", []))
+
         self.filename = (
-            self.output_dir / f"{rb_num}" / f"{INSTRUMENT}_{axes}_{title_format_datetime}Z.txt"
+            self.output_dir
+            / f"{rb_num}"
+            / f"{INSTRUMENT}{'_' + '_'.join(motors) if motors else ''}_{title_format_datetime}Z.txt"
         )
         if rb_num == "Unknown RB":
             logger.warning('No RB number found, saving to "Unknown RB"')
@@ -74,14 +80,14 @@ def start(self, doc: RunStart) -> None:
         ]
         header_data = {k: v for k, v in doc.items() if k not in exclude_list}
 
-        formatted_time = datetime_obj.astimezone(ZoneInfo("Europe/London")).strftime(
-            "%Y-%m-%d %H:%M:%S"
-        )
+        formatted_time = datetime_obj.astimezone(ZoneInfo("UTC")).strftime("%Y-%m-%d %H:%M:%S")
         header_data[START_TIME] = formatted_time
 
-        with open(self.filename, "a", newline="", encoding="utf-8") as outfile:
-            for key, value in header_data.items():
-                outfile.write(f"{key}: {value}\n")
+        # make sure the parent directory exists, create it if not
+        os.makedirs(self.filename.parent, exist_ok=True)
+
+        with open(self.filename, "a", newline="\n", encoding="utf-8") as outfile:
+            outfile.writelines([f"{key}: {value}\n" for key, value in header_data.items()])
 
         logger.debug("successfully wrote header in %s", self.filename)
 

src/ibex_bluesky_core/callbacks/fitting/livefit_logger.py

@@ -1,4 +1,4 @@
-"""Creates a readable .csv file of Bluesky fitting metrics."""
+"""Creates a readable text file, with comma separated values of Bluesky fitting metrics."""
 
 import csv
 import logging
@@ -22,7 +22,7 @@
 DATA = "data"
 RB = "rb_number"
 INSTRUMENT = node()
-DEFAULT_PATH = Path("//isis/inst$") / INSTRUMENT / "user" / "TEST" / "scans"
+DEFAULT_PATH = Path("//isis.cclrc.ac.uk/inst$") / INSTRUMENT / "user" / "TEST" / "scans"
 logger = logging.getLogger(__name__)
 
 
@@ -72,12 +72,12 @@ def start(self, doc: RunStart) -> None:
 
         """
         datetime_obj = datetime.fromtimestamp(doc[TIME])
-        title_format_datetime = datetime_obj.astimezone(ZoneInfo("Europe/London")).strftime(
+        title_format_datetime = datetime_obj.astimezone(ZoneInfo("UTC")).strftime(
             "%Y-%m-%d_%H-%M-%S"
         )
         self.output_dir.mkdir(parents=True, exist_ok=True)
         self.current_start_document = doc[UID]
-        file = f"{INSTRUMENT}_{self.x}_{self.y}_{title_format_datetime}Z{self.postfix}.csv"
+        file = f"{INSTRUMENT}_{self.x}_{self.y}_{title_format_datetime}Z{self.postfix}.txt"
         rb_num = doc.get("rb_number", "Unknown RB")
         if rb_num == "Unknown RB":
             logger.warning('No RB number found, will save to "Unknown RB"')
@@ -126,13 +126,15 @@ def stop(self, doc: RunStop) -> None:
 
         self.stats = self.livefit.result.fit_report().split("\n")
 
+        # make sure the parent directory exists, create it if not
+        os.makedirs(self.filename.parent, exist_ok=True)
+
         # Writing to csv file
         with open(self.filename, "w", newline="", encoding="utf-8") as csvfile:
             # Writing the data
             self.csvwriter = csv.writer(csvfile)
 
-            for row in self.stats:
-                csvfile.write(row + os.linesep)
+            csvfile.writelines([row + os.linesep for row in self.stats])
 
             csvfile.write(os.linesep)  # Space out file
             csvfile.write(os.linesep)

tests/callbacks/fitting/test_fit_logging_callback.py

@@ -28,21 +28,25 @@ def test_after_fitting_callback_writes_to_file_successfully_no_y_uncertainty(
 
     lf = LiveFit(Linear.fit(), y="invariant", x="motor", update_every=50)
     lfl = LiveFitLogger(lf, y="invariant", x="motor", postfix=postfix, output_dir=filepath)
-    with patch("ibex_bluesky_core.callbacks.fitting.livefit_logger.open", m):
+    with (
+        patch("ibex_bluesky_core.callbacks.fitting.livefit_logger.open", m),
+        patch("ibex_bluesky_core.callbacks.fitting.livefit_logger.os.makedirs"),
+    ):
         with patch("time.time", MagicMock(return_value=time)):
             RE(scan([invariant], mot, -1, 1, 3), [lf, lfl], rb_number="0")
 
     assert m.call_args_list[0].args == (
-        filepath / "0" / f"{node()}_motor_invariant_2024-10-04_14-43-43Z{postfix}.csv",
+        filepath / "0" / f"{node()}_motor_invariant_2024-10-04_13-43-43Z{postfix}.txt",
         "w",
     )  # type: ignore
 
     handle = m()
+    rows_writelines = next(i.args[0] for i in handle.writelines.call_args_list)
     rows = [i.args[0] for i in handle.write.call_args_list]
 
     # Check that it starts writing to the file in the expected way
 
-    assert f"    Model({Linear.__name__}  [{Linear.equation}])" + os.linesep in rows
+    assert f"    Model({Linear.__name__}  [{Linear.equation}])" + os.linesep in rows_writelines
     assert "x,y,modelled y\r\n" in rows
 
 
@@ -58,12 +62,15 @@ def test_fitting_callback_handles_no_rb_number_save(
 
     lf = LiveFit(Linear.fit(), y="invariant", x="motor", update_every=50)
     lfl = LiveFitLogger(lf, y="invariant", x="motor", postfix=postfix, output_dir=filepath)
-    with patch("ibex_bluesky_core.callbacks.fitting.livefit_logger.open", m):
+    with (
+        patch("ibex_bluesky_core.callbacks.fitting.livefit_logger.open", m),
+        patch("ibex_bluesky_core.callbacks.fitting.livefit_logger.os.makedirs"),
+    ):
         with patch("time.time", MagicMock(return_value=time)):
             RE(scan([invariant], mot, -1, 1, 3), [lf, lfl])
 
     assert m.call_args_list[0].args == (
-        filepath / "Unknown RB" / f"{node()}_motor_invariant_2024-10-04_14-43-43Z{postfix}.csv",
+        filepath / "Unknown RB" / f"{node()}_motor_invariant_2024-10-04_13-43-43Z{postfix}.txt",
         "w",
     )  # type: ignore
 
@@ -84,21 +91,25 @@ def test_after_fitting_callback_writes_to_file_successfully_with_y_uncertainty(
         lf, y="invariant", x="motor", postfix=postfix, output_dir=filepath, yerr="uncertainty"
     )
 
-    with patch("ibex_bluesky_core.callbacks.fitting.livefit_logger.open", m):
+    with (
+        patch("ibex_bluesky_core.callbacks.fitting.livefit_logger.open", m),
+        patch("ibex_bluesky_core.callbacks.fitting.livefit_logger.os.makedirs"),
+    ):
         with patch("time.time", MagicMock(return_value=time)):
             RE(scan([invariant, uncertainty], mot, -1, 1, 3), [lf, lfl], rb_number="0")
 
     assert m.call_args_list[0].args == (
-        filepath / "0" / f"{node()}_motor_invariant_2024-10-04_14-43-43Z{postfix}.csv",
+        filepath / "0" / f"{node()}_motor_invariant_2024-10-04_13-43-43Z{postfix}.txt",
         "w",
     )  # type: ignore
 
     handle = m()
     rows = [i.args[0] for i in handle.write.call_args_list]
+    rows_writelines = next(i.args[0] for i in handle.writelines.call_args_list)
 
     # Check that it starts writing to the file in the expected way
 
-    assert f"    Model({Linear.__name__}  [{Linear.equation}])" + os.linesep in rows
+    assert f"    Model({Linear.__name__}  [{Linear.equation}])" + os.linesep in rows_writelines
     assert "x,y,y uncertainty,modelled y\r\n" in rows
 
 
@@ -117,7 +128,10 @@ def test_file_not_written_if_no_fitting_result(RE: run_engine.RunEngine):
     lf = LiveFit(method, y="invariant", x="motor")
     lfl = LiveFitLogger(lf, y="invariant", x="motor", postfix=postfix, output_dir=filepath)
 
-    with patch("ibex_bluesky_core.callbacks.fitting.livefit_logger.open", m):
+    with (
+        patch("ibex_bluesky_core.callbacks.fitting.livefit_logger.open", m),
+        patch("ibex_bluesky_core.callbacks.fitting.livefit_logger.os.makedirs"),
+    ):
         RE(scan([invariant], mot, -1, 1, 3), [lf, lfl], rb_number="0")
 
     assert not m.called

tests/callbacks/test_write_log_callback.py

@@ -22,34 +22,67 @@ def test_header_data_all_available_on_start(cb):
     time = 1728049423.5860472
     uid = "test123"
     scan_id = 1234
-    run_start = RunStart(time=time, uid=uid, scan_id=scan_id, rb_number="0")
-    with patch("ibex_bluesky_core.callbacks.file_logger.open", mock_open()) as mock_file:
+    run_start = RunStart(
+        time=time, uid=uid, scan_id=scan_id, rb_number="0", detectors=["dae"], motors=("block",)
+    )
+    with (
+        patch("ibex_bluesky_core.callbacks.file_logger.open", mock_open()) as mock_file,
+        patch("ibex_bluesky_core.callbacks.file_logger.os.makedirs"),
+    ):
         cb.start(run_start)
         result = (
             save_path
             / f"{run_start.get('rb_number', None)}"
-            / f"{node()}_block_dae_2024-10-04_14-43-43Z.txt"
+            / f"{node()}_block_2024-10-04_14-43-43Z.txt"
         )
 
-    mock_file.assert_called_with(result, "a", newline="", encoding="utf-8")
+    mock_file.assert_called_with(result, "a", newline="\n", encoding="utf-8")
+    writelines_call_args = mock_file().writelines.call_args[0][0]
     # time should have been renamed to start_time and converted to human readable
-    mock_file().write.assert_any_call("start_time: 2024-10-04 14:43:43\n")
-    mock_file().write.assert_any_call(f"uid: {uid}\n")
+    assert "start_time: 2024-10-04 13:43:43\n" in writelines_call_args
+    assert f"uid: {uid}\n" in writelines_call_args
 
 
 def test_no_rb_number_folder(cb):
     time = 1728049423.5860472
     uid = "test123"
     scan_id = 1234
-    run_start = RunStart(time=time, uid=uid, scan_id=scan_id)
-    with patch("ibex_bluesky_core.callbacks.file_logger.open", mock_open()) as mock_file:
+    run_start = RunStart(time=time, uid=uid, scan_id=scan_id, detectors=["dae"], motors=("block",))
+
+    with (
+        patch("ibex_bluesky_core.callbacks.file_logger.open", mock_open()) as mock_file,
+        patch("ibex_bluesky_core.callbacks.file_logger.os.makedirs") as mock_mkdir,
+    ):
+        cb.start(run_start)
+        result = save_path / "Unknown RB" / f"{node()}_block_2024-10-04_14-43-43Z.txt"
+        assert mock_mkdir.called
+
+    mock_file.assert_called_with(result, "a", newline="\n", encoding="utf-8")
+    # time should have been renamed to start_time and converted to human readable
+    writelines_call_args = mock_file().writelines.call_args[0][0]
+    assert "start_time: 2024-10-04 13:43:43\n" in writelines_call_args
+    assert f"uid: {uid}\n" in writelines_call_args
+
+
+def test_no_motors_doesnt_append_to_filename(cb):
+    time = 1728049423.5860472
+    uid = "test123"
+    scan_id = 1234
+    run_start = RunStart(time=time, uid=uid, scan_id=scan_id, detectors=["dae"])
+
+    with (
+        patch("ibex_bluesky_core.callbacks.file_logger.open", mock_open()) as mock_file,
+        patch("ibex_bluesky_core.callbacks.file_logger.os.makedirs") as mock_mkdir,
+    ):
         cb.start(run_start)
-        result = save_path / "Unknown RB" / f"{node()}_block_dae_2024-10-04_14-43-43Z.txt"
+        result = save_path / "Unknown RB" / f"{node()}_2024-10-04_14-43-43Z.txt"
+        assert mock_mkdir.called
 
-    mock_file.assert_called_with(result, "a", newline="", encoding="utf-8")
+    mock_file.assert_called_with(result, "a", newline="\n", encoding="utf-8")
     # time should have been renamed to start_time and converted to human readable
-    mock_file().write.assert_any_call("start_time: 2024-10-04 14:43:43\n")
-    mock_file().write.assert_any_call(f"uid: {uid}\n")
+    writelines_call_args = mock_file().writelines.call_args[0][0]
+    assert "start_time: 2024-10-04 13:43:43\n" in writelines_call_args
+    assert f"uid: {uid}\n" in writelines_call_args
 
 
 def test_descriptor_data_does_nothing_if_doc_not_called_primary(cb):