Move tomography processing into Relion 5 format (#331)

Author: stephen-riggs

src/murfey/cli/inject_spa_processing.py

@@ -189,7 +189,7 @@ def run():
                     "autoproc_program_id": detached_ids[3],
                     "movie": m.path,
                     "mrc_out": str(mrc_out),
-                    "pix_size": proc_params["angpix"],
+                    "pixel_size": proc_params["angpix"],
                     "image_number": m.image_number,
                     "microscope": get_microscope(),
                     "mc_uuid": m.murfey_id,

src/murfey/cli/spa_ispyb_messages.py

@@ -413,7 +413,7 @@ def run():
                     "autoproc_program_id": collected_ids[3].id,
                     "movie": f.file_path,
                     "mrc_out": f.mrc_out,
-                    "pix_size": float(metadata["pixel_size_on_image"]) * 1e10,
+                    "pixel_size": float(metadata["pixel_size_on_image"]) * 1e10,
                     "image_number": f.image_number,
                     "microscope": get_microscope(),
                     "mc_uuid": murfey_ids[2 * i],

src/murfey/server/__init__.py

@@ -159,11 +159,15 @@ def _mc_path(mov_path: Path) -> str:
         core = Path(*Path(mov_path).parts[: visit_idx + 1])
         ppath = Path(mov_path)
         sub_dataset = "/".join(ppath.relative_to(core).parts[:-1])
+        extra_path = machine_config.processed_extra_directory
         mrc_out = (
             core
             / machine_config.processed_directory_name
             / sub_dataset
+            / extra_path
             / "MotionCorr"
+            / "job002"
+            / "Movies"
             / str(ppath.stem + "_motion_corrected.mrc")
         )
         return str(mrc_out)
@@ -691,7 +695,7 @@ def _register_picked_particles_use_diameter(
                         "micrographs_file": saved_message.micrographs_file,
                         "coord_list_file": saved_message.coord_list_file,
                         "output_file": saved_message.extract_file,
-                        "pix_size": (
+                        "pixel_size": (
                             relion_options["angpix"]
                             * relion_options["motion_corr_binning"]
                         ),
@@ -727,7 +731,7 @@ def _register_picked_particles_use_diameter(
                     "micrographs_file": params_to_forward["micrographs_file"],
                     "coord_list_file": params_to_forward["coord_list_file"],
                     "output_file": params_to_forward["extract_file"],
-                    "pix_size": (
+                    "pixel_size": (
                         relion_options["angpix"] * relion_options["motion_corr_binning"]
                     ),
                     "ctf_image": params_to_forward["ctf_values"]["CtfImage"],
@@ -843,7 +847,7 @@ def _register_picked_particles_use_boxsize(message: dict, _db=murfey_db):
             "micrographs_file": params_to_forward["micrographs_file"],
             "coord_list_file": params_to_forward["coord_list_file"],
             "output_file": params_to_forward["extract_file"],
-            "pix_size": relion_params.angpix * relion_params.motion_corr_binning,
+            "pixel_size": relion_params.angpix * relion_params.motion_corr_binning,
             "ctf_image": params_to_forward["ctf_values"]["CtfImage"],
             "ctf_max_resolution": params_to_forward["ctf_values"]["CtfMaxResolution"],
             "ctf_figure_of_merit": params_to_forward["ctf_values"]["CtfFigureOfMerit"],
@@ -1934,7 +1938,7 @@ def _flush_spa_preprocessing(message: dict):
                 "autoproc_program_id": collected_ids[3].id,
                 "movie": f.file_path,
                 "mrc_out": f.mrc_out,
-                "pix_size": proc_params.angpix,
+                "pixel_size": proc_params.angpix,
                 "image_number": f.image_number,
                 "microscope": get_microscope(),
                 "mc_uuid": murfey_ids[2 * i],
@@ -2038,7 +2042,7 @@ def _flush_tomography_preprocessing(message: dict):
                 "autoproc_program_id": detached_ids[3],
                 "movie": f.file_path,
                 "mrc_out": f.mrc_out,
-                "pix_size": proc_params.pixel_size,
+                "pixel_size": proc_params.pixel_size,
                 "kv": proc_params.voltage,
                 "image_number": f.image_number,
                 "microscope": get_microscope(),
@@ -2407,7 +2411,9 @@ def feedback_callback(header: dict, message: dict) -> None:
                 preproc_params = get_tomo_preproc_params(ids.dcgid)
                 stack_file = (
                     Path(message["mrc_out"]).parents[1]
-                    / "align_output"
+                    / "Tomograms"
+                    / "job006"
+                    / "tomograms"
                     / f"{relevant_tilt_series.tag}_stack.mrc"
                 )
                 if not stack_file.parent.exists():
@@ -2421,7 +2427,7 @@ def feedback_callback(header: dict, message: dict) -> None:
                         "dcid": ids.dcid,
                         "appid": ids.appid,
                         "stack_file": str(stack_file),
-                        "pix_size": preproc_params.pixel_size,
+                        "pixel_size": preproc_params.pixel_size,
                         "manual_tilt_offset": -tilt_offset,
                         "node_creator_queue": machine_config.node_creator_queue,
                     },

src/murfey/server/api/__init__.py

@@ -651,11 +651,15 @@ def register_completed_tilt_series(
                 "/".join(secure_filename(p) for p in Path(first_tilt.movie_path).parts)
             )
             sub_dataset = "/".join(ppath.relative_to(core).parts[:-1])
+            extra_path = machine_config.processed_extra_directory
             stack_file = (
                 core
                 / machine_config.processed_directory_name
                 / sub_dataset
-                / "align_output"
+                / extra_path
+                / "Tomograms"
+                / "job006"
+                / "tomograms"
                 / f"{ts.tag}_stack.mrc"
             )
             if not stack_file.parent.exists():
@@ -669,7 +673,7 @@ def register_completed_tilt_series(
                     "dcid": ids.dcid,
                     "appid": ids.appid,
                     "stack_file": str(stack_file),
-                    "pix_size": preproc_params.pixel_size,
+                    "pixel_size": preproc_params.pixel_size,
                     "manual_tilt_offset": -tilt_offset,
                     "node_creator_queue": machine_config.node_creator_queue,
                 },
@@ -1011,7 +1015,7 @@ async def request_spa_preprocessing(
                 "autoproc_program_id": detached_ids[3],
                 "movie": proc_file.path,
                 "mrc_out": str(mrc_out),
-                "pix_size": proc_params["angpix"],
+                "pixel_size": proc_params["angpix"],
                 "image_number": proc_file.image_number,
                 "microscope": get_microscope(),
                 "mc_uuid": murfey_ids[0],
@@ -1060,22 +1064,18 @@ async def request_tomography_preprocessing(
     core = Path(*Path(proc_file.path).parts[: visit_idx + 1])
     ppath = Path("/".join(secure_filename(p) for p in Path(proc_file.path).parts))
     sub_dataset = "/".join(ppath.relative_to(core).parts[:-1])
+    extra_path = machine_config.processed_extra_directory
     mrc_out = (
         core
         / machine_config.processed_directory_name
         / sub_dataset
+        / extra_path
         / "MotionCorr"
+        / "job002"
+        / "Movies"
         / str(ppath.stem + "_motion_corrected.mrc")
     )
     mrc_out = Path("/".join(secure_filename(p) for p in mrc_out.parts))
-    ctf_out = (
-        core
-        / machine_config.processed_directory_name
-        / sub_dataset
-        / "CTF"
-        / str(ppath.stem + "_ctf.mrc")
-    )
-    ctf_out = Path("/".join(secure_filename(p) for p in ctf_out.parts))
     session_id = (
         db.exec(
             select(ClientEnvironment).where(ClientEnvironment.client_id == client_id)
@@ -1105,8 +1105,6 @@ async def request_tomography_preprocessing(
         murfey_ids = _murfey_id(appid, db, number=1, close=False)
         if not mrc_out.parent.exists():
             mrc_out.parent.mkdir(parents=True, exist_ok=True)
-        if not ctf_out.parent.exists():
-            ctf_out.parent.mkdir(parents=True, exist_ok=True)
         zocalo_message = {
             "recipes": ["em-tomo-preprocess"],
             "parameters": {
@@ -1117,8 +1115,7 @@ async def request_tomography_preprocessing(
                 "autoproc_program_id": appid,
                 "movie": proc_file.path,
                 "mrc_out": str(mrc_out),
-                "pix_size": (proc_file.pixel_size) * 10**10,
-                "output_image": str(ctf_out),
+                "pixel_size": (proc_file.pixel_size) * 10**10,
                 "image_number": proc_file.image_number,
                 "kv": int(proc_file.voltage),
                 "microscope": get_microscope(),

src/murfey/server/demo_api.py

@@ -777,7 +777,7 @@ def flush_spa_processing(visit_name: str, client_id: int, tag: Tag, db=murfey_db
                 "autoproc_program_id": detached_ids[3],
                 "movie": f.file_path,
                 "mrc_out": f.mrc_out,
-                "pix_size": proc_params["angpix"],
+                "pixel_size": proc_params["angpix"],
                 "image_number": f.image_number,
                 "microscope": get_microscope(),
                 "mc_uuid": murfey_ids[2 * i],