Remove the manual tilt offset from the user parameters, and the table which contains it (#572)

We had a `TomographyProcessingParameters` table in the database which just contained the tilt offset. This removes that table and renames the `TomographyPreprocessingParameters` to just be `TomographyProcessingParameters`.

I've also removed `manual_tilt_offset` from the client side entirely, as it is calculated server-side. I expect that will mean it disappears from the TUI display.

The renaming in particular is a high-risk change, which needs a test build and run before we can say this works

Author: stephen-riggs

src/murfey/client/analyser.py

@@ -24,7 +24,7 @@
 from murfey.client.tui.forms import FormDependency
 from murfey.util.client import Observer, get_machine_config_client
 from murfey.util.mdoc import get_block
-from murfey.util.models import PreprocessingParametersTomo, ProcessingParametersSPA
+from murfey.util.models import ProcessingParametersSPA, ProcessingParametersTomo
 
 logger = logging.getLogger("murfey.client.analyser")
 
@@ -65,7 +65,7 @@ def __init__(
         self._environment = environment
         self._force_mdoc_metadata = force_mdoc_metadata
         self.parameters_model: (
-            Type[ProcessingParametersSPA] | Type[PreprocessingParametersTomo] | None
+            Type[ProcessingParametersSPA] | Type[ProcessingParametersTomo] | None
         ) = None
 
         self.queue: queue.Queue = queue.Queue()
@@ -183,7 +183,7 @@ def _find_context(self, file_path: Path) -> bool:
                 if not self._context:
                     logger.info("Acquisition software: tomo")
                     self._context = TomographyContext("tomo", self._basepath)
-                    self.parameters_model = PreprocessingParametersTomo
+                    self.parameters_model = ProcessingParametersTomo
                 return True
 
             # Files with these suffixes belong to the serial EM tomography workflow
@@ -207,7 +207,7 @@ def _find_context(self, file_path: Path) -> bool:
                 ):
                     return False
                 self._context = TomographyContext("serialem", self._basepath)
-                self.parameters_model = PreprocessingParametersTomo
+                self.parameters_model = ProcessingParametersTomo
                 return True
         return False
 

src/murfey/client/contexts/tomo.py

@@ -23,7 +23,6 @@
     get_machine_config_client,
 )
 from murfey.util.mdoc import get_block, get_global_data, get_num_blocks
-from murfey.util.tomo import midpoint
 
 logger = logging.getLogger("murfey.client.contexts.tomo")
 
@@ -72,7 +71,6 @@ class TomographyContext(Context):
         ProcessingParameter(
             "dose_per_frame", "Dose Per Frame (e- / Angstrom^2 / frame)", default=1
         ),
-        ProcessingParameter("manual_tilt_offset", "Tilt Offset", default=0),
         ProcessingParameter("gain_ref", "Gain Reference"),
         ProcessingParameter("eer_fractionation", "EER Fractionation", default=20),
     ]
@@ -570,7 +568,6 @@ def gather_metadata(
                         if environment
                         else None
                     )
-                    metadata["manual_tilt_offset"] = 0
                     metadata["source"] = str(self._basepath)
                 except KeyError:
                     return OrderedDict({})
@@ -631,9 +628,6 @@ def gather_metadata(
                 if environment
                 else None
             )
-            mdoc_metadata["manual_tilt_offset"] = -midpoint(
-                [float(b["TiltAngle"]) for b in blocks]
-            )
             mdoc_metadata["source"] = str(self._basepath)
             mdoc_metadata["tag"] = str(self._basepath)
             mdoc_metadata["tilt_series_tag"] = metadata_file.stem

src/murfey/client/multigrid_control.py

@@ -410,12 +410,6 @@ def _start_dc(self, json, from_form: bool = False):
         source = Path(json["source"])
         context = self.analysers[source]._context
         if isinstance(context, TomographyContext):
-            if from_form:
-                capture_post(
-                    f"{self._environment.url.geturl()}/clients/{self._environment.client_id}/tomography_processing_parameters",
-                    json=json,
-                )
-
             source = Path(json["source"])
             context.register_tomography_data_collections(
                 file_extension=json["file_extension"],
@@ -443,7 +437,7 @@ def _start_dc(self, json, from_form: bool = False):
                 eer_fractionation_file = eer_response.json()["eer_fractionation_file"]
                 json.update({"eer_fractionation_file": eer_fractionation_file})
             capture_post(
-                f"{self._environment.url.geturl()}/sessions/{self._environment.murfey_session}/tomography_preprocessing_parameters",
+                f"{self._environment.url.geturl()}/sessions/{self._environment.murfey_session}/tomography_processing_parameters",
                 json=json,
             )
             capture_post(

src/murfey/client/tui/app.py

@@ -504,7 +504,7 @@ def _start_dc(self, json, from_form: bool = False):
                 eer_fractionation_file = eer_response.json()["eer_fractionation_file"]
                 json.update({"eer_fractionation_file": eer_fractionation_file})
             requests.post(
-                f"{self.app._environment.url.geturl()}/sessions/{self.app._environment.murfey_session}/tomography_preprocessing_parameters",
+                f"{self.app._environment.url.geturl()}/sessions/{self.app._environment.murfey_session}/tomography_processing_parameters",
                 json=json,
             )
             capture_post(

src/murfey/client/tui/screens.py

@@ -58,7 +58,7 @@
 from murfey.client.tui.forms import FormDependency
 from murfey.util import posix_path
 from murfey.util.client import capture_post, get_machine_config_client, read_config
-from murfey.util.models import PreprocessingParametersTomo, ProcessingParametersSPA
+from murfey.util.models import ProcessingParametersSPA, ProcessingParametersTomo
 
 log = logging.getLogger("murfey.tui.screens")
 
@@ -464,7 +464,7 @@ def compose(self):
     def _write_params(
         self,
         params: dict | None = None,
-        model: PreprocessingParametersTomo | ProcessingParametersSPA | None = None,
+        model: ProcessingParametersTomo | ProcessingParametersSPA | None = None,
     ):
         if params:
             try:
@@ -474,9 +474,9 @@ def _write_params(
             for k in analyser._context.user_params + analyser._context.metadata_params:
                 self.app.query_one("#info").write(f"{k.label}: {params.get(k.name)}")
             self.app._start_dc(params)
-            if model == PreprocessingParametersTomo:
+            if model == ProcessingParametersTomo:
                 requests.post(
-                    f"{self.app._environment.url.geturl()}/sessions/{self.app._environment.murfey_session}/tomography_preprocessing_parameters",
+                    f"{self.app._environment.url.geturl()}/sessions/{self.app._environment.murfey_session}/tomography_processing_parameters",
                     json=params,
                 )
             elif model == ProcessingParametersSPA:
@@ -1157,9 +1157,7 @@ def on_input_changed(self, event):
     def on_button_pressed(self, event):
         if self.app._multigrid and self.app._processing_enabled:
             if self._context == TomographyContext:
-                valid = validate_form(
-                    self._user_params, PreprocessingParametersTomo.Base
-                )
+                valid = validate_form(self._user_params, ProcessingParametersTomo.Base)
             else:
                 valid = validate_form(self._user_params, ProcessingParametersSPA.Base)
             if not valid:

src/murfey/server/__init__.py

@@ -189,19 +189,10 @@ def get_job_ids(tilt_series_id: int, appid: int) -> JobIDs:
     )
 
 
-def get_tomo_proc_params(pj_id: int, *args) -> db.TomographyProcessingParameters:
+def get_tomo_proc_params(dcg_id: int, *args) -> db.TomographyProcessingParameters:
     results = murfey_db.exec(
         select(db.TomographyProcessingParameters).where(
-            db.TomographyProcessingParameters.pj_id == pj_id
-        )
-    ).one()
-    return results
-
-
-def get_tomo_preproc_params(dcg_id: int, *args) -> db.TomographyPreprocessingParameters:
-    results = murfey_db.exec(
-        select(db.TomographyPreprocessingParameters).where(
-            db.TomographyPreprocessingParameters.dcg_id == dcg_id
+            db.TomographyProcessingParameters.dcg_id == dcg_id
         )
     ).one()
     return results
@@ -1660,7 +1651,7 @@ def _flush_tomography_preprocessing(message: dict):
         .where(db.DataCollectionGroup.session_id == session_id)
         .where(db.DataCollectionGroup.tag == message["data_collection_group_tag"])
     ).first()
-    proc_params = get_tomo_preproc_params(collected_ids.id)
+    proc_params = get_tomo_proc_params(collected_ids.id)
     if not proc_params:
         visit_name = message["visit_name"].replace("\r\n", "").replace("\n", "")
         logger.warning(
@@ -2160,7 +2151,7 @@ def feedback_callback(header: dict, message: dict) -> None:
                 ]
                 tilts = get_all_tilts(relevant_tilt_series.id)
                 ids = get_job_ids(relevant_tilt_series.id, alignment_ids[2].id)
-                preproc_params = get_tomo_preproc_params(ids.dcgid)
+                preproc_params = get_tomo_proc_params(ids.dcgid)
                 stack_file = (
                     Path(message["mrc_out"]).parents[3]
                     / "Tomograms"
@@ -2534,11 +2525,11 @@ def feedback_callback(header: dict, message: dict) -> None:
                 .where(db.ProcessingJob.recipe == "em-tomo-preprocess")
             ).one()
             if not murfey_db.exec(
-                select(func.count(db.TomographyPreprocessingParameters.dcg_id)).where(
-                    db.TomographyPreprocessingParameters.dcg_id == collected_ids[0].id
+                select(func.count(db.TomographyProcessingParameters.dcg_id)).where(
+                    db.TomographyProcessingParameters.dcg_id == collected_ids[0].id
                 )
             ).one():
-                params = db.TomographyPreprocessingParameters(
+                params = db.TomographyProcessingParameters(
                     dcg_id=collected_ids[0].id,
                     pixel_size=float(message["pixel_size_on_image"]) * 10**10,
                     voltage=message["voltage"],

src/murfey/server/api/__init__.py

@@ -42,7 +42,7 @@
     get_job_ids,
     get_machine_config,
     get_microscope,
-    get_tomo_preproc_params,
+    get_tomo_proc_params,
     sanitise,
     templates,
 )
@@ -70,7 +70,6 @@
     SPARelionParameters,
     Tilt,
     TiltSeries,
-    TomographyProcessingParameters,
 )
 from murfey.util.models import (
     BLSampleImageParameters,
@@ -86,7 +85,6 @@
     GridSquareParameters,
     MillingParameters,
     PostInfo,
-    PreprocessingParametersTomo,
     ProcessingJobParameters,
     ProcessingParametersSPA,
     ProcessingParametersTomo,
@@ -618,9 +616,9 @@ def post_foil_hole(
     return register_foil_hole(session_id, gs_name, foil_hole_params, db)
 
 
-@router.post("/sessions/{session_id}/tomography_preprocessing_parameters")
-def register_tomo_preproc_params(
-    session_id: MurfeySessionID, proc_params: PreprocessingParametersTomo, db=murfey_db
+@router.post("/sessions/{session_id}/tomography_processing_parameters")
+def register_tomo_proc_params(
+    session_id: MurfeySessionID, proc_params: ProcessingParametersTomo, db=murfey_db
 ):
     session_processing_parameters = db.exec(
         select(SessionProcessingParameters).where(
@@ -643,41 +641,6 @@ def register_tomo_preproc_params(
         _transport_object.send(_transport_object.feedback_queue, zocalo_message)
 
 
-@router.post("/sessions/{session_id}/tomography_processing_parameters")
-def register_tomo_proc_params(
-    session_id: MurfeySessionID, proc_params: ProcessingParametersTomo, db=murfey_db
-):
-    log.info(
-        f"Registering tomography processing parameters {sanitise(proc_params.tag)}, {sanitise(proc_params.tilt_series_tag)}"
-    )
-    collected_ids = db.exec(
-        select(
-            DataCollectionGroup,
-            DataCollection,
-            ProcessingJob,
-            AutoProcProgram,
-        )
-        .where(DataCollectionGroup.session_id == session_id)
-        .where(DataCollectionGroup.tag == proc_params.tag)
-        .where(DataCollection.tag == proc_params.tilt_series_tag)
-        .where(DataCollection.dcg_id == DataCollectionGroup.id)
-        .where(ProcessingJob.dc_id == DataCollection.id)
-        .where(AutoProcProgram.pj_id == ProcessingJob.id)
-        .where(ProcessingJob.recipe == "em-tomo-preprocess")
-    ).one()
-    if not db.exec(
-        select(func.count(TomographyProcessingParameters.pj_id)).where(
-            TomographyProcessingParameters.pj_id == collected_ids[2].id
-        )
-    ).one():
-        tomogram_params = TomographyProcessingParameters(
-            pj_id=collected_ids[2].id, manual_tilt_offset=proc_params.manual_tilt_offset
-        )
-        db.add(tomogram_params)
-    db.commit()
-    db.close()
-
-
 class Tag(BaseModel):
     tag: str
 
@@ -804,7 +767,7 @@ def register_completed_tilt_series(
             ]
             tilts = get_all_tilts(ts.id)
             ids = get_job_ids(ts.id, collected_ids[3].id)
-            preproc_params = get_tomo_preproc_params(ids.dcgid)
+            preproc_params = get_tomo_proc_params(ids.dcgid)
 
             first_tilt = db.exec(
                 select(Tilt).where(Tilt.tilt_series_id == ts.id)

src/murfey/server/demo_api.py

@@ -57,7 +57,6 @@
     SPARelionParameters,
     Tilt,
     TiltSeries,
-    TomographyPreprocessingParameters,
     TomographyProcessingParameters,
 )
 from murfey.util.models import (
@@ -70,7 +69,6 @@
     GainReference,
     GridSquareParameters,
     PostInfo,
-    PreprocessingParametersTomo,
     ProcessingJobParameters,
     ProcessingParametersSPA,
     ProcessingParametersTomo,
@@ -476,9 +474,9 @@ def register_spa_proc_params(
     db.commit()
 
 
-@router.post("/sessions/{session_id}/tomography_preprocessing_parameters")
-def register_tomo_preproc_params(
-    session_id: MurfeySessionID, proc_params: PreprocessingParametersTomo, db=murfey_db
+@router.post("/sessions/{session_id}/tomography_processing_parameters")
+def register_tomo_proc_params(
+    session_id: MurfeySessionID, proc_params: ProcessingParametersTomo, db=murfey_db
 ):
     log.info(
         f"Registering tomography preprocessing parameters {sanitise(proc_params.tag)}, {sanitise(proc_params.tilt_series_tag)}"
@@ -499,68 +497,19 @@ def register_tomo_preproc_params(
         .where(ProcessingJob.recipe == "em-tomo-preprocess")
     ).one()
     if not db.exec(
-        select(func.count(TomographyPreprocessingParameters.dcg_id)).where(
-            TomographyPreprocessingParameters.dcg_id == collected_ids[0].id
+        select(func.count(TomographyProcessingParameters.dcg_id)).where(
+            TomographyProcessingParameters.dcg_id == collected_ids[0].id
         )
     ).one():
-        params = TomographyPreprocessingParameters(
+        params = TomographyProcessingParameters(
             dcg_id=collected_ids[0].id,
             pixel_size=proc_params.pixel_size_on_image,
             dose_per_frame=proc_params.dose_per_frame,
             gain_ref=proc_params.gain_ref,
             motion_corr_binning=proc_params.motion_corr_binning,
             voltage=proc_params.voltage,
-            # manual_tilt_offset=proc_params.manual_tilt_offset,
         )
         db.add(params)
-    if not db.exec(
-        select(func.count(TomographyProcessingParameters.pj_id)).where(
-            TomographyProcessingParameters.pj_id == collected_ids[2].id
-        )
-    ).one():
-        tomogram_params = TomographyProcessingParameters(
-            pj_id=collected_ids[2].id, manual_tilt_offset=proc_params.manual_tilt_offset
-        )
-        db.add(tomogram_params)
-    db.commit()
-    db.close()
-
-
-@router.post("/clients/{client_id}/tomography_processing_parameters")
-def register_tomo_proc_params(
-    client_id: int, proc_params: ProcessingParametersTomo, db=murfey_db
-):
-    client = db.exec(
-        select(ClientEnvironment).where(ClientEnvironment.client_id == client_id)
-    ).one()
-    session_id = client.session_id
-    log.info(
-        f"Registering tomography processing parameters {sanitise(proc_params.tag)}, {sanitise(proc_params.tilt_series_tag)}, {session_id}"
-    )
-    collected_ids = db.exec(
-        select(
-            DataCollectionGroup,
-            DataCollection,
-            ProcessingJob,
-            AutoProcProgram,
-        )
-        .where(DataCollectionGroup.session_id == session_id)
-        .where(DataCollectionGroup.tag == proc_params.tag)
-        .where(DataCollection.tag == proc_params.tilt_series_tag)
-        .where(DataCollection.dcg_id == DataCollectionGroup.id)
-        .where(ProcessingJob.dc_id == DataCollection.id)
-        .where(AutoProcProgram.pj_id == ProcessingJob.id)
-        .where(ProcessingJob.recipe == "em-tomo-preprocess")
-    ).one()
-    if not db.exec(
-        select(func.count(TomographyProcessingParameters.pj_id)).where(
-            TomographyProcessingParameters.pj_id == collected_ids[2].id
-        )
-    ).one():
-        tomogram_params = TomographyProcessingParameters(
-            pj_id=collected_ids[2].id, manual_tilt_offset=proc_params.manual_tilt_offset
-        )
-        db.add(tomogram_params)
     db.commit()
     db.close()