Resolved merge conflicts with 'main' branch

tieneupin · tieneupin · commit 446f6c0fb8ef · 2025-07-17T18:27:22.000+01:00
diff --git a/.bumpclient.toml b/.bumpclient.toml
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.19.2"
+current_version = "0.19.4"
 commit = true
 tag = false
 
diff --git a/.bumpversion.toml b/.bumpversion.toml
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.19.2"
+current_version = "0.19.4"
 commit = true
 tag = true
 
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -3,7 +3,7 @@ name: Build and test
 on: [push, pull_request]
 
 env:
-  ISPYB_DATABASE_SCHEMA: 4.6.0
+  ISPYB_DATABASE_SCHEMA: 4.8.0
   # Installs from GitHub
   # Versions: https://github.com/DiamondLightSource/ispyb-database/tags
   # Previous version(s):
diff --git a/Helm/Chart.yaml b/Helm/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v1
 name: murfey-services
 description: Umbrella Helm chart for deploying the servers and daemons needed to enable Murfey to transfer and process data
-version: 0.19.2
+version: 0.19.4
 dependencies:
   - name: murfey-instrument-server-clem
   - name: murfey-instrument-server-tem
diff --git a/Helm/charts/murfey-instrument-server-clem/Chart.yaml b/Helm/charts/murfey-instrument-server-clem/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v1
 name: murfey-instrument-server-clem
 description: Helm chart for deploying a Murfey instrument server, which executes orders to detect, modify, and transfer files on the instrument PC, and notifies the backend server about transferred files
-version: 0.19.2
+version: 0.19.4
diff --git a/Helm/charts/murfey-instrument-server-tem/Chart.yaml b/Helm/charts/murfey-instrument-server-tem/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v1
 name: murfey-instrument-server-tem
 description: Helm chart for deploying a Murfey instrument server, which executes orders to detect, modify, and transfer files on the instrument PC, and notifies the backend server about transferred files
-version: 0.19.2
+version: 0.19.4
diff --git a/Helm/charts/murfey-rsync/Chart.yaml b/Helm/charts/murfey-rsync/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v1
 name: murfey-rsync
 description: Helm chart for deploying an rsync daemon, which is responsible for executing the transfer of files from the client storage directory to the server storage system
-version: 0.19.2
+version: 0.19.4
diff --git a/Helm/charts/murfey-server/Chart.yaml b/Helm/charts/murfey-server/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v1
 name: murfey-server
 description: Helm chart for deploying a Murfey backend server, which is responsible for orchestrating the data transfer and processing workflow between the client PC and the storage system
-version: 0.19.2
+version: 0.19.4
diff --git a/pyproject.toml b/pyproject.toml
@@ -7,7 +7,7 @@ requires = [
 
 [project]
 name = "murfey"
-version = "0.19.2"
+version = "0.19.4"
 description = "Client-Server architecture hauling Cryo-EM data"
 readme = "README.md"
 keywords = [
diff --git a/src/murfey/__init__.py b/src/murfey/__init__.py
@@ -1,4 +1,4 @@
 from __future__ import annotations
 
-__version__ = "0.19.2"
-__supported_client_version__ = "0.19.2"
+__version__ = "0.19.4"
+__supported_client_version__ = "0.19.4"
diff --git a/src/murfey/client/contexts/tomo_metadata.py b/src/murfey/client/contexts/tomo_metadata.py
@@ -25,8 +25,8 @@ def ensure_dcg_exists(transferred_file: Path, environment: MurfeyInstanceEnviron
     dcg_tag = str(source).replace(f"/{environment.visit}", "")
     url = f"{str(environment.url.geturl())}{url_path_for('workflow.router', 'register_dc_group', visit_name=environment.visit, session_id=environment.murfey_session)}"
     dcg_data = {
-        "experiment_type": "single particle",
-        "experiment_type_id": 37,
+        "experiment_type": "tomo",
+        "experiment_type_id": 36,
         "tag": dcg_tag,
     }
     capture_post(url, json=dcg_data)
@@ -57,6 +57,9 @@ def post_transfer(
 
             windows_path = session_data["TomographySession"]["AtlasId"]
             logger.info(f"Windows path to atlas metadata found: {windows_path}")
+            if not windows_path:
+                logger.warning("No atlas metadata path found")
+                return
             visit_index = windows_path.split("\\").index(environment.visit)
             partial_path = "/".join(windows_path.split("\\")[visit_index + 1 :])
             logger.info("Partial Linux path successfully constructed from Windows path")
@@ -257,9 +260,12 @@ def post_transfer(
                 for_parsing = xml.read()
             batch_xml = xmltodict.parse(for_parsing)
 
-            batch_positions_list = batch_xml["BatchPositionsList"]["BatchPositions"][
-                "BatchPositionParameters"
-            ]
+            batch_positions_from_xml = batch_xml["BatchPositionsList"]["BatchPositions"]
+            if not batch_positions_from_xml:
+                logger.info("No batch positions yet")
+                return
+
+            batch_positions_list = batch_positions_from_xml["BatchPositionParameters"]
             if isinstance(batch_positions_list, dict):
                 # Case of a single batch
                 batch_positions_list = [batch_positions_list]
diff --git a/src/murfey/instrument_server/api.py b/src/murfey/instrument_server/api.py
@@ -215,6 +215,15 @@ def stop_multigrid_watcher(session_id: MurfeySessionID, label: str):
     return {"success": True}
 
 
+@router.get("/sessions/{session_id}/multigrid_controller/status")
+def check_multigrid_controller_exists(
+    session_id: MurfeySessionID,
+):
+    if controllers.get(session_id, None) is not None:
+        return {"exists": True}
+    return {"exists": False}
+
+
 @router.post("/sessions/{session_id}/multigrid_controller/visit_end_time")
 def update_multigrid_controller_visit_end_time(
     session_id: MurfeySessionID, end_time: datetime
diff --git a/src/murfey/server/api/instrument.py b/src/murfey/server/api/instrument.py
@@ -4,7 +4,7 @@
 import datetime
 import logging
 from pathlib import Path
-from typing import Annotated, List, Optional
+from typing import Annotated, Any, List, Optional
 from urllib.parse import quote
 
 import aiohttp
@@ -101,6 +101,31 @@ async def check_if_session_is_active(
                 return {"active": response.status == 200}
 
 
+@router.get("/sessions/{session_id}/multigrid_controller/status")
+async def check_multigrid_controller_exists(session_id: MurfeySessionID, db=murfey_db):
+    session = db.exec(select(Session).where(Session.id == session_id)).one()
+    instrument_name = session.instrument_name
+    machine_config = get_machine_config(instrument_name=instrument_name)[
+        instrument_name
+    ]
+    if machine_config.instrument_server_url:
+        log.debug(
+            f"Submitting request to inspect multigrid controller for session {session_id}"
+        )
+        async with aiohttp.ClientSession() as clientsession:
+            async with clientsession.get(
+                f"{machine_config.instrument_server_url}{url_path_for('api.router', 'check_multigrid_controller_exists', session_id=session_id)}",
+                headers={
+                    "Authorization": f"Bearer {instrument_server_tokens[session_id]['access_token']}"
+                },
+            ) as resp:
+                data: dict[str, Any] = await resp.json()
+    else:
+        data = {"detail": "No instrument server URL found"}
+    log.debug(f"Received response: {data}")
+    return data
+
+
 @router.post("/sessions/{session_id}/multigrid_watcher")
 async def setup_multigrid_watcher(
     session_id: MurfeySessionID, watcher_spec: MultigridWatcherSetup, db=murfey_db
@@ -165,6 +190,36 @@ async def start_multigrid_watcher(session_id: MurfeySessionID, db=murfey_db):
     return data
 
 
+@router.post("/sessions/{session_id}/multigrid_controller/visit_end_time")
+async def update_visit_end_time(
+    session_id: MurfeySessionID, end_time: datetime.datetime, db=murfey_db
+):
+    # Load data for session
+    session_entry = db.exec(select(Session).where(Session.id == session_id)).one()
+    instrument_name = session_entry.instrument_name
+
+    # Update visit end time in database
+    session_entry.visit_end_time = end_time
+    db.add(session_entry)
+    db.commit()
+
+    # Update the multigrid controller
+    data = {}
+    machine_config = get_machine_config(instrument_name=instrument_name)[
+        instrument_name
+    ]
+    if machine_config.instrument_server_url:
+        async with aiohttp.ClientSession() as clientsession:
+            async with clientsession.post(
+                f"{machine_config.instrument_server_url}{url_path_for('api.router', 'update_multigrid_controller_visit_end_time', session_id=session_id)}?end_time={quote(end_time.isoformat())}",
+                headers={
+                    "Authorization": f"Bearer {instrument_server_tokens[session_id]['access_token']}"
+                },
+            ) as resp:
+                data = await resp.json()
+    return data
+
+
 class ProvidedProcessingParameters(BaseModel):
     dose_per_frame: float
     extract_downscale: bool = True
@@ -397,36 +452,6 @@ async def finalise_session(session_id: MurfeySessionID, db=murfey_db):
     return data
 
 
-@router.post("/sessions/{session_id}/multigrid_controller/visit_end_time")
-async def update_visit_end_time(
-    session_id: MurfeySessionID, end_time: datetime.datetime, db=murfey_db
-):
-    # Load data for session
-    session_entry = db.exec(select(Session).where(Session.id == session_id)).one()
-    instrument_name = session_entry.instrument_name
-
-    # Update visit end time in database
-    session_entry.visit_end_time = end_time
-    db.add(session_entry)
-    db.commit()
-
-    # Update the multigrid controller
-    data = {}
-    machine_config = get_machine_config(instrument_name=instrument_name)[
-        instrument_name
-    ]
-    if machine_config.instrument_server_url:
-        async with aiohttp.ClientSession() as clientsession:
-            async with clientsession.post(
-                f"{machine_config.instrument_server_url}{url_path_for('api.router', 'update_multigrid_controller_visit_end_time', session_id=session_id)}?end_time={quote(end_time.isoformat())}",
-                headers={
-                    "Authorization": f"Bearer {instrument_server_tokens[session_id]['access_token']}"
-                },
-            ) as resp:
-                data = await resp.json()
-    return data
-
-
 @router.post("/sessions/{session_id}/abandon_session")
 async def abandon_session(session_id: MurfeySessionID, db=murfey_db):
     data = {}
diff --git a/src/murfey/server/feedback.py b/src/murfey/server/feedback.py
@@ -1982,6 +1982,7 @@ def feedback_callback(header: dict, message: dict) -> None:
                         session_id=message["session_id"],
                         tag=message.get("tag"),
                     )
+                    dcgid = murfey_dcg.id
                 else:
                     record = DataCollectionGroup(
                         sessionId=ispyb_session_id,
@@ -1999,9 +2000,14 @@ def feedback_callback(header: dict, message: dict) -> None:
                         atlas_id = murfey.server._transport_object.do_insert_atlas(
                             atlas_record
                         )["return_value"]
+                    else:
+                        atlas_id = None
                     murfey_dcg = db.DataCollectionGroup(
                         id=dcgid,
                         atlas_id=atlas_id,
+                        atlas=message.get("atlas", ""),
+                        atlas_pixel_size=message.get("atlas_pixel_size"),
+                        sample=message.get("sample"),
                         session_id=message["session_id"],
                         tag=message.get("tag"),
                     )
diff --git a/src/murfey/server/ispyb.py b/src/murfey/server/ispyb.py
@@ -405,13 +405,34 @@ def do_insert_search_map(
             search_map_parameters.pixel_size *= (
                 search_map_parameters.height / search_map_parameters.height_on_atlas
             )
+        search_map_parameters.height = (
+            int(search_map_parameters.height / 7.8)
+            if search_map_parameters.height
+            else None
+        )
+        search_map_parameters.width = (
+            int(search_map_parameters.width / 7.8)
+            if search_map_parameters.width
+            else None
+        )
+        search_map_parameters.x_location = (
+            int(search_map_parameters.x_location / 7.8)
+            if search_map_parameters.x_location
+            else None
+        )
+        search_map_parameters.y_location = (
+            int(search_map_parameters.y_location / 7.8)
+            if search_map_parameters.y_location
+            else None
+        )
         record = GridSquare(
             atlasId=atlas_id,
             gridSquareImage=search_map_parameters.image,
             pixelLocationX=search_map_parameters.x_location,
             pixelLocationY=search_map_parameters.y_location,
             height=search_map_parameters.height_on_atlas,
             width=search_map_parameters.width_on_atlas,
+            angle=0,
             stageLocationX=search_map_parameters.x_stage_position,
             stageLocationY=search_map_parameters.y_stage_position,
             pixelSize=search_map_parameters.pixel_size,
@@ -452,13 +473,19 @@ def do_update_search_map(
                 if search_map_parameters.image:
                     grid_square.gridSquareImage = search_map_parameters.image
                 if search_map_parameters.x_location:
-                    grid_square.pixelLocationX = search_map_parameters.x_location
+                    grid_square.pixelLocationX = int(
+                        search_map_parameters.x_location / 7.8
+                    )
                 if search_map_parameters.y_location:
-                    grid_square.pixelLocationY = search_map_parameters.y_location
+                    grid_square.pixelLocationY = int(
+                        search_map_parameters.y_location / 7.8
+                    )
                 if search_map_parameters.height_on_atlas:
-                    grid_square.height = search_map_parameters.height_on_atlas
+                    grid_square.height = int(
+                        search_map_parameters.height_on_atlas / 7.8
+                    )
                 if search_map_parameters.width_on_atlas:
-                    grid_square.width = search_map_parameters.width_on_atlas
+                    grid_square.width = int(search_map_parameters.width_on_atlas / 7.8)
                 if search_map_parameters.x_stage_position:
                     grid_square.stageLocationX = search_map_parameters.x_stage_position
                 if search_map_parameters.y_stage_position:
diff --git a/src/murfey/util/route_manifest.yaml b/src/murfey/util/route_manifest.yaml
@@ -43,6 +43,11 @@ murfey.instrument_server.api.router:
     path_params: []
     methods:
       - POST
+  - path: /sessions/{session_id}/multigrid_controller/status
+    function: check_multigrid_controller_exists
+    path_params: []
+    methods:
+      - GET
   - path: /sessions/{session_id}/stop_rsyncer
     function: stop_rsyncer
     path_params: []
@@ -503,6 +508,11 @@ murfey.server.api.instrument.router:
     path_params: []
     methods:
       - POST
+  - path: /instrument_server/sessions/{session_id}/multigrid_controller/status
+    function: check_multigrid_controller_exists
+    path_params: []
+    methods:
+      - GET
   - path: /instrument_server/sessions/{session_id}/provided_processing_parameters
     function: pass_proc_params_to_instrument_server
     path_params: []
diff --git a/src/murfey/workflows/tomo/tomo_metadata.py b/src/murfey/workflows/tomo/tomo_metadata.py
@@ -339,10 +339,20 @@ def register_batch_position_in_database(
             search_map.height / 2,
         ]
         tilt_series.x_location = (
-            centre_batch_pixel[0] - batch_parameters.x_beamshift / search_map.pixel_size
+            (
+                centre_batch_pixel[0]
+                - batch_parameters.x_beamshift / search_map.pixel_size
+            )
+            * 512
+            / search_map.width
         )
         tilt_series.y_location = (
-            centre_batch_pixel[1] - batch_parameters.y_beamshift / search_map.pixel_size
+            (
+                centre_batch_pixel[1]
+                - batch_parameters.y_beamshift / search_map.pixel_size
+            )
+            * 512
+            / search_map.width
         )
     else:
         logger.warning(
diff --git a/tests/instrument_server/test_api.py b/tests/instrument_server/test_api.py
diff --git a/tests/server/api/test_instrument.py b/tests/server/api/test_instrument.py
diff --git a/tests/workflows/tomo/test_tomo_metadata.py b/tests/workflows/tomo/test_tomo_metadata.py
