Cover the case where the .dm file is seen before the grid square XML so that the foil hole positions can't be determined until the XML is seen, at which point there needs to be an update

Author: d-j-hatton

src/murfey/client/contexts/spa.py

@@ -456,10 +456,7 @@ def _position_analysis(
         foil_hole = foil_hole_from_file(transferred_file)
         if foil_hole not in self._foil_holes[grid_square]:
             fh_url = f"{str(environment.url.geturl())}/sessions/{environment.murfey_session}/grid_square/{grid_square}/foil_hole"
-            if (
-                grid_square_metadata_file.is_file()
-                and environment.murfey_session is not None
-            ):
+            if environment.murfey_session is not None:
                 fh = foil_hole_data(
                     grid_square_metadata_file,
                     foil_hole,

src/murfey/server/ispyb.py

@@ -253,7 +253,7 @@ def do_update_grid_square(
     def do_insert_foil_hole(
         self,
         grid_square_id: int,
-        scale_factor: float,
+        scale_factor: Optional[float],
         foil_hole_parameters: FoilHoleParameters,
     ):
         if (
@@ -265,21 +265,22 @@ def do_insert_foil_hole(
                 foil_hole_parameters.readout_area_x
                 / foil_hole_parameters.thumbnail_size_x
             )
-        foil_hole_parameters.diameter = (
-            int(foil_hole_parameters.diameter * scale_factor)
-            if foil_hole_parameters.diameter
-            else None
-        )
-        foil_hole_parameters.x_location = (
-            int(foil_hole_parameters.x_location * scale_factor)
-            if foil_hole_parameters.x_location
-            else None
-        )
-        foil_hole_parameters.y_location = (
-            int(foil_hole_parameters.y_location * scale_factor)
-            if foil_hole_parameters.y_location
-            else None
-        )
+        if scale_factor:
+            foil_hole_parameters.diameter = (
+                int(foil_hole_parameters.diameter * scale_factor)
+                if foil_hole_parameters.diameter
+                else None
+            )
+            foil_hole_parameters.x_location = (
+                int(foil_hole_parameters.x_location * scale_factor)
+                if foil_hole_parameters.x_location
+                else None
+            )
+            foil_hole_parameters.y_location = (
+                int(foil_hole_parameters.y_location * scale_factor)
+                if foil_hole_parameters.y_location
+                else None
+            )
         record = FoilHole(
             gridSquareId=grid_square_id,
             foilHoleLabel=foil_hole_parameters.name,

src/murfey/util/spa_metadata.py

@@ -148,36 +148,38 @@ def grid_square_data(xml_path: Path, grid_square: int) -> GridSquareInfo:
 
 
 def foil_hole_data(xml_path: Path, foil_hole: int, grid_square: int) -> FoilHoleInfo:
-    with open(xml_path, "r") as xml:
-        for_parsing = xml.read()
-        data = xmltodict.parse(for_parsing)
-    data = data["GridSquareXml"]
-    serialization_array = data["TargetLocations"]["TargetLocationsEfficient"][
-        "a:m_serializationArray"
-    ]
     required_key = ""
-    for key in serialization_array.keys():
-        if key.startswith("b:KeyValuePairOfintTargetLocation"):
-            required_key = key
-            break
-    if required_key:
-        image_paths = list(
-            (xml_path.parent.parent).glob(
-                f"Images-Disc*/GridSquare_{grid_square}/FoilHoles/FoilHole_{foil_hole}_*.jpg"
-            )
+    if xml_path.is_file():
+        with open(xml_path, "r") as xml:
+            for_parsing = xml.read()
+            data = xmltodict.parse(for_parsing)
+        data = data["GridSquareXml"]
+        serialization_array = data["TargetLocations"]["TargetLocationsEfficient"][
+            "a:m_serializationArray"
+        ]
+        required_key = ""
+        for key in serialization_array.keys():
+            if key.startswith("b:KeyValuePairOfintTargetLocation"):
+                required_key = key
+                break
+    image_paths = list(
+        (xml_path.parent.parent).glob(
+            f"Images-Disc*/GridSquare_{grid_square}/FoilHoles/FoilHole_{foil_hole}_*.jpg"
         )
-        image_paths.sort(key=lambda x: x.stat().st_ctime)
-        image_path: Union[Path, str] = image_paths[-1] if image_paths else ""
-        if image_path:
-            with open(Path(image_path).with_suffix(".xml")) as fh_xml:
-                fh_xml_data = xmltodict.parse(fh_xml.read())
-            readout_area = fh_xml_data["MicroscopeImage"]["microscopeData"][
-                "acquisition"
-            ]["camera"]["ReadoutArea"]
-            pixel_size = fh_xml_data["MicroscopeImage"]["SpatialScale"]["pixelSize"][
-                "x"
-            ]["numericValue"]
-            full_size = (int(readout_area["a:width"]), int(readout_area["a:height"]))
+    )
+    image_paths.sort(key=lambda x: x.stat().st_ctime)
+    image_path: Union[Path, str] = image_paths[-1] if image_paths else ""
+    if image_path:
+        with open(Path(image_path).with_suffix(".xml")) as fh_xml:
+            fh_xml_data = xmltodict.parse(fh_xml.read())
+        readout_area = fh_xml_data["MicroscopeImage"]["microscopeData"]["acquisition"][
+            "camera"
+        ]["ReadoutArea"]
+        pixel_size = fh_xml_data["MicroscopeImage"]["SpatialScale"]["pixelSize"]["x"][
+            "numericValue"
+        ]
+        full_size = (int(readout_area["a:width"]), int(readout_area["a:height"]))
+    if required_key:
         for fh_block in serialization_array[required_key]:
             pix = fh_block["b:value"]["PixelCenter"]
             stage = fh_block["b:value"]["StagePosition"]
@@ -198,6 +200,17 @@ def foil_hole_data(xml_path: Path, foil_hole: int, grid_square: int) -> FoilHole
                     image=str(image_path),
                     diameter=diameter,
                 )
+    elif image_path:
+        return FoilHoleInfo(
+            id=foil_hole,
+            grid_square_id=grid_square,
+            readout_area_x=full_size[0] if image_path else None,
+            readout_area_y=full_size[1] if image_path else None,
+            thumbnail_size_x=None,
+            thumbnail_size_y=None,
+            pixel_size=float(pixel_size) if image_path else None,
+            image=str(image_path),
+        )
     logger.warning(
         f"Foil hole positions could not be determined from metadata file {xml_path} for foil hole {foil_hole}"
     )

src/murfey/workflows/spa/flush_spa_preprocess.py

@@ -122,23 +122,37 @@ def register_foil_hole(
             .where(FoilHole.grid_square_id == gsid)
             .where(FoilHole.session_id == session_id)
         ).one()
-        foil_hole.x_location = foil_hole_params.x_location
-        foil_hole.y_location = foil_hole_params.y_location
-        foil_hole.x_stage_position = foil_hole_params.x_stage_position
-        foil_hole.y_stage_position = foil_hole_params.y_stage_position
-        foil_hole.readout_area_x = foil_hole_params.readout_area_x
-        foil_hole.readout_area_y = foil_hole_params.readout_area_y
-        foil_hole.thumbnail_size_x = foil_hole_params.thumbnail_size_x or jpeg_size[0]
-        foil_hole.thumbnail_size_y = foil_hole_params.thumbnail_size_y or jpeg_size[1]
-        foil_hole.pixel_size = foil_hole_params.pixel_size
-        if _transport_object:
+        foil_hole.x_location = foil_hole_params.x_location or foil_hole.x_location
+        foil_hole.y_location = foil_hole_params.y_location or foil_hole.y_location
+        foil_hole.x_stage_position = (
+            foil_hole_params.x_stage_position or foil_hole.x_stage_position
+        )
+        foil_hole.y_stage_position = (
+            foil_hole_params.y_stage_position or foil_hole.y_stage_position
+        )
+        foil_hole.readout_area_x = (
+            foil_hole_params.readout_area_x or foil_hole.readout_area_x
+        )
+        foil_hole.readout_area_y = (
+            foil_hole_params.readout_area_y or foil_hole.readout_area_y
+        )
+        foil_hole.thumbnail_size_x = (
+            foil_hole_params.thumbnail_size_x or foil_hole.thumbnail_size_x
+        ) or jpeg_size[0]
+        foil_hole.thumbnail_size_y = (
+            foil_hole_params.thumbnail_size_y or foil_hole.thumbnail_size_y
+        ) or jpeg_size[1]
+        foil_hole.pixel_size = foil_hole_params.pixel_size or foil_hole.pixel_size
+        if _transport_object and gs.readout_area_x:
             _transport_object.do_update_foil_hole(
                 foil_hole.id, gs.thumbnail_size_x / gs.readout_area_x, foil_hole_params
             )
     except Exception:
         if _transport_object:
             fh_ispyb_response = _transport_object.do_insert_foil_hole(
-                gs.id, gs.thumbnail_size_x / gs.readout_area_x, foil_hole_params
+                gs.id,
+                gs.thumbnail_size_x / gs.readout_area_x if gs.readout_area_x else None,
+                foil_hole_params,
             )
         else:
             fh_ispyb_response = {"success": False, "return_value": None}
@@ -334,7 +348,8 @@ def flush_spa_preprocessing(message: dict, db: Session, demo: bool = False):
                 )
             except Exception as e:
                 logger.error(
-                    f"Flushing position analysis for {f.file_path} caused exception {e}", exc_info=True
+                    f"Flushing position analysis for {f.file_path} caused exception {e}",
+                    exc_info=True,
                 )
                 foil_hole_id = None