Fixes for grid square and foil hole registration (#459)

Author: d-j-hatton

src/murfey/client/contexts/spa.py

@@ -456,10 +456,7 @@ def _position_analysis(
         foil_hole = foil_hole_from_file(transferred_file)
         if foil_hole not in self._foil_holes[grid_square]:
             fh_url = f"{str(environment.url.geturl())}/sessions/{environment.murfey_session}/grid_square/{grid_square}/foil_hole"
-            if (
-                grid_square_metadata_file.is_file()
-                and environment.murfey_session is not None
-            ):
+            if environment.murfey_session is not None:
                 fh = foil_hole_data(
                     grid_square_metadata_file,
                     foil_hole,

src/murfey/client/contexts/spa_metadata.py

@@ -21,13 +21,6 @@ def _foil_hole_positions(xml_path: Path, grid_square: int) -> Dict[str, FoilHole
         for_parsing = xml.read()
         data = xmltodict.parse(for_parsing)
     data = data["GridSquareXml"]
-    readout_area = data["MicroscopeImage"]["microscopeData"]["acquisition"]["camera"][
-        "ReadoutArea"
-    ]
-    pixel_size = data["MicroscopeImage"]["SpatialScale"]["pixelSize"]["x"][
-        "numericValue"
-    ]
-    full_size = (int(readout_area["a:width"]), int(readout_area["a:height"]))
     serialization_array = data["TargetLocations"]["TargetLocationsEfficient"][
         "a:m_serializationArray"
     ]
@@ -58,11 +51,6 @@ def _foil_hole_positions(xml_path: Path, grid_square: int) -> Dict[str, FoilHole
                 y_location=int(float(pix_loc["c:y"])),
                 x_stage_position=float(stage["c:X"]),
                 y_stage_position=float(stage["c:Y"]),
-                readout_area_x=full_size[0] if image_path else None,
-                readout_area_y=full_size[1] if image_path else None,
-                thumbnail_size_x=None,
-                thumbnail_size_y=None,
-                pixel_size=float(pixel_size) if image_path else None,
                 image=str(image_path),
                 diameter=int(float(diameter)),
             )
@@ -128,17 +116,21 @@ def post_transfer(
             source_visit_dir = source.parent
 
             logger.info(
-                f"Looking for atlas XML file in metadata directory {str((source_visit_dir / partial_path).parent)}"
+                f"Looking for atlas XML file in metadata directory {str((source_visit_dir / environment.visit / partial_path).parent)}"
             )
             atlas_xml_path = list(
-                (source_visit_dir / partial_path).parent.glob("Atlas_*.xml")
+                (source_visit_dir / environment.visit / partial_path).parent.glob(
+                    "Atlas_*.xml"
+                )
             )[0]
             logger.info(f"Atlas XML path {str(atlas_xml_path)} found")
             with open(atlas_xml_path, "rb") as atlas_xml:
                 atlas_xml_data = xmltodict.parse(atlas_xml)
-                atlas_original_pixel_size = atlas_xml_data["MicroscopeImage"][
-                    "SpatialScale"
-                ]["pixelSize"]["x"]["numericValue"]
+                atlas_original_pixel_size = float(
+                    atlas_xml_data["MicroscopeImage"]["SpatialScale"]["pixelSize"]["x"][
+                        "numericValue"
+                    ]
+                )
 
             # need to calculate the pixel size of the downscaled image
             atlas_pixel_size = atlas_original_pixel_size * 7.8
@@ -190,7 +182,7 @@ def post_transfer(
                         )
 
         elif transferred_file.suffix == ".dm" and environment:
-            gs_name = transferred_file.name.split("_")[1]
+            gs_name = transferred_file.stem.split("_")[1]
             fh_positions = _foil_hole_positions(transferred_file, int(gs_name))
             source = _get_source(transferred_file, environment=environment)
             visitless_source = str(source).replace(f"/{environment.visit}", "")

src/murfey/server/ispyb.py

@@ -253,7 +253,7 @@ def do_update_grid_square(
     def do_insert_foil_hole(
         self,
         grid_square_id: int,
-        scale_factor: float,
+        scale_factor: Optional[float],
         foil_hole_parameters: FoilHoleParameters,
     ):
         if (
@@ -265,21 +265,22 @@ def do_insert_foil_hole(
                 foil_hole_parameters.readout_area_x
                 / foil_hole_parameters.thumbnail_size_x
             )
-        foil_hole_parameters.diameter = (
-            int(foil_hole_parameters.diameter * scale_factor)
-            if foil_hole_parameters.diameter
-            else None
-        )
-        foil_hole_parameters.x_location = (
-            int(foil_hole_parameters.x_location * scale_factor)
-            if foil_hole_parameters.x_location
-            else None
-        )
-        foil_hole_parameters.y_location = (
-            int(foil_hole_parameters.y_location * scale_factor)
-            if foil_hole_parameters.y_location
-            else None
-        )
+        if scale_factor:
+            foil_hole_parameters.diameter = (
+                int(foil_hole_parameters.diameter * scale_factor)
+                if foil_hole_parameters.diameter
+                else None
+            )
+            foil_hole_parameters.x_location = (
+                int(foil_hole_parameters.x_location * scale_factor)
+                if foil_hole_parameters.x_location
+                else None
+            )
+            foil_hole_parameters.y_location = (
+                int(foil_hole_parameters.y_location * scale_factor)
+                if foil_hole_parameters.y_location
+                else None
+            )
         record = FoilHole(
             gridSquareId=grid_square_id,
             foilHoleLabel=foil_hole_parameters.name,

src/murfey/util/spa_metadata.py

@@ -148,36 +148,37 @@ def grid_square_data(xml_path: Path, grid_square: int) -> GridSquareInfo:
 
 
 def foil_hole_data(xml_path: Path, foil_hole: int, grid_square: int) -> FoilHoleInfo:
-    with open(xml_path, "r") as xml:
-        for_parsing = xml.read()
-        data = xmltodict.parse(for_parsing)
-    data = data["GridSquareXml"]
-    serialization_array = data["TargetLocations"]["TargetLocationsEfficient"][
-        "a:m_serializationArray"
-    ]
     required_key = ""
-    for key in serialization_array.keys():
-        if key.startswith("b:KeyValuePairOfintTargetLocation"):
-            required_key = key
-            break
-    if required_key:
-        image_paths = list(
-            (xml_path.parent.parent).glob(
-                f"Images-Disc*/GridSquare_{grid_square}/FoilHoles/FoilHole_{foil_hole}_*.jpg"
-            )
+    if xml_path.is_file():
+        with open(xml_path, "r") as xml:
+            for_parsing = xml.read()
+            data = xmltodict.parse(for_parsing)
+        data = data["GridSquareXml"]
+        serialization_array = data["TargetLocations"]["TargetLocationsEfficient"][
+            "a:m_serializationArray"
+        ]
+        for key in serialization_array.keys():
+            if key.startswith("b:KeyValuePairOfintTargetLocation"):
+                required_key = key
+                break
+    image_paths = list(
+        (xml_path.parent.parent).glob(
+            f"Images-Disc*/GridSquare_{grid_square}/FoilHoles/FoilHole_{foil_hole}_*.jpg"
         )
-        image_paths.sort(key=lambda x: x.stat().st_ctime)
-        image_path: Union[Path, str] = image_paths[-1] if image_paths else ""
-        if image_path:
-            with open(Path(image_path).with_suffix(".xml")) as fh_xml:
-                fh_xml_data = xmltodict.parse(fh_xml.read())
-            readout_area = fh_xml_data["MicroscopeImage"]["microscopeData"][
-                "acquisition"
-            ]["camera"]["ReadoutArea"]
-            pixel_size = fh_xml_data["MicroscopeImage"]["SpatialScale"]["pixelSize"][
-                "x"
-            ]["numericValue"]
-            full_size = (int(readout_area["a:width"]), int(readout_area["a:height"]))
+    )
+    image_paths.sort(key=lambda x: x.stat().st_ctime)
+    image_path: Union[Path, str] = image_paths[-1] if image_paths else ""
+    if image_path:
+        with open(Path(image_path).with_suffix(".xml")) as fh_xml:
+            fh_xml_data = xmltodict.parse(fh_xml.read())
+        readout_area = fh_xml_data["MicroscopeImage"]["microscopeData"]["acquisition"][
+            "camera"
+        ]["ReadoutArea"]
+        pixel_size = fh_xml_data["MicroscopeImage"]["SpatialScale"]["pixelSize"]["x"][
+            "numericValue"
+        ]
+        full_size = (int(readout_area["a:width"]), int(readout_area["a:height"]))
+    if required_key:
         for fh_block in serialization_array[required_key]:
             pix = fh_block["b:value"]["PixelCenter"]
             stage = fh_block["b:value"]["StagePosition"]
@@ -198,6 +199,17 @@ def foil_hole_data(xml_path: Path, foil_hole: int, grid_square: int) -> FoilHole
                     image=str(image_path),
                     diameter=diameter,
                 )
+    elif image_path:
+        return FoilHoleInfo(
+            id=foil_hole,
+            grid_square_id=grid_square,
+            readout_area_x=full_size[0] if image_path else None,
+            readout_area_y=full_size[1] if image_path else None,
+            thumbnail_size_x=None,
+            thumbnail_size_y=None,
+            pixel_size=float(pixel_size) if image_path else None,
+            image=str(image_path),
+        )
     logger.warning(
         f"Foil hole positions could not be determined from metadata file {xml_path} for foil hole {foil_hole}"
     )

src/murfey/workflows/spa/flush_spa_preprocess.py

@@ -39,10 +39,14 @@ def register_grid_square(
             .where(GridSquare.tag == grid_square_params.tag)
             .where(GridSquare.session_id == session_id)
         ).one()
-        grid_square.x_location = grid_square_params.x_location
-        grid_square.y_location = grid_square_params.y_location
-        grid_square.x_stage_position = grid_square_params.x_stage_position
-        grid_square.y_stage_position = grid_square_params.y_stage_position
+        grid_square.x_location = grid_square_params.x_location or grid_square.x_location
+        grid_square.y_location = grid_square_params.y_location or grid_square.y_location
+        grid_square.x_stage_position = (
+            grid_square_params.x_stage_position or grid_square.x_stage_position
+        )
+        grid_square.y_stage_position = (
+            grid_square_params.y_stage_position or grid_square.y_stage_position
+        )
         if _transport_object:
             _transport_object.do_update_grid_square(grid_square.id, grid_square_params)
     except Exception:
@@ -122,23 +126,37 @@ def register_foil_hole(
             .where(FoilHole.grid_square_id == gsid)
             .where(FoilHole.session_id == session_id)
         ).one()
-        foil_hole.x_location = foil_hole_params.x_location
-        foil_hole.y_location = foil_hole_params.y_location
-        foil_hole.x_stage_position = foil_hole_params.x_stage_position
-        foil_hole.y_stage_position = foil_hole_params.y_stage_position
-        foil_hole.readout_area_x = foil_hole_params.readout_area_x
-        foil_hole.readout_area_y = foil_hole_params.readout_area_y
-        foil_hole.thumbnail_size_x = foil_hole_params.thumbnail_size_x or jpeg_size[0]
-        foil_hole.thumbnail_size_y = foil_hole_params.thumbnail_size_y or jpeg_size[1]
-        foil_hole.pixel_size = foil_hole_params.pixel_size
-        if _transport_object:
+        foil_hole.x_location = foil_hole_params.x_location or foil_hole.x_location
+        foil_hole.y_location = foil_hole_params.y_location or foil_hole.y_location
+        foil_hole.x_stage_position = (
+            foil_hole_params.x_stage_position or foil_hole.x_stage_position
+        )
+        foil_hole.y_stage_position = (
+            foil_hole_params.y_stage_position or foil_hole.y_stage_position
+        )
+        foil_hole.readout_area_x = (
+            foil_hole_params.readout_area_x or foil_hole.readout_area_x
+        )
+        foil_hole.readout_area_y = (
+            foil_hole_params.readout_area_y or foil_hole.readout_area_y
+        )
+        foil_hole.thumbnail_size_x = (
+            foil_hole_params.thumbnail_size_x or foil_hole.thumbnail_size_x
+        ) or jpeg_size[0]
+        foil_hole.thumbnail_size_y = (
+            foil_hole_params.thumbnail_size_y or foil_hole.thumbnail_size_y
+        ) or jpeg_size[1]
+        foil_hole.pixel_size = foil_hole_params.pixel_size or foil_hole.pixel_size
+        if _transport_object and gs.readout_area_x:
             _transport_object.do_update_foil_hole(
                 foil_hole.id, gs.thumbnail_size_x / gs.readout_area_x, foil_hole_params
             )
     except Exception:
         if _transport_object:
             fh_ispyb_response = _transport_object.do_insert_foil_hole(
-                gs.id, gs.thumbnail_size_x / gs.readout_area_x, foil_hole_params
+                gs.id,
+                gs.thumbnail_size_x / gs.readout_area_x if gs.readout_area_x else None,
+                foil_hole_params,
             )
         else:
             fh_ispyb_response = {"success": False, "return_value": None}
@@ -334,7 +352,8 @@ def flush_spa_preprocessing(message: dict, db: Session, demo: bool = False):
                 )
             except Exception as e:
                 logger.error(
-                    f"Flushing position analysis for {f.file_path} caused exception {e}", exc_info=True
+                    f"Flushing position analysis for {f.file_path} caused exception {e}",
+                    exc_info=True,
                 )
                 foil_hole_id = None