Get the longest source, and log failed Images-Disc searches (#480)

The globs for `Images-Disc*` were not working, and this attempts to fix them again. If they fail, it logs the directory being searched.
This also includes a few fixes to the database inserts for grid squares and foil holes

Author: stephen-riggs

src/murfey/client/contexts/spa.py

@@ -79,10 +79,19 @@ def _grid_square_metadata_file(
 
 
 def _get_source(file_path: Path, environment: MurfeyInstanceEnvironment) -> Path | None:
+    possible_sources = []
     for s in environment.sources:
         if file_path.is_relative_to(s):
-            return s
-    return None
+            possible_sources.append(s)
+    if not possible_sources:
+        return None
+    elif len(possible_sources) == 1:
+        return possible_sources[0]
+    source = possible_sources[0]
+    for extra_source in possible_sources[1:]:
+        if extra_source.is_relative_to(source):
+            source = extra_source
+    return source
 
 
 def _get_xml_list_index(key: str, xml_list: list) -> int:

src/murfey/client/contexts/spa_metadata.py

@@ -66,11 +66,13 @@ def _atlas_destination(
         instrument_name=environment.instrument_name,
         demo=environment.demo,
     )
-    if environment.visit in environment.default_destinations[source]:
-        return (
-            Path(machine_config.get("rsync_basepath", ""))
-            / Path(environment.default_destinations[source]).parent
-        )
+    for i, destination_part in enumerate(
+        Path(environment.default_destinations[source]).parts
+    ):
+        if destination_part == environment.visit:
+            return Path(machine_config.get("rsync_basepath", "")) / "/".join(
+                Path(environment.default_destinations[source]).parent.parts[: i + 1]
+            )
     return (
         Path(machine_config.get("rsync_basepath", ""))
         / Path(environment.default_destinations[source]).parent
@@ -117,12 +119,10 @@ def post_transfer(
             source_visit_dir = source.parent
 
             logger.info(
-                f"Looking for atlas XML file in metadata directory {str((source_visit_dir / environment.visit / partial_path).parent)}"
+                f"Looking for atlas XML file in metadata directory {str((source_visit_dir / partial_path).parent)}"
             )
             atlas_xml_path = list(
-                (source_visit_dir / environment.visit / partial_path).parent.glob(
-                    "Atlas_*.xml"
-                )
+                (source_visit_dir / partial_path).parent.glob("Atlas_*.xml")
             )[0]
             logger.info(f"Atlas XML path {str(atlas_xml_path)} found")
             with open(atlas_xml_path, "rb") as atlas_xml:
@@ -137,7 +137,6 @@ def post_transfer(
             atlas_pixel_size = atlas_original_pixel_size * 7.8
             logger.info(f"Atlas image pixel size determined to be {atlas_pixel_size}")
 
-            sample = None
             for p in partial_path.split("/"):
                 if p.startswith("Sample"):
                     sample = int(p.replace("Sample", ""))
@@ -150,31 +149,34 @@ def post_transfer(
                     atlas=Path(partial_path), sample=sample
                 )
                 url = f"{str(environment.url.geturl())}/visits/{environment.visit}/{environment.murfey_session}/register_data_collection_group"
-                dcg_search_dir = "/".join(
+                dcg_search_dir = "/" + "/".join(
                     p
                     for p in transferred_file.parent.parts[1:]
                     if p != environment.visit
                 )
-                dcg_tag = str(
-                    sorted(
-                        Path(dcg_search_dir).glob("Images-Disc*"),
-                        key=lambda x: x.stat().st_ctime,
-                    )[-1]
+                dcg_images_dirs = sorted(
+                    Path(dcg_search_dir).glob("Images-Disc*"),
+                    key=lambda x: x.stat().st_ctime,
                 )
+                if not dcg_images_dirs:
+                    logger.warning(f"Cannot find Images-Disc* in {dcg_search_dir}")
+                    return
+                dcg_tag = str(dcg_images_dirs[-1])
                 dcg_data = {
                     "experiment_type": "single particle",
                     "experiment_type_id": 37,
                     "tag": dcg_tag,
                     "atlas": str(
                         _atlas_destination(environment, source, transferred_file)
-                        / environment.samples[source].atlas
+                        / environment.samples[source].atlas.parent
+                        / atlas_xml_path.with_suffix(".jpg").name
                     ),
                     "sample": environment.samples[source].sample,
                     "atlas_pixel_size": atlas_pixel_size,
                 }
                 capture_post(url, json=dcg_data)
                 gs_pix_positions = get_grid_square_atlas_positions(
-                    source_visit_dir / environment.visit / partial_path
+                    source_visit_dir / partial_path
                 )
                 for gs, pos_data in gs_pix_positions.items():
                     if pos_data:
@@ -206,12 +208,16 @@ def post_transfer(
             visitless_source_search_dir = str(source).replace(
                 f"/{environment.visit}", ""
             )
-            visitless_source = str(
-                sorted(
-                    Path(visitless_source_search_dir).glob("Images-Disc*"),
-                    key=lambda x: x.stat().st_ctime,
-                )[-1]
+            visitless_source_images_dirs = sorted(
+                Path(visitless_source_search_dir).glob("Images-Disc*"),
+                key=lambda x: x.stat().st_ctime,
             )
+            if not visitless_source_images_dirs:
+                logger.warning(
+                    f"Cannot find Images-Disc* in {visitless_source_search_dir}"
+                )
+                return
+            visitless_source = str(visitless_source_images_dirs[-1])
             for fh, fh_data in fh_positions.items():
                 capture_post(
                     f"{str(environment.url.geturl())}/sessions/{environment.murfey_session}/grid_square/{gs_name}/foil_hole",

src/murfey/server/__init__.py

@@ -2566,6 +2566,7 @@ def feedback_callback(header: dict, message: dict) -> None:
                     message["atlas_pixel_size"],
                     message["sample"],
                 )
+                _transport_object.transport.ack(header)
             return None
         elif message["register"] == "data_collection":
             murfey_session_id = message["session_id"]

src/murfey/server/ispyb.py

@@ -213,32 +213,28 @@ def do_update_grid_square(
                         grid_square_parameters.readout_area_x
                         / grid_square_parameters.thumbnail_size_x
                     )
-                grid_square.gridSquareImage = grid_square_parameters.image
-                grid_square.pixelLocationX = (
-                    int(grid_square_parameters.x_location / 7.8)
-                    if grid_square_parameters.x_location
-                    else None
-                )
-                grid_square.pixelLocationY = (
-                    int(grid_square_parameters.y_location / 7.8)
-                    if grid_square_parameters.y_location
-                    else None
-                )
-                grid_square.pixelLocationY = grid_square_parameters.y_location
-                grid_square.height = (
-                    int(grid_square_parameters.height / 7.8)
-                    if grid_square_parameters.height is not None
-                    else None
-                )
-                grid_square.width = (
-                    int(grid_square_parameters.width / 7.8)
-                    if grid_square_parameters.width is not None
-                    else None
-                )
-                grid_square.angle = grid_square_parameters.angle
-                grid_square.stageLocationX = grid_square_parameters.x_stage_position
-                grid_square.stageLocationY = grid_square_parameters.y_stage_position
-                grid_square.pixelSize = grid_square_parameters.pixel_size
+                if grid_square_parameters.image:
+                    grid_square.gridSquareImage = grid_square_parameters.image
+                if grid_square_parameters.x_location:
+                    grid_square.pixelLocationX = int(
+                        grid_square_parameters.x_location / 7.8
+                    )
+                if grid_square_parameters.y_location:
+                    grid_square.pixelLocationY = int(
+                        grid_square_parameters.y_location / 7.8
+                    )
+                if grid_square_parameters.height is not None:
+                    grid_square.height = int(grid_square_parameters.height / 7.8)
+                if grid_square_parameters.width is not None:
+                    grid_square.width = int(grid_square_parameters.width / 7.8)
+                if grid_square_parameters.angle:
+                    grid_square.angle = grid_square_parameters.angle
+                if grid_square_parameters.x_stage_position:
+                    grid_square.stageLocationX = grid_square_parameters.x_stage_position
+                if grid_square_parameters.y_stage_position:
+                    grid_square.stageLocationY = grid_square_parameters.y_stage_position
+                if grid_square_parameters.pixel_size:
+                    grid_square.pixelSize = grid_square_parameters.pixel_size
                 db.add(grid_square)
                 db.commit()
                 return {"success": True, "return_value": grid_square.gridSquareId}
@@ -296,7 +292,7 @@ def do_insert_foil_hole(
             with Session() as db:
                 db.add(record)
                 db.commit()
-                log.info(f"Created FoilHole {record.gridSquareId}")
+                log.info(f"Created FoilHole {record.foilHoleId}")
                 return {"success": True, "return_value": record.foilHoleId}
         except ispyb.ISPyBException as e:
             log.error(
@@ -317,24 +313,22 @@ def do_update_foil_hole(
                 foil_hole = (
                     db.query(FoilHole).filter(FoilHole.foilHoleId == foil_hole_id).one()
                 )
-                foil_hole.foilHoleImage = foil_hole_parameters.image
-                foil_hole.pixelLocationX = (
-                    int(foil_hole_parameters.x_location * scale_factor)
-                    if foil_hole_parameters.x_location
-                    else None
-                )
-                foil_hole.pixelLocationY = (
-                    int(foil_hole_parameters.y_location * scale_factor)
-                    if foil_hole_parameters.y_location
-                    else None
-                )
-                foil_hole.diameter = (
-                    foil_hole_parameters.diameter * scale_factor
-                    if foil_hole_parameters.diameter is not None
-                    else None
-                )
-                foil_hole.stageLocationX = foil_hole_parameters.x_stage_position
-                foil_hole.stageLocationY = foil_hole_parameters.y_stage_position
+                if foil_hole_parameters.image:
+                    foil_hole.foilHoleImage = foil_hole_parameters.image
+                if foil_hole_parameters.x_location:
+                    foil_hole.pixelLocationX = int(
+                        foil_hole_parameters.x_location * scale_factor
+                    )
+                if foil_hole_parameters.y_location:
+                    foil_hole.pixelLocationY = int(
+                        foil_hole_parameters.y_location * scale_factor
+                    )
+                if foil_hole_parameters.diameter is not None:
+                    foil_hole.diameter = foil_hole_parameters.diameter * scale_factor
+                if foil_hole_parameters.x_stage_position:
+                    foil_hole.stageLocationX = foil_hole_parameters.x_stage_position
+                if foil_hole_parameters.y_stage_position:
+                    foil_hole.stageLocationY = foil_hole_parameters.y_stage_position
                 if (
                     foil_hole_parameters.readout_area_x is not None
                     and foil_hole_parameters.thumbnail_size_x is not None