Merge branch 'main' into flush-test

Author: stephen-riggs

src/murfey/client/contexts/spa.py

@@ -66,13 +66,16 @@ def _grid_square_metadata_file(
             break
     else:
         raise ValueError(f"Could not determine grid square metadata path for {f}")
-    return (
+    metadata_file = (
         base_dir
         / visit
         / mid_dir.parent.parent.parent
         / "Metadata"
         / f"GridSquare_{grid_square}.dm"
     )
+    if not metadata_file.is_file():
+        logger.warning(f"Grid square metadata file {str(metadata_file)} does not exist")
+    return metadata_file
 
 
 def _get_source(file_path: Path, environment: MurfeyInstanceEnvironment) -> Path | None:

src/murfey/client/contexts/spa_metadata.py

@@ -30,6 +30,7 @@ def _foil_hole_positions(xml_path: Path, grid_square: int) -> Dict[str, FoilHole
             required_key = key
             break
     if not required_key:
+        logger.warning(f"Required key not found for {str(xml_path)}")
         return {}
     foil_holes = {}
     for fh_block in serialization_array[required_key]:
@@ -149,10 +150,13 @@ def post_transfer(
                     atlas=Path(partial_path), sample=sample
                 )
                 url = f"{str(environment.url.geturl())}/visits/{environment.visit}/{environment.murfey_session}/register_data_collection_group"
+                dcg_tag = "/".join(
+                    p for p in transferred_file.parent.parts if p != environment.visit
+                )
                 dcg_data = {
                     "experiment_type": "single particle",
                     "experiment_type_id": 37,
-                    "tag": str(source),
+                    "tag": dcg_tag,
                     "atlas": str(
                         _atlas_destination(environment, source, transferred_file)
                         / environment.samples[source].atlas
@@ -169,7 +173,7 @@ def post_transfer(
                         capture_post(
                             f"{str(environment.url.geturl())}/sessions/{environment.murfey_session}/grid_square/{gs}",
                             json={
-                                "tag": str(source),
+                                "tag": dcg_tag,
                                 "x_location": pos_data[0],
                                 "y_location": pos_data[1],
                                 "x_stage_position": pos_data[2],
@@ -186,6 +190,9 @@ def post_transfer(
             and environment
         ):
             gs_name = transferred_file.stem.split("_")[1]
+            logger.info(
+                f"Collecting foil hole positions for {str(transferred_file)} and grid square {int(gs_name)}"
+            )
             fh_positions = _foil_hole_positions(transferred_file, int(gs_name))
             source = _get_source(transferred_file, environment=environment)
             visitless_source = str(source).replace(f"/{environment.visit}", "")

src/murfey/util/spa_metadata.py

@@ -123,6 +123,9 @@ def grid_square_data(xml_path: Path, grid_square: int) -> GridSquareInfo:
             f"Images-Disc*/GridSquare_{grid_square}/GridSquare_*.jpg"
         )
     )
+    logger.info(
+        f"{len(image_paths)} images found when searching {str(xml_path.parent.parent)}"
+    )
     if image_paths:
         image_paths.sort(key=lambda x: x.stat().st_ctime)
         image_path = image_paths[-1]

src/murfey/workflows/spa/flush_spa_preprocess.py

@@ -107,7 +107,7 @@ def register_foil_hole(
     gs_name: int,
     foil_hole_params: FoilHoleParameters,
     murfey_db: Session,
-):
+) -> Optional[int]:
     try:
         gs = murfey_db.exec(
             select(GridSquare)
@@ -117,10 +117,10 @@ def register_foil_hole(
         ).one()
         gsid = gs.id
     except NoResultFound:
-        logger.debug(
+        logger.warning(
             f"Foil hole {sanitise(str(foil_hole_params.name))} could not be registered as grid square {sanitise(str(gs_name))} was not found"
         )
-        return
+        return None
     secured_foil_hole_image_path = secure_path(Path(foil_hole_params.image))
     if foil_hole_params.image and secured_foil_hole_image_path.is_file():
         jpeg_size = Image.open(secured_foil_hole_image_path).size
@@ -190,6 +190,7 @@ def register_foil_hole(
     murfey_db.add(foil_hole)
     murfey_db.commit()
     murfey_db.close()
+    return foil_hole.id
 
 
 def _grid_square_metadata_file(f: Path, grid_square: int) -> Optional[Path]:
@@ -284,8 +285,7 @@ def _flush_position_analysis(
             name=foil_hole,
         )
     # Insert or update this foil hole in the database
-    register_foil_hole(session_id, gs.id, foil_hole_parameters, murfey_db)
-    return foil_hole
+    return register_foil_hole(session_id, gs.id, foil_hole_parameters, murfey_db)
 
 
 def flush_spa_preprocess(message: dict, murfey_db: Session, demo: bool = False) -> bool:
@@ -346,23 +346,29 @@ def flush_spa_preprocess(message: dict, murfey_db: Session, demo: bool = False)
         murfey_db.add(feedback_params)
 
     for i, f in enumerate(stashed_files):
-        if f.foil_hole_id:
-            foil_hole_id = f.foil_hole_id
-        else:
-            # Register grid square and foil hole if not present
-            try:
+        try:
+            foil_hole_id = None
+            if f.foil_hole_id:
+                # Check if the foil hole id has been registered in the database
+                db_foil_hole = murfey_db.exec(
+                    select(FoilHole).where(FoilHole.id == f.foil_hole_id)
+                ).all()
+                if db_foil_hole:
+                    foil_hole_id = f.foil_hole_id
+            if not foil_hole_id:
+                # Register grid square and foil hole if not present
                 foil_hole_id = _flush_position_analysis(
                     movie_path=Path(f.file_path),
                     dcg_id=collected_ids[0].id,
                     session_id=session_id,
                     murfey_db=murfey_db,
                 )
-            except Exception as e:
-                logger.error(
-                    f"Flushing position analysis for {f.file_path} caused exception {e}",
-                    exc_info=True,
-                )
-                foil_hole_id = None
+        except Exception as e:
+            logger.error(
+                f"Flushing position analysis for {f.file_path} caused exception {e}",
+                exc_info=True,
+            )
+            foil_hole_id = None
 
         mrcp = Path(f.mrc_out)
         ppath = Path(f.file_path)