Migration of CLEM functions from murfey to cryoem-services (#323)

* Removed command line entry points for image stack creation
* Removed local CLEM image processing functions and 'murfey.workflows' usage of them
* Removed CLEM package dependencies
* Refactored and renamed functions still used by Murfey for its part of the CLEM workflow

Author: tieneupin

pyproject.toml

@@ -65,10 +65,8 @@ server = [
     "pillow",
     "prometheus_client",
     "python-jose[cryptography]",
-    "readlif", # Specific to cryo-CLEM workflow
     "sqlmodel",
     "stomp-py<=8.1.0", # 8.1.1 (released 2024-04-06) doesn't work with our project
-    "tifffile", # Specific to cryo-CLEM workflow
     "uvicorn[standard]",
     "zocalo",
 ]
@@ -84,16 +82,14 @@ murfey = "murfey.client:run"
 "murfey.decrypt_password" = "murfey.cli.decrypt_db_password:run"
 "murfey.generate_key" = "murfey.cli.generate_crypto_key:run"
 "murfey.generate_password" = "murfey.cli.generate_db_password:run"
-"murfey.lif_to_tiff" = "murfey.cli.lif_to_tiff:run"
 "murfey.server" = "murfey.server:run"
 "murfey.sessions" = "murfey.cli.db_sessions:run"
 "murfey.simulate" = "murfey.cli.dummy:run"
 "murfey.spa_inject" = "murfey.cli.inject_spa_processing:run"
 "murfey.spa_ispyb_entries" = "murfey.cli.spa_ispyb_messages:run"
-"murfey.tiff_to_stack" = "murfey.cli.tiff_to_stack:run"
 "murfey.transfer" = "murfey.cli.transfer:run"
 [project.entry-points."murfey.workflows"]
-"lif_to_tiff" = "murfey.workflows.lif_to_tiff:zocalo_cluster_request"
+"lif_to_stack" = "murfey.workflows.lif_to_stack:zocalo_cluster_request"
 "tiff_to_stack" = "murfey.workflows.tiff_to_stack:zocalo_cluster_request"
 
 [tool.setuptools]

src/murfey/cli/lif_to_tiff.py

@@ -6,14 +6,14 @@
 import logging
 from datetime import datetime
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import Dict, Generator, List, Optional
+from xml.etree import ElementTree as ET
 
 from defusedxml.ElementTree import parse
 
 from murfey.client.context import Context
 from murfey.client.instance_environment import MurfeyInstanceEnvironment
 from murfey.util import capture_post, get_machine_config
-from murfey.util.clem.xml import get_image_elements
 
 # Create logger object
 logger = logging.getLogger("murfey.client.contexts.clem")
@@ -52,7 +52,40 @@ def _get_source(
     return None
 
 
-# WORK IN PROGRESS
+def _get_image_elements(root: ET.Element) -> List[ET.Element]:
+    """
+    Searches the XML metadata recursively to find the nodes tagged as "Element" that
+    have image-related tags. Some LIF datasets have layers of nested elements, so a
+    recursive approach is needed to avoid certain datasets breaking it.
+    """
+
+    # Nested function which generates list of elements
+    def _find_elements_recursively(
+        node: ET.Element,
+    ) -> Generator[ET.Element, None, None]:
+
+        # Find items labelled "Element" under current node
+        elem_list = node.findall("./Children/Element")
+        if len(elem_list) < 1:  # Try alternative path for top-level of XML tree
+            elem_list = node.findall("./Element")
+
+        # Recursively search for items tagged as Element under child branches
+        for elem in elem_list:
+            yield elem
+            new_node = elem  # New starting point for the search
+            new_elem_list = _find_elements_recursively(new_node)  # Call self
+            for new_elem in new_elem_list:
+                yield new_elem
+
+    # Get initial list of elements
+    elem_list = list(_find_elements_recursively(root))
+
+    # Keep only the element nodes that have image-related tags
+    elem_list = [elem for elem in elem_list if elem.find("./Data/Image")]
+
+    return elem_list
+
+
 class CLEMContext(Context):
     def __init__(self, acquisition_software: str, basepath: Path):
         super().__init__("CLEM", acquisition_software)
@@ -205,7 +238,7 @@ def post_transfer(
 
                 # Extract metadata to get the expected size of the series
                 metadata = parse(transferred_file).getroot()
-                metadata = get_image_elements(metadata)[0]
+                metadata = _get_image_elements(metadata)[0]
 
                 # Get channel and dimension information
                 channels = metadata.findall(
@@ -294,7 +327,7 @@ def post_transfer(
             )
 
             # Construct the URL for the Murfey server to communicate with
-            url = f"{str(environment.url.geturl())}/sessions/{environment.murfey_session}/lif_to_tiff"
+            url = f"{str(environment.url.geturl())}/sessions/{environment.murfey_session}/lif_to_stack"
             # Type checking to satisfy MyPy
             if not url:
                 logger.warning("No URL found for the environment")

src/murfey/cli/tiff_to_stack.py

@@ -5,8 +5,6 @@
 from fastapi import APIRouter
 
 from murfey.server import _transport_object
-from murfey.util.clem.lif import convert_lif_to_tiff
-from murfey.util.clem.tiff import convert_tiff_to_stack
 from murfey.util.models import LifFileInfo, TiffSeriesInfo
 
 # Use backport from importlib_metadata for Python <3.10
@@ -19,30 +17,27 @@
 router = APIRouter()
 
 
-@router.post("/sessions/{session_id}/lif_to_tiff")  # API posts to this URL
-def lif_to_tiff(
+@router.post("/sessions/{session_id}/lif_to_stack")  # API posts to this URL
+def lif_to_stack(
     session_id: int,  # Used by the decorator
     lif_info: LifFileInfo,
 ):
     # Get command line entry point
     murfey_workflows = entry_points().select(
-        group="murfey.workflows", name="lif_to_tiff"
+        group="murfey.workflows", name="lif_to_stack"
     )
 
     # Use entry point if found
     if murfey_workflows:
         murfey_workflows[0].load()(
-            # Match the arguments found in murfey.workflows.lif_to_tiff
+            # Match the arguments found in murfey.workflows.lif_to_stack
             file=lif_info.name,
             root_folder="images",
             messenger=_transport_object,
         )
-    # Call function directly otherwise
+    # Raise error if Murfey workflow not found
     else:
-        convert_lif_to_tiff(
-            file=lif_info.name,
-            root_folder="images",
-        )
+        raise RuntimeError("The relevant Murfey workflow was not found")
 
 
 @router.post("/sessions/{session_id}/tiff_to_stack")
@@ -52,7 +47,7 @@ def tiff_to_stack(
 ):
     # Get command line entry point
     murfey_workflows = entry_points().select(
-        group="murfey.workflows", name="lif_to_tiff"
+        group="murfey.workflows", name="tiff_to_stack"
     )
 
     # Use entry point if found
@@ -64,10 +59,6 @@ def tiff_to_stack(
             metadata=tiff_info.series_metadata,
             messenger=_transport_object,
         )
-    # Call function directly otherwise
+    # Raise error if Murfey workflow not found
     else:
-        convert_tiff_to_stack(
-            tiff_list=tiff_info.tiff_files,
-            root_folder="images",
-            metadata_file=tiff_info.series_metadata,
-        )
+        raise RuntimeError("The relevant Murfey workflow was not found")