Further CLEM fixes (#308)

* Corrected CLEM context logic to ignore non-image series files and better discern between image series.

---------

Co-authored-by: Daniel Hatton <[email protected]>

Author: tieneupin

src/murfey/client/analyser.py

@@ -113,7 +113,9 @@ def _find_extension(self, file_path: Path):
     def _find_context(self, file_path: Path) -> bool:
         """
         Using various conditionals, identifies what workflow the file is part of, and
-        assigns the necessary context class to it for subsequent stages of processing
+        assigns the correct context class to that batch of rsync files for subsequent
+        stages of processing. Actions to take for individual files will be determined
+        in the Context classes themselves.
         """
 
         # CLEM workflow checks
@@ -155,7 +157,7 @@ def _find_context(self, file_path: Path) -> bool:
                                 demo=self._environment.demo,
                             )
                         except Exception as e:
-                            logger.warning(f"exception encountered: {e}")
+                            logger.error(f"Exception encountered: {e}")
                             cfg = {}
                     else:
                         cfg = {}
@@ -300,7 +302,7 @@ def _analyse(self):
                                 environment=self._environment,
                             )
                         except Exception as e:
-                            logger.warning(f"exception encountered {e}")
+                            logger.error(f"Exception encountered: {e}")
                         if self._role == "detector":
                             if not dc_metadata:
                                 try:
@@ -347,7 +349,7 @@ def _analyse(self):
 
                 # If a file with a CLEM context is identified, immediately post it
                 elif isinstance(self._context, CLEMContext):
-                    logger.info(
+                    logger.debug(
                         f"File {transferred_file.name!r} will be processed as part of CLEM workflow"
                     )
                     self.post_transfer(transferred_file)
@@ -366,7 +368,7 @@ def _analyse(self):
                                 environment=self._environment,
                             )
                         except Exception as e:
-                            logger.info(f"exception encountered {e}")
+                            logger.error(f"Exception encountered: {e}")
                         if self._role == "detector":
                             if not dc_metadata:
                                 try:
@@ -452,5 +454,5 @@ def stop(self):
                 self.queue.put(None)
                 self.thread.join()
         except Exception as e:
-            logger.debug(f"Exception encountered while stopping analyser: {e}")
+            logger.error(f"Exception encountered while stopping analyser: {e}")
         logger.debug("Analyser thread stop completed")

src/murfey/client/contexts/clem.py

@@ -116,7 +116,6 @@ def post_transfer(
 
             # Process TIF/TIFF files
             if transferred_file.suffix in (".tif", ".tiff"):
-                logger.debug("Detected a TIFF file")
 
                 # Files should be named "PositionX--ZXX--CXX.tif" by default
                 # If Position is repeated, it will add an additional --00X to the end
@@ -126,6 +125,8 @@ def post_transfer(
                     )
                     return False
 
+                logger.debug(f"File {transferred_file.name!r} is part of a TIFF image series")
+
                 # Create a unique name for the series
                 # For standard file name
                 if len(transferred_file.stem.split("--")) == 3:
@@ -149,7 +150,7 @@ def post_transfer(
                     )
                     return False
                 logger.debug(
-                    f"File {transferred_file.name!r} given the series identifier {series_name!r}"
+                    f"File {transferred_file.name!r} given the series name {series_name!r}"
                 )
 
                 # Create key-value pairs containing empty list if not already present
@@ -171,21 +172,31 @@ def post_transfer(
 
             # Process XLIF files
             if transferred_file.suffix == ".xlif":
-                logger.debug("Detected an XLIF file")
 
                 # Skip processing of "_histo" histogram XLIF files
                 if transferred_file.stem.endswith("_histo"):
                     logger.debug(
                         f"File {transferred_file.name!r} contains histogram metadata; skipping processing"
                     )
+                    return True
+
+                # Skip processing of "IOManagerConfiguation.xlif" files (yes, the typo IS part of the file name)
+                if "IOManagerConfiguation" in transferred_file.stem:
+                    logger.debug(
+                        f"File {transferred_file.name!r} is a Leica configuration file; skipping processing"
+                    )
+                    return True
 
+                logger.debug(f"File {transferred_file.name!r} contains metadata for an image series")
+
+                # Create series name for XLIF file
                 # XLIF files don't have the "--ZXX--CXX" additions in the file name
                 # But they have "/Metadata/" as the immediate parent
                 series_name = "/".join(
                     [*file_path.parent.parent.parts[-2:], file_path.stem]
                 )  # The previous 2 parent directories should be unique enough
                 logger.debug(
-                    f"File {transferred_file.name!r} given the series identifier {series_name!r}"
+                    f"File {transferred_file.name!r} given the series name {series_name!r}"
                 )
 
                 # Extract metadata to get the expected size of the series
@@ -221,17 +232,21 @@ def post_transfer(
 
             # Post message if all files for the associated series have been collected
             # .get(series_name, 0) returns 0 if no associated key is found
-            if len(self._tiff_series[series_name]) == 0:
+            if not len(self._tiff_series.get(series_name, [])):
                 logger.debug(f"TIFF series {series_name!r} not yet loaded")
                 return True
             elif self._files_in_series.get(series_name, 0) == 0:
                 logger.debug(
                     f"Metadata for TIFF series {series_name!r} not yet processed"
                 )
                 return True
-            elif len(self._tiff_series[series_name]) == self._files_in_series.get(
-                series_name, 0
-            ):
+            elif len(
+                self._tiff_series.get(series_name, [])
+            ) == self._files_in_series.get(series_name, 0):
+                logger.debug(
+                    f"Collected expected number of TIFF files for series {series_name!r}; posting job to server"
+                )
+
                 # Construct URL for Murfey server to communicate with
                 url = f"{str(environment.url.geturl())}/sessions/{environment.murfey_session}/tiff_to_stack"
                 if not url:
@@ -270,6 +285,8 @@ def post_transfer(
                 logger.warning(f"No source found for file {transferred_file}")
                 return True
 
+            logger.debug(f"File {transferred_file.name!r} is a valid LIF file; posting job to server")
+
             # Construct the URL for the Murfey server to communicate with
             url = f"{str(environment.url.geturl())}/sessions/{environment.murfey_session}/lif_to_tiff"
             # Type checking to satisfy MyPy

src/murfey/client/contexts/tomo.py

@@ -1000,7 +1000,7 @@ def gather_metadata(
                     metadata_file.parent / data_file
                 )
         except Exception as e:
-            logger.warning(f"Exception encountered in metadata gathering: {str(e)}")
+            logger.error(f"Exception encountered in metadata gathering: {str(e)}")
             return OrderedDict({})
 
         return mdoc_metadata

src/murfey/util/clem/tiff.py

@@ -99,11 +99,18 @@ def process_tiff_files(
             f
             for f in tiff_list
             if (f"C{str(c).zfill(2)}" in f.stem or f"C{str(c).zfill(3)}" in f.stem)
-            and (img_name in f.stem)
+            and (img_name == f.stem.split("--")[0])
         ]
         tiff_sublist.sort(
             key=lambda f: (int(f.stem.split("--")[1].replace("Z", "")),)
-        )  # Increasing order of Z
+        )  # Sort by Z as an int, not str
+
+        # Return error message if the list of TIFFs is empty for some reason
+        if not tiff_sublist:
+            logger.error(
+                f"Error processing {color} channel for {img_name}; no TIFF files found"
+            )
+            return False
 
         # Load image stack
         logger.info("Loading image stack")
@@ -216,7 +223,7 @@ def convert_tiff_to_stack(
             part.lower() == root_folder.lower() and counter < 1
         ):  # Remove case-sensitivity
             path_parts[p] = new_root_folder
-            counter += 0  # Do for first instance only
+            counter += 1  # Do for first instance only
         # Remove last level in path if same as previous one (redundancy)
         if p == len(path_parts) - 1:
             if part.replace(" ", "_") == path_parts[p - 1].replace(" ", "_"):

src/murfey/workflows/lif_to_tiff.py

@@ -12,10 +12,12 @@
 
 
 def zocalo_cluster_request(
-    file: Path, root_folder: str, messenger: TransportManager | None = None
+    file: Path,
+    root_folder: str,
+    messenger: TransportManager | None = None,
 ):
     if messenger:
-        # Set working directory to be the parent of the designated root folder
+        # Construct path to session directory
         path_parts = list(file.parts)
         new_path = []
         for p in range(len(path_parts)):
@@ -27,16 +29,16 @@ def zocalo_cluster_request(
             if part.lower() == root_folder.lower():
                 break
             new_path.append(part)
-        working_dir = Path("/".join(new_path))
+        session_dir = Path("/".join(new_path))
 
         messenger.send(
             "processing_recipe",
             {
                 "recipes": ["lif-to-tiff"],
                 "parameters": {
-                    # Where the cluster generates and saves log files
-                    "working_dir": str(working_dir),
-                    "lif_path": str(file),
+                    # Represent file paths canonically
+                    "session_dir": repr(str(session_dir)),
+                    "lif_path": repr(str(file)),
                     "root_dir": root_folder,
                 },
             },

src/murfey/workflows/tiff_to_stack.py

@@ -19,8 +19,7 @@ def zocalo_cluster_request(
     messenger: TransportManager | None = None,
 ):
     if messenger:
-
-        # Set working directory to be the parent of the designated root folder
+        # Construct path to session directory
         path_parts = list(file.parts)
         new_path = []
         for p in range(len(path_parts)):
@@ -32,14 +31,19 @@ def zocalo_cluster_request(
             if part.lower() == root_folder.lower():
                 break
             new_path.append(part)
-        working_dir = Path("/".join(new_path))
+        session_dir = Path("/".join(new_path))
+
+        # If no metadata file provided, generate path to one
+        if metadata is None:
+            series_name = file.stem.split("--")[0]
+            metadata = file.parent / "Metadata" / (series_name + ".xlif")
 
         messenger.send(
             "processing_recipe",
             {
                 "recipes": ["tiff-to-stack"],
                 "parameters": {
-                    "working_dir": str(working_dir),
+                    "session_dir": str(session_dir),
                     "tiff_path": str(file),
                     "root_dir": root_folder,
                     "metadata": str(metadata),