Merged recent changes from 'main' branch

Author: tieneupin

src/murfey/cli/inject_spa_processing.py

@@ -21,7 +21,7 @@
     SPAFeedbackParameters,
     SPARelionParameters,
 )
-from murfey.util.spa_params import default_spa_parameters
+from murfey.util.processing_params import default_spa_parameters
 
 
 def run():

src/murfey/server/__init__.py

@@ -56,7 +56,7 @@
     get_machine_config,
     get_microscope,
 )
-from murfey.util.spa_params import default_spa_parameters
+from murfey.util.processing_params import default_spa_parameters
 from murfey.util.state import global_state
 
 try:

src/murfey/server/api/__init__.py

@@ -105,7 +105,7 @@
     TiltSeriesInfo,
     Visit,
 )
-from murfey.util.spa_params import default_spa_parameters
+from murfey.util.processing_params import default_spa_parameters
 from murfey.util.state import global_state
 
 log = logging.getLogger("murfey.server.api")
@@ -1293,7 +1293,31 @@ def suggest_path(
         )
     if not machine_config.rsync_basepath:
         raise ValueError("No rsync basepath set")
+
+    # Construct the full path to where the dataset is to be saved
     check_path = machine_config.rsync_basepath / base_path
+
+    # Check previous year to account for the year rolling over during data collection
+    if not check_path.exists():
+        base_path_parts = base_path.split("/")
+        for part in base_path_parts:
+            # Find the path part corresponding to the year
+            if len(part) == 4 and part.isdigit():
+                year_idx = base_path_parts.index(part)
+                base_path_parts[year_idx] = str(int(part) - 1)
+        base_path = "/".join(base_path_parts)
+        check_path_prev = check_path
+        check_path = machine_config.rsync_basepath / base_path
+
+        # If it's not in the previous year either, it's a genuine error
+        if not check_path.exists():
+            log_message = (
+                "Unable to find current visit folder under "
+                f"{str(check_path_prev)!r} or {str(check_path)!r}"
+            )
+            log.error(log_message)
+            raise FileNotFoundError(log_message)
+
     check_path_name = check_path.name
     while check_path.exists():
         count = count + 1 if count else 2
@@ -1490,6 +1514,26 @@ async def process_gain(
         / secure_filename(visit_name)
         / machine_config.gain_directory_name
     )
+
+    # Check under previous year if the folder doesn't exist
+    if not filepath.exists():
+        filepath_prev = filepath
+        filepath = (
+            Path(machine_config.rsync_basepath)
+            / (machine_config.rsync_module or "data")
+            / str(datetime.datetime.now().year - 1)
+            / secure_filename(visit_name)
+            / machine_config.gain_directory_name
+        )
+        # If it's not in the previous year, it's a genuine error
+        if not filepath.exists():
+            log_message = (
+                "Unable to find gain reference directory under "
+                f"{str(filepath_prev)!r} or {str(filepath)}"
+            )
+            log.error(log_message)
+            raise FileNotFoundError(log_message)
+
     if gain_reference_params.eer:
         new_gain_ref, new_gain_ref_superres = await prepare_eer_gain(
             filepath / safe_path_name,

src/murfey/server/api/clem.py

@@ -2,13 +2,15 @@
 
 import re
 import traceback
+from ast import literal_eval
 from importlib.metadata import EntryPoint  # type hinting only
 from logging import getLogger
 from pathlib import Path
-from typing import Optional, Type, Union
+from typing import Literal, Optional, Type, Union
 
 from backports.entry_points_selectable import entry_points
 from fastapi import APIRouter
+from pydantic import BaseModel, validator
 from sqlalchemy.exc import NoResultFound
 from sqlmodel import Session, select
 
@@ -23,7 +25,6 @@
     CLEMTIFFFile,
 )
 from murfey.util.db import Session as MurfeySession
-from murfey.util.models import TIFFSeriesInfo
 
 # Set up logger
 logger = getLogger("murfey.server.api.clem")
@@ -624,7 +625,7 @@ def register_image_stack(
     "/sessions/{session_id}/clem/preprocessing/process_raw_lifs"
 )  # API posts to this URL
 def process_raw_lifs(
-    session_id: int,  # Used by the decorator
+    session_id: int,
     lif_file: Path,
     db: Session = murfey_db,
 ):
@@ -656,9 +657,15 @@ def process_raw_lifs(
     return True
 
 
+class TIFFSeriesInfo(BaseModel):
+    series_name: str
+    tiff_files: list[Path]
+    series_metadata: Path
+
+
 @router.post("/sessions/{session_id}/clem/preprocessing/process_raw_tiffs")
 def process_raw_tiffs(
-    session_id: int,  # Used by the decorator
+    session_id: int,
     tiff_info: TIFFSeriesInfo,
     db: Session = murfey_db,
 ):
@@ -689,3 +696,72 @@ def process_raw_tiffs(
         messenger=_transport_object,
     )
     return True
+
+
+class AlignAndMergeParams(BaseModel):
+    # Processing parameters
+    series_name: str
+    images: list[Path]
+    metadata: Path
+    # Optional processing parameters
+    crop_to_n_frames: Optional[int] = None
+    align_self: Literal["enabled", ""] = ""
+    flatten: Literal["mean", "min", "max", ""] = ""
+    align_across: Literal["enabled", ""] = ""
+
+    @validator(
+        "images",
+        pre=True,
+    )
+    def parse_stringified_list(cls, value):
+        if isinstance(value, str):
+            try:
+                eval_result = literal_eval(value)
+                if isinstance(eval_result, list):
+                    parent_tiffs = [Path(p) for p in eval_result]
+                    return parent_tiffs
+            except (SyntaxError, ValueError):
+                raise ValueError("Unable to parse input")
+        # Return value as-is; if it fails, it fails
+        return value
+
+
+@router.post("/sessions/{session_id}/clem/processing/align_and_merge_stacks")
+def align_and_merge_stacks(
+    session_id: int,
+    align_and_merge_params: AlignAndMergeParams,
+    db: Session = murfey_db,
+):
+    try:
+        # Try and load relevant Murfey workflow
+        workflow: EntryPoint = list(
+            entry_points().select(group="murfey.workflows", name="clem.align_and_merge")
+        )[0]
+    except IndexError:
+        raise RuntimeError("The relevant Murfey workflow was not found")
+
+    # Get instrument name from the database to load the correct config file
+    session_row: MurfeySession = db.exec(
+        select(MurfeySession).where(MurfeySession.id == session_id)
+    ).one()
+    instrument_name = session_row.instrument_name
+
+    # Pass arguments to correct workflow
+    workflow.load()(
+        # Match the arguments found in murfey.workflows.clem.align_and_merge
+        # Session parameters
+        session_id=session_id,
+        instrument_name=instrument_name,
+        # Processing parameters
+        series_name=align_and_merge_params.series_name,
+        images=align_and_merge_params.images,
+        metadata=align_and_merge_params.metadata,
+        # Optional processing parameters
+        crop_to_n_frames=align_and_merge_params.crop_to_n_frames,
+        align_self=align_and_merge_params.align_self,
+        flatten=align_and_merge_params.flatten,
+        align_across=align_and_merge_params.align_across,
+        # Optional session parameters
+        messenger=_transport_object,
+    )
+    return True

src/murfey/server/demo_api.py

@@ -90,7 +90,7 @@
     TiltSeriesInfo,
     Visit,
 )
-from murfey.util.spa_params import default_spa_parameters
+from murfey.util.processing_params import default_spa_parameters
 from murfey.util.state import global_state
 
 log = logging.getLogger("murfey.server.demo_api")
@@ -110,7 +110,7 @@ class Settings(BaseSettings):
 
 settings = Settings()
 
-machine_config: dict = {}
+machine_config: dict[str, MachineConfig] = {}
 if settings.murfey_machine_configuration:
     microscope = get_microscope()
     machine_config = machine_config_from_file(
@@ -1292,12 +1292,46 @@ def suggest_path(
     instrument_name = (
         db.exec(select(Session).where(Session.id == session_id)).one().instrument_name
     )
-    check_path = (
-        machine_config[instrument_name].rsync_basepath / params.base_path
+    rsync_basepath = (
+        machine_config[instrument_name].rsync_basepath
         if machine_config
-        else Path(f"/dls/{get_microscope()}") / params.base_path
+        else Path(f"/dls/{get_microscope()}")
     )
+    check_path = rsync_basepath / params.base_path
     check_path = check_path.parent / f"{check_path.stem}{count}{check_path.suffix}"
+    check_path = check_path.resolve()
+
+    # Check for path traversal attempt
+    if not str(check_path).startswith(str(rsync_basepath)):
+        raise Exception(f"Path traversal attempt detected: {str(check_path)!r}")
+
+    # Check previous year to account for the year rolling over during data collection
+    if not sanitise_path(check_path).exists():
+        base_path_parts = list(params.base_path.parts)
+        for part in base_path_parts:
+            # Find the path part corresponding to the year
+            if len(part) == 4 and part.isdigit():
+                year_idx = base_path_parts.index(part)
+                base_path_parts[year_idx] = str(int(part) - 1)
+        base_path = "/".join(base_path_parts)
+        check_path_prev = check_path
+        check_path = rsync_basepath / base_path
+        check_path = check_path.parent / f"{check_path.stem}{count}{check_path.suffix}"
+        check_path = check_path.resolve()
+
+        # Check for path traversal attempt
+        if not str(check_path).startswith(str(rsync_basepath)):
+            raise Exception(f"Path traversal attempt detected: {str(check_path)!r}")
+
+        # If visit is not in the previous year either, it's a genuine error
+        if not check_path.exists():
+            log_message = sanitise(
+                "Unable to find current visit folder under "
+                f"{str(check_path_prev)!r} or {str(check_path)!r}"
+            )
+            log.error(log_message)
+            raise FileNotFoundError(log_message)
+
     check_path_name = check_path.name
     while sanitise_path(check_path).exists():
         count = count + 1 if count else 2

src/murfey/util/models.py

@@ -147,19 +147,6 @@ class FractionationParameters(BaseModel):
     fractionation_file_name: str = "eer_fractionation.txt"
 
 
-"""
-Cryo-CLEM
-=========
-Models related to the cryo-CLEM workflow.
-"""
-
-
-class TIFFSeriesInfo(BaseModel):
-    series_name: str
-    tiff_files: List[Path]
-    series_metadata: Path
-
-
 """
 FIB
 ===

src/murfey/util/spa_params.py src/murfey/util/processing_params.pysrc/murfey/util/spa_params.py renamed to src/murfey/util/processing_params.py

@@ -1,6 +1,18 @@
+from typing import Literal, Optional
+
 from pydantic import BaseModel
 
 
+class CLEMAlignAndMergeParameters(BaseModel):
+    crop_to_n_frames: Optional[int] = 50
+    align_self: Literal["enabled", ""] = "enabled"
+    flatten: Literal["mean", "min", "max", ""] = "mean"
+    align_across: Literal["enabled", ""] = "enabled"
+
+
+default_clem_align_and_merge_parameters = CLEMAlignAndMergeParameters()
+
+
 class SPAParameters(BaseModel):
     nr_iter_2d: int = 25
     nr_iter_3d: int = 25

src/murfey/workflows/clem/align_and_merge.py

@@ -25,9 +25,10 @@ def submit_cluster_request(
     images: list[Path],
     metadata: Path,
     # Optional processing parameters
-    align_self: Optional[str] = None,
-    flatten: Optional[Literal["min", "max", "mean"]] = "mean",
-    align_across: Optional[str] = None,
+    crop_to_n_frames: Optional[int] = None,
+    align_self: Literal["enabled", ""] = "",
+    flatten: Literal["mean", "min", "max", ""] = "mean",
+    align_across: Literal["enabled", ""] = "",
     # Optional session parameters
     messenger: Optional[TransportManager] = None,
 ):
@@ -64,6 +65,7 @@ def submit_cluster_request(
                 "series_name": series_name,
                 "images": [str(file) for file in images],
                 "metadata": str(metadata),
+                "crop_to_n_frames": crop_to_n_frames,
                 "align_self": align_self,
                 "flatten": flatten,
                 "align_across": align_across,

src/murfey/workflows/clem/register_preprocessing_results.py

@@ -25,6 +25,9 @@
     CLEMTIFFFile,
 )
 from murfey.util.db import Session as MurfeySession
+from murfey.util.processing_params import (
+    default_clem_align_and_merge_parameters as processing_params,
+)
 from murfey.workflows.clem import get_db_entry
 from murfey.workflows.clem.align_and_merge import submit_cluster_request
 
@@ -187,9 +190,10 @@ def register_lif_preprocessing_result(
             series_name=result.series_name,
             images=image_stacks,
             metadata=result.metadata,
-            align_self=None,
-            flatten="mean",
-            align_across=None,
+            crop_to_n_frames=processing_params.crop_to_n_frames,
+            align_self=processing_params.align_self,
+            flatten=processing_params.flatten,
+            align_across=processing_params.align_across,
             messenger=_transport_object,
         )
         if cluster_response is False:
@@ -369,9 +373,10 @@ def register_tiff_preprocessing_result(
             series_name=result.series_name,
             images=image_stacks,
             metadata=result.metadata,
-            align_self=None,
-            flatten="mean",
-            align_across=None,
+            crop_to_n_frames=processing_params.crop_to_n_frames,
+            align_self=processing_params.align_self,
+            flatten=processing_params.flatten,
+            align_across=processing_params.align_across,
             messenger=_transport_object,
         )
         if cluster_response is False: