Merge branch 'main' into tomo-metadata

stephen-riggs · stephen-riggs · commit 82ee1b16d67c · 2025-07-02T16:31:23.000+01:00
diff --git a/.bumpclient.toml b/.bumpclient.toml
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.18.2"
+current_version = "0.19.0"
 commit = true
 tag = false
 
diff --git a/.bumpversion.toml b/.bumpversion.toml
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.18.2"
+current_version = "0.19.0"
 commit = true
 tag = true
 
diff --git a/Helm/Chart.yaml b/Helm/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v1
 name: murfey-services
 description: Umbrella Helm chart for deploying the servers and daemons needed to enable Murfey to transfer and process data
-version: 0.18.2
+version: 0.19.0
 dependencies:
   - name: murfey-instrument-server-clem
   - name: murfey-instrument-server-tem
diff --git a/Helm/charts/murfey-instrument-server-clem/Chart.yaml b/Helm/charts/murfey-instrument-server-clem/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v1
 name: murfey-instrument-server-clem
 description: Helm chart for deploying a Murfey instrument server, which executes orders to detect, modify, and transfer files on the instrument PC, and notifies the backend server about transferred files
-version: 0.18.2
+version: 0.19.0
diff --git a/Helm/charts/murfey-instrument-server-tem/Chart.yaml b/Helm/charts/murfey-instrument-server-tem/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v1
 name: murfey-instrument-server-tem
 description: Helm chart for deploying a Murfey instrument server, which executes orders to detect, modify, and transfer files on the instrument PC, and notifies the backend server about transferred files
-version: 0.18.2
+version: 0.19.0
diff --git a/Helm/charts/murfey-rsync/Chart.yaml b/Helm/charts/murfey-rsync/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v1
 name: murfey-rsync
 description: Helm chart for deploying an rsync daemon, which is responsible for executing the transfer of files from the client storage directory to the server storage system
-version: 0.18.2
+version: 0.19.0
diff --git a/Helm/charts/murfey-server/Chart.yaml b/Helm/charts/murfey-server/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v1
 name: murfey-server
 description: Helm chart for deploying a Murfey backend server, which is responsible for orchestrating the data transfer and processing workflow between the client PC and the storage system
-version: 0.18.2
+version: 0.19.0
diff --git a/pyproject.toml b/pyproject.toml
@@ -7,7 +7,7 @@ requires = [
 
 [project]
 name = "murfey"
-version = "0.18.2"
+version = "0.19.0"
 description = "Client-Server architecture hauling Cryo-EM data"
 readme = "README.md"
 keywords = [
@@ -32,7 +32,8 @@ classifiers = [
 dependencies = [
     "backports.entry_points_selectable",
     "defusedxml", # For safely parsing XML files
-    "pydantic<2", # Locked to <2 by cygwin terminal
+    "pydantic>=2",
+    "pydantic-settings",
     "requests",
     "rich",
     "werkzeug",
@@ -47,7 +48,7 @@ client = [
     "websocket-client",
 ]
 developer = [
-    "bump-my-version<0.11.0", # Version control
+    "bump-my-version", # Version control
     "ipykernel", # Enable interactive coding with VS Code and Jupyter Notebook
     "pre-commit", # Formatting, linting, type checking, etc.
     "pytest", # Test code functionality
@@ -61,7 +62,7 @@ server = [
     "aiohttp",
     "cryptography",
     "fastapi[standard]",
-    "ispyb", # Responsible for setting requirements for SQLAlchemy and mysql-connector-python; v10.0.0: sqlalchemy <2, mysql-connector-python >=8.0.32
+    "ispyb>=10.2.4", # Responsible for setting requirements for SQLAlchemy and mysql-connector-python;
     "jinja2",
     "mrcfile",
     "numpy<2",
@@ -73,7 +74,7 @@ server = [
     "sqlalchemy[postgresql]", # Add as explicit dependency
     "sqlmodel",
     "stomp-py<=8.1.0", # 8.1.1 (released 2024-04-06) doesn't work with our project
-    "zocalo",
+    "zocalo>=1",
 ]
 [project.urls]
 Bug-Tracker = "https://github.com/DiamondLightSource/python-murfey/issues"
diff --git a/src/murfey/__init__.py b/src/murfey/__init__.py
@@ -1,4 +1,4 @@
 from __future__ import annotations
 
-__version__ = "0.18.2"
-__supported_client_version__ = "0.18.2"
+__version__ = "0.19.0"
+__supported_client_version__ = "0.19.0"
diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py
@@ -261,7 +261,7 @@ def _analyse(self):
                     valid_extension = self._find_extension(transferred_file)
                     if not valid_extension:
                         logger.error(f"No extension found for {transferred_file}")
-                        break
+                        continue
                     found = self._find_context(transferred_file)
                     if not found:
                         logger.debug(
@@ -332,7 +332,7 @@ def _analyse(self):
                     valid_extension = self._find_extension(transferred_file)
                     if not valid_extension:
                         logger.error(f"No extension found for {transferred_file}")
-                        break
+                        continue
                     if self._extension:
                         logger.info(
                             f"Extension found successfully for {transferred_file}"
diff --git a/src/murfey/client/contexts/spa.py b/src/murfey/client/contexts/spa.py
@@ -506,7 +506,8 @@ def post_transfer(
                         except Exception as e:
                             # try to continue if position information gathering fails so that movie is processed anyway
                             logger.warning(
-                                f"Unable to register foil hole for {str(file_transferred_to)}. Exception: {str(e)}"
+                                f"Unable to register foil hole for {str(file_transferred_to)}. Exception: {str(e)}",
+                                exc_info=True,
                             )
                             foil_hole = None
 
diff --git a/src/murfey/client/contexts/tomo.py b/src/murfey/client/contexts/tomo.py
@@ -173,7 +173,7 @@ def register_tomography_data_collections(
                         )
 
         except Exception as e:
-            logger.error(f"ERROR {e}, {self.data_collection_parameters}")
+            logger.error(f"ERROR {e}, {self.data_collection_parameters}", exc_info=True)
 
     def _file_transferred_to(
         self, environment: MurfeyInstanceEnvironment, source: Path, file_path: Path
@@ -533,7 +533,9 @@ def gather_metadata(
                     try:
                         for_parsing = xml.read()
                     except Exception:
-                        logger.warning(f"Failed to parse file {metadata_file}")
+                        logger.warning(
+                            f"Failed to parse file {metadata_file}", exc_info=True
+                        )
                         return OrderedDict({})
                     data = xmltodict.parse(for_parsing)
                 try:
@@ -628,7 +630,9 @@ def gather_metadata(
                     / int(mdoc_metadata["num_eer_frames"])
                 )
         except Exception as e:
-            logger.error(f"Exception encountered in metadata gathering: {str(e)}")
+            logger.error(
+                f"Exception encountered in metadata gathering: {str(e)}", exc_info=True
+            )
             return OrderedDict({})
 
         return mdoc_metadata
diff --git a/src/murfey/client/instance_environment.py b/src/murfey/client/instance_environment.py
@@ -8,7 +8,7 @@
 from typing import Dict, List, NamedTuple, Optional
 from urllib.parse import ParseResult
 
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 from murfey.client.watchdir import DirWatcher
 
@@ -56,9 +56,7 @@ class MurfeyInstanceEnvironment(BaseModel):
     samples: Dict[Path, SampleInfo] = {}
     rsync_url: str = ""
 
-    class Config:
-        validate_assignment: bool = True
-        arbitrary_types_allowed: bool = True
+    model_config = ConfigDict(arbitrary_types_allowed=True)
 
     def clear(self):
         self.sources = []
diff --git a/src/murfey/client/tui/screens.py b/src/murfey/client/tui/screens.py
@@ -196,7 +196,7 @@ def validate_form(form: dict, model: BaseModel) -> bool:
     try:
         convert = lambda x: None if x == "None" else x
         validated = model(**{k: convert(v) for k, v in form.items()})
-        log.info(validated.dict())
+        log.info(validated.model_dump())
         return True
     except (AttributeError, ValidationError) as e:
         log.warning(f"Form validation failed: {str(e)}")
diff --git a/src/murfey/client/websocket.py b/src/murfey/client/websocket.py
@@ -25,7 +25,7 @@ def __init__(
     ):
         self.id = str(uuid.uuid4()) if id is None else id
         log.info(f"Opening websocket connection for Client {self.id}")
-        websocket.enableTrace(True)
+        websocket.enableTrace(False)
 
         # Parse server URL and get proxy path used, if any
         url = urllib.parse.urlparse(server)._replace(
diff --git a/src/murfey/instrument_server/api.py b/src/murfey/instrument_server/api.py
@@ -335,14 +335,14 @@ def register_processing_parameters(
     session_id: MurfeySessionID, proc_param_block: ProcessingParameterBlock
 ):
     data_collection_parameters[proc_param_block.label] = {}
-    for k, v in proc_param_block.params.dict().items():
+    for k, v in proc_param_block.params.model_dump().items():
         if v is not None:
             data_collection_parameters[proc_param_block.label][k] = v
     if controllers.get(session_id):
         controllers[session_id].data_collection_parameters.update(
             data_collection_parameters[proc_param_block.label]
         )
-        for k, v in proc_param_block.params.dict().items():
+        for k, v in proc_param_block.params.model_dump().items():
             if v is not None and hasattr(controllers[session_id]._environment, k):
                 setattr(controllers[session_id]._environment, k, v)
     return {"success": True}
diff --git a/src/murfey/server/api/clem.py b/src/murfey/server/api/clem.py
@@ -10,7 +10,7 @@
 
 from backports.entry_points_selectable import entry_points
 from fastapi import APIRouter
-from pydantic import BaseModel, validator
+from pydantic import BaseModel, field_validator
 from sqlalchemy.exc import NoResultFound
 from sqlmodel import Session, select
 
@@ -820,10 +820,8 @@ class AlignAndMergeParams(BaseModel):
     flatten: Literal["mean", "min", "max", ""] = ""
     align_across: Literal["enabled", ""] = ""
 
-    @validator(
-        "images",
-        pre=True,
-    )
+    @field_validator("images", mode="before")
+    @classmethod
     def parse_stringified_list(cls, value):
         if isinstance(value, str):
             try:
diff --git a/src/murfey/server/api/prometheus.py b/src/murfey/server/api/prometheus.py
@@ -39,7 +39,7 @@ def increment_rsync_file_count(
         logger.error(
             f"Failed to find rsync instance for visit {sanitise(visit_name)} "
             "with the following properties: \n"
-            f"{rsyncer_info.dict()}",
+            f"{rsyncer_info.model_dump()}",
             exc_info=True,
         )
         return None
diff --git a/src/murfey/server/api/session_control.py b/src/murfey/server/api/session_control.py
@@ -102,7 +102,7 @@ def get_current_visits(instrument_name: str, db=ispyb_db):
 
 
 class SessionInfo(BaseModel):
-    session_id: Optional[int]
+    session_id: Optional[int] = None
     session_name: str = ""
     rescale: bool = True
 
diff --git a/src/murfey/server/api/workflow.py b/src/murfey/server/api/workflow.py
@@ -88,8 +88,6 @@ class DCGroupParameters(BaseModel):
     atlas: str = ""
     sample: Optional[int] = None
     atlas_pixel_size: float = 0
-    atlas_size_x: int = 0
-    atlas_size_y: int = 0
 
 
 @router.post("/visits/{visit_name}/{session_id}/register_data_collection_group")
@@ -124,8 +122,6 @@ def register_dc_group(
                         "atlas": dcg_params.atlas,
                         "sample": dcg_params.sample,
                         "atlas_pixel_size": dcg_params.atlas_pixel_size,
-                        "atlas_size_x": dcg_params.atlas_size_x,
-                        "atlas_size_y": dcg_params.atlas_size_y,
                         "dcgid": dcg_murfey[0].id,
                         "session_id": session_id,
                     },
@@ -368,13 +364,13 @@ class SPAProcessFile(BaseModel):
     tag: str
     path: str
     description: str
-    processing_job: Optional[int]
-    data_collection_id: Optional[int]
+    processing_job: Optional[int] = None
+    data_collection_id: Optional[int] = None
     image_number: int
-    autoproc_program_id: Optional[int]
-    foil_hole_id: Optional[int]
-    pixel_size: Optional[float]
-    dose_per_frame: Optional[float]
+    autoproc_program_id: Optional[int] = None
+    foil_hole_id: Optional[int] = None
+    pixel_size: Optional[float] = None
+    dose_per_frame: Optional[float] = None
     mc_binning: Optional[int] = 1
     gain_ref: Optional[str] = None
     extract_downscale: bool = True
@@ -630,9 +626,9 @@ class TomoProcessFile(BaseModel):
     tag: str
     image_number: int
     pixel_size: float
-    dose_per_frame: Optional[float]
+    dose_per_frame: Optional[float] = None
     frame_count: int
-    tilt_axis: Optional[float]
+    tilt_axis: Optional[float] = None
     mc_uuid: Optional[int] = None
     voltage: float = 300
     mc_binning: int = 1
@@ -919,7 +915,7 @@ class Sample(BaseModel):
     sample_group_id: int
     sample_id: int
     subsample_id: int
-    image_path: Optional[Path]
+    image_path: Optional[Path] = None
 
 
 @correlative_router.get("/visit/{visit_name}/samples")
diff --git a/src/murfey/server/demo_api.py b/src/murfey/server/demo_api.py
@@ -13,32 +13,38 @@
 from fastapi.responses import FileResponse, HTMLResponse
 from ispyb.sqlalchemy import BLSession
 from PIL import Image
-from pydantic import BaseModel, BaseSettings
+from pydantic import BaseModel
+from pydantic_settings import BaseSettings
 from sqlalchemy import func
 from sqlmodel import select
 from werkzeug.utils import secure_filename
 
 import murfey.server.api.bootstrap
 import murfey.server.prometheus as prom
-from murfey.server import (
-    _flush_grid_square_records,
-    _murfey_id,
-    get_hostname,
-    get_microscope,
-    sanitise,
-    sanitise_path,
-)
-from murfey.server import shutdown as _shutdown
-from murfey.server import templates
-from murfey.server.api.auth import MurfeySessionID, validate_token
+from murfey.server.api import templates
+from murfey.server.api.auth import MurfeySessionIDFrontend as MurfeySessionID
+from murfey.server.api.auth import validate_token
 from murfey.server.api.session_info import Visit
 from murfey.server.api.workflow import (
     DCGroupParameters,
     DCParameters,
     ProcessingJobParameters,
 )
+from murfey.server.feedback import (
+    _flush_grid_square_records,
+    _murfey_id,
+    get_microscope,
+    sanitise,
+)
 from murfey.server.murfey_db import murfey_db
-from murfey.util.config import MachineConfig, from_file, security_from_file
+from murfey.server.run import shutdown as _shutdown
+from murfey.util import sanitise_path
+from murfey.util.config import (
+    MachineConfig,
+    from_file,
+    get_hostname,
+    security_from_file,
+)
 from murfey.util.db import (
     AutoProcProgram,
     ClientEnvironment,
diff --git a/src/murfey/server/main.py b/src/murfey/server/main.py
@@ -7,7 +7,7 @@
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.staticfiles import StaticFiles
 from prometheus_client import make_asgi_app
-from pydantic import BaseSettings
+from pydantic_settings import BaseSettings
 
 import murfey.server
 import murfey.server.api.auth
diff --git a/src/murfey/util/config.py b/src/murfey/util/config.py
diff --git a/src/murfey/util/models.py b/src/murfey/util/models.py
diff --git a/src/murfey/util/processing_params.py b/src/murfey/util/processing_params.py
diff --git a/src/murfey/util/rsync.py b/src/murfey/util/rsync.py
diff --git a/src/murfey/workflows/clem/register_align_and_merge_results.py b/src/murfey/workflows/clem/register_align_and_merge_results.py
diff --git a/src/murfey/workflows/clem/register_preprocessing_results.py b/src/murfey/workflows/clem/register_preprocessing_results.py
diff --git a/tests/cli/test_decrypt_password.py b/tests/cli/test_decrypt_password.py
diff --git a/tests/cli/test_generate_password.py b/tests/cli/test_generate_password.py
