Merged recent changes from 'main' branch

tieneupin · tieneupin · commit 106a91f99979 · 2025-03-19T14:21:11.000Z
diff --git a/.bumpclient.toml b/.bumpclient.toml
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.17.0"
+current_version = "0.17.1"
 commit = true
 tag = false
 
diff --git a/.bumpversion.toml b/.bumpversion.toml
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.17.0"
+current_version = "0.17.1"
 commit = true
 tag = true
 
diff --git a/Helm/Chart.yaml b/Helm/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v1
 name: murfey-services
 description: Umbrella Helm chart for deploying the servers and daemons needed to enable Murfey to transfer and process data
-version: 0.17.0
+version: 0.17.1
 dependencies:
   - name: murfey-instrument-server-clem
   - name: murfey-instrument-server-tem
diff --git a/Helm/charts/murfey-instrument-server-clem/Chart.yaml b/Helm/charts/murfey-instrument-server-clem/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v1
 name: murfey-instrument-server-clem
 description: Helm chart for deploying a Murfey instrument server, which executes orders to detect, modify, and transfer files on the instrument PC, and notifies the backend server about transferred files
-version: 0.17.0
+version: 0.17.1
diff --git a/Helm/charts/murfey-rsync/Chart.yaml b/Helm/charts/murfey-rsync/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v1
 name: murfey-rsync
 description: Helm chart for deploying an rsync daemon, which is responsible for executing the transfer of files from the client storage directory to the server storage system
-version: 0.17.0
+version: 0.17.1
diff --git a/Helm/charts/murfey-server/Chart.yaml b/Helm/charts/murfey-server/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v1
 name: murfey-server
 description: Helm chart for deploying a Murfey backend server, which is responsible for orchestrating the data transfer and processing workflow between the client PC and the storage system
-version: 0.17.0
+version: 0.17.1
diff --git a/pyproject.toml b/pyproject.toml
@@ -7,7 +7,7 @@ requires = [
 
 [project]
 name = "murfey"
-version = "0.17.0"
+version = "0.17.1"
 description = "Client-Server architecture hauling Cryo-EM data"
 readme = "README.md"
 keywords = [
diff --git a/src/murfey/__init__.py b/src/murfey/__init__.py
@@ -1,4 +1,4 @@
 from __future__ import annotations
 
-__version__ = "0.17.0"
-__supported_client_version__ = "0.17.0"
+__version__ = "0.17.1"
+__supported_client_version__ = "0.17.1"
diff --git a/src/murfey/client/contexts/spa.py b/src/murfey/client/contexts/spa.py
@@ -431,12 +431,15 @@ def _position_analysis(
                 grid_square_metadata_file,
                 grid_square,
             )
+            metadata_source_as_str = (
+                "/".join(source.parts[:-2])
+                + f"/{environment.visit}/"
+                + source.parts[-2]
+            )
             metadata_source = Path(
-                (
-                    "/".join(source.parts[:-2])
-                    + f"/{environment.visit}/"
-                    + source.parts[-2]
-                )[1:]
+                metadata_source_as_str[1:]
+                if metadata_source_as_str.startswith("//")
+                else metadata_source_as_str
             )
             image_path = (
                 _file_transferred_to(environment, metadata_source, Path(gs.image))
@@ -471,12 +474,15 @@ def _position_analysis(
                     foil_hole,
                     grid_square,
                 )
+                metadata_source_as_str = (
+                    "/".join(source.parts[:-2])
+                    + f"/{environment.visit}/"
+                    + source.parts[-2]
+                )
                 metadata_source = Path(
-                    (
-                        "/".join(source.parts[:-2])
-                        + f"/{environment.visit}/"
-                        + source.parts[-2]
-                    )[1:]
+                    metadata_source_as_str[1:]
+                    if metadata_source_as_str.startswith("//")
+                    else metadata_source_as_str
                 )
                 image_path = (
                     _file_transferred_to(environment, metadata_source, Path(fh.image))
diff --git a/src/murfey/client/contexts/spa_metadata.py b/src/murfey/client/contexts/spa_metadata.py
@@ -44,7 +44,7 @@ def _foil_hole_positions(xml_path: Path, grid_square: int) -> Dict[str, FoilHole
             required_key = key
             break
     if not required_key:
-        logger.warning(f"Required key not found for {str(xml_path)}")
+        logger.info(f"Required key not found for {str(xml_path)}")
         return {}
     foil_holes = {}
     for fh_block in serialization_array[required_key]:
@@ -163,10 +163,13 @@ def post_transfer(
                     atlas=Path(partial_path), sample=sample
                 )
                 url = f"{str(environment.url.geturl())}/visits/{environment.visit}/{environment.murfey_session}/register_data_collection_group"
-                dcg_search_dir = "/" + "/".join(
-                    p
-                    for p in transferred_file.parent.parts[1:]
-                    if p != environment.visit
+                dcg_search_dir = "/".join(
+                    p for p in transferred_file.parent.parts if p != environment.visit
+                )
+                dcg_search_dir = (
+                    dcg_search_dir[1:]
+                    if dcg_search_dir.startswith("//")
+                    else dcg_search_dir
                 )
                 dcg_images_dirs = sorted(
                     Path(dcg_search_dir).glob("Images-Disc*"),
@@ -215,11 +218,16 @@ def post_transfer(
         ):
             # Make sure we have a data collection group before trying to register grid square
             url = f"{str(environment.url.geturl())}/visits/{environment.visit}/{environment.murfey_session}/register_data_collection_group"
-            dcg_search_dir = "/" + "/".join(
+            dcg_search_dir = "/".join(
                 p
-                for p in transferred_file.parent.parent.parts[1:]
+                for p in transferred_file.parent.parent.parts
                 if p != environment.visit
             )
+            dcg_search_dir = (
+                dcg_search_dir[1:]
+                if dcg_search_dir.startswith("//")
+                else dcg_search_dir
+            )
             dcg_images_dirs = sorted(
                 Path(dcg_search_dir).glob("Images-Disc*"),
                 key=lambda x: x.stat().st_ctime,
@@ -241,6 +249,8 @@ def post_transfer(
             )
             fh_positions = _foil_hole_positions(transferred_file, int(gs_name))
             source = _get_source(transferred_file, environment=environment)
+            if source is None:
+                return None
             visitless_source_search_dir = str(source).replace(
                 f"/{environment.visit}", ""
             )
@@ -261,12 +271,15 @@ def post_transfer(
                     transferred_file,
                     int(gs_name),
                 )
+                metadata_source_as_str = (
+                    "/".join(source.parts[:-2])
+                    + f"/{environment.visit}/"
+                    + source.parts[-2]
+                )
                 metadata_source = Path(
-                    (
-                        "/".join(Path(visitless_source).parts[:-2])
-                        + f"/{environment.visit}/"
-                        + Path(visitless_source).parts[-2]
-                    )[1:]
+                    metadata_source_as_str[1:]
+                    if metadata_source_as_str.startswith("//")
+                    else metadata_source_as_str
                 )
                 image_path = (
                     _file_transferred_to(
diff --git a/src/murfey/client/multigrid_control.py b/src/murfey/client/multigrid_control.py
@@ -143,7 +143,7 @@ def _start_rsyncer_multigrid(
             source,
             destination,
             force_metadata=self.processing_enabled,
-            analyse=not extra_directory and use_suggested_path and analyse,
+            analyse=analyse,
             remove_files=remove_files,
             tag=tag,
             limited=limited,
diff --git a/src/murfey/instrument_server/__init__.py b/src/murfey/instrument_server/__init__.py
@@ -5,6 +5,8 @@
 from rich.logging import RichHandler
 
 import murfey
+from murfey.client import read_config
+from murfey.client.customlogging import CustomHandler
 from murfey.util import LogFilter
 
 logger = logging.getLogger("murfey.instrument_server")
@@ -33,6 +35,14 @@ def run():
     logging.getLogger("fastapi").addHandler(rich_handler)
     logging.getLogger("uvicorn").addHandler(rich_handler)
 
+    ws = murfey.client.websocket.WSApp(
+        server=read_config()["Murfey"].get("server"),
+        id=0,
+    )
+
+    handler = CustomHandler(ws.send)
+    logging.getLogger().addHandler(handler)
+
     logger.info(
         f"Starting Murfey server version {murfey.__version__}, listening on {args.host}:{args.port}"
     )
diff --git a/src/murfey/instrument_server/api.py b/src/murfey/instrument_server/api.py
@@ -21,7 +21,7 @@
 from murfey.client.multigrid_control import MultigridController
 from murfey.client.rsync import RSyncer
 from murfey.client.watchdir_multigrid import MultigridDirWatcher
-from murfey.util import sanitise, sanitise_nonpath, secure_path
+from murfey.util import posix_path, sanitise, sanitise_nonpath, secure_path
 from murfey.util.instrument_models import MultigridWatcherSpec
 from murfey.util.models import File, Token
 
@@ -291,7 +291,7 @@ def upload_gain_reference(
     ).json()
     cmd = [
         "rsync",
-        safe_gain_path,
+        posix_path(Path(safe_gain_path)),
         f"{urlparse(_get_murfey_url(), allow_fragments=False).hostname}::{machine_config.get('rsync_module', 'data')}/{safe_visit_path}/{safe_destination_dir}/{secure_filename(gain_reference.gain_path.name)}",
     ]
     gain_rsync = subprocess.run(cmd)
diff --git a/src/murfey/server/api/__init__.py b/src/murfey/server/api/__init__.py
@@ -59,6 +59,7 @@
     DataCollectionGroup,
     FoilHole,
     GridSquare,
+    MagnificationLookup,
     Movie,
     PreprocessStash,
     ProcessingJob,
@@ -166,6 +167,27 @@ def machine_info_by_name(instrument_name: str) -> Optional[MachineConfig]:
     return None
 
 
+@router.get("/mag_table/")
+def get_mag_table(db=murfey_db) -> List[MagnificationLookup]:
+    return db.exec(select(MagnificationLookup)).all()
+
+
+@router.post("/mag_table/")
+def add_to_mag_table(rows: List[MagnificationLookup], db=murfey_db):
+    for r in rows:
+        db.add(r)
+    db.commit()
+
+
+@router.delete("/mag_table/{mag}")
+def remove_mag_table_row(mag: int, db=murfey_db):
+    row = db.exec(
+        select(MagnificationLookup).where(MagnificationLookup.magnification == mag)
+    ).one()
+    db.delete(row)
+    db.commit()
+
+
 @router.get("/instruments/{instrument_name}/instrument_name")
 def get_instrument_display_name(instrument_name: str) -> str:
     machine_config = get_machine_config(instrument_name=instrument_name)[
@@ -396,6 +418,61 @@ def increment_rsync_transferred_files_prometheus(
     ).inc(rsyncer_info.data_bytes)
 
 
+class ProcessingDetails(BaseModel):
+    data_collection_group: DataCollectionGroup
+    data_collections: List[DataCollection]
+    processing_jobs: List[ProcessingJob]
+    relion_params: SPARelionParameters
+    feedback_params: SPAFeedbackParameters
+
+
+@router.get("/sessions/{session_id}/spa_processing_parameters")
+def get_spa_proc_param_details(
+    session_id: MurfeySessionID, db=murfey_db
+) -> Optional[List[ProcessingDetails]]:
+    params = db.exec(
+        select(
+            DataCollectionGroup,
+            DataCollection,
+            ProcessingJob,
+            SPARelionParameters,
+            SPAFeedbackParameters,
+        )
+        .where(DataCollectionGroup.session_id == session_id)
+        .where(DataCollectionGroup.id == DataCollection.dcg_id)
+        .where(DataCollection.id == ProcessingJob.dc_id)
+        .where(SPARelionParameters.pj_id == ProcessingJob.id)
+        .where(SPAFeedbackParameters.pj_id == ProcessingJob.id)
+    ).all()
+    if not params:
+        return None
+    unique_dcg_indices = []
+    dcg_ids = []
+    for i, p in enumerate(params):
+        if p[0].id not in dcg_ids:
+            dcg_ids.append(p[0].id)
+            unique_dcg_indices.append(i)
+
+    def _parse(ps, i, dcg_id):
+        res = []
+        for p in ps:
+            if p[0].id == dcg_id:
+                if p[i] not in res:
+                    res.append(p[i])
+        return res
+
+    return [
+        ProcessingDetails(
+            data_collection_group=params[i][0],
+            data_collections=_parse(params, 1, d),
+            processing_jobs=_parse(params, 2, d),
+            relion_params=_parse(params, 3, d)[0],
+            feedback_params=_parse(params, 4, d)[0],
+        )
+        for i, d in zip(unique_dcg_indices, dcg_ids)
+    ]
+
+
 @router.post("/sessions/{session_id}/spa_processing_parameters")
 def register_spa_proc_params(
     session_id: MurfeySessionID, proc_params: ProcessingParametersSPA, db=murfey_db
diff --git a/src/murfey/util/__init__.py b/src/murfey/util/__init__.py
@@ -74,9 +74,14 @@ def sanitise_nonpath(in_string: str) -> str:
 
 def secure_path(in_path: Path, keep_spaces: bool = False) -> Path:
     if keep_spaces:
-        secured_parts = [
-            secure_filename(p) if " " not in p else p for p in in_path.parts
-        ]
+        secured_parts = []
+        for p, part in enumerate(in_path.parts):
+            if " " in part:
+                secured_parts.append(part)
+            elif ":" in part and not p:
+                secured_parts.append(secure_filename(part) + ":")
+            else:
+                secured_parts.append(secure_filename(part))
     else:
         secured_parts = [
             (
diff --git a/src/murfey/workflows/notifications/__init__.py b/src/murfey/workflows/notifications/__init__.py
@@ -35,15 +35,16 @@ def notification_setup(
                 enp.max_value = v[1]
                 break
         else:
-            new_notification_parameters.append(
-                NotificationParameter(
-                    dcg_id=dcgid,
-                    name=k,
-                    min_value=v[0],
-                    max_value=v[1],
-                    num_instances_since_triggered=num_instances_between_triggers,
+            if v[0] is not None and v[1] is not None:
+                new_notification_parameters.append(
+                    NotificationParameter(
+                        dcg_id=dcgid,
+                        name=k,
+                        min_value=v[0],
+                        max_value=v[1],
+                        num_instances_since_triggered=num_instances_between_triggers,
+                    )
                 )
-            )
     murfey_db.add_all(existing_notification_parameters + new_notification_parameters)
     murfey_db.commit()
     murfey_db.close()
