Merged recent changes from 'main' branch

Author: tieneupin

.bumpclient.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.15.1"
+current_version = "0.15.3"
 commit = true
 tag = false
 

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.15.1"
+current_version = "0.15.3"
 commit = true
 tag = true
 

README.md

@@ -2,7 +2,7 @@
 
 A transporter for data from Diamond eBIC microscope and detector machines onto the Diamond network.
 
-### Who is Murfey?
+## Who is Murfey?
 
 Murfey, the package, is named after [Eliza Murfey, the inventor](https://nationalrrmuseum.org/blog/mother-of-invention-women-railroad-innovators/):
 
@@ -11,38 +11,38 @@ Murfey, the package, is named after [Eliza Murfey, the inventor](https://nationa
 > it was Murfey who designed the packings that would lubricate the axles with oil, aiding
 > in the reduction of derailments caused by seized axles and bearings.
 
-### How do I set up a development environment?
+## How do I set up a development environment?
 
 We suggest you start with your favourite virtual environment (mamba/conda/python virtualenv/...),
 then install using the following command.
 
-#### From Git
-
-```bash
+```text
 $ git clone [email protected]:DiamondLightSource/python-murfey.git
 $ cd python-murfey
 $ pip install -e .[client,server,developer]
 ```
 
 The packages included under the `[developer]` installation key contain some helpful tools to aid you with developing Murfey further:
 
+- `bump-my-version` - Simplifies version control.
 - `ipykernel` - Enables interactive code development via Jupyter Notebooks.
 - `pre-commit` - Allows for the installation and running of hooks to help with linting, formatting, and type checking your code.
 - `pytest` - Used in conjunction with test functions to evaluate the reliability of your code.
-- `bump2version` - A nice little script to simplify version control.
+
+Instructions for setting up the database for Murfey to register files to can be found [here](src/murfey/server/MURFEY_DB.md).
 
 Finally, you may want to set up an ISPyB mock database server and a Zocalo
 development environment. The instructions for this are out of scope here.
 
 You can then start the Murfey server with
 
-```bash
+```text
 $ murfey.server
 ```
 
 and connect the client with
 
-```bash
+```text
 $ murfey --server http://127.0.0.1:8000
 ```
 

pyproject.toml

@@ -7,7 +7,7 @@ requires = [
 
 [project]
 name = "murfey"
-version = "0.15.1"
+version = "0.15.3"
 description = "Client-Server architecture hauling Cryo-EM data"
 readme = "README.md"
 keywords = [
@@ -30,6 +30,7 @@ classifiers = [
     "Programming Language :: Python :: 3.12",
 ]
 dependencies = [
+    "backports.entry_points_selectable",
     "defusedxml", # For safely parsing XML files
     "pydantic<2", # Locked to <2 by zocalo
     "requests",
@@ -55,7 +56,6 @@ developer = [
 server = [
     # "matplotlib", # For visual statistical analysis of images
     "aiohttp",
-    "backports.entry_points_selectable",
     "cryptography",
     "fastapi[standard]",
     "ispyb", # Responsible for setting requirements for SQLAlchemy and mysql-connector-python; v10.0.0: sqlalchemy <2, mysql-connector-python >=8.0.32
@@ -96,9 +96,13 @@ murfey = "murfey.client:run"
 "murfey.transfer" = "murfey.cli.transfer:run"
 [project.entry-points."murfey.auth.token_validation"]
 "password" = "murfey.server.api.auth:password_token_validation"
+[project.entry-points."murfey.config.extraction"]
+"murfey_machine" = "murfey.util.config:get_extended_machine_config"
 [project.entry-points."murfey.workflows"]
-"lif_to_stack" = "murfey.workflows.lif_to_stack:zocalo_cluster_request"
-"tiff_to_stack" = "murfey.workflows.tiff_to_stack:zocalo_cluster_request"
+"process_raw_lifs" = "murfey.workflows.clem.process_raw_lifs:zocalo_cluster_request"
+"process_raw_tiffs" = "murfey.workflows.clem.process_raw_tiffs:zocalo_cluster_request"
+"register_lif_preprocessing_result" = "murfey.workflows.clem.register_preprocessing_results:register_lif_preprocessing_result"
+"register_tiff_preprocessing_result" = "murfey.workflows.clem.register_preprocessing_results:register_tiff_preprocessing_result"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

src/murfey/__init__.py

@@ -1,4 +1,4 @@
 from __future__ import annotations
 
-__version__ = "0.15.1"
-__supported_client_version__ = "0.15.1"
+__version__ = "0.15.3"
+__supported_client_version__ = "0.15.3"

src/murfey/client/__init__.py

@@ -65,6 +65,31 @@ def read_config() -> configparser.ConfigParser:
 requests.delete = partial(requests.delete, headers={"Authorization": f"Bearer {token}"})
 
 
+def write_config(config: configparser.ConfigParser):
+    mcch = os.environ.get("MURFEY_CLIENT_CONFIG_HOME")
+    murfey_client_config_home = Path(mcch) if mcch else Path.home()
+    with open(murfey_client_config_home / ".murfey", "w") as configfile:
+        config.write(configfile)
+
+
+def main_loop(
+    source_watchers: List[murfey.client.watchdir.DirWatcher],
+    appearance_time: float,
+    transfer_all: bool,
+):
+    log.info(
+        f"Murfey {murfey.__version__} on Python {'.'.join(map(str, sys.version_info[0:3]))} entering main loop"
+    )
+    if appearance_time > 0:
+        modification_time: float | None = time.time() - appearance_time * 3600
+    else:
+        modification_time = None
+    while True:
+        for sw in source_watchers:
+            sw.scan(modification_time=modification_time, transfer_all=transfer_all)
+        time.sleep(15)
+
+
 def _enable_webbrowser_in_cygwin():
     """Helper function to make webbrowser.open() work in CygWin"""
     if "cygwin" in platform.system().lower() and shutil.which("cygstart"):
@@ -288,7 +313,9 @@ def run():
 
     status_bar = StatusBar()
 
-    machine_data = requests.get(f"{murfey_url.geturl()}/machine").json()
+    machine_data = requests.get(
+        f"{murfey_url.geturl()}/instruments/{instrument_name}/machine"
+    ).json()
     gain_ref: Path | None = None
 
     instance_environment = MurfeyInstanceEnvironment(
@@ -323,28 +350,3 @@ def run():
     )
     app.run()
     rich_handler.redirect = False
-
-
-def main_loop(
-    source_watchers: List[murfey.client.watchdir.DirWatcher],
-    appearance_time: float,
-    transfer_all: bool,
-):
-    log.info(
-        f"Murfey {murfey.__version__} on Python {'.'.join(map(str, sys.version_info[0:3]))} entering main loop"
-    )
-    if appearance_time > 0:
-        modification_time: float | None = time.time() - appearance_time * 3600
-    else:
-        modification_time = None
-    while True:
-        for sw in source_watchers:
-            sw.scan(modification_time=modification_time, transfer_all=transfer_all)
-        time.sleep(15)
-
-
-def write_config(config: configparser.ConfigParser):
-    mcch = os.environ.get("MURFEY_CLIENT_CONFIG_HOME")
-    murfey_client_config_home = Path(mcch) if mcch else Path.home()
-    with open(murfey_client_config_home / ".murfey", "w") as configfile:
-        config.write(configfile)

src/murfey/client/context.py

@@ -4,7 +4,7 @@
 from pathlib import Path
 from typing import Any, Dict, List, NamedTuple
 
-import importlib_metadata
+from backports.entry_points_selectable import entry_points
 
 from murfey.client.instance_environment import MurfeyInstanceEnvironment
 
@@ -41,7 +41,8 @@ def __init__(self, name: str, acquisition_software: str):
         self.name = name
 
     def post_transfer(self, transferred_file: Path, role: str = "", **kwargs):
-        for h in importlib_metadata.entry_points(group="murfey.post_transfer_hooks"):
+        # Search external packages for additional hooks to include in Murfey
+        for h in entry_points(group="murfey.post_transfer_hooks"):
             if h.name == self.name:
                 h.load()(transferred_file, role=role, **kwargs)
 

src/murfey/client/contexts/clem.py

@@ -386,7 +386,7 @@ def process_lif_file(
 
         try:
             # Construct the URL to post the request to
-            url = f"{str(environment.url.geturl())}/sessions/{environment.murfey_session}/lif_to_stack?lif_file={quote(str(lif_file), safe='')}"
+            url = f"{str(environment.url.geturl())}/sessions/{environment.murfey_session}/clem/preprocessing/process_raw_lifs?lif_file={quote(str(lif_file), safe='')}"
             # Validate
             if not url:
                 logger.error(
@@ -442,7 +442,7 @@ def process_tiff_series(
 
         try:
             # Construct URL for Murfey server to communicate with
-            url = f"{str(environment.url.geturl())}/sessions/{environment.murfey_session}/tiff_to_stack"
+            url = f"{str(environment.url.geturl())}/sessions/{environment.murfey_session}/clem/preprocessing/process_raw_tiffs"
             if not url:
                 logger.error(
                     "URL could not be constructed from the environment and file path"