Catch up main commits

Author: stephen-riggs

.bumpclient.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.15.3"
+current_version = "0.16.9"
 commit = true
 tag = false
 

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.15.3"
+current_version = "0.16.9"
 commit = true
 tag = true
 

.github/workflows/ci.yml

@@ -17,7 +17,7 @@ jobs:
         python-version: ["3.9", "3.10", "3.11"]
     services:
       mariadb:
-        image: mariadb:11.3.2 # released 2024-05-06
+        image: mariadb:11.6.2 # released 2024-05-06
         # Pulls image from DockerHub
         # Docker images: https://hub.docker.com/_/mariadb
         # Previous version(s):
@@ -29,6 +29,16 @@ jobs:
           - 3306:3306
         options: --health-cmd="healthcheck.sh --connect --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=3
 
+      postgres:
+        image: postgres:latest
+        env:
+          POSTGRES_DB: murfey_test_db
+          POSTGRES_PASSWORD: psql_pwd
+          POSTGRES_USER: psql_user
+        ports:
+          - 5432:5432
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+
     steps:
       - uses: actions/checkout@v4
       - name: Use Python ${{ matrix.python-version }}
@@ -57,7 +67,7 @@ jobs:
           docker run --detach --name rabbitmq -p 127.0.0.1:5672:5672 -p 127.0.0.1:15672:15672 test-rabbitmq
           docker container list -a
 
-      - name: Get database
+      - name: Get ispyb database
         uses: actions/download-artifact@v4
         with:
           name: database
@@ -74,7 +84,7 @@ jobs:
           mysql-version: "11.3"
           auto-start: false
 
-      - name: Set up test database
+      - name: Set up test ipsyb database
         run: |
           set -eu
           cp ".github/workflows/config/my.cnf" .my.cnf
@@ -103,12 +113,18 @@ jobs:
         run: wget -t 10 -w 1 http://127.0.0.1:15672 -O -
 
       - name: Run tests
+        env:
+          POSTGRES_HOST: localhost
+          POSTGRES_PORT: 5432
+          POSTGRES_DB: murfey_test_db
+          POSTGRES_PASSWORD: psql_pwd
+          POSTGRES_USER: psql_user
         run: |
           export ISPYB_CREDENTIALS=".github/workflows/config/ispyb.cfg"
           PYTHONDEVMODE=1 pytest -v -ra --cov=murfey --cov-report=xml --cov-branch
 
       - name: Upload to Codecov
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           name: ${{ matrix.python-version }}
           files: coverage.xml

.github/workflows/publish-version.yml

@@ -128,7 +128,7 @@ jobs:
           name: package-distributions
           path: dist/
       - name: Sign the dists with Sigstore
-        uses: sigstore/gh-action-sigstore-python@v2.1.1
+        uses: sigstore/gh-action-sigstore-python@v3.0.0
         with:
           inputs: >-
             ./dist/*.tar.gz

README.md

@@ -29,7 +29,7 @@ The packages included under the `[developer]` installation key contain some help
 - `pre-commit` - Allows for the installation and running of hooks to help with linting, formatting, and type checking your code.
 - `pytest` - Used in conjunction with test functions to evaluate the reliability of your code.
 
-Instructions for setting up the database for Murfey to register files to can be found [here](src/murfey/server/MURFEY_DB.md).
+Instructions for setting up the database for Murfey to register files to can be found [here](src/murfey/server/README.md).
 
 Finally, you may want to set up an ISPyB mock database server and a Zocalo
 development environment. The instructions for this are out of scope here.

pyproject.toml

@@ -7,7 +7,7 @@ requires = [
 
 [project]
 name = "murfey"
-version = "0.15.3"
+version = "0.16.9"
 description = "Client-Server architecture hauling Cryo-EM data"
 readme = "README.md"
 keywords = [
@@ -30,31 +30,35 @@ classifiers = [
     "Programming Language :: Python :: 3.12",
 ]
 dependencies = [
+    "backports.entry_points_selectable",
     "defusedxml", # For safely parsing XML files
     "pydantic>=2",
     "requests",
     "rich",
     "werkzeug",
+    "xmltodict",
 ]
 [project.optional-dependencies]
 cicd = [
-    "pytest-cov", # Used by Azure Pipelines for PyTest coverage reports
+    "pytest-cov", # Used for generating PyTest coverage reports
 ]
 client = [
     "textual==0.42.0",
     "websocket-client",
-    "xmltodict",
 ]
 developer = [
     "bump-my-version", # Version control
     "ipykernel", # Enable interactive coding with VS Code and Jupyter Notebook
     "pre-commit", # Formatting, linting, type checking, etc.
     "pytest", # Test code functionality
 ]
+instrument-server = [
+    "fastapi[standard]",
+    "python-jose[cryptography]",
+    "uvicorn[standard]",
+]
 server = [
-    # "matplotlib", # For visual statistical analysis of images
     "aiohttp",
-    "backports.entry_points_selectable",
     "cryptography",
     "fastapi[standard]",
     "ispyb>=10.2.4", # Responsible for setting requirements for SQLAlchemy and mysql-connector-python;
@@ -97,8 +101,13 @@ murfey = "murfey.client:run"
 [project.entry-points."murfey.config.extraction"]
 "murfey_machine" = "murfey.util.config:get_extended_machine_config"
 [project.entry-points."murfey.workflows"]
-"lif_to_stack" = "murfey.workflows.lif_to_stack:zocalo_cluster_request"
-"tiff_to_stack" = "murfey.workflows.tiff_to_stack:zocalo_cluster_request"
+"clem.align_and_merge" = "murfey.workflows.clem.align_and_merge:submit_cluster_request"
+"clem.process_raw_lifs" = "murfey.workflows.clem.process_raw_lifs:zocalo_cluster_request"
+"clem.process_raw_tiffs" = "murfey.workflows.clem.process_raw_tiffs:zocalo_cluster_request"
+"clem.register_align_and_merge_result" = "murfey.workflows.clem.register_align_and_merge_results:register_align_and_merge_result"
+"clem.register_lif_preprocessing_result" = "murfey.workflows.clem.register_preprocessing_results:register_lif_preprocessing_result"
+"clem.register_tiff_preprocessing_result" = "murfey.workflows.clem.register_preprocessing_results:register_tiff_preprocessing_result"
+"spa.flush_spa_preprocess" = "murfey.workflows.spa.flush_spa_preprocess:flush_spa_preprocess"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

src/murfey/__init__.py

@@ -1,4 +1,4 @@
 from __future__ import annotations
 
-__version__ = "0.15.3"
-__supported_client_version__ = "0.15.3"
+__version__ = "0.16.9"
+__supported_client_version__ = "0.16.9"

src/murfey/cli/inject_spa_processing.py

@@ -21,7 +21,7 @@
     SPAFeedbackParameters,
     SPARelionParameters,
 )
-from murfey.util.spa_params import default_spa_parameters
+from murfey.util.processing_params import default_spa_parameters
 
 
 def run():

src/murfey/cli/transfer.py

@@ -10,11 +10,13 @@
 from rich.prompt import Confirm
 
 from murfey.client import read_config
+from murfey.util.config import MachineConfig
 
 
 def run():
     config = read_config()
-    known_server = config["Murfey"].get("server")
+    known_server = config["Murfey"].get("server", "")
+    instrument_name = config["Murfey"].get("instrument_name", "")
 
     parser = argparse.ArgumentParser(description="Transfer using a remote rsync daemon")
 
@@ -35,11 +37,13 @@ def run():
     console = Console()
     murfey_url = urlparse(args.server, allow_fragments=False)
 
-    machine_data = requests.get(f"{murfey_url.geturl()}/machine").json()
-    if Path(args.source or ".").resolve() in machine_data.data_directories.keys():
-        console.print(
-            f"[red]Source directory is the base directory for the {machine_data.data_directories[Path(args.source or '.').resolve()]}, exiting"
-        )
+    machine_data = MachineConfig(
+        requests.get(
+            f"{murfey_url.geturl()}/instruments/{instrument_name}/machine"
+        ).json()
+    )
+    if Path(args.source or ".").resolve() in machine_data.data_directories:
+        console.print("[red]Source directory is the base directory, exiting")
         return
 
     cmd = [

src/murfey/client/__init__.py

@@ -65,6 +65,31 @@ def read_config() -> configparser.ConfigParser:
 requests.delete = partial(requests.delete, headers={"Authorization": f"Bearer {token}"})
 
 
+def write_config(config: configparser.ConfigParser):
+    mcch = os.environ.get("MURFEY_CLIENT_CONFIG_HOME")
+    murfey_client_config_home = Path(mcch) if mcch else Path.home()
+    with open(murfey_client_config_home / ".murfey", "w") as configfile:
+        config.write(configfile)
+
+
+def main_loop(
+    source_watchers: List[murfey.client.watchdir.DirWatcher],
+    appearance_time: float,
+    transfer_all: bool,
+):
+    log.info(
+        f"Murfey {murfey.__version__} on Python {'.'.join(map(str, sys.version_info[0:3]))} entering main loop"
+    )
+    if appearance_time > 0:
+        modification_time: float | None = time.time() - appearance_time * 3600
+    else:
+        modification_time = None
+    while True:
+        for sw in source_watchers:
+            sw.scan(modification_time=modification_time, transfer_all=transfer_all)
+        time.sleep(15)
+
+
 def _enable_webbrowser_in_cygwin():
     """Helper function to make webbrowser.open() work in CygWin"""
     if "cygwin" in platform.system().lower() and shutil.which("cygstart"):
@@ -190,12 +215,6 @@ def run():
         default=False,
         help="Remove source files immediately after their transfer",
     )
-    parser.add_argument(
-        "--relax",
-        action="store_true",
-        default=False,
-        help="Relax the condition that the source directory needs to be recognised from the configuration",
-    )
     parser.add_argument(
         "--name",
         type=str,
@@ -319,34 +338,8 @@ def run():
         gain_ref=gain_ref,
         redirected_logger=rich_handler,
         force_mdoc_metadata=not args.ignore_mdoc_metadata,
-        strict=not args.relax,
         processing_enabled=machine_data.get("processing_enabled", True),
         skip_existing_processing=args.skip_existing_processing,
     )
     app.run()
     rich_handler.redirect = False
-
-
-def main_loop(
-    source_watchers: List[murfey.client.watchdir.DirWatcher],
-    appearance_time: float,
-    transfer_all: bool,
-):
-    log.info(
-        f"Murfey {murfey.__version__} on Python {'.'.join(map(str, sys.version_info[0:3]))} entering main loop"
-    )
-    if appearance_time > 0:
-        modification_time: float | None = time.time() - appearance_time * 3600
-    else:
-        modification_time = None
-    while True:
-        for sw in source_watchers:
-            sw.scan(modification_time=modification_time, transfer_all=transfer_all)
-        time.sleep(15)
-
-
-def write_config(config: configparser.ConfigParser):
-    mcch = os.environ.get("MURFEY_CLIENT_CONFIG_HOME")
-    murfey_client_config_home = Path(mcch) if mcch else Path.home()
-    with open(murfey_client_config_home / ".murfey", "w") as configfile:
-        config.write(configfile)