Merged recent changes from main branch

Author: tieneupin

.bumpclient.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.15.3"
+current_version = "0.15.5"
 commit = true
 tag = false
 

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.15.3"
+current_version = "0.15.5"
 commit = true
 tag = true
 

.github/workflows/ci.yml

@@ -17,7 +17,7 @@ jobs:
         python-version: ["3.9", "3.10", "3.11"]
     services:
       mariadb:
-        image: mariadb:11.3.2 # released 2024-05-06
+        image: mariadb:11.6.2 # released 2024-05-06
         # Pulls image from DockerHub
         # Docker images: https://hub.docker.com/_/mariadb
         # Previous version(s):
@@ -108,7 +108,7 @@ jobs:
           PYTHONDEVMODE=1 pytest -v -ra --cov=murfey --cov-report=xml --cov-branch
 
       - name: Upload to Codecov
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           name: ${{ matrix.python-version }}
           files: coverage.xml

.github/workflows/publish-version.yml

@@ -128,7 +128,7 @@ jobs:
           name: package-distributions
           path: dist/
       - name: Sign the dists with Sigstore
-        uses: sigstore/gh-action-sigstore-python@v2.1.1
+        uses: sigstore/gh-action-sigstore-python@v3.0.0
         with:
           inputs: >-
             ./dist/*.tar.gz

README.md

@@ -29,7 +29,7 @@ The packages included under the `[developer]` installation key contain some help
 - `pre-commit` - Allows for the installation and running of hooks to help with linting, formatting, and type checking your code.
 - `pytest` - Used in conjunction with test functions to evaluate the reliability of your code.
 
-Instructions for setting up the database for Murfey to register files to can be found [here](src/murfey/server/MURFEY_DB.md).
+Instructions for setting up the database for Murfey to register files to can be found [here](src/murfey/server/README.md).
 
 Finally, you may want to set up an ISPyB mock database server and a Zocalo
 development environment. The instructions for this are out of scope here.

pyproject.toml

@@ -7,7 +7,7 @@ requires = [
 
 [project]
 name = "murfey"
-version = "0.15.3"
+version = "0.15.5"
 description = "Client-Server architecture hauling Cryo-EM data"
 readme = "README.md"
 keywords = [
@@ -99,10 +99,12 @@ murfey = "murfey.client:run"
 [project.entry-points."murfey.config.extraction"]
 "murfey_machine" = "murfey.util.config:get_extended_machine_config"
 [project.entry-points."murfey.workflows"]
-"process_raw_lifs" = "murfey.workflows.clem.process_raw_lifs:zocalo_cluster_request"
-"process_raw_tiffs" = "murfey.workflows.clem.process_raw_tiffs:zocalo_cluster_request"
-"register_lif_preprocessing_result" = "murfey.workflows.clem.register_preprocessing_results:register_lif_preprocessing_result"
-"register_tiff_preprocessing_result" = "murfey.workflows.clem.register_preprocessing_results:register_tiff_preprocessing_result"
+"clem.align_and_merge" = "murfey.workflows.clem.align_and_merge:submit_cluster_request"
+"clem.process_raw_lifs" = "murfey.workflows.clem.process_raw_lifs:zocalo_cluster_request"
+"clem.process_raw_tiffs" = "murfey.workflows.clem.process_raw_tiffs:zocalo_cluster_request"
+"clem.register_align_and_merge_result" = "murfey.workflows.clem.register_align_and_merge_results:register_align_and_merge_result"
+"clem.register_lif_preprocessing_result" = "murfey.workflows.clem.register_preprocessing_results:register_lif_preprocessing_result"
+"clem.register_tiff_preprocessing_result" = "murfey.workflows.clem.register_preprocessing_results:register_tiff_preprocessing_result"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

src/murfey/__init__.py

@@ -1,4 +1,4 @@
 from __future__ import annotations
 
-__version__ = "0.15.3"
-__supported_client_version__ = "0.15.3"
+__version__ = "0.15.5"
+__supported_client_version__ = "0.15.5"

src/murfey/cli/transfer.py

@@ -36,10 +36,8 @@ def run():
     murfey_url = urlparse(args.server, allow_fragments=False)
 
     machine_data = requests.get(f"{murfey_url.geturl()}/machine").json()
-    if Path(args.source or ".").resolve() in machine_data.data_directories.keys():
-        console.print(
-            f"[red]Source directory is the base directory for the {machine_data.data_directories[Path(args.source or '.').resolve()]}, exiting"
-        )
+    if Path(args.source or ".").resolve() in machine_data.data_directories:
+        console.print("[red]Source directory is the base directory, exiting")
         return
 
     cmd = [

src/murfey/client/__init__.py

@@ -215,12 +215,6 @@ def run():
         default=False,
         help="Remove source files immediately after their transfer",
     )
-    parser.add_argument(
-        "--relax",
-        action="store_true",
-        default=False,
-        help="Relax the condition that the source directory needs to be recognised from the configuration",
-    )
     parser.add_argument(
         "--name",
         type=str,
@@ -344,7 +338,6 @@ def run():
         gain_ref=gain_ref,
         redirected_logger=rich_handler,
         force_mdoc_metadata=not args.ignore_mdoc_metadata,
-        strict=not args.relax,
         processing_enabled=machine_data.get("processing_enabled", True),
         skip_existing_processing=args.skip_existing_processing,
     )

src/murfey/client/analyser.py

@@ -57,7 +57,6 @@ def __init__(
         self._limited = limited
         self._experiment_type = ""
         self._acquisition_software = ""
-        self._role = ""
         self._extension: str = ""
         self._unseen_xml: list = []
         self._context: Context | None = None
@@ -126,14 +125,12 @@ def _find_context(self, file_path: Path) -> bool:
         in the Context classes themselves.
         """
         if "atlas" in file_path.parts:
-            self._role = "detector"
             self._context = SPAMetadataContext("epu", self._basepath)
             return True
 
         # CLEM workflow checks
         # Look for LIF and XLIF files
         if file_path.suffix in (".lif", ".xlif"):
-            self._role = "detector"
             self._context = CLEMContext("leica", self._basepath)
             return True
         # Look for TIFF files associated with CLEM workflow
@@ -152,7 +149,6 @@ def _find_context(self, file_path: Path) -> bool:
                 ).get("analyse_created_directories", [])
             )
             if created_directories.intersection(set(file_path.parts)):
-                self._role = "detector"
                 self._context = CLEMContext("leica", self._basepath)
                 return True
 
@@ -181,9 +177,6 @@ def _find_context(self, file_path: Path) -> bool:
                         else SPAContext("epu", self._basepath)
                     )
                 self.parameters_model = ProcessingParametersSPA
-                # Assign it the detector attribute if not already present
-                if not self._role:
-                    self._role = "detector"
                 return True
 
             # Files starting with "Position" belong to the standard tomography workflow
@@ -198,23 +191,6 @@ def _find_context(self, file_path: Path) -> bool:
                     logger.info("Acquisition software: tomo")
                     self._context = TomographyContext("tomo", self._basepath)
                     self.parameters_model = PreprocessingParametersTomo
-                # Assign role if not already present
-                if not self._role:
-                    # Fractions files attributed to the detector
-                    if (
-                        "Fractions" in split_file_name[-1]
-                        or "fractions" in split_file_name[-1]
-                    ):
-                        self._role = "detector"
-                    # MDOC files attributed to the microscope
-                    elif (
-                        file_path.suffix == ".mdoc"
-                        or file_path.with_suffix(".mdoc").is_file()
-                    ):
-                        self._role = "microscope"
-                    # Attribute all other files to the detector
-                    else:
-                        self._role = "detector"
                 return True
 
             # Files with these suffixes belong to the serial EM tomography workflow
@@ -239,19 +215,14 @@ def _find_context(self, file_path: Path) -> bool:
                     return False
                 self._context = TomographyContext("serialem", self._basepath)
                 self.parameters_model = PreprocessingParametersTomo
-                if not self._role:
-                    if "Frames" in file_path.parts:
-                        self._role = "detector"
-                    else:
-                        self._role = "microscope"
                 return True
         return False
 
     def post_transfer(self, transferred_file: Path):
         try:
             if self._context:
                 self._context.post_transfer(
-                    transferred_file, role=self._role, environment=self._environment
+                    transferred_file, environment=self._environment
                 )
         except Exception as e:
             logger.error(f"An exception was encountered post transfer: {e}")
@@ -309,19 +280,17 @@ def _analyse(self):
                         self.queue.task_done()
                         continue
                     elif self._extension:
-                        logger.info(f"Context found successfully: {self._role}")
+                        logger.info(
+                            f"Context found successfully for {transferred_file}"
+                        )
                         try:
                             self._context.post_first_transfer(
                                 transferred_file,
-                                role=self._role,
                                 environment=self._environment,
                             )
                         except Exception as e:
                             logger.error(f"Exception encountered: {e}")
-                        if (
-                            self._role == "detector"
-                            and "atlas" not in transferred_file.parts
-                        ):
+                        if "atlas" not in transferred_file.parts:
                             if not dc_metadata:
                                 try:
                                     dc_metadata = self._context.gather_metadata(
@@ -377,20 +346,16 @@ def _analyse(self):
                     self._find_extension(transferred_file)
                     if self._extension:
                         logger.info(
-                            f"Context found successfully: {self._role}, {transferred_file}"
+                            f"Extension found successfully for {transferred_file}"
                         )
                         try:
                             self._context.post_first_transfer(
                                 transferred_file,
-                                role=self._role,
                                 environment=self._environment,
                             )
                         except Exception as e:
                             logger.error(f"Exception encountered: {e}")
-                        if (
-                            self._role == "detector"
-                            and "atlas" not in transferred_file.parts
-                        ):
+                        if "atlas" not in transferred_file.parts:
                             if not dc_metadata:
                                 try:
                                     dc_metadata = self._context.gather_metadata(
@@ -443,8 +408,8 @@ def _xml_file(self, data_file: Path) -> Path:
         if not self._environment:
             return data_file.with_suffix(".xml")
         file_name = f"{'_'.join(p for p in data_file.stem.split('_')[:-1])}.xml"
-        data_directories = self._murfey_config.get("data_directories", {})
-        for dd in data_directories.keys():
+        data_directories = self._murfey_config.get("data_directories", [])
+        for dd in data_directories:
             if str(data_file).startswith(dd):
                 base_dir = Path(dd)
                 mid_dir = data_file.relative_to(dd).parent