Merged recent changes from 'main' branch

Author: tieneupin

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.16.6"
+current_version = "0.16.7"
 commit = true
 tag = true
 

.github/workflows/ci.yml

@@ -29,6 +29,16 @@ jobs:
           - 3306:3306
         options: --health-cmd="healthcheck.sh --connect --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=3
 
+      postgres:
+        image: postgres:latest
+        env:
+          POSTGRES_DB: murfey_test_db
+          POSTGRES_PASSWORD: psql_pwd
+          POSTGRES_USER: psql_user
+        ports:
+          - 5432:5432
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+
     steps:
       - uses: actions/checkout@v4
       - name: Use Python ${{ matrix.python-version }}
@@ -57,7 +67,7 @@ jobs:
           docker run --detach --name rabbitmq -p 127.0.0.1:5672:5672 -p 127.0.0.1:15672:15672 test-rabbitmq
           docker container list -a
 
-      - name: Get database
+      - name: Get ispyb database
         uses: actions/download-artifact@v4
         with:
           name: database
@@ -74,7 +84,7 @@ jobs:
           mysql-version: "11.3"
           auto-start: false
 
-      - name: Set up test database
+      - name: Set up test ipsyb database
         run: |
           set -eu
           cp ".github/workflows/config/my.cnf" .my.cnf
@@ -103,6 +113,12 @@ jobs:
         run: wget -t 10 -w 1 http://127.0.0.1:15672 -O -
 
       - name: Run tests
+        env:
+          POSTGRES_HOST: localhost
+          POSTGRES_PORT: 5432
+          POSTGRES_DB: murfey_test_db
+          POSTGRES_PASSWORD: psql_pwd
+          POSTGRES_USER: psql_user
         run: |
           export ISPYB_CREDENTIALS=".github/workflows/config/ispyb.cfg"
           PYTHONDEVMODE=1 pytest -v -ra --cov=murfey --cov-report=xml --cov-branch

pyproject.toml

@@ -7,7 +7,7 @@ requires = [
 
 [project]
 name = "murfey"
-version = "0.16.6"
+version = "0.16.7"
 description = "Client-Server architecture hauling Cryo-EM data"
 readme = "README.md"
 keywords = [

src/murfey/__init__.py

@@ -1,4 +1,4 @@
 from __future__ import annotations
 
-__version__ = "0.16.6"
+__version__ = "0.16.7"
 __supported_client_version__ = "0.16.6"

src/murfey/client/analyser.py

@@ -16,7 +16,7 @@
 
 from murfey.client.context import Context
 from murfey.client.contexts.clem import CLEMContext
-from murfey.client.contexts.spa import SPAContext, SPAModularContext
+from murfey.client.contexts.spa import SPAModularContext
 from murfey.client.contexts.spa_metadata import SPAMetadataContext
 from murfey.client.contexts.tomo import TomographyContext
 from murfey.client.instance_environment import MurfeyInstanceEnvironment
@@ -159,23 +159,7 @@ def _find_context(self, file_path: Path) -> bool:
             if split_file_name[0].startswith("FoilHole"):
                 if not self._context:
                     logger.info("Acquisition software: EPU")
-                    if self._environment:
-                        try:
-                            cfg = get_machine_config_client(
-                                str(self._environment.url.geturl()),
-                                instrument_name=self._environment.instrument_name,
-                                demo=self._environment.demo,
-                            )
-                        except Exception as e:
-                            logger.error(f"Exception encountered: {e}")
-                            cfg = {}
-                    else:
-                        cfg = {}
-                    self._context = (
-                        SPAModularContext("epu", self._basepath)
-                        if cfg.get("modular_spa")
-                        else SPAContext("epu", self._basepath)
-                    )
+                    self._context = SPAModularContext("epu", self._basepath)
                 self.parameters_model = ProcessingParametersSPA
                 return True
 
@@ -325,8 +309,7 @@ def _analyse(self):
                                         "form": dc_metadata,
                                         "dependencies": (
                                             spa_form_dependencies
-                                            if isinstance(self._context, SPAContext)
-                                            or isinstance(
+                                            if isinstance(
                                                 self._context, SPAModularContext
                                             )
                                             else {}
@@ -385,8 +368,7 @@ def _analyse(self):
                                         "form": dc_metadata,
                                         "dependencies": (
                                             spa_form_dependencies
-                                            if isinstance(self._context, SPAContext)
-                                            or isinstance(
+                                            if isinstance(
                                                 self._context, SPAModularContext
                                             )
                                             else {}

src/murfey/client/contexts/spa.py

@@ -66,13 +66,16 @@ def _grid_square_metadata_file(
             break
     else:
         raise ValueError(f"Could not determine grid square metadata path for {f}")
-    return (
+    metadata_file = (
         base_dir
         / visit
         / mid_dir.parent.parent.parent
         / "Metadata"
         / f"GridSquare_{grid_square}.dm"
     )
+    if not metadata_file.is_file():
+        logger.warning(f"Grid square metadata file {str(metadata_file)} does not exist")
+    return metadata_file
 
 
 def _get_source(file_path: Path, environment: MurfeyInstanceEnvironment) -> Path | None:
@@ -89,7 +92,7 @@ def _get_xml_list_index(key: str, xml_list: list) -> int:
     raise ValueError(f"Key not found in XML list: {key}")
 
 
-class _SPAContext(Context):
+class SPAModularContext(Context):
     user_params = [
         ProcessingParameter(
             "dose_per_frame",
@@ -130,7 +133,6 @@ def __init__(self, acquisition_software: str, basepath: Path):
         super().__init__("SPA", acquisition_software)
         self._basepath = basepath
         self._processing_job_stash: dict = {}
-        self._preprocessing_triggers: dict = {}
         self._foil_holes: Dict[int, List[int]] = {}
 
     def gather_metadata(
@@ -362,8 +364,6 @@ def gather_metadata(
         ) or True
         return metadata
 
-
-class SPAModularContext(_SPAContext):
     def _position_analysis(
         self,
         transferred_file: Path,
@@ -658,104 +658,3 @@ def _launch_spa_pipeline(
         url: str = "",
     ):
         return
-
-
-class SPAContext(_SPAContext):
-    def _register_data_collection(
-        self,
-        tag: str,
-        url: str,
-        data: dict,
-        environment: MurfeyInstanceEnvironment,
-    ):
-        logger.info(f"registering data collection with data {data}")
-        environment.id_tag_registry["data_collection"].append(tag)
-        image_directory = str(environment.default_destinations[Path(tag)])
-        json = {
-            "voltage": data["voltage"],
-            "pixel_size_on_image": data["pixel_size_on_image"],
-            "experiment_type": data["experiment_type"],
-            "image_size_x": data["image_size_x"],
-            "image_size_y": data["image_size_y"],
-            "file_extension": data["file_extension"],
-            "acquisition_software": data["acquisition_software"],
-            "image_directory": image_directory,
-            "tag": tag,
-            "source": tag,
-            "magnification": data["magnification"],
-            "total_exposed_dose": data.get("total_exposed_dose"),
-            "c2aperture": data.get("c2aperture"),
-            "exposure_time": data.get("exposure_time"),
-            "slit_width": data.get("slit_width"),
-            "phase_plate": data.get("phase_plate", False),
-        }
-        capture_post(url, json=json)
-
-    def post_transfer(
-        self,
-        transferred_file: Path,
-        environment: MurfeyInstanceEnvironment | None = None,
-        **kwargs,
-    ) -> bool:
-        return True
-
-    def _register_processing_job(
-        self,
-        tag: str,
-        environment: MurfeyInstanceEnvironment,
-        parameters: Dict[str, Any] | None = None,
-    ):
-        logger.info(f"registering processing job with parameters: {parameters}")
-        parameters = parameters or {}
-        environment.id_tag_registry["processing_job"].append(tag)
-        proc_url = f"{str(environment.url.geturl())}/visits/{environment.visit}/{environment.murfey_session}/register_processing_job"
-        machine_config = get_machine_config_client(
-            str(environment.url.geturl()),
-            instrument_name=environment.instrument_name,
-            demo=environment.demo,
-        )
-        image_directory = str(
-            Path(machine_config.get("rsync_basepath", "."))
-            / environment.default_destinations[Path(tag)]
-        )
-        if self._acquisition_software == "epu":
-            import_images = f"{Path(image_directory).resolve()}/GridSquare*/Data/*{parameters['file_extension']}"
-        else:
-            import_images = (
-                f"{Path(image_directory).resolve()}/*{parameters['file_extension']}"
-            )
-        msg: Dict[str, Any] = {
-            "tag": tag,
-            "source": tag,
-            "recipe": "ispyb-relion",
-            "parameters": {
-                "acquisition_software": parameters["acquisition_software"],
-                "voltage": parameters["voltage"],
-                "gain_ref": parameters["gain_ref"],
-                "dose_per_frame": parameters["dose_per_frame"],
-                "eer_grouping": parameters["eer_fractionation"],
-                "import_images": import_images,
-                "angpix": float(parameters["pixel_size_on_image"]) * 1e10,
-                "symmetry": parameters["symmetry"],
-                "boxsize": parameters["boxsize"],
-                "downscale": parameters["downscale"],
-                "small_boxsize": parameters["small_boxsize"],
-                "mask_diameter": parameters["mask_diameter"],
-                "use_cryolo": parameters["use_cryolo"],
-                "estimate_particle_diameter": parameters["estimate_particle_diameter"],
-            },
-        }
-        if parameters["particle_diameter"]:
-            msg["parameters"]["particle_diameter"] = parameters["particle_diameter"]
-        capture_post(proc_url, json=msg)
-
-    def _launch_spa_pipeline(
-        self,
-        tag: str,
-        jobid: int,
-        environment: MurfeyInstanceEnvironment,
-        url: str = "",
-    ):
-        environment.id_tag_registry["auto_proc_program"].append(tag)
-        data = {"job_id": jobid}
-        capture_post(url, json=data)

src/murfey/client/contexts/spa_metadata.py

@@ -30,6 +30,7 @@ def _foil_hole_positions(xml_path: Path, grid_square: int) -> Dict[str, FoilHole
             required_key = key
             break
     if not required_key:
+        logger.warning(f"Required key not found for {str(xml_path)}")
         return {}
     foil_holes = {}
     for fh_block in serialization_array[required_key]:
@@ -149,10 +150,13 @@ def post_transfer(
                     atlas=Path(partial_path), sample=sample
                 )
                 url = f"{str(environment.url.geturl())}/visits/{environment.visit}/{environment.murfey_session}/register_data_collection_group"
+                dcg_tag = "/".join(
+                    p for p in transferred_file.parent.parts if p != environment.visit
+                )
                 dcg_data = {
                     "experiment_type": "single particle",
                     "experiment_type_id": 37,
-                    "tag": str(source),
+                    "tag": dcg_tag,
                     "atlas": str(
                         _atlas_destination(environment, source, transferred_file)
                         / environment.samples[source].atlas
@@ -169,7 +173,7 @@ def post_transfer(
                         capture_post(
                             f"{str(environment.url.geturl())}/sessions/{environment.murfey_session}/grid_square/{gs}",
                             json={
-                                "tag": str(source),
+                                "tag": dcg_tag,
                                 "x_location": pos_data[0],
                                 "y_location": pos_data[1],
                                 "x_stage_position": pos_data[2],
@@ -186,6 +190,9 @@ def post_transfer(
             and environment
         ):
             gs_name = transferred_file.stem.split("_")[1]
+            logger.info(
+                f"Collecting foil hole positions for {str(transferred_file)} and grid square {int(gs_name)}"
+            )
             fh_positions = _foil_hole_positions(transferred_file, int(gs_name))
             source = _get_source(transferred_file, environment=environment)
             visitless_source = str(source).replace(f"/{environment.visit}", "")