Merged version bump from 'main'

Author: tieneupin

.bumpclient.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.16.8"
+current_version = "0.16.9"
 commit = true
 tag = false
 

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.16.8"
+current_version = "0.16.9"
 commit = true
 tag = true
 

pyproject.toml

@@ -7,7 +7,7 @@ requires = [
 
 [project]
 name = "murfey"
-version = "0.16.8"
+version = "0.16.9"
 description = "Client-Server architecture hauling Cryo-EM data"
 readme = "README.md"
 keywords = [

src/murfey/__init__.py

@@ -1,4 +1,4 @@
 from __future__ import annotations
 
-__version__ = "0.16.8"
-__supported_client_version__ = "0.16.8"
+__version__ = "0.16.9"
+__supported_client_version__ = "0.16.9"

src/murfey/cli/transfer.py

@@ -10,11 +10,13 @@
 from rich.prompt import Confirm
 
 from murfey.client import read_config
+from murfey.util.config import MachineConfig
 
 
 def run():
     config = read_config()
-    known_server = config["Murfey"].get("server")
+    known_server = config["Murfey"].get("server", "")
+    instrument_name = config["Murfey"].get("instrument_name", "")
 
     parser = argparse.ArgumentParser(description="Transfer using a remote rsync daemon")
 
@@ -35,7 +37,11 @@ def run():
     console = Console()
     murfey_url = urlparse(args.server, allow_fragments=False)
 
-    machine_data = requests.get(f"{murfey_url.geturl()}/machine").json()
+    machine_data = MachineConfig(
+        requests.get(
+            f"{murfey_url.geturl()}/instruments/{instrument_name}/machine"
+        ).json()
+    )
     if Path(args.source or ".").resolve() in machine_data.data_directories:
         console.print("[red]Source directory is the base directory, exiting")
         return

src/murfey/client/contexts/clem.py

@@ -34,9 +34,6 @@ def _file_transferred_to(
     # rsync basepath and modules are set in the microscope's configuration YAML file
     return (
         Path(machine_config.get("rsync_basepath", ""))
-        / (
-            machine_config.get("rsync_module", "data") or "data"
-        )  # Add "data" if it wasn't set
         / str(datetime.now().year)
         / source.name
         / file_path.relative_to(source)

src/murfey/client/contexts/spa.py

@@ -46,7 +46,7 @@ def _file_transferred_to(
         return (
             Path(machine_config.get("rsync_basepath", ""))
             / Path(environment.default_destinations[source])
-            / file_path.relative_to(source)
+            / file_path.relative_to(source)  # need to strip out the rsync_module name
         )
     return (
         Path(machine_config.get("rsync_basepath", ""))

src/murfey/client/contexts/tomo.py

@@ -78,6 +78,7 @@ class TomographyContext(Context):
         ProcessingParameter("image_size_y", "Image Size Y"),
         ProcessingParameter("pixel_size_on_image", "Pixel Size"),
         ProcessingParameter("motion_corr_binning", "Motion Correction Binning"),
+        ProcessingParameter("frame_count", "Number of image frames"),
         ProcessingParameter("num_eer_frames", "Number of EER Frames"),
     ]
 
@@ -92,31 +93,6 @@ def __init__(self, acquisition_software: str, basepath: Path):
         self._processing_job_stash: dict = {}
         self._lock: RLock = RLock()
 
-    def _flush_data_collections(self):
-        logger.info(
-            f"Flushing {len(self._data_collection_stash)} data collection API calls"
-        )
-        for dc_data in self._data_collection_stash:
-            data = {
-                **dc_data[2],
-                **{
-                    k: v
-                    for k, v in dc_data[1].data_collection_parameters.items()
-                    if k != "tag"
-                },
-            }
-            capture_post(dc_data[0], json=data)
-        self._data_collection_stash = []
-
-    def _flush_processing_jobs(self):
-        logger.info(
-            f"Flushing {len(self._processing_job_stash.keys())} processing job API calls"
-        )
-        for v in self._processing_job_stash.values():
-            for pd in v:
-                requests.post(pd[0], json=pd[1])
-        self._processing_job_stash = {}
-
     def _file_transferred_to(
         self, environment: MurfeyInstanceEnvironment, source: Path, file_path: Path
     ):
@@ -227,8 +203,18 @@ def _add_tilt(
                 self._tilt_series_sizes[tilt_series] = 0
             try:
                 if environment:
-                    url = f"{str(environment.url.geturl())}/visits/{environment.visit}/{environment.murfey_session}/start_data_collection"
-                    data = {
+                    dcg_url = f"{str(environment.url.geturl())}/visits/{str(environment.visit)}/{environment.murfey_session}/register_data_collection_group"
+                    dcg_data = {
+                        "experiment_type": "tomo",
+                        "experiment_type_id": 36,
+                        "tag": str(self._basepath),
+                        "atlas": "",
+                        "sample": None,
+                    }
+                    capture_post(dcg_url, json=dcg_data)
+
+                    dc_url = f"{str(environment.url.geturl())}/visits/{environment.visit}/{environment.murfey_session}/start_data_collection"
+                    dc_data = {
                         "experiment_type": "tomography",
                         "file_extension": file_path.suffix,
                         "acquisition_software": self._acquisition_software,
@@ -245,7 +231,7 @@ def _add_tilt(
                         environment.data_collection_parameters
                         and environment.data_collection_parameters.get("voltage")
                     ):
-                        data.update(
+                        dc_data.update(
                             {
                                 "voltage": environment.data_collection_parameters[
                                     "voltage"
@@ -264,51 +250,16 @@ def _add_tilt(
                                 ],
                             }
                         )
+                    capture_post(dc_url, json=dc_data)
+
                     proc_url = f"{str(environment.url.geturl())}/visits/{environment.visit}/{environment.murfey_session}/register_processing_job"
-                    if (
-                        environment.data_collection_group_ids.get(str(self._basepath))
-                        is None
-                    ):
-                        self._data_collection_stash.append((url, environment, data))
-                        self._processing_job_stash[tilt_series] = [
-                            (
-                                proc_url,
-                                {
-                                    "tag": tilt_series,
-                                    "source": str(self._basepath),
-                                    "recipe": "em-tomo-preprocess",
-                                    "experiment_type": "tomography",
-                                },
-                            )
-                        ]
-                        self._processing_job_stash[tilt_series].append(
-                            (
-                                proc_url,
-                                {
-                                    "tag": tilt_series,
-                                    "source": str(self._basepath),
-                                    "recipe": "em-tomo-align",
-                                    "experiment_type": "tomography",
-                                },
-                            )
-                        )
-                    else:
-                        capture_post(url, json=data)
-                        capture_post(
-                            proc_url,
-                            json={
-                                "tag": tilt_series,
-                                "source": str(self._basepath),
-                                "recipe": "em-tomo-preprocess",
-                                "experiment_type": "tomography",
-                            },
-                        )
+                    for recipe in ("em-tomo-preprocess", "em-tomo-align"):
                         capture_post(
                             proc_url,
                             json={
                                 "tag": tilt_series,
                                 "source": str(self._basepath),
-                                "recipe": "em-tomo-align",
+                                "recipe": recipe,
                                 "experiment_type": "tomography",
                             },
                         )
@@ -361,6 +312,9 @@ def _add_tilt(
                 "dose_per_frame": environment.data_collection_parameters.get(
                     "dose_per_frame", 0
                 ),
+                "frame_count": environment.data_collection_parameters.get(
+                    "frame_count", 0
+                ),
                 "mc_binning": environment.data_collection_parameters.get(
                     "motion_corr_binning", 1
                 ),
@@ -591,6 +545,7 @@ def gather_metadata(
             mdoc_metadata: OrderedDict = OrderedDict({})
             mdoc_metadata["experiment_type"] = "tomography"
             mdoc_metadata["voltage"] = float(mdoc_data["Voltage"])
+            mdoc_metadata["frame_count"] = int(mdoc_data_block["NumSubFrames"])
             mdoc_metadata["image_size_x"] = int(mdoc_data["ImageSize"][0])
             mdoc_metadata["image_size_y"] = int(mdoc_data["ImageSize"][1])
             mdoc_metadata["magnification"] = int(mdoc_data_block["Magnification"])
@@ -658,6 +613,9 @@ def gather_metadata(
                 mdoc_metadata["num_eer_frames"] = murfey.util.eer.num_frames(
                     metadata_file.parent / data_file
                 )
+                mdoc_metadata["frame_count"] = int(
+                    mdoc_metadata["eer_fractionation"] / mdoc_metadata["num_eer_frames"]
+                )
         except Exception as e:
             logger.error(f"Exception encountered in metadata gathering: {str(e)}")
             return OrderedDict({})

src/murfey/client/instance_environment.py

@@ -41,7 +41,6 @@ class MurfeyInstanceEnvironment(BaseModel):
     destination_registry: Dict[str, str] = {}
     watchers: Dict[Path, DirWatcher] = {}
     demo: bool = False
-    data_collection_group_ids: Dict[str, int] = {}
     data_collection_parameters: dict = {}
     movies: Dict[Path, MovieTracker] = {}
     movie_tilt_pair: Dict[Path, str] = {}
@@ -64,7 +63,6 @@ def clear(self):
         for w in self.watchers.values():
             w.stop()
         self.watchers = {}
-        self.data_collection_group_ids = {}
         self.data_collection_parameters = {}
         self.movies = {}
         self.movie_tilt_pair = {}

src/murfey/client/multigrid_control.py

@@ -32,6 +32,7 @@ class MultigridController:
     session_id: int
     murfey_url: str = "http://localhost:8000"
     rsync_url: str = ""
+    rsync_module: str = "data"
     demo: bool = False
     processing_enabled: bool = True
     do_transfer: bool = True
@@ -59,12 +60,13 @@ def __post_init__(self):
             f"{self.murfey_url}/instruments/{self.instrument_name}/machine"
         ).json()
         self.rsync_url = machine_data.get("rsync_url", "")
+        self.rsync_module = machine_data.get("rsync_module", "data")
         self._environment = MurfeyInstanceEnvironment(
             url=urlparse(self.murfey_url, allow_fragments=False),
             client_id=0,
             murfey_session=self.session_id,
             software_versions=machine_data.get("software_versions", {}),
-            default_destination=f"{machine_data.get('rsync_module') or 'data'}/{datetime.now().year}",
+            default_destination=f"{datetime.now().year}",
             demo=self.demo,
             visit=self.visit,
             data_collection_parameters=self.data_collection_parameters,
@@ -123,7 +125,7 @@ def _start_rsyncer_multigrid(
                     break
             else:
                 self._environment.default_destinations[source] = (
-                    f"{machine_data.get('rsync_module') or 'data'}/{datetime.now().year}"
+                    f"{datetime.now().year}"
                 )
                 destination = determine_default_destination(
                     self._environment.visit,
@@ -205,7 +207,7 @@ def _start_rsyncer(
                 rsync_cmd = [
                     "rsync",
                     f"{posix_path(self._environment.gain_ref)!r}",  # '!r' will print strings in ''
-                    f"{self._environment.url.hostname}::{visit_path}/processing",
+                    f"{self._environment.url.hostname}::{self.rsync_module}/{visit_path}/processing",
                 ]
                 # Wrap in bash shell
                 cmd = [
@@ -223,6 +225,7 @@ def _start_rsyncer(
             self.rsync_processes[source] = RSyncer(
                 source,
                 basepath_remote=Path(destination),
+                rsync_module=self.rsync_module,
                 server_url=(
                     urlparse(self.rsync_url)
                     if self.rsync_url
@@ -374,46 +377,6 @@ def _start_dc(self, json, from_form: bool = False):
                 )
 
             source = Path(json["source"])
-
-            url = f"{str(self._environment.url.geturl())}/visits/{str(self._environment.visit)}/{self.session_id}/register_data_collection_group"
-            dcg_data = {
-                "experiment_type": "tomo",
-                "experiment_type_id": 36,
-                "tag": str(source),
-            }
-            requests.post(url, json=dcg_data)
-
-            data = {
-                "voltage": json["voltage"],
-                "pixel_size_on_image": json["pixel_size_on_image"],
-                "experiment_type": json["experiment_type"],
-                "image_size_x": json["image_size_x"],
-                "image_size_y": json["image_size_y"],
-                "file_extension": json["file_extension"],
-                "acquisition_software": json["acquisition_software"],
-                "image_directory": str(self._environment.default_destinations[source]),
-                "tag": json["tilt_series_tag"],
-                "source": str(source),
-                "magnification": json["magnification"],
-                "total_exposed_dose": json.get("total_exposed_dose"),
-                "c2aperture": json.get("c2aperture"),
-                "exposure_time": json.get("exposure_time"),
-                "slit_width": json.get("slit_width"),
-                "phase_plate": json.get("phase_plate", False),
-            }
-            capture_post(
-                f"{str(self._environment.url.geturl())}/visits/{str(self._environment.visit)}/{self._environment.murfey_session}/start_data_collection",
-                json=data,
-            )
-            for recipe in ("em-tomo-preprocess", "em-tomo-align"):
-                capture_post(
-                    f"{str(self._environment.url.geturl())}/visits/{str(self._environment.visit)}/{self._environment.murfey_session}/register_processing_job",
-                    json={
-                        "tag": json["tilt_series_tag"],
-                        "source": str(source),
-                        "recipe": recipe,
-                    },
-                )
             log.info("Registering tomography processing parameters")
             if self._environment.data_collection_parameters.get("num_eer_frames"):
                 eer_response = requests.post(
@@ -437,8 +400,6 @@ def _start_dc(self, json, from_form: bool = False):
                 f"{self._environment.url.geturl()}/sessions/{self._environment.murfey_session}/tomography_preprocessing_parameters",
                 json=json,
             )
-            context._flush_data_collections()
-            context._flush_processing_jobs()
             capture_post(
                 f"{self._environment.url.geturl()}/visits/{self._environment.visit}/{self._environment.murfey_session}/flush_tomography_processing",
                 json={"rsync_source": str(source)},