Convert data directories to a list by removing the role (#412)

Author: stephen-riggs

src/murfey/cli/transfer.py

@@ -36,10 +36,8 @@ def run():
     murfey_url = urlparse(args.server, allow_fragments=False)
 
     machine_data = requests.get(f"{murfey_url.geturl()}/machine").json()
-    if Path(args.source or ".").resolve() in machine_data.data_directories.keys():
-        console.print(
-            f"[red]Source directory is the base directory for the {machine_data.data_directories[Path(args.source or '.').resolve()]}, exiting"
-        )
+    if Path(args.source or ".").resolve() in machine_data.data_directories:
+        console.print("[red]Source directory is the base directory, exiting")
         return
 
     cmd = [

src/murfey/client/__init__.py

@@ -215,12 +215,6 @@ def run():
         default=False,
         help="Remove source files immediately after their transfer",
     )
-    parser.add_argument(
-        "--relax",
-        action="store_true",
-        default=False,
-        help="Relax the condition that the source directory needs to be recognised from the configuration",
-    )
     parser.add_argument(
         "--name",
         type=str,
@@ -344,7 +338,6 @@ def run():
         gain_ref=gain_ref,
         redirected_logger=rich_handler,
         force_mdoc_metadata=not args.ignore_mdoc_metadata,
-        strict=not args.relax,
         processing_enabled=machine_data.get("processing_enabled", True),
         skip_existing_processing=args.skip_existing_processing,
     )

src/murfey/client/analyser.py

@@ -57,7 +57,6 @@ def __init__(
         self._limited = limited
         self._experiment_type = ""
         self._acquisition_software = ""
-        self._role = ""
         self._extension: str = ""
         self._unseen_xml: list = []
         self._context: Context | None = None
@@ -126,14 +125,12 @@ def _find_context(self, file_path: Path) -> bool:
         in the Context classes themselves.
         """
         if "atlas" in file_path.parts:
-            self._role = "detector"
             self._context = SPAMetadataContext("epu", self._basepath)
             return True
 
         # CLEM workflow checks
         # Look for LIF and XLIF files
         if file_path.suffix in (".lif", ".xlif"):
-            self._role = "detector"
             self._context = CLEMContext("leica", self._basepath)
             return True
         # Look for TIFF files associated with CLEM workflow
@@ -152,7 +149,6 @@ def _find_context(self, file_path: Path) -> bool:
                 ).get("analyse_created_directories", [])
             )
             if created_directories.intersection(set(file_path.parts)):
-                self._role = "detector"
                 self._context = CLEMContext("leica", self._basepath)
                 return True
 
@@ -181,9 +177,6 @@ def _find_context(self, file_path: Path) -> bool:
                         else SPAContext("epu", self._basepath)
                     )
                 self.parameters_model = ProcessingParametersSPA
-                # Assign it the detector attribute if not already present
-                if not self._role:
-                    self._role = "detector"
                 return True
 
             # Files starting with "Position" belong to the standard tomography workflow
@@ -198,23 +191,6 @@ def _find_context(self, file_path: Path) -> bool:
                     logger.info("Acquisition software: tomo")
                     self._context = TomographyContext("tomo", self._basepath)
                     self.parameters_model = PreprocessingParametersTomo
-                # Assign role if not already present
-                if not self._role:
-                    # Fractions files attributed to the detector
-                    if (
-                        "Fractions" in split_file_name[-1]
-                        or "fractions" in split_file_name[-1]
-                    ):
-                        self._role = "detector"
-                    # MDOC files attributed to the microscope
-                    elif (
-                        file_path.suffix == ".mdoc"
-                        or file_path.with_suffix(".mdoc").is_file()
-                    ):
-                        self._role = "microscope"
-                    # Attribute all other files to the detector
-                    else:
-                        self._role = "detector"
                 return True
 
             # Files with these suffixes belong to the serial EM tomography workflow
@@ -239,19 +215,14 @@ def _find_context(self, file_path: Path) -> bool:
                     return False
                 self._context = TomographyContext("serialem", self._basepath)
                 self.parameters_model = PreprocessingParametersTomo
-                if not self._role:
-                    if "Frames" in file_path.parts:
-                        self._role = "detector"
-                    else:
-                        self._role = "microscope"
                 return True
         return False
 
     def post_transfer(self, transferred_file: Path):
         try:
             if self._context:
                 self._context.post_transfer(
-                    transferred_file, role=self._role, environment=self._environment
+                    transferred_file, environment=self._environment
                 )
         except Exception as e:
             logger.error(f"An exception was encountered post transfer: {e}")
@@ -309,19 +280,17 @@ def _analyse(self):
                         self.queue.task_done()
                         continue
                     elif self._extension:
-                        logger.info(f"Context found successfully: {self._role}")
+                        logger.info(
+                            f"Context found successfully for {transferred_file}"
+                        )
                         try:
                             self._context.post_first_transfer(
                                 transferred_file,
-                                role=self._role,
                                 environment=self._environment,
                             )
                         except Exception as e:
                             logger.error(f"Exception encountered: {e}")
-                        if (
-                            self._role == "detector"
-                            and "atlas" not in transferred_file.parts
-                        ):
+                        if "atlas" not in transferred_file.parts:
                             if not dc_metadata:
                                 try:
                                     dc_metadata = self._context.gather_metadata(
@@ -377,20 +346,16 @@ def _analyse(self):
                     self._find_extension(transferred_file)
                     if self._extension:
                         logger.info(
-                            f"Context found successfully: {self._role}, {transferred_file}"
+                            f"Extension found successfully for {transferred_file}"
                         )
                         try:
                             self._context.post_first_transfer(
                                 transferred_file,
-                                role=self._role,
                                 environment=self._environment,
                             )
                         except Exception as e:
                             logger.error(f"Exception encountered: {e}")
-                        if (
-                            self._role == "detector"
-                            and "atlas" not in transferred_file.parts
-                        ):
+                        if "atlas" not in transferred_file.parts:
                             if not dc_metadata:
                                 try:
                                     dc_metadata = self._context.gather_metadata(
@@ -443,8 +408,8 @@ def _xml_file(self, data_file: Path) -> Path:
         if not self._environment:
             return data_file.with_suffix(".xml")
         file_name = f"{'_'.join(p for p in data_file.stem.split('_')[:-1])}.xml"
-        data_directories = self._murfey_config.get("data_directories", {})
-        for dd in data_directories.keys():
+        data_directories = self._murfey_config.get("data_directories", [])
+        for dd in data_directories:
             if str(data_file).startswith(dd):
                 base_dir = Path(dd)
                 mid_dir = data_file.relative_to(dd).parent

src/murfey/client/context.py

@@ -40,14 +40,24 @@ def __init__(self, name: str, acquisition_software: str):
         self._acquisition_software = acquisition_software
         self.name = name
 
-    def post_transfer(self, transferred_file: Path, role: str = "", **kwargs):
+    def post_transfer(
+        self,
+        transferred_file: Path,
+        environment: MurfeyInstanceEnvironment | None = None,
+        **kwargs,
+    ):
         # Search external packages for additional hooks to include in Murfey
         for h in entry_points(group="murfey.post_transfer_hooks"):
             if h.name == self.name:
-                h.load()(transferred_file, role=role, **kwargs)
+                h.load()(transferred_file, environment=environment, **kwargs)
 
-    def post_first_transfer(self, transferred_file: Path, role: str = "", **kwargs):
-        self.post_transfer(transferred_file, role=role, **kwargs)
+    def post_first_transfer(
+        self,
+        transferred_file: Path,
+        environment: MurfeyInstanceEnvironment | None = None,
+        **kwargs,
+    ):
+        self.post_transfer(transferred_file, environment=environment, **kwargs)
 
     def gather_metadata(
         self, metadata_file: Path, environment: MurfeyInstanceEnvironment | None = None

src/murfey/client/contexts/clem.py

@@ -101,14 +101,11 @@ def __init__(self, acquisition_software: str, basepath: Path):
     def post_transfer(
         self,
         transferred_file: Path,
-        role: str = "",
         environment: Optional[MurfeyInstanceEnvironment] = None,
         **kwargs,
     ) -> bool:
 
-        super().post_transfer(
-            transferred_file, role=role, environment=environment, **kwargs
-        )
+        super().post_transfer(transferred_file, environment=environment, **kwargs)
 
         # Process files generated by "auto-save" acquisition mode
         # These include TIF/TIFF and XLIF files

src/murfey/client/contexts/fib.py

@@ -44,13 +44,10 @@ def __init__(self, acquisition_software: str, basepath: Path):
     def post_transfer(
         self,
         transferred_file: Path,
-        role: str = "",
         environment: MurfeyInstanceEnvironment | None = None,
         **kwargs,
     ):
-        super().post_transfer(
-            transferred_file, role=role, environment=environment, **kwargs
-        )
+        super().post_transfer(transferred_file, environment=environment, **kwargs)
         if self._acquisition_software == "autotem":
             parts = transferred_file.parts
             if "DCImages" in parts and transferred_file.suffix == ".png":

src/murfey/client/contexts/spa.py

@@ -136,9 +136,9 @@ def _foil_hole_from_file(f: Path) -> int:
 
 
 def _grid_square_metadata_file(
-    f: Path, data_directories: Dict[Path, str], visit: str, grid_square: int
+    f: Path, data_directories: List[Path], visit: str, grid_square: int
 ) -> Path:
-    for dd in data_directories.keys():
+    for dd in data_directories:
         if str(f).startswith(str(dd)):
             base_dir = dd
             mid_dir = f.relative_to(dd).parent
@@ -542,7 +542,7 @@ def _position_analysis(
         grid_square = _grid_square_from_file(transferred_file)
         grid_square_metadata_file = _grid_square_metadata_file(
             transferred_file,
-            {Path(d): l for d, l in machine_config["data_directories"].items()},
+            machine_config["data_directories"],
             environment.visit,
             grid_square,
         )
@@ -669,18 +669,16 @@ def _position_analysis(
     def post_transfer(
         self,
         transferred_file: Path,
-        role: str = "",
         environment: MurfeyInstanceEnvironment | None = None,
         **kwargs,
     ) -> bool:
         super().post_transfer(
             transferred_file=transferred_file,
-            role=role,
             environment=environment,
             **kwargs,
         )
         data_suffixes = (".mrc", ".tiff", ".tif", ".eer")
-        if role == "detector" and "gain" not in transferred_file.name:
+        if "gain" not in transferred_file.name:
             if transferred_file.suffix in data_suffixes:
                 if self._acquisition_software == "epu":
                     if environment:
@@ -856,7 +854,6 @@ def _register_data_collection(
     def post_transfer(
         self,
         transferred_file: Path,
-        role: str = "",
         environment: MurfeyInstanceEnvironment | None = None,
         **kwargs,
     ) -> bool:

src/murfey/client/contexts/spa_metadata.py

@@ -43,13 +43,11 @@ def __init__(self, acquisition_software: str, basepath: Path):
     def post_transfer(
         self,
         transferred_file: Path,
-        role: str = "",
         environment: Optional[MurfeyInstanceEnvironment] = None,
         **kwargs,
     ):
         super().post_transfer(
             transferred_file=transferred_file,
-            role=role,
             environment=environment,
             **kwargs,
         )

src/murfey/client/contexts/tomo.py

@@ -549,20 +549,19 @@ def _extract_tilt_series(p: Path) -> str:
     def post_transfer(
         self,
         transferred_file: Path,
-        role: str = "",
         environment: MurfeyInstanceEnvironment | None = None,
         **kwargs,
     ) -> List[str]:
         super().post_transfer(
             transferred_file=transferred_file,
-            role=role,
             environment=environment,
             **kwargs,
         )
 
         data_suffixes = (".mrc", ".tiff", ".tif", ".eer")
         completed_tilts = []
-        if role == "detector" and "gain" not in transferred_file.name:
+
+        if "gain" not in transferred_file.name:
             if transferred_file.suffix in data_suffixes:
                 if self._acquisition_software == "tomo":
                     if environment:
@@ -632,13 +631,10 @@ def post_transfer(
     def post_first_transfer(
         self,
         transferred_file: Path,
-        role: str = "",
         environment: MurfeyInstanceEnvironment | None = None,
         **kwargs,
     ):
-        self.post_transfer(
-            transferred_file, role=role, environment=environment, **kwargs
-        )
+        self.post_transfer(transferred_file, environment=environment, **kwargs)
 
     def gather_metadata(
         self, metadata_file: Path, environment: MurfeyInstanceEnvironment | None = None

src/murfey/client/multigrid_control.py

@@ -257,13 +257,6 @@ def rsync_result(update: RSyncerUpdate):
                 force_mdoc_metadata=self.force_mdoc_metadata,
                 limited=limited,
             )
-            for data_dir in self._machine_config["data_directories"].keys():
-                if source.resolve().is_relative_to(Path(data_dir)):
-                    self.analysers[source]._role = self._machine_config[
-                        "data_directories"
-                    ][data_dir]
-                    log.info(f"role found for {source}")
-                    break
             if force_metadata:
                 self.analysers[source].subscribe(
                     partial(self._start_dc, from_form=True)