Merge main and fix conflict

Author: stephen-riggs

.bumpclient.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.20.2"
+current_version = "0.20.3"
 commit = true
 tag = false
 

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.20.2"
+current_version = "0.20.3"
 commit = true
 tag = true
 

Helm/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 name: murfey-services
 description: Umbrella Helm chart for deploying the servers and daemons needed to enable Murfey to transfer and process data
-version: 0.20.2
+version: 0.20.3
 dependencies:
   - name: murfey-instrument-server-clem
   - name: murfey-instrument-server-tem

Helm/charts/murfey-instrument-server-clem/Chart.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 name: murfey-instrument-server-clem
 description: Helm chart for deploying a Murfey instrument server, which executes orders to detect, modify, and transfer files on the instrument PC, and notifies the backend server about transferred files
-version: 0.20.2
+version: 0.20.3

Helm/charts/murfey-instrument-server-tem/Chart.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 name: murfey-instrument-server-tem
 description: Helm chart for deploying a Murfey instrument server, which executes orders to detect, modify, and transfer files on the instrument PC, and notifies the backend server about transferred files
-version: 0.20.2
+version: 0.20.3

Helm/charts/murfey-rsync/Chart.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 name: murfey-rsync
 description: Helm chart for deploying an rsync daemon, which is responsible for executing the transfer of files from the client storage directory to the server storage system
-version: 0.20.2
+version: 0.20.3

Helm/charts/murfey-server/Chart.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 name: murfey-server
 description: Helm chart for deploying a Murfey backend server, which is responsible for orchestrating the data transfer and processing workflow between the client PC and the storage system
-version: 0.20.2
+version: 0.20.3

pyproject.toml

@@ -7,7 +7,7 @@ requires = [
 
 [project]
 name = "murfey"
-version = "0.20.2"
+version = "0.20.3"
 description = "Client-Server architecture hauling Cryo-EM data"
 readme = "README.md"
 keywords = [

src/murfey/__init__.py

@@ -1,4 +1,4 @@
 from __future__ import annotations
 
-__version__ = "0.20.2"
-__supported_client_version__ = "0.20.2"
+__version__ = "0.20.3"
+__supported_client_version__ = "0.20.3"

src/murfey/client/watchdir_multigrid.py

@@ -51,6 +51,55 @@ def stop(self):
             self.thread.join()
         log.debug("MultigridDirWatcher thread stop completed")
 
+    def _handle_metadata(self, directory: Path):
+        self.notify(
+            directory,
+            extra_directory=f"metadata_{directory.name}",
+            include_mid_path=False,
+            analyse=self._analyse,
+            limited=True,
+            tag="metadata",
+        )
+        self._seen_dirs.append(directory)
+
+    def _handle_fractions(self, directory: Path, first_loop: bool):
+        processing_started = False
+        for d02 in directory.glob("Images-Disc*"):
+            if d02 not in self._seen_dirs:
+                # If 'skip_existing_processing' is set, do not process for
+                # any data directories found on the first loop.
+                # This allows you to avoid triggering processing again if Murfey is restarted
+                self.notify(
+                    d02,
+                    include_mid_path=False,
+                    remove_files=True,
+                    analyse=(
+                        not (first_loop and self._skip_existing_processing)
+                        if self._analyse
+                        else False
+                    ),
+                    tag="fractions",
+                )
+                self._seen_dirs.append(d02)
+            processing_started = d02 in self._seen_dirs
+        if not processing_started:
+            if (
+                directory.is_dir()
+                and directory not in self._seen_dirs
+                and list(directory.iterdir())
+            ):
+                self.notify(
+                    directory,
+                    include_mid_path=False,
+                    analyse=(
+                        not (first_loop and self._skip_existing_processing)
+                        if self._analyse
+                        else False
+                    ),
+                    tag="fractions",
+                )
+                self._seen_dirs.append(directory)
+
     def _process(self):
         first_loop = True
         while not self._stopping:
@@ -75,53 +124,23 @@ def _process(self):
                         )
                         self._seen_dirs.append(d)
                 else:
-                    if d.is_dir() and d not in self._seen_dirs:
-                        self.notify(
-                            d,
-                            extra_directory=f"metadata_{d.name}",
-                            include_mid_path=False,
-                            analyse=self._analyse,
-                            limited=True,
-                            tag="metadata",
-                        )
-                        self._seen_dirs.append(d)
-                    processing_started = False
-                    for d02 in (d.parent.parent / d.name).glob("Images-Disc*"):
-                        if d02 not in self._seen_dirs:
-                            # If 'skip_existing_processing' is set, do not process for
-                            # any data directories found on the first loop.
-                            # This allows you to avoid triggering processing again if Murfey is restarted
-                            self.notify(
-                                d02,
-                                include_mid_path=False,
-                                remove_files=True,
-                                analyse=(
-                                    not (first_loop and self._skip_existing_processing)
-                                    if self._analyse
-                                    else False
-                                ),
-                                tag="fractions",
-                            )
-                            self._seen_dirs.append(d02)
-                        processing_started = d02 in self._seen_dirs
-                    if not processing_started:
-                        d02 = d.parent.parent / d.name
-                        if (
-                            d02.is_dir()
-                            and d02 not in self._seen_dirs
-                            and list((d.parent.parent / d.name).iterdir())
-                        ):
-                            self.notify(
-                                d02,
-                                include_mid_path=False,
-                                analyse=(
-                                    not (first_loop and self._skip_existing_processing)
-                                    if self._analyse
-                                    else False
-                                ),
-                                tag="fractions",
-                            )
-                            self._seen_dirs.append(d02)
+                    # hack for tomo multigrid metadata structure
+                    sample_dirs = list(d.glob("Sample*"))
+                    if d.is_dir() and len(sample_dirs):
+                        for sample in sample_dirs:
+                            if len(list(sample.glob("*.mdoc"))):
+                                if sample not in self._seen_dirs:
+                                    self._handle_metadata(sample)
+                                self._handle_fractions(
+                                    d.parent.parent.parent
+                                    / f"{d.parent.name}_{d.name}",
+                                    first_loop,
+                                )
+
+                    else:
+                        if d.is_dir() and d not in self._seen_dirs:
+                            self._handle_metadata(d)
+                        self._handle_fractions(d.parent.parent / d.name, first_loop)
 
             if first_loop:
                 first_loop = False