Container setup for TEM workflow + tomography/SPA bugfixes (#503)

* Set up Helm charts needed to create an instrument server container to test TEM workflow with
* Added an rsync_url key to the MurfeyInstanceEnvironment class, now that we are supporting externally run rsync daemons
* Set microscope name using instrument_name from database lookup, instead of using the get_microscope() function
* Added logs to the prepare_(eer_)gain() function to see why the subprocess fails
* Fixed logic for why the first tomography position is not processed
* Added logic to handle cases in the feedback_callback function where no workflows are found
* Locked NumPy to <2

Author: tieneupin

Helm/Chart.yaml

@@ -4,5 +4,6 @@ description: Umbrella Helm chart for deploying the servers and daemons needed to
 version: 0.16.12
 dependencies:
   - name: murfey-instrument-server-clem
+  - name: murfey-instrument-server-tem
   - name: murfey-server
   - name: murfey-rsync

Helm/charts/murfey-instrument-server-tem/Chart.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+name: murfey-instrument-server-tem
+description: Helm chart for deploying a Murfey instrument server, which executes orders to detect, modify, and transfer files on the instrument PC, and notifies the backend server about transferred files
+version: 0.16.12

Helm/charts/murfey-instrument-server-tem/templates/deployment.yaml

@@ -0,0 +1,68 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.appName }}
+  namespace: {{ .Values.global.namespace }}
+  labels:
+    app: {{ .Values.appName }}
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Cluster
+  ports:
+    {{- toYaml .Values.servicePorts | nindent 2 }}
+  selector:
+    app: {{ .Values.appName }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Values.appName }}
+  namespace: {{ .Values.global.namespace }}
+  labels:
+    app: {{ .Values.appName }}
+spec:
+  replicas: {{ .Values.replicas }}
+  selector:
+    matchLabels:
+      app: {{ .Values.appName }}
+  template:
+    metadata:
+      labels:
+        app: {{ .Values.appName }}
+    spec:
+      securityContext:
+        runAsUser: {{ .Values.global.runAsUser }}
+        runAsGroup: {{ .Values.global.runAsGroup }}
+      volumes:
+        # Mount config files from secrets
+        - name: murfey-client-config
+          secret:
+            secretName: {{ .Values.global.murfeyClientConfigTEMSecretName }}
+            items:
+              - key: {{ .Values.global.murfeyClientConfigTEMFileName }}
+                path: .murfey
+        # Mount data directories
+        {{- toYaml .Values.extraVolumes | nindent 8 }}
+      containers:
+        - name: {{ .Values.appName }}
+          image: {{ .Values.image }}
+          imagePullPolicy: Always
+          securityContext:
+            privileged: false
+          volumeMounts:
+            # Mount Murfey client config
+            - name: murfey-client-config
+              mountPath: /murfey/config/.murfey
+              subPath: .murfey
+              readOnly: false
+            # Mount data directories
+            {{- toYaml .Values.extraVolumeMounts | nindent 12 }}
+          env:
+            - name: MURFEY_CLIENT_CONFIG_HOME
+              value: "/tmp"
+          ports:
+            - containerPort: {{ .Values.containerPort }}
+          command:
+            {{- toYaml .Values.command | nindent 12 }}
+          args:
+            {{- toYaml .Values.args | nindent 12 }}

pyproject.toml

@@ -64,7 +64,7 @@ server = [
     "ispyb", # Responsible for setting requirements for SQLAlchemy and mysql-connector-python; v10.0.0: sqlalchemy <2, mysql-connector-python >=8.0.32
     "jinja2",
     "mrcfile",
-    "numpy",
+    "numpy<2",
     "packaging",
     "passlib",
     "pillow",

src/murfey/client/__init__.py

@@ -322,6 +322,11 @@ def run():
         default_destination=args.destination or str(datetime.now().year),
         demo=args.demo,
         processing_only_mode=server_routing_prefix_found,
+        rsync_url=(
+            urlparse(machine_data["rsync_url"]).hostname
+            if machine_data.get("rsync_url")
+            else ""
+        ),
     )
 
     ws.environment = instance_environment

src/murfey/client/analyser.py

@@ -23,6 +23,7 @@
 from murfey.client.rsync import RSyncerUpdate, TransferResult
 from murfey.client.tui.forms import FormDependency
 from murfey.util import Observer, get_machine_config_client
+from murfey.util.mdoc import get_block
 from murfey.util.models import PreprocessingParametersTomo, ProcessingParametersSPA
 
 logger = logging.getLogger("murfey.client.analyser")
@@ -113,6 +114,13 @@ def _find_extension(self, file_path: Path):
         ):
             logger.info(f"File extension re-evaluated: {file_path.suffix}")
             self._extension = file_path.suffix
+        # If we see an .mdoc file first, use that to determine the file extensions
+        elif file_path.suffix == ".mdoc":
+            with open(file_path, "r") as md:
+                md.seek(0)
+                mdoc_data_block = get_block(md)
+            if subframe_path := mdoc_data_block.get("SubFramePath"):
+                self._extension = Path(subframe_path).suffix
         # Check for LIF files separately
         elif file_path.suffix == ".lif":
             self._extension = file_path.suffix
@@ -124,6 +132,7 @@ def _find_context(self, file_path: Path) -> bool:
         stages of processing. Actions to take for individual files will be determined
         in the Context classes themselves.
         """
+        logger.debug(f"Finding context using file {str(file_path)!r}")
         if "atlas" in file_path.parts:
             self._context = SPAMetadataContext("epu", self._basepath)
             return True
@@ -258,9 +267,9 @@ def _analyse(self):
                     self._find_extension(transferred_file)
                     found = self._find_context(transferred_file)
                     if not found:
-                        # logger.warning(
-                        #     f"Context not understood for {transferred_file}, stopping analysis"
-                        # )
+                        logger.debug(
+                            f"Couldn't find context for {str(transferred_file)!r}"
+                        )
                         self.queue.task_done()
                         continue
                     elif self._extension:
@@ -383,6 +392,10 @@ def _analyse(self):
                         SPAMetadataContext,
                     ),
                 ):
+                    context = str(self._context).split(" ")[0].split(".")[-1]
+                    logger.debug(
+                        f"Transferring file {str(transferred_file)} with context {context!r}"
+                    )
                     self.post_transfer(transferred_file)
             self.queue.task_done()
 

src/murfey/client/contexts/tomo.py

@@ -366,13 +366,19 @@ def _check_tilt_series(
         newly_completed_series: List[str] = []
         mdoc_tilt_series_size = self._tilt_series_sizes.get(tilt_series, 0)
         if not self._tilt_series or not mdoc_tilt_series_size:
+            logger.debug(f"Tilt series size not yet set for {tilt_series!r}")
             return newly_completed_series
 
         counted_tilts = len(self._tilt_series.get(tilt_series, []))
         tilt_series_size_check = counted_tilts >= mdoc_tilt_series_size
         if tilt_series_size_check and tilt_series not in self._completed_tilt_series:
             self._completed_tilt_series.append(tilt_series)
             newly_completed_series.append(tilt_series)
+        else:
+            logger.debug(
+                f"{tilt_series!r} not complete yet. Counted {counted_tilts} tilts. "
+                f"Expected number of tilts was {mdoc_tilt_series_size}"
+            )
         return newly_completed_series
 
     def _add_tomo_tilt(

src/murfey/client/instance_environment.py

@@ -52,6 +52,7 @@ class MurfeyInstanceEnvironment(BaseModel):
     superres: bool = True
     murfey_session: Optional[int] = None
     samples: Dict[Path, SampleInfo] = {}
+    rsync_url: str = ""
 
     class Config:
         validate_assignment: bool = True

src/murfey/client/multigrid_control.py

@@ -410,7 +410,7 @@ def _start_dc(self, json, from_form: bool = False):
                 f"{self._environment.url.geturl()}/visits/{self._environment.visit}/{self._environment.murfey_session}/flush_tomography_processing",
                 json={"rsync_source": str(source)},
             )
-            log.info("tomography processing flushed")
+            log.info("Tomography processing flushed")
 
         elif isinstance(context, SPAModularContext):
             url = f"{str(self._environment.url.geturl())}/visits/{str(self._environment.visit)}/{self.session_id}/register_data_collection_group"

src/murfey/client/tui/app.py

@@ -511,7 +511,7 @@ def _start_dc(self, json, from_form: bool = False):
                 f"{self.app._environment.url.geturl()}/visits/{self._visit}/{self.app._environment.murfey_session}/flush_tomography_processing",
                 json={"rsync_source": str(source)},
             )
-            log.info("tomography processing flushed")
+            log.info("Tomography processing flushed")
         elif isinstance(context, SPAModularContext):
             url = f"{str(self._url.geturl())}/visits/{str(self._visit)}/{self._environment.murfey_session}/register_data_collection_group"
             dcg_data = {