Change how we use the rsync module specified in the machine configuration (#476)

Currently we assume that the rsync module name forms part of the path but this isn't necessarily the case. We should specify the whole rsync module path as the rsync_basepath in the configuration and insert the module name directly where we need it (at the point of construction of the rsync command).

Author: d-j-hatton

src/murfey/client/contexts/clem.py

@@ -34,9 +34,6 @@ def _file_transferred_to(
     # rsync basepath and modules are set in the microscope's configuration YAML file
     return (
         Path(machine_config.get("rsync_basepath", ""))
-        / (
-            machine_config.get("rsync_module", "data") or "data"
-        )  # Add "data" if it wasn't set
         / str(datetime.now().year)
         / source.name
         / file_path.relative_to(source)

src/murfey/client/contexts/spa.py

@@ -46,7 +46,7 @@ def _file_transferred_to(
         return (
             Path(machine_config.get("rsync_basepath", ""))
             / Path(environment.default_destinations[source])
-            / file_path.relative_to(source)
+            / file_path.relative_to(source)  # need to strip out the rsync_module name
         )
     return (
         Path(machine_config.get("rsync_basepath", ""))

src/murfey/client/multigrid_control.py

@@ -32,6 +32,7 @@ class MultigridController:
     session_id: int
     murfey_url: str = "http://localhost:8000"
     rsync_url: str = ""
+    rsync_module: str = "data"
     demo: bool = False
     processing_enabled: bool = True
     do_transfer: bool = True
@@ -59,12 +60,13 @@ def __post_init__(self):
             f"{self.murfey_url}/instruments/{self.instrument_name}/machine"
         ).json()
         self.rsync_url = machine_data.get("rsync_url", "")
+        self.rsync_module = machine_data.get("rsync_module", "data")
         self._environment = MurfeyInstanceEnvironment(
             url=urlparse(self.murfey_url, allow_fragments=False),
             client_id=0,
             murfey_session=self.session_id,
             software_versions=machine_data.get("software_versions", {}),
-            default_destination=f"{machine_data.get('rsync_module') or 'data'}/{datetime.now().year}",
+            default_destination=f"{datetime.now().year}",
             demo=self.demo,
             visit=self.visit,
             data_collection_parameters=self.data_collection_parameters,
@@ -123,7 +125,7 @@ def _start_rsyncer_multigrid(
                     break
             else:
                 self._environment.default_destinations[source] = (
-                    f"{machine_data.get('rsync_module') or 'data'}/{datetime.now().year}"
+                    f"{datetime.now().year}"
                 )
                 destination = determine_default_destination(
                     self._environment.visit,
@@ -205,7 +207,7 @@ def _start_rsyncer(
                 rsync_cmd = [
                     "rsync",
                     f"{posix_path(self._environment.gain_ref)!r}",  # '!r' will print strings in ''
-                    f"{self._environment.url.hostname}::{visit_path}/processing",
+                    f"{self._environment.url.hostname}::{self.rsync_module}/{visit_path}/processing",
                 ]
                 # Wrap in bash shell
                 cmd = [
@@ -223,6 +225,7 @@ def _start_rsyncer(
             self.rsync_processes[source] = RSyncer(
                 source,
                 basepath_remote=Path(destination),
+                rsync_module=self.rsync_module,
                 server_url=(
                     urlparse(self.rsync_url)
                     if self.rsync_url

src/murfey/client/rsync.py

@@ -54,6 +54,7 @@ def __init__(
         self,
         basepath_local: Path,
         basepath_remote: Path,
+        rsync_module: str,
         server_url: ParseResult,
         stop_callback: Callable = lambda *args, **kwargs: None,
         local: bool = False,
@@ -66,6 +67,7 @@ def __init__(
         super().__init__()
         self._basepath = basepath_local.absolute()
         self._basepath_remote = basepath_remote
+        self._rsync_module = rsync_module
         self._do_transfer = do_transfer
         self._remove_files = remove_files
         self._required_substrings_for_removal = required_substrings_for_removal
@@ -76,7 +78,9 @@ def __init__(
         if local:
             self._remote = str(basepath_remote)
         else:
-            self._remote = f"{server_url.hostname}::{basepath_remote}/"
+            self._remote = (
+                f"{server_url.hostname}::{self._rsync_module}/{basepath_remote}/"
+            )
         # For local tests you can use something along the lines of
         # self._remote = f"wra62962@ws133:/dls/tmp/wra62962/junk/{basepath_remote}"
         # to avoid having to set up an rsync daemon
@@ -116,6 +120,7 @@ def from_rsyncer(cls, rsyncer: RSyncer, **kwargs):
         return cls(
             rsyncer._basepath,
             rsyncer._basepath_remote,
+            rsyncer._rsync_module,
             rsyncer._server_url,
             local=kwarguments_from_rsyncer["local"],
             status_bar=kwarguments_from_rsyncer["status_bar"],

src/murfey/client/tui/app.py

@@ -164,7 +164,7 @@ def _start_rsyncer_multigrid(
                     break
             else:
                 self._environment.default_destinations[source] = (
-                    f"{machine_data.get('rsync_module') or 'data'}/{datetime.now().year}"
+                    f"{datetime.now().year}"
                 )
                 destination = determine_default_destination(
                     self._visit,
@@ -214,7 +214,7 @@ def _start_rsyncer(
                 rsync_cmd = [
                     "rsync",
                     f"{posix_path(self._environment.gain_ref)!r}",
-                    f"{self._url.hostname}::{visit_path}/processing",
+                    f"{self._url.hostname}::{self._machine_config.get('rsync_module', 'data')}/{visit_path}/processing",
                 ]
                 # Encase in bash shell
                 cmd = [
@@ -232,6 +232,7 @@ def _start_rsyncer(
             self.rsync_processes[source] = RSyncer(
                 source,
                 basepath_remote=Path(destination),
+                rsync_module=self._machine_config.get("rsync_module", "data"),
                 server_url=urlparse(rsync_url) if rsync_url else self._url,
                 # local=self._environment.demo,
                 status_bar=self._statusbar,

src/murfey/client/tui/screens.py

@@ -91,7 +91,10 @@ def determine_default_destination(
     elif machine_data.get("data_directories"):
         for data_dir in machine_data["data_directories"]:
             if source.resolve() == Path(data_dir):
-                _default = destination + f"/{visit}"
+                _default = (
+                    destination
+                    + f"{machine_data.get('rsync_module') or 'data'}/{visit}"
+                )
                 break
             else:
                 try:
@@ -310,12 +313,7 @@ def _add_directory(self, directory: str, add_destination: bool = True):
                 if source.is_relative_to(s):
                     return
             self.app._environment.sources.append(source)
-            machine_data = requests.get(
-                f"{self.app._environment.url.geturl()}/machine"
-            ).json()
-            self.app._default_destinations[source] = (
-                f"{machine_data.get('rsync_module') or 'data'}/{datetime.now().year}"
-            )
+            self.app._default_destinations[source] = f"{datetime.now().year}"
         if self._launch_btn:
             self._launch_btn.disabled = False
         self.query_one("#selected-directories").write(str(source) + "\n")
@@ -889,7 +887,7 @@ def on_button_pressed(self, event):
             rsync_cmd = [
                 "rsync",
                 f"{posix_path(self._dir_tree._gain_reference)!r}",
-                f"{self.app._environment.url.hostname}::{visit_path}/processing/{secure_filename(self._dir_tree._gain_reference.name)}",
+                f"{self.app._environment.url.hostname}::{self.app._machine_config.get('rsync_module', 'data')}/{visit_path}/processing/{secure_filename(self._dir_tree._gain_reference.name)}",
             ]
             # Encase in bash shell
             cmd = ["bash", "-c", " ".join(rsync_cmd)]
@@ -1011,13 +1009,10 @@ def compose(self):
                             and d.name
                             not in machine_config["create_directories"].values()
                         ):
-                            machine_data = requests.get(
-                                f"{self.app._environment.url.geturl()}/machine"
-                            ).json()
                             dest = determine_default_destination(
                                 self.app._visit,
                                 s,
-                                f"{machine_data.get('rsync_module') or 'data'}/{datetime.now().year}",
+                                f"{datetime.now().year}",
                                 self.app._environment,
                                 self.app.analysers,
                                 touch=True,

src/murfey/instrument_server/api.py

@@ -276,15 +276,23 @@ class GainReference(BaseModel):
     gain_destination_dir: str = "processing"
 
 
-@router.post("/sessions/{session_id}/upload_gain_reference")
-def upload_gain_reference(session_id: MurfeySessionID, gain_reference: GainReference):
+@router.post(
+    "/instruments/{instrument_name}/sessions/{session_id}/upload_gain_reference"
+)
+def upload_gain_reference(
+    instrument_name: str, session_id: MurfeySessionID, gain_reference: GainReference
+):
     safe_gain_path = sanitise(str(gain_reference.gain_path))
     safe_visit_path = sanitise(gain_reference.visit_path)
     safe_destination_dir = sanitise(gain_reference.gain_destination_dir)
+    machine_config = requests.get(
+        f"{_get_murfey_url()}/instruments/{sanitise_nonpath(instrument_name)}/machine",
+        headers={"Authorization": f"Bearer {tokens[session_id]}"},
+    ).json()
     cmd = [
         "rsync",
         safe_gain_path,
-        f"{urlparse(_get_murfey_url(), allow_fragments=False).hostname}::{safe_visit_path}/{safe_destination_dir}/{secure_filename(gain_reference.gain_path.name)}",
+        f"{urlparse(_get_murfey_url(), allow_fragments=False).hostname}::{machine_config.get('rsync_module', 'data')}/{safe_visit_path}/{safe_destination_dir}/{secure_filename(gain_reference.gain_path.name)}",
     ]
     gain_rsync = subprocess.run(cmd)
     if gain_rsync.returncode:

src/murfey/server/__init__.py

@@ -1628,10 +1628,7 @@ def _register_class_selection(message: dict, _db=murfey_db, demo: bool = False):
 def _find_initial_model(visit: str, machine_config: MachineConfig) -> Path | None:
     if machine_config.initial_model_search_directory:
         visit_directory = (
-            machine_config.rsync_basepath
-            / (machine_config.rsync_module or "data")
-            / str(datetime.now().year)
-            / visit
+            machine_config.rsync_basepath / str(datetime.now().year) / visit
         )
         possible_models = [
             p

src/murfey/server/api/__init__.py

@@ -1453,7 +1453,6 @@ async def process_gain(
     safe_path_name = secure_filename(gain_reference_params.gain_ref.name)
     filepath = (
         Path(machine_config.rsync_basepath)
-        / (machine_config.rsync_module or "data")
         / str(datetime.datetime.now().year)
         / secure_filename(visit_name)
         / machine_config.gain_directory_name
@@ -1464,7 +1463,6 @@ async def process_gain(
         filepath_prev = filepath
         filepath = (
             Path(machine_config.rsync_basepath)
-            / (machine_config.rsync_module or "data")
             / str(datetime.datetime.now().year - 1)
             / secure_filename(visit_name)
             / machine_config.gain_directory_name
@@ -1541,7 +1539,6 @@ async def write_eer_fractionation_file(
     else:
         file_path = (
             Path(machine_config.rsync_basepath)
-            / (machine_config.rsync_module or "data")
             / str(datetime.datetime.now().year)
             / secure_filename(visit_name)
             / "processing"
@@ -1586,7 +1583,6 @@ async def make_gif(
     ]
     output_dir = (
         Path(machine_config.rsync_basepath)
-        / (machine_config.rsync_module or "data")
         / secure_filename(year)
         / secure_filename(visit_name)
         / "processed"

src/murfey/server/api/instrument.py

@@ -234,12 +234,12 @@ async def request_gain_reference_upload(
         instrument_name
     ]
     visit = db.exec(select(Session).where(Session.id == session_id)).one().visit
-    visit_path = f"{machine_config.rsync_module or 'data'}/{datetime.datetime.now().year}/{visit}"
+    visit_path = f"{datetime.datetime.now().year}/{visit}"
     data = {}
     if machine_config.instrument_server_url:
         async with aiohttp.ClientSession() as session:
             async with session.post(
-                f"{machine_config.instrument_server_url}/sessions/{session_id}/upload_gain_reference",
+                f"{machine_config.instrument_server_url}/instruments/{instrument_name}/sessions/{session_id}/upload_gain_reference",
                 json={
                     "gain_path": str(gain_reference_request.gain_path),
                     "visit_path": visit_path,