Skip to content

Commit 2d3e030

Browse files
Separate server endpoints into different routers (#595)
* Refactored 'murfey.server' API endpoints so that they can be covered by different authentication methods * Replaced hard-coded URLs with function that constructs URL based on a provided router and function name * Added CLI 'murfey.generate_route_manifest' to generate up-to-date route manifests * Migrated server startup and feedback functions to separate modules * Fixed CLEM workflow to run on instrument server (see PR #594) --------- Co-authored-by: Eu Pin Tien <[email protected]>
1 parent f0ff051 commit 2d3e030

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

64 files changed

+7262
-6055
lines changed

Dockerfiles/murfey-instrument-server

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# podman build --build-arg groupid=<groupid> --build-arg userid=<userid> --build-arg groupname=<groupname> --no-cache -f path/to/Dockerfiles/murfey-instrument-server -t murfey-instrument-server:<version> path/to/python-murfey
33

44
# Set up the base image to build with
5-
FROM docker.io/library/python:3.12.8-slim-bullseye AS base
5+
FROM docker.io/library/python:3.12.10-slim-bookworm AS base
66

77
# Install Vim in base image
88
RUN apt-get update && \

Dockerfiles/murfey-rsync

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# Template build command
22
# podman build --build-arg groupid=<groupid> --build-arg userid=<userid> --build-arg groupname=<groupname> --no-cache -f path/to/Dockerfiles/murfey-rsync
33

4-
FROM docker.io/library/alpine:3.20
4+
FROM docker.io/library/alpine:3.21
55
# FROM alpine:3.14
66

77
ARG groupid

Dockerfiles/murfey-server

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# podman build --build-arg groupid=<groupid> --build-arg userid=<userid> --build-arg groupname=<groupname> --no-cache -f path/to/Dockerfiles/murfey-server -t murfey-server:<version> path/to/python-murfey
33

44
# Set up the base image to build with
5-
FROM docker.io/library/python:3.12.8-slim-bullseye AS base
5+
FROM docker.io/library/python:3.12.10-slim-bookworm AS base
66

77
# Install Vim and PostgreSQL dependencies in base image
88
RUN apt-get update && \

pyproject.toml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,9 +87,10 @@ GitHub = "https://github.com/DiamondLightSource/python-murfey"
8787
"murfey.decrypt_password" = "murfey.cli.decrypt_db_password:run"
8888
"murfey.generate_key" = "murfey.cli.generate_crypto_key:run"
8989
"murfey.generate_password" = "murfey.cli.generate_db_password:run"
90+
"murfey.generate_route_manifest" = "murfey.cli.generate_route_manifest:run"
9091
"murfey.instrument_server" = "murfey.instrument_server:run"
9192
"murfey.repost_failed_calls" = "murfey.cli.repost_failed_calls:run"
92-
"murfey.server" = "murfey.server:run"
93+
"murfey.server" = "murfey.server.run:run"
9394
"murfey.sessions" = "murfey.cli.db_sessions:run"
9495
"murfey.simulate" = "murfey.cli.dummy:run"
9596
"murfey.spa_inject" = "murfey.cli.inject_spa_processing:run"
@@ -117,6 +118,7 @@ zip-safe = false
117118

118119
[tool.setuptools.package-data]
119120
"murfey.client.tui" = ["*.css"]
121+
"murfey.util" = ["route_manifest.yaml"]
120122

121123
[tool.setuptools.packages.find]
122124
where = ["src", "tests"]

src/murfey/bootstrap/__main__.py

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,11 @@
77
import pathlib
88
import subprocess
99
import sys
10-
from urllib.parse import urlparse
10+
from urllib.parse import ParseResult, urlparse
1111
from urllib.request import urlopen
1212

13+
from murfey.util.api import url_path_for
14+
1315
"""
1416
A script to simplify installing Murfey on a network-isolated machine.
1517
This could in theory be invoked by
@@ -64,10 +66,13 @@ def _download_to_file(url: str, outfile: str):
6466
# Construct a minimal base path string
6567
# Extract the host name for pip installation purposes
6668
try:
67-
murfey_url = urlparse(args.server)
69+
murfey_url: ParseResult = urlparse(args.server)
6870
except Exception:
6971
exit(f"{args.server} is not a valid URL")
72+
murfey_proxy_path = murfey_url.path.rstrip("/")
7073
murfey_base = f"{murfey_url.scheme}://{murfey_url.netloc}"
74+
if murfey_proxy_path:
75+
murfey_base = f"{murfey_base}{murfey_proxy_path}"
7176
murfey_hostname = murfey_url.netloc.split(":")[0]
7277

7378
# Check that Python version is supported
@@ -82,7 +87,10 @@ def _download_to_file(url: str, outfile: str):
8287
# Step 1: Download pip wheel
8388
print()
8489
print(f"1/4 -- Connecting to murfey server on {murfey_base}...")
85-
_download_to_file(f"{murfey_base}/bootstrap/pip.whl", "pip.whl")
90+
_download_to_file(
91+
f"{murfey_base}{url_path_for('bootstrap.bootstrap', 'get_pip_wheel')}",
92+
"pip.whl",
93+
)
8694

8795
# Step 2: Get pip to install itself
8896
print()
@@ -96,7 +104,7 @@ def _download_to_file(url: str, outfile: str):
96104
"--trusted-host",
97105
murfey_hostname,
98106
"-i",
99-
f"{murfey_base}/pypi",
107+
f"{murfey_base}{url_path_for('bootstrap.pypi', 'get_pypi_index')}",
100108
"pip",
101109
]
102110
)
@@ -116,7 +124,7 @@ def _download_to_file(url: str, outfile: str):
116124
"--trusted-host",
117125
murfey_hostname,
118126
"-i",
119-
f"{murfey_base}/pypi",
127+
f"{murfey_base}{url_path_for('bootstrap.pypi', 'get_pypi_index')}",
120128
"--upgrade",
121129
"pip",
122130
]
@@ -135,7 +143,7 @@ def _download_to_file(url: str, outfile: str):
135143
"--trusted-host",
136144
murfey_hostname,
137145
"-i",
138-
f"{murfey_base}/pypi",
146+
f"{murfey_base}{url_path_for('bootstrap.pypi', 'get_pypi_index')}",
139147
"murfey[client]",
140148
]
141149
)
Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
"""
2+
CLI to generate a manifest of the FastAPI router paths present in both the instrument
3+
server and backend server to enable lookup of the URLs based on function name.
4+
"""
5+
6+
import importlib
7+
import inspect
8+
import pkgutil
9+
from argparse import ArgumentParser
10+
from pathlib import Path
11+
from types import ModuleType
12+
from typing import Any
13+
14+
import yaml
15+
from fastapi import APIRouter
16+
17+
import murfey
18+
19+
20+
def find_routers(name: str) -> dict[str, APIRouter]:
21+
22+
def _extract_routers_from_module(module: ModuleType):
23+
routers = {}
24+
for name, obj in inspect.getmembers(module):
25+
if isinstance(obj, APIRouter):
26+
module_path = module.__name__
27+
key = f"{module_path}.{name}"
28+
routers[key] = obj
29+
return routers
30+
31+
routers = {}
32+
33+
# Import the module or package
34+
try:
35+
root = importlib.import_module(name)
36+
except ImportError:
37+
raise ImportError(
38+
f"Cannot import '{name}'. Please ensure that you've installed all the "
39+
"dependencies for the client, instrument server, and backend server "
40+
"before running this command."
41+
)
42+
43+
# If it's a package, walk through submodules and extract routers from each
44+
if hasattr(root, "__path__"):
45+
module_list = pkgutil.walk_packages(root.__path__, prefix=name + ".")
46+
for _, module_name, _ in module_list:
47+
try:
48+
module = importlib.import_module(module_name)
49+
except ImportError:
50+
raise ImportError(
51+
f"Cannot import '{module_name}'. Please ensure that you've "
52+
"installed all the dependencies for the client, instrument "
53+
"server, and backend server before running this command."
54+
)
55+
56+
routers.update(_extract_routers_from_module(module))
57+
58+
# Extract directly from single module
59+
else:
60+
routers.update(_extract_routers_from_module(root))
61+
62+
return routers
63+
64+
65+
def get_route_manifest(routers: dict[str, APIRouter]):
66+
67+
manifest = {}
68+
69+
for router_name, router in routers.items():
70+
routes = []
71+
for route in router.routes:
72+
path_params = []
73+
for param in route.dependant.path_params:
74+
param_type = param.type_ if param.type_ is not None else Any
75+
param_info = {
76+
"name": param.name if hasattr(param, "name") else "",
77+
"type": (
78+
param_type.__name__
79+
if hasattr(param_type, "__name__")
80+
else str(param_type)
81+
),
82+
}
83+
path_params.append(param_info)
84+
route_info = {
85+
"path": route.path if hasattr(route, "path") else "",
86+
"function": route.name if hasattr(route, "name") else "",
87+
"path_params": path_params,
88+
"methods": list(route.methods) if hasattr(route, "methods") else [],
89+
}
90+
routes.append(route_info)
91+
manifest[router_name] = routes
92+
return manifest
93+
94+
95+
def run():
96+
# Set up additional args
97+
parser = ArgumentParser()
98+
parser.add_argument(
99+
"--debug",
100+
action="store_true",
101+
default=False,
102+
help=("Outputs the modules being inspected when creating the route manifest"),
103+
)
104+
args = parser.parse_args()
105+
106+
# Find routers
107+
print("Finding routers...")
108+
routers = {
109+
**find_routers("murfey.instrument_server.api"),
110+
**find_routers("murfey.server.api"),
111+
}
112+
# Generate the manifest
113+
print("Extracting route information")
114+
manifest = get_route_manifest(routers)
115+
116+
# Verify
117+
if args.debug:
118+
for router_name, routes in manifest.items():
119+
print(f"Routes found in {router_name!r}")
120+
for route in routes:
121+
for key, value in route.items():
122+
print(f"\t{key}: {value}")
123+
print()
124+
125+
# Save the manifest
126+
murfey_dir = Path(murfey.__path__[0])
127+
manifest_file = murfey_dir / "util" / "route_manifest.yaml"
128+
with open(manifest_file, "w") as file:
129+
yaml.dump(manifest, file, default_flow_style=False, sort_keys=False)
130+
print(
131+
"Route manifest for instrument and backend servers saved to "
132+
f"{str(manifest_file)!r}"
133+
)
134+
exit()
135+
136+
137+
if __name__ == "__main__":
138+
run()

src/murfey/cli/spa_ispyb_messages.py

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,11 @@
1818
from sqlmodel import create_engine, select
1919

2020
from murfey.client.contexts.spa import _get_xml_list_index
21-
from murfey.server import _murfey_id, _register
21+
from murfey.server.feedback import _murfey_id, _register
2222
from murfey.server.ispyb import ISPyBSession, TransportManager, get_session_id
2323
from murfey.server.murfey_db import url
2424
from murfey.util import db
25+
from murfey.util.api import url_path_for
2526
from murfey.util.config import get_machine_config, get_microscope, get_security_config
2627

2728

@@ -69,7 +70,7 @@ def run():
6970
help="Path to directory containing image files",
7071
)
7172
parser.add_argument(
72-
"--suffic",
73+
"--suffix",
7374
dest="suffix",
7475
required=True,
7576
type=str,
@@ -203,7 +204,9 @@ def run():
203204
]
204205
)
205206
binning_factor = 1
206-
server_config = requests.get(f"{args.url}/machine").json()
207+
server_config = requests.get(
208+
f"{args.url}{url_path_for('session_control.router', 'machine_info_by_instrument', instrument_name=args.microscope)}"
209+
).json()
207210
if server_config.get("superres"):
208211
# If camera is capable of superres and collection is in superres
209212
binning_factor = 2

src/murfey/cli/transfer.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,14 @@
33
import argparse
44
import subprocess
55
from pathlib import Path
6-
from urllib.parse import urlparse
6+
from urllib.parse import ParseResult, urlparse
77

88
import requests
99
from rich.console import Console
1010
from rich.prompt import Confirm
1111

1212
from murfey.client import read_config
13+
from murfey.util.api import url_path_for
1314
from murfey.util.config import MachineConfig
1415

1516

@@ -35,11 +36,11 @@ def run():
3536
args = parser.parse_args()
3637

3738
console = Console()
38-
murfey_url = urlparse(args.server, allow_fragments=False)
39+
murfey_url: ParseResult = urlparse(args.server, allow_fragments=False)
3940

4041
machine_data = MachineConfig(
4142
requests.get(
42-
f"{murfey_url.geturl()}/instruments/{instrument_name}/machine"
43+
f"{murfey_url.geturl()}{url_path_for('session_control.router', 'machine_info_by_instrument', instrument_name=instrument_name)}"
4344
).json()
4445
)
4546
if Path(args.source or ".").resolve() in machine_data.data_directories:

src/murfey/client/__init__.py

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,26 @@
2626
from murfey.client.instance_environment import MurfeyInstanceEnvironment
2727
from murfey.client.tui.app import MurfeyTUI
2828
from murfey.client.tui.status_bar import StatusBar
29-
from murfey.util.client import _get_visit_list, authorised_requests, read_config
29+
from murfey.util.api import url_path_for
30+
from murfey.util.client import authorised_requests, read_config
31+
from murfey.util.models import Visit
3032

3133
log = logging.getLogger("murfey.client")
3234

3335
requests.get, requests.post, requests.put, requests.delete = authorised_requests()
3436

3537

38+
def _get_visit_list(api_base: ParseResult, instrument_name: str):
39+
proxy_path = api_base.path.rstrip("/")
40+
get_visits_url = api_base._replace(
41+
path=f"{proxy_path}{url_path_for('session_control.router', 'get_current_visits', instrument_name=instrument_name)}"
42+
)
43+
server_reply = requests.get(get_visits_url.geturl())
44+
if server_reply.status_code != 200:
45+
raise ValueError(f"Server unreachable ({server_reply.status_code})")
46+
return [Visit.parse_obj(v) for v in server_reply.json()]
47+
48+
3649
def write_config(config: configparser.ConfigParser):
3750
mcch = os.environ.get("MURFEY_CLIENT_CONFIG_HOME")
3851
murfey_client_config_home = Path(mcch) if mcch else Path.home()
@@ -262,7 +275,9 @@ def run():
262275
rich_handler.setLevel(logging.DEBUG if args.debug else logging.INFO)
263276

264277
# Set up websocket app and handler
265-
client_id = requests.get(f"{murfey_url.geturl()}/new_client_id/").json()
278+
client_id = requests.get(
279+
f"{murfey_url.geturl()}{url_path_for('session_control.router', 'new_client_id')}"
280+
).json()
266281
ws = murfey.client.websocket.WSApp(
267282
server=args.server,
268283
id=client_id["new_id"],
@@ -279,7 +294,7 @@ def run():
279294

280295
# Load machine data for subsequent sections
281296
machine_data = requests.get(
282-
f"{murfey_url.geturl()}/instruments/{instrument_name}/machine"
297+
f"{murfey_url.geturl()}{url_path_for('session_control.router', 'machine_info_by_instrument', instrument_name=instrument_name)}"
283298
).json()
284299
gain_ref: Path | None = None
285300

src/murfey/client/analyser.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,9 @@ def __init__(
8282
else {}
8383
)
8484

85+
def __repr__(self) -> str:
86+
return f"<Analyser ({self._basepath})>"
87+
8588
def _find_extension(self, file_path: Path):
8689
"""
8790
Identifies the file extension and stores that information in the class.

0 commit comments

Comments
 (0)