Skip to content

Commit bf9d567

Browse files
tieneupintieneupin
committed
Merged recent changes from 'main'
2 parents 76a4911 + 59d2aa4 commit bf9d567

File tree

12 files changed

+1286
-3
lines changed

12 files changed

+1286
-3
lines changed

‎src/murfey/client/analyser.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
from murfey.client.contexts.spa import SPAModularContext
2020
from murfey.client.contexts.spa_metadata import SPAMetadataContext
2121
from murfey.client.contexts.tomo import TomographyContext
22+
from murfey.client.contexts.tomo_metadata import TomographyMetadataContext
2223
from murfey.client.instance_environment import MurfeyInstanceEnvironment
2324
from murfey.client.rsync import RSyncerUpdate, TransferResult
2425
from murfey.util.client import Observer, get_machine_config_client
@@ -226,6 +227,13 @@ def _analyse(self):
226227
and not self._context
227228
):
228229
self._context = SPAMetadataContext("epu", self._basepath)
230+
elif (
231+
"Batch" in transferred_file.parts
232+
or "SearchMaps" in transferred_file.parts
233+
or transferred_file.name == "Session.dm"
234+
and not self._context
235+
):
236+
self._context = TomographyMetadataContext("tomo", self._basepath)
229237
self.post_transfer(transferred_file)
230238
else:
231239
dc_metadata = {}
@@ -369,9 +377,10 @@ def _analyse(self):
369377
elif isinstance(
370378
self._context,
371379
(
372-
TomographyContext,
373380
SPAModularContext,
374381
SPAMetadataContext,
382+
TomographyContext,
383+
TomographyMetadataContext,
375384
),
376385
):
377386
context = str(self._context).split(" ")[0].split(".")[-1]

‎src/murfey/client/contexts/tomo_metadata.py‎

Lines changed: 324 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,324 @@
1+
import logging
2+
from pathlib import Path
3+
from typing import Optional
4+
5+
import requests
6+
import xmltodict
7+
8+
from murfey.client.context import Context
9+
from murfey.client.contexts.spa import _file_transferred_to, _get_source
10+
from murfey.client.contexts.spa_metadata import _atlas_destination
11+
from murfey.client.instance_environment import MurfeyInstanceEnvironment, SampleInfo
12+
from murfey.util.api import url_path_for
13+
from murfey.util.client import authorised_requests, capture_post
14+
15+
logger = logging.getLogger("murfey.client.contexts.tomo_metadata")
16+
17+
requests.get, requests.post, requests.put, requests.delete = authorised_requests()
18+
19+
20+
def ensure_dcg_exists(transferred_file: Path, environment: MurfeyInstanceEnvironment):
21+
# Make sure we have a data collection group
22+
source = _get_source(transferred_file, environment=environment)
23+
if not source:
24+
return None
25+
dcg_tag = str(source).replace(f"/{environment.visit}", "")
26+
url = f"{str(environment.url.geturl())}{url_path_for('workflow.router', 'register_dc_group', visit_name=environment.visit, session_id=environment.murfey_session)}"
27+
dcg_data = {
28+
"experiment_type": "single particle",
29+
"experiment_type_id": 37,
30+
"tag": dcg_tag,
31+
}
32+
capture_post(url, json=dcg_data)
33+
return dcg_tag
34+
35+
36+
class TomographyMetadataContext(Context):
37+
def __init__(self, acquisition_software: str, basepath: Path):
38+
super().__init__("Tomography_metadata", acquisition_software)
39+
self._basepath = basepath
40+
41+
def post_transfer(
42+
self,
43+
transferred_file: Path,
44+
environment: Optional[MurfeyInstanceEnvironment] = None,
45+
**kwargs,
46+
):
47+
super().post_transfer(
48+
transferred_file=transferred_file,
49+
environment=environment,
50+
**kwargs,
51+
)
52+
53+
if transferred_file.name == "Session.dm" and environment:
54+
logger.info("Tomography session metadata found")
55+
with open(transferred_file, "r") as session_xml:
56+
session_data = xmltodict.parse(session_xml.read())
57+
58+
windows_path = session_data["TomographySession"]["AtlasId"]
59+
logger.info(f"Windows path to atlas metadata found: {windows_path}")
60+
visit_index = windows_path.split("\\").index(environment.visit)
61+
partial_path = "/".join(windows_path.split("\\")[visit_index + 1 :])
62+
logger.info("Partial Linux path successfully constructed from Windows path")
63+
64+
source = _get_source(transferred_file, environment)
65+
if not source:
66+
logger.warning(
67+
f"Source could not be identified for {str(transferred_file)}"
68+
)
69+
return
70+
71+
source_visit_dir = source.parent
72+
73+
logger.info(
74+
f"Looking for atlas XML file in metadata directory {str((source_visit_dir / partial_path).parent)}"
75+
)
76+
atlas_xml_path = list(
77+
(source_visit_dir / partial_path).parent.glob("Atlas_*.xml")
78+
)[0]
79+
logger.info(f"Atlas XML path {str(atlas_xml_path)} found")
80+
with open(atlas_xml_path, "rb") as atlas_xml:
81+
atlas_xml_data = xmltodict.parse(atlas_xml)
82+
atlas_pixel_size = float(
83+
atlas_xml_data["MicroscopeImage"]["SpatialScale"]["pixelSize"]["x"][
84+
"numericValue"
85+
]
86+
)
87+
88+
for p in partial_path.split("/"):
89+
if p.startswith("Sample"):
90+
sample = int(p.replace("Sample", ""))
91+
break
92+
else:
93+
logger.warning(f"Sample could not be identified for {transferred_file}")
94+
return
95+
environment.samples[source] = SampleInfo(
96+
atlas=Path(partial_path), sample=sample
97+
)
98+
url = f"{str(environment.url.geturl())}{url_path_for('workflow.router', 'register_dc_group', visit_name=environment.visit, session_id=environment.murfey_session)}"
99+
dcg_tag = "/".join(
100+
p for p in transferred_file.parent.parts if p != environment.visit
101+
).replace("//", "/")
102+
dcg_data = {
103+
"experiment_type": "tomo",
104+
"experiment_type_id": 36,
105+
"tag": dcg_tag,
106+
"atlas": str(
107+
_atlas_destination(environment, source, transferred_file)
108+
/ environment.samples[source].atlas.parent
109+
/ atlas_xml_path.with_suffix(".jpg").name
110+
),
111+
"sample": environment.samples[source].sample,
112+
"atlas_pixel_size": atlas_pixel_size,
113+
}
114+
capture_post(url, json=dcg_data)
115+
116+
elif transferred_file.name == "SearchMap.xml" and environment:
117+
logger.info("Tomography session search map xml found")
118+
dcg_tag = ensure_dcg_exists(transferred_file, environment)
119+
with open(transferred_file, "r") as sm_xml:
120+
sm_data = xmltodict.parse(sm_xml.read())
121+
122+
# This bit gets SearchMap location on Atlas
123+
sm_pixel_size = float(
124+
sm_data["MicroscopeImage"]["SpatialScale"]["pixelSize"]["x"][
125+
"numericValue"
126+
]
127+
)
128+
stage_position = sm_data["MicroscopeImage"]["microscopeData"]["stage"][
129+
"Position"
130+
]
131+
sm_binning = float(
132+
sm_data["MicroscopeImage"]["microscopeData"]["acquisition"]["camera"][
133+
"Binning"
134+
]["a:x"]
135+
)
136+
137+
# Get the stage transformation
138+
sm_transformations = sm_data["MicroscopeImage"]["CustomData"][
139+
"a:KeyValueOfstringanyType"
140+
]
141+
stage_matrix: dict[str, float] = {}
142+
image_matrix: dict[str, float] = {}
143+
for key_val in sm_transformations:
144+
if key_val["a:Key"] == "ReferenceCorrectionForStage":
145+
stage_matrix = {
146+
"m11": float(key_val["a:Value"]["b:_m11"]),
147+
"m12": float(key_val["a:Value"]["b:_m12"]),
148+
"m21": float(key_val["a:Value"]["b:_m21"]),
149+
"m22": float(key_val["a:Value"]["b:_m22"]),
150+
}
151+
elif key_val["a:Key"] == "ReferenceCorrectionForImageShift":
152+
image_matrix = {
153+
"m11": float(key_val["a:Value"]["b:_m11"]),
154+
"m12": float(key_val["a:Value"]["b:_m12"]),
155+
"m21": float(key_val["a:Value"]["b:_m21"]),
156+
"m22": float(key_val["a:Value"]["b:_m22"]),
157+
}
158+
if not stage_matrix or not image_matrix:
159+
logger.error(
160+
f"No stage or image shift matrix found for {transferred_file}"
161+
)
162+
163+
ref_matrix = {
164+
"m11": float(
165+
sm_data["MicroscopeImage"]["ReferenceTransformation"]["matrix"][
166+
"a:_m11"
167+
]
168+
),
169+
"m12": float(
170+
sm_data["MicroscopeImage"]["ReferenceTransformation"]["matrix"][
171+
"a:_m12"
172+
]
173+
),
174+
"m21": float(
175+
sm_data["MicroscopeImage"]["ReferenceTransformation"]["matrix"][
176+
"a:_m21"
177+
]
178+
),
179+
"m22": float(
180+
sm_data["MicroscopeImage"]["ReferenceTransformation"]["matrix"][
181+
"a:_m22"
182+
]
183+
),
184+
}
185+
186+
source = _get_source(transferred_file, environment=environment)
187+
image_path = (
188+
_file_transferred_to(
189+
environment, source, transferred_file.parent / "SearchMap.jpg"
190+
)
191+
if source
192+
else ""
193+
)
194+
195+
sm_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomo_router', 'register_search_map', session_id=environment.murfey_session, sm_name=transferred_file.parent.name)}"
196+
capture_post(
197+
sm_url,
198+
json={
199+
"tag": dcg_tag,
200+
"x_stage_position": float(stage_position["X"]),
201+
"y_stage_position": float(stage_position["Y"]),
202+
"pixel_size": sm_pixel_size,
203+
"image": str(image_path),
204+
"binning": sm_binning,
205+
"reference_matrix": ref_matrix,
206+
"stage_correction": stage_matrix,
207+
"image_shift_correction": image_matrix,
208+
},
209+
)
210+
211+
elif transferred_file.name == "SearchMap.dm" and environment:
212+
logger.info("Tomography session search map dm found")
213+
dcg_tag = ensure_dcg_exists(transferred_file, environment)
214+
with open(transferred_file, "r") as sm_xml:
215+
sm_data = xmltodict.parse(sm_xml.read())
216+
217+
# This bit gets SearchMap size
218+
try:
219+
sm_width = int(sm_data["TileSetXml"]["ImageSize"]["a:width"])
220+
sm_height = int(sm_data["TileSetXml"]["ImageSize"]["a:height"])
221+
except KeyError:
222+
logger.warning(f"Unable to find size for SearchMap {transferred_file}")
223+
readout_width = int(
224+
sm_data["TileSetXml"]["AcquisitionSettings"]["a:camera"][
225+
"a:ReadoutArea"
226+
]["b:width"]
227+
)
228+
readout_height = int(
229+
sm_data["TileSetXml"]["AcquisitionSettings"]["a:camera"][
230+
"a:ReadoutArea"
231+
]["b:height"]
232+
)
233+
sm_width = int(
234+
8005 * readout_width / max(readout_height, readout_width)
235+
)
236+
sm_height = int(
237+
8005 * readout_height / max(readout_height, readout_width)
238+
)
239+
logger.warning(
240+
f"Inserting incorrect width {sm_width}, height {sm_height} for SearchMap display"
241+
)
242+
243+
sm_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomo_router', 'register_search_map', session_id=environment.murfey_session, sm_name=transferred_file.parent.name)}"
244+
capture_post(
245+
sm_url,
246+
json={
247+
"tag": dcg_tag,
248+
"height": sm_height,
249+
"width": sm_width,
250+
},
251+
)
252+
253+
elif transferred_file.name == "BatchPositionsList.xml" and environment:
254+
logger.info("Tomography session batch positions list found")
255+
dcg_tag = ensure_dcg_exists(transferred_file, environment)
256+
with open(transferred_file) as xml:
257+
for_parsing = xml.read()
258+
batch_xml = xmltodict.parse(for_parsing)
259+
260+
batch_positions_list = batch_xml["BatchPositionsList"]["BatchPositions"][
261+
"BatchPositionParameters"
262+
]
263+
if isinstance(batch_positions_list, dict):
264+
# Case of a single batch
265+
batch_positions_list = [batch_positions_list]
266+
267+
for batch_position in batch_positions_list:
268+
batch_name = batch_position["Name"]
269+
search_map_name = batch_position["PositionOnTileSet"]["TileSetName"]
270+
batch_stage_location_x = float(
271+
batch_position["PositionOnTileSet"]["StagePositionX"]
272+
)
273+
batch_stage_location_y = float(
274+
batch_position["PositionOnTileSet"]["StagePositionY"]
275+
)
276+
277+
# Always need search map before batch position
278+
sm_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomo_router', 'register_search_map', session_id=environment.murfey_session, sm_name=search_map_name)}"
279+
capture_post(
280+
sm_url,
281+
json={
282+
"tag": dcg_tag,
283+
},
284+
)
285+
286+
# Then register batch position
287+
bp_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomo_router', 'register_batch_position', session_id=environment.murfey_session, batch_name=batch_name)}"
288+
capture_post(
289+
bp_url,
290+
json={
291+
"tag": dcg_tag,
292+
"x_stage_position": batch_stage_location_x,
293+
"y_stage_position": batch_stage_location_y,
294+
"x_beamshift": 0,
295+
"y_beamshift": 0,
296+
"search_map_name": search_map_name,
297+
},
298+
)
299+
300+
# Beamshifts
301+
if batch_position.get("AdditionalExposureTemplateAreas"):
302+
beamshifts = batch_position["AdditionalExposureTemplateAreas"][
303+
"ExposureTemplateAreaParameters"
304+
]
305+
if type(beamshifts) is dict:
306+
beamshifts = [beamshifts]
307+
for beamshift in beamshifts:
308+
beamshift_name = beamshift["Name"]
309+
beamshift_position_x = float(beamshift["PositionX"])
310+
beamshift_position_y = float(beamshift["PositionY"])
311+
312+
# Registration of beamshifted position
313+
bp_url = f"{str(environment.url.geturl())}{url_path_for('session_control.tomo_router', 'register_batch_position', session_id=environment.murfey_session, batch_name=beamshift_name)}"
314+
capture_post(
315+
bp_url,
316+
json={
317+
"tag": dcg_tag,
318+
"x_stage_position": batch_stage_location_x,
319+
"y_stage_position": batch_stage_location_y,
320+
"x_beamshift": beamshift_position_x,
321+
"y_beamshift": beamshift_position_y,
322+
"search_map_name": search_map_name,
323+
},
324+
)

0 commit comments

Comments
 (0)