Skip to content

Commit a69aa2a

Browse files
tieneupintieneupin
committed
Merged recent changes from 'main' branch and resolved merge conflicts
2 parents 9167512 + 4a8210a commit a69aa2a

File tree

20 files changed

+624
-388
lines changed

20 files changed

+624
-388
lines changed

‎.bumpclient.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
[tool.bumpversion]
2-
current_version = "0.15.5"
2+
current_version = "0.16.1"
33
commit = true
44
tag = false
55

‎.bumpversion.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
[tool.bumpversion]
2-
current_version = "0.15.5"
2+
current_version = "0.16.1"
33
commit = true
44
tag = true
55

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ requires = [
77

88
[project]
99
name = "murfey"
10-
version = "0.15.5"
10+
version = "0.16.1"
1111
description = "Client-Server architecture hauling Cryo-EM data"
1212
readme = "README.md"
1313
keywords = [

‎src/murfey/__init__.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
from __future__ import annotations
22

3-
__version__ = "0.15.5"
4-
__supported_client_version__ = "0.15.5"
3+
__version__ = "0.16.1"
4+
__supported_client_version__ = "0.16.1"

‎src/murfey/client/contexts/spa.py‎

Lines changed: 42 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ class FoilHole(NamedTuple):
4141
thumbnail_size_y: Optional[int] = None
4242
pixel_size: Optional[float] = None
4343
image: str = ""
44+
diameter: Optional[float] = None
4445

4546

4647
class GridSquare(NamedTuple):
@@ -59,9 +60,18 @@ class GridSquare(NamedTuple):
5960
tag: str = ""
6061

6162

62-
def _get_grid_square_atlas_positions(
63-
xml_path: Path, grid_square: str = ""
64-
) -> Dict[str, Tuple[Optional[int], Optional[int], Optional[float], Optional[float]]]:
63+
def _get_grid_square_atlas_positions(xml_path: Path, grid_square: str = "") -> Dict[
64+
str,
65+
Tuple[
66+
Optional[int],
67+
Optional[int],
68+
Optional[float],
69+
Optional[float],
70+
Optional[int],
71+
Optional[int],
72+
Optional[float],
73+
],
74+
]:
6575
with open(
6676
xml_path,
6777
"r",
@@ -71,7 +81,16 @@ def _get_grid_square_atlas_positions(
7181
"TileXml"
7282
]
7383
gs_pix_positions: Dict[
74-
str, Tuple[Optional[int], Optional[int], Optional[float], Optional[float]]
84+
str,
85+
Tuple[
86+
Optional[int],
87+
Optional[int],
88+
Optional[float],
89+
Optional[float],
90+
Optional[int],
91+
Optional[int],
92+
Optional[float],
93+
],
7594
] = {}
7695
for ti in tile_info:
7796
try:
@@ -96,6 +115,13 @@ def _get_grid_square_atlas_positions(
96115
* 1e9,
97116
float(gs["value"]["b:PositionOnTheAtlas"]["c:Physical"]["d:y"])
98117
* 1e9,
118+
int(
119+
float(gs["value"]["b:PositionOnTheAtlas"]["c:Size"]["d:width"])
120+
),
121+
int(
122+
float(gs["value"]["b:PositionOnTheAtlas"]["c:Size"]["d:height"])
123+
),
124+
float(gs["value"]["b:PositionOnTheAtlas"]["c:Rotation"]),
99125
)
100126
if grid_square:
101127
break
@@ -221,6 +247,7 @@ def _foil_hole_data(
221247
for fh_block in serialization_array[required_key]:
222248
pix = fh_block["b:value"]["PixelCenter"]
223249
stage = fh_block["b:value"]["StagePosition"]
250+
diameter = fh_block["b:value"]["PixelWidthHeight"]["c:width"]
224251
if int(fh_block["b:key"]) == foil_hole:
225252
return FoilHole(
226253
id=foil_hole,
@@ -236,6 +263,7 @@ def _foil_hole_data(
236263
thumbnail_size_y=None,
237264
pixel_size=float(pixel_size) if image_path else None,
238265
image=str(image_path),
266+
diameter=diameter,
239267
)
240268
logger.warning(
241269
f"Foil hole positions could not be determined from metadata file {xml_path} for foil hole {foil_hole}"
@@ -542,7 +570,7 @@ def _position_analysis(
542570
grid_square = _grid_square_from_file(transferred_file)
543571
grid_square_metadata_file = _grid_square_metadata_file(
544572
transferred_file,
545-
machine_config["data_directories"],
573+
[Path(p) for p in machine_config["data_directories"]],
546574
environment.visit,
547575
grid_square,
548576
)
@@ -557,7 +585,10 @@ def _position_analysis(
557585
Optional[int],
558586
Optional[float],
559587
Optional[float],
560-
] = (None, None, None, None)
588+
Optional[int],
589+
Optional[int],
590+
Optional[float],
591+
] = (None, None, None, None, None, None, None)
561592
data_collection_group = (
562593
requests.get(
563594
f"{str(environment.url.geturl())}/sessions/{environment.murfey_session}/data_collection_groups"
@@ -611,6 +642,9 @@ def _position_analysis(
611642
"y_location": gs_pix_position[1],
612643
"x_stage_position": gs_pix_position[2],
613644
"y_stage_position": gs_pix_position[3],
645+
"width": gs_pix_position[4],
646+
"height": gs_pix_position[5],
647+
"angle": gs_pix_position[6],
614648
},
615649
)
616650
foil_hole = _foil_hole_from_file(transferred_file)
@@ -651,6 +685,7 @@ def _position_analysis(
651685
"thumbnail_size_x": fh.thumbnail_size_x,
652686
"thumbnail_size_y": fh.thumbnail_size_y,
653687
"pixel_size": fh.pixel_size,
688+
"diameter": fh.diameter,
654689
"tag": str(source),
655690
"image": str(image_path),
656691
},
@@ -886,6 +921,7 @@ def _register_processing_job(
886921
)
887922
msg: Dict[str, Any] = {
888923
"tag": tag,
924+
"source": tag,
889925
"recipe": "ispyb-relion",
890926
"parameters": {
891927
"acquisition_software": parameters["acquisition_software"],

‎src/murfey/client/contexts/spa_metadata.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,30 @@ def post_transfer(
6464
visitless_path = Path(
6565
str(transferred_file).replace(f"/{environment.visit}", "")
6666
)
67+
visit_index_of_transferred_file = transferred_file.parts.index(
68+
environment.visit
69+
)
70+
atlas_xml_path = list(
71+
(
72+
Path(
73+
"/".join(
74+
transferred_file.parts[
75+
: visit_index_of_transferred_file + 1
76+
]
77+
)
78+
)
79+
/ partial_path
80+
).parent.glob("Atlas_*.xml")
81+
)[0]
82+
with open(atlas_xml_path, "rb") as atlas_xml:
83+
atlas_xml_data = xmltodict.parse(atlas_xml)
84+
atlas_original_pixel_size = atlas_xml_data["MicroscopeImage"][
85+
"SpatialScale"
86+
]["pixelSize"]["x"]["numericValue"]
87+
88+
# need to calculate the pixel size of the downscaled image
89+
atlas_pixel_size = atlas_original_pixel_size * 7.8
90+
6791
source = _get_source(
6892
visitless_path.parent / "Images-Disc1" / visitless_path.name,
6993
environment,
@@ -90,6 +114,7 @@ def post_transfer(
90114
/ environment.samples[source].atlas
91115
),
92116
"sample": environment.samples[source].sample,
117+
"atlas_pixel_size": atlas_pixel_size,
93118
}
94119
capture_post(url, json=dcg_data)
95120
registered_grid_squares = (
@@ -121,5 +146,8 @@ def post_transfer(
121146
"y_location": pos_data[1],
122147
"x_stage_position": pos_data[2],
123148
"y_stage_position": pos_data[3],
149+
"width": pos_data[4],
150+
"height": pos_data[5],
151+
"angle": pos_data[6],
124152
},
125153
)

‎src/murfey/client/contexts/tomo.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -375,6 +375,7 @@ def _add_tilt(
375375
proc_url,
376376
{
377377
"tag": tilt_series,
378+
"source": str(self._basepath),
378379
"recipe": "em-tomo-preprocess",
379380
"experiment_type": "tomography",
380381
},
@@ -385,6 +386,7 @@ def _add_tilt(
385386
proc_url,
386387
{
387388
"tag": tilt_series,
389+
"source": str(self._basepath),
388390
"recipe": "em-tomo-align",
389391
"experiment_type": "tomography",
390392
},
@@ -396,6 +398,7 @@ def _add_tilt(
396398
proc_url,
397399
json={
398400
"tag": tilt_series,
401+
"source": str(self._basepath),
399402
"recipe": "em-tomo-preprocess",
400403
"experiment_type": "tomography",
401404
},
@@ -404,6 +407,7 @@ def _add_tilt(
404407
proc_url,
405408
json={
406409
"tag": tilt_series,
410+
"source": str(self._basepath),
407411
"recipe": "em-tomo-align",
408412
"experiment_type": "tomography",
409413
},

‎src/murfey/client/multigrid_control.py‎

Lines changed: 80 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -366,16 +366,9 @@ def _start_dc(self, json, from_form: bool = False):
366366
f"{self._environment.url.geturl()}/clients/{self._environment.client_id}/tomography_processing_parameters",
367367
json=json,
368368
)
369+
369370
source = Path(json["source"])
370-
self._environment.listeners["data_collection_group_ids"] = {
371-
context._flush_data_collections
372-
}
373-
self._environment.listeners["data_collection_ids"] = {
374-
context._flush_processing_job
375-
}
376-
self._environment.listeners["autoproc_program_ids"] = {
377-
context._flush_preprocess
378-
}
371+
379372
self._environment.id_tag_registry["data_collection_group"].append(
380373
str(source)
381374
)
@@ -386,12 +379,85 @@ def _start_dc(self, json, from_form: bool = False):
386379
"tag": str(source),
387380
}
388381
requests.post(url, json=dcg_data)
382+
383+
data = {
384+
"voltage": json["voltage"],
385+
"pixel_size_on_image": json["pixel_size_on_image"],
386+
"experiment_type": json["experiment_type"],
387+
"image_size_x": json["image_size_x"],
388+
"image_size_y": json["image_size_y"],
389+
"file_extension": json["file_extension"],
390+
"acquisition_software": json["acquisition_software"],
391+
"image_directory": str(self._environment.default_destinations[source]),
392+
"tag": json["tilt_series_tag"],
393+
"source": str(source),
394+
"magnification": json["magnification"],
395+
"total_exposed_dose": json.get("total_exposed_dose"),
396+
"c2aperture": json.get("c2aperture"),
397+
"exposure_time": json.get("exposure_time"),
398+
"slit_width": json.get("slit_width"),
399+
"phase_plate": json.get("phase_plate", False),
400+
}
401+
capture_post(
402+
f"{str(self._environment.url.geturl())}/visits/{str(self._environment.visit)}/{self._environment.murfey_session}/start_data_collection",
403+
json=data,
404+
)
405+
for recipe in ("em-tomo-preprocess", "em-tomo-align"):
406+
capture_post(
407+
f"{str(self._environment.url.geturl())}/visits/{str(self._environment.visit)}/{self._environment.murfey_session}/register_processing_job",
408+
json={
409+
"tag": json["tilt_series_tag"],
410+
"source": str(source),
411+
"recipe": recipe,
412+
},
413+
)
414+
log.info("Registering tomography processing parameters")
415+
if self._environment.data_collection_parameters.get("num_eer_frames"):
416+
eer_response = requests.post(
417+
f"{str(self._environment.url.geturl())}/visits/{self._environment.visit}/{self._environment.murfey_session}/eer_fractionation_file",
418+
json={
419+
"num_frames": self._environment.data_collection_parameters[
420+
"num_eer_frames"
421+
],
422+
"fractionation": self._environment.data_collection_parameters[
423+
"eer_fractionation"
424+
],
425+
"dose_per_frame": self._environment.data_collection_parameters[
426+
"dose_per_frame"
427+
],
428+
"fractionation_file_name": "eer_fractionation_tomo.txt",
429+
},
430+
)
431+
eer_fractionation_file = eer_response.json()["eer_fractionation_file"]
432+
json.update({"eer_fractionation_file": eer_fractionation_file})
433+
requests.post(
434+
f"{self._environment.url.geturl()}/sessions/{self._environment.murfey_session}/tomography_preprocessing_parameters",
435+
json=json,
436+
)
437+
context._flush_data_collections()
438+
context._flush_processing_jobs()
439+
capture_post(
440+
f"{self._environment.url.geturl()}/visits/{self._environment.visit}/{self._environment.murfey_session}/flush_tomography_processing",
441+
json={"rsync_source": str(source)},
442+
)
443+
log.info("tomography processing flushed")
444+
389445
elif isinstance(context, SPAContext) or isinstance(context, SPAModularContext):
390446
url = f"{str(self._environment.url.geturl())}/visits/{str(self._environment.visit)}/{self.session_id}/register_data_collection_group"
391447
dcg_data = {
392448
"experiment_type": "single particle",
393449
"experiment_type_id": 37,
394450
"tag": str(source),
451+
"atlas": (
452+
str(self._environment.samples[source].atlas)
453+
if self._environment.samples.get(source)
454+
else ""
455+
),
456+
"sample": (
457+
self._environment.samples[source].sample
458+
if self._environment.samples.get(source)
459+
else None
460+
),
395461
}
396462
capture_post(url, json=dcg_data)
397463
if from_form:
@@ -428,7 +494,11 @@ def _start_dc(self, json, from_form: bool = False):
428494
):
429495
capture_post(
430496
f"{str(self._environment.url.geturl())}/visits/{str(self._environment.visit)}/{self.session_id}/register_processing_job",
431-
json={"tag": str(source), "recipe": recipe},
497+
json={
498+
"tag": str(source),
499+
"source": str(source),
500+
"recipe": recipe,
501+
},
432502
)
433503
log.info(f"Posting SPA processing parameters: {json}")
434504
response = capture_post(

‎src/murfey/client/tui/app.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -516,7 +516,11 @@ def _start_dc(self, json, from_form: bool = False):
516516
for recipe in ("em-tomo-preprocess", "em-tomo-align"):
517517
capture_post(
518518
f"{str(self._url.geturl())}/visits/{str(self._visit)}/{self._environment.murfey_session}/register_processing_job",
519-
json={"tag": json["tilt_series_tag"], "recipe": recipe},
519+
json={
520+
"tag": json["tilt_series_tag"],
521+
"source": str(source),
522+
"recipe": recipe,
523+
},
520524
)
521525
log.info("Registering tomography processing parameters")
522526
if self.app._environment.data_collection_parameters.get("num_eer_frames"):
@@ -600,7 +604,11 @@ def _start_dc(self, json, from_form: bool = False):
600604
):
601605
capture_post(
602606
f"{str(self._url.geturl())}/visits/{str(self._visit)}/{self._environment.murfey_session}/register_processing_job",
603-
json={"tag": str(source), "recipe": recipe},
607+
json={
608+
"tag": str(source),
609+
"source": str(source),
610+
"recipe": recipe,
611+
},
604612
)
605613
log.info(f"Posting SPA processing parameters: {json}")
606614
response = capture_post(

‎src/murfey/client/tui/screens.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -717,7 +717,11 @@ def on_button_pressed(self, event: Button.Pressed):
717717
if self._switch_status:
718718
self.app.install_screen(
719719
DirectorySelection(
720-
[p for p in machine_data.get("data_directories", []) if p.exists()]
720+
[
721+
p
722+
for p in machine_data.get("data_directories", [])
723+
if Path(p).exists()
724+
]
721725
),
722726
"directory-select",
723727
)

0 commit comments

Comments
 (0)