Skip to content

Commit 443fec9

Browse files
stephen-riggsstephen-riggs
authored
Feedback from tomogram picking (#682)
Ability to receive messages back from tomogram picking and then send on a request to run tomogram 2d extraction/classification. Registers new pj recipe for the first tomogram Changes SPAFeedbackParameters to a more general ClassificationFeedbackParameters for reuse Add table for tomogram picks Currently then set to run 2D classification in batches of 10000 particles with 5 classes. Needs deciding if that is the right value. Also probably isn't sending exactly the right set of parameters onwards to the next recipe as the recipe hasn't yet been written, but the right items should be stored in the database here.
1 parent 8687bba commit 443fec9

File tree

14 files changed

+668
-57
lines changed

14 files changed

+668
-57
lines changed

‎pyproject.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ GitHub = "https://github.com/DiamondLightSource/python-murfey"
107107
"clem.register_preprocessing_result" = "murfey.workflows.clem.register_preprocessing_results:run"
108108
"pato" = "murfey.workflows.notifications:notification_setup"
109109
"picked_particles" = "murfey.workflows.spa.picking:particles_picked"
110+
"picked_tomogram" = "murfey.workflows.tomo.picking:picked_tomogram"
110111
"spa.flush_spa_preprocess" = "murfey.workflows.spa.flush_spa_preprocess:flush_spa_preprocess"
111112

112113
[tool.setuptools]

‎src/murfey/cli/inject_spa_processing.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,12 @@
1313
from murfey.util.config import get_machine_config, get_microscope, get_security_config
1414
from murfey.util.db import (
1515
AutoProcProgram,
16+
ClassificationFeedbackParameters,
1617
ClientEnvironment,
1718
DataCollection,
1819
DataCollectionGroup,
1920
Movie,
2021
ProcessingJob,
21-
SPAFeedbackParameters,
2222
SPARelionParameters,
2323
)
2424
from murfey.util.processing_params import default_spa_parameters
@@ -137,9 +137,9 @@ def run():
137137
.where(ProcessingJob.recipe == "em-spa-preprocess")
138138
).one()
139139
params = murfey_db.exec(
140-
select(SPARelionParameters, SPAFeedbackParameters)
140+
select(SPARelionParameters, ClassificationFeedbackParameters)
141141
.where(SPARelionParameters.pj_id == collected_ids[2].id)
142-
.where(SPAFeedbackParameters.pj_id == SPARelionParameters.pj_id)
142+
.where(ClassificationFeedbackParameters.pj_id == SPARelionParameters.pj_id)
143143
).one()
144144
proc_params: dict | None = dict(params[0])
145145
feedback_params = params[1]

‎src/murfey/cli/spa_ispyb_messages.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -363,7 +363,7 @@ def run():
363363
small_boxsize=metadata["small_boxsize"],
364364
mask_diameter=metadata["mask_diameter"],
365365
)
366-
feedback_params = db.SPAFeedbackParameters(
366+
feedback_params = db.ClassificationFeedbackParameters(
367367
pj_id=collected_ids[2].id,
368368
estimate_particle_diameter=not bool(metadata["particle_diameter"]),
369369
hold_class2d=False,

‎src/murfey/client/contexts/tomo.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,13 @@ def register_tomography_data_collections(
161161
data=dc_data,
162162
)
163163

164-
for recipe in ("em-tomo-preprocess", "em-tomo-align"):
164+
recipes_to_assign_pjids = [
165+
"em-tomo-preprocess",
166+
"em-tomo-align",
167+
]
168+
if not self._tilt_series_with_pjids:
169+
recipes_to_assign_pjids.append("em-tomo-class2d")
170+
for recipe in recipes_to_assign_pjids:
165171
capture_post(
166172
base_url=str(environment.url.geturl()),
167173
router_name="workflow.router",

‎src/murfey/server/api/session_info.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
from murfey.util import sanitise
3535
from murfey.util.config import MachineConfig
3636
from murfey.util.db import (
37+
ClassificationFeedbackParameters,
3738
ClientEnvironment,
3839
DataCollection,
3940
DataCollectionGroup,
@@ -44,7 +45,6 @@
4445
RsyncInstance,
4546
Session,
4647
SessionProcessingParameters,
47-
SPAFeedbackParameters,
4848
SPARelionParameters,
4949
Tilt,
5050
TiltSeries,
@@ -283,7 +283,7 @@ class ProcessingDetails(BaseModel):
283283
data_collections: List[DataCollection]
284284
processing_jobs: List[ProcessingJob]
285285
relion_params: SPARelionParameters
286-
feedback_params: SPAFeedbackParameters
286+
feedback_params: ClassificationFeedbackParameters
287287

288288

289289
@spa_router.get("/sessions/{session_id}/spa_processing_parameters")
@@ -296,13 +296,13 @@ def get_spa_proc_param_details(
296296
DataCollection,
297297
ProcessingJob,
298298
SPARelionParameters,
299-
SPAFeedbackParameters,
299+
ClassificationFeedbackParameters,
300300
)
301301
.where(DataCollectionGroup.session_id == session_id)
302302
.where(DataCollectionGroup.id == DataCollection.dcg_id)
303303
.where(DataCollection.id == ProcessingJob.dc_id)
304304
.where(SPARelionParameters.pj_id == ProcessingJob.id)
305-
.where(SPAFeedbackParameters.pj_id == ProcessingJob.id)
305+
.where(ClassificationFeedbackParameters.pj_id == ProcessingJob.id)
306306
).all()
307307
if not params:
308308
return None

‎src/murfey/server/api/workflow.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
from murfey.util.config import get_machine_config
4545
from murfey.util.db import (
4646
AutoProcProgram,
47+
ClassificationFeedbackParameters,
4748
DataCollection,
4849
DataCollectionGroup,
4950
FoilHole,
@@ -54,7 +55,6 @@
5455
SearchMap,
5556
Session,
5657
SessionProcessingParameters,
57-
SPAFeedbackParameters,
5858
SPARelionParameters,
5959
Tilt,
6060
TiltSeries,
@@ -409,9 +409,9 @@ async def request_spa_preprocessing(
409409
.where(ProcessingJob.recipe == "em-spa-preprocess")
410410
).one()
411411
params = db.exec(
412-
select(SPARelionParameters, SPAFeedbackParameters)
412+
select(SPARelionParameters, ClassificationFeedbackParameters)
413413
.where(SPARelionParameters.pj_id == collected_ids[2].id)
414-
.where(SPAFeedbackParameters.pj_id == SPARelionParameters.pj_id)
414+
.where(ClassificationFeedbackParameters.pj_id == SPARelionParameters.pj_id)
415415
).one()
416416
proc_params: Optional[dict] = dict(params[0])
417417
feedback_params = params[1]

‎src/murfey/server/demo_api.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
)
5050
from murfey.util.db import (
5151
AutoProcProgram,
52+
ClassificationFeedbackParameters,
5253
ClientEnvironment,
5354
DataCollection,
5455
DataCollectionGroup,
@@ -60,7 +61,6 @@
6061
ProcessingJob,
6162
RsyncInstance,
6263
Session,
63-
SPAFeedbackParameters,
6464
SPARelionParameters,
6565
Tilt,
6666
TiltSeries,
@@ -244,7 +244,7 @@ class ProcessingDetails(BaseModel):
244244
data_collections: List[DataCollection]
245245
processing_jobs: List[ProcessingJob]
246246
relion_params: SPARelionParameters
247-
feedback_params: SPAFeedbackParameters
247+
feedback_params: ClassificationFeedbackParameters
248248

249249

250250
@router.get("/sessions/{session_id}/spa_processing_parameters")
@@ -257,13 +257,13 @@ def get_spa_proc_param_details(
257257
DataCollection,
258258
ProcessingJob,
259259
SPARelionParameters,
260-
SPAFeedbackParameters,
260+
ClassificationFeedbackParameters,
261261
)
262262
.where(DataCollectionGroup.session_id == session_id)
263263
.where(DataCollectionGroup.id == DataCollection.dcg_id)
264264
.where(DataCollection.id == ProcessingJob.dc_id)
265265
.where(SPARelionParameters.pj_id == ProcessingJob.id)
266-
.where(SPAFeedbackParameters.pj_id == ProcessingJob.id)
266+
.where(ClassificationFeedbackParameters.pj_id == ProcessingJob.id)
267267
).all()
268268
if not params:
269269
return None
@@ -560,9 +560,9 @@ def flush_spa_processing(
560560
.where(ProcessingJob.recipe == "em-spa-preprocess")
561561
).one()
562562
params = db.exec(
563-
select(SPARelionParameters, SPAFeedbackParameters)
563+
select(SPARelionParameters, ClassificationFeedbackParameters)
564564
.where(SPARelionParameters.pj_id == collected_ids[2].id)
565-
.where(SPAFeedbackParameters.pj_id == SPARelionParameters.pj_id)
565+
.where(ClassificationFeedbackParameters.pj_id == SPARelionParameters.pj_id)
566566
).one()
567567
proc_params = dict(params[0])
568568
feedback_params = params[1]

‎src/murfey/server/feedback.py‎

Lines changed: 37 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -317,13 +317,15 @@ def _pj_id(app_id: int, _db, recipe: str = "") -> int:
317317

318318
def _get_spa_params(
319319
app_id: int, _db
320-
) -> Tuple[db.SPARelionParameters, db.SPAFeedbackParameters]:
320+
) -> Tuple[db.SPARelionParameters, db.ClassificationFeedbackParameters]:
321321
pj_id = _pj_id(app_id, _db, recipe="em-spa-preprocess")
322322
relion_params = _db.exec(
323323
select(db.SPARelionParameters).where(db.SPARelionParameters.pj_id == pj_id)
324324
).one()
325325
feedback_params = _db.exec(
326-
select(db.SPAFeedbackParameters).where(db.SPAFeedbackParameters.pj_id == pj_id)
326+
select(db.ClassificationFeedbackParameters).where(
327+
db.ClassificationFeedbackParameters.pj_id == pj_id
328+
)
327329
).one()
328330
_db.expunge(relion_params)
329331
_db.expunge(feedback_params)
@@ -416,8 +418,8 @@ def _release_3d_hold(message: dict, _db):
416418
)
417419
).one()
418420
feedback_params = _db.exec(
419-
select(db.SPAFeedbackParameters).where(
420-
db.SPAFeedbackParameters.pj_id == pj_id_params
421+
select(db.ClassificationFeedbackParameters).where(
422+
db.ClassificationFeedbackParameters.pj_id == pj_id_params
421423
)
422424
).one()
423425
class3d_params = _db.exec(
@@ -494,8 +496,8 @@ def _release_refine_hold(message: dict, _db):
494496
)
495497
).one()
496498
feedback_params = _db.exec(
497-
select(db.SPAFeedbackParameters).where(
498-
db.SPAFeedbackParameters.pj_id == pj_id_params
499+
select(db.ClassificationFeedbackParameters).where(
500+
db.ClassificationFeedbackParameters.pj_id == pj_id_params
499501
)
500502
).one()
501503
refine_params = _db.exec(
@@ -586,8 +588,8 @@ def _register_incomplete_2d_batch(message: dict, _db, demo: bool = False):
586588
)
587589
).one()
588590
feedback_params = _db.exec(
589-
select(db.SPAFeedbackParameters).where(
590-
db.SPAFeedbackParameters.pj_id == pj_id_params
591+
select(db.ClassificationFeedbackParameters).where(
592+
db.ClassificationFeedbackParameters.pj_id == pj_id_params
591593
)
592594
).one()
593595
if feedback_params.hold_class2d:
@@ -712,8 +714,8 @@ def _register_complete_2d_batch(message: dict, _db, demo: bool = False):
712714
)
713715
).one()
714716
feedback_params = _db.exec(
715-
select(db.SPAFeedbackParameters).where(
716-
db.SPAFeedbackParameters.pj_id == pj_id_params
717+
select(db.ClassificationFeedbackParameters).where(
718+
db.ClassificationFeedbackParameters.pj_id == pj_id_params
717719
)
718720
).one()
719721
_db.expunge(relion_params)
@@ -933,7 +935,7 @@ def _flush_class2d(
933935
app_id: int,
934936
_db,
935937
relion_params: db.SPARelionParameters | None = None,
936-
feedback_params: db.SPAFeedbackParameters | None = None,
938+
feedback_params: db.ClassificationFeedbackParameters | None = None,
937939
):
938940
instrument_name = (
939941
_db.exec(select(db.Session).where(db.Session.id == session_id))
@@ -954,8 +956,8 @@ def _flush_class2d(
954956
_db.expunge(relion_params)
955957
if not feedback_params:
956958
feedback_params = _db.exec(
957-
select(db.SPAFeedbackParameters).where(
958-
db.SPAFeedbackParameters.pj_id == pj_id_params
959+
select(db.ClassificationFeedbackParameters).where(
960+
db.ClassificationFeedbackParameters.pj_id == pj_id_params
959961
)
960962
).one()
961963
_db.expunge(feedback_params)
@@ -1032,8 +1034,8 @@ def _register_class_selection(message: dict, _db, demo: bool = False):
10321034
).all()
10331035
# Add the class selection score to the database
10341036
feedback_params = _db.exec(
1035-
select(db.SPAFeedbackParameters).where(
1036-
db.SPAFeedbackParameters.pj_id == pj_id_params
1037+
select(db.ClassificationFeedbackParameters).where(
1038+
db.ClassificationFeedbackParameters.pj_id == pj_id_params
10371039
)
10381040
).one()
10391041
_db.expunge(feedback_params)
@@ -1253,8 +1255,8 @@ def _register_3d_batch(message: dict, _db, demo: bool = False):
12531255
).one()
12541256
relion_options = dict(relion_params)
12551257
feedback_params = _db.exec(
1256-
select(db.SPAFeedbackParameters).where(
1257-
db.SPAFeedbackParameters.pj_id == pj_id_params
1258+
select(db.ClassificationFeedbackParameters).where(
1259+
db.ClassificationFeedbackParameters.pj_id == pj_id_params
12581260
)
12591261
).one()
12601262
other_options = dict(feedback_params)
@@ -1431,8 +1433,8 @@ def _register_initial_model(message: dict, _db, demo: bool = False):
14311433
pj_id_params = _pj_id(message["program_id"], _db, recipe="em-spa-preprocess")
14321434
# Add the initial model file to the database
14331435
feedback_params = _db.exec(
1434-
select(db.SPAFeedbackParameters).where(
1435-
db.SPAFeedbackParameters.pj_id == pj_id_params
1436+
select(db.ClassificationFeedbackParameters).where(
1437+
db.ClassificationFeedbackParameters.pj_id == pj_id_params
14361438
)
14371439
).one()
14381440
feedback_params.initial_model = message.get("initial_model")
@@ -1598,8 +1600,8 @@ def _register_refinement(message: dict, _db, demo: bool = False):
15981600
).one()
15991601
relion_options = dict(relion_params)
16001602
feedback_params = _db.exec(
1601-
select(db.SPAFeedbackParameters).where(
1602-
db.SPAFeedbackParameters.pj_id == pj_id_params
1603+
select(db.ClassificationFeedbackParameters).where(
1604+
db.ClassificationFeedbackParameters.pj_id == pj_id_params
16031605
)
16041606
).one()
16051607
other_options = dict(feedback_params)
@@ -1746,8 +1748,8 @@ def _register_bfactors(message: dict, _db, demo: bool = False):
17461748
).one()
17471749
relion_options = dict(relion_params)
17481750
feedback_params = _db.exec(
1749-
select(db.SPAFeedbackParameters).where(
1750-
db.SPAFeedbackParameters.pj_id == pj_id_params
1751+
select(db.ClassificationFeedbackParameters).where(
1752+
db.ClassificationFeedbackParameters.pj_id == pj_id_params
17511753
)
17521754
).one()
17531755

@@ -2309,7 +2311,7 @@ def feedback_callback(header: dict, message: dict, _db=murfey_db) -> None:
23092311
eer_fractionation_file=message["eer_fractionation_file"],
23102312
symmetry=message["symmetry"],
23112313
)
2312-
feedback_params = db.SPAFeedbackParameters(
2314+
feedback_params = db.ClassificationFeedbackParameters(
23132315
pj_id=collected_ids[2].id,
23142316
estimate_particle_diameter=True,
23152317
hold_class2d=False,
@@ -2366,7 +2368,18 @@ def feedback_callback(header: dict, message: dict, _db=murfey_db) -> None:
23662368
gain_ref=message["gain_ref"],
23672369
eer_fractionation_file=message["eer_fractionation_file"],
23682370
)
2371+
feedback_params = db.ClassificationFeedbackParameters(
2372+
pj_id=collected_ids[2].id,
2373+
estimate_particle_diameter=True,
2374+
hold_class2d=False,
2375+
hold_class3d=False,
2376+
class_selection_score=0,
2377+
star_combination_job=0,
2378+
initial_model="",
2379+
next_job=0,
2380+
)
23692381
_db.add(params)
2382+
_db.add(feedback_params)
23702383
_db.commit()
23712384
_db.close()
23722385
if murfey.server._transport_object:

0 commit comments

Comments
 (0)