Skip to content

Commit 0801794

Browse files
bencapbencap
authored
Merge pull request #328 from VariantEffect/tests/bencap/298/mapping-job
Tests for Mapping Job
2 parents bc29fc3 + b6299ba commit 0801794

File tree

5 files changed

+1080
-163
lines changed

5 files changed

+1080
-163
lines changed

‎src/mavedb/worker/jobs.py‎

Lines changed: 21 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -375,15 +375,8 @@ async def map_variants_for_score_set(
375375
else:
376376
raise NonexistentMappingReferenceError()
377377

378-
target_gene = db.scalars(
379-
select(TargetGene)
380-
.join(ScoreSet)
381-
.join(TargetSequence)
382-
.where(
383-
ScoreSet.id == score_set_id,
384-
# TargetSequence.sequence == target_sequence,
385-
)
386-
).one()
378+
# TODO(VariantEffect/dcd_mapping2#2): Handle variant mappings for score sets with more than 1 target.
379+
target_gene = score_set.target_genes[0]
387380

388381
excluded_pre_mapped_keys = {"sequence"}
389382
if computed_genomic_ref and mapped_genomic_ref:
@@ -500,8 +493,6 @@ async def map_variants_for_score_set(
500493
except Exception as backoff_e:
501494
score_set.mapping_state = MappingState.failed
502495
score_set.mapping_errors = {"error_message": "Encountered an internal server error during mapping"}
503-
db.add(score_set)
504-
db.commit()
505496
send_slack_message(backoff_e)
506497
logging_context = {**logging_context, **format_raised_exception_info_as_dict(backoff_e)}
507498
logger.critical(
@@ -511,35 +502,31 @@ async def map_variants_for_score_set(
511502
else:
512503
if new_job_id and not max_retries_exceeded:
513504
score_set.mapping_state = MappingState.queued
514-
db.add(score_set)
515-
db.commit()
516505
logger.info(
517506
msg="After encountering an error while parsing mapped variants, another mapping job was queued.",
518507
extra=logging_context,
519508
)
520509
elif new_job_id is None and not max_retries_exceeded:
521510
score_set.mapping_state = MappingState.failed
522511
score_set.mapping_errors = {"error_message": "Encountered an internal server error during mapping"}
523-
db.add(score_set)
524-
db.commit()
525512
logger.error(
526513
msg="After encountering an error while parsing mapped variants, another mapping job was unable to be queued. This score set will not be mapped.",
527514
extra=logging_context,
528515
)
529516
else:
530517
score_set.mapping_state = MappingState.failed
531518
score_set.mapping_errors = {"error_message": "Encountered an internal server error during mapping"}
532-
db.add(score_set)
533-
db.commit()
534519
logger.error(
535520
msg="After encountering an error while parsing mapped variants, the maximum retries for this job were exceeded. This score set will not be mapped.",
536521
extra=logging_context,
537522
)
538523
finally:
524+
db.add(score_set)
525+
db.commit()
539526
return {"success": False, "retried": (not max_retries_exceeded and new_job_id is not None)}
540527

541528
ctx["state"][ctx["job_id"]] = logging_context.copy()
542-
return {"success": True}
529+
return {"success": True, "retried": False}
543530

544531

545532
async def variant_mapper_manager(ctx: dict, correlation_id: str, updater_id: int, attempt: int = 1) -> dict:
@@ -583,8 +570,17 @@ async def variant_mapper_manager(ctx: dict, correlation_id: str, updater_id: int
583570

584571
except Exception as e:
585572
send_slack_message(e)
573+
574+
# Attempt to remove this item from the mapping queue.
575+
try:
576+
await redis.lrem(MAPPING_QUEUE_NAME, 1, queued_id) # type: ignore
577+
logger.warning(msg="Removed un-queueable score set from the queue.", extra=logging_context)
578+
except Exception:
579+
pass
580+
586581
logging_context = {**logging_context, **format_raised_exception_info_as_dict(e)}
587582
logger.error(msg="Variant mapper manager encountered an unexpected error during setup.", extra=logging_context)
583+
588584
return {"success": False, "enqueued_job": None}
589585

590586
new_job = None
@@ -646,6 +642,13 @@ async def variant_mapper_manager(ctx: dict, correlation_id: str, updater_id: int
646642
if not queued_score_set:
647643
return {"success": False, "enqueued_job": new_job_id}
648644

645+
# Attempt to remove this item from the mapping queue.
646+
try:
647+
await redis.lrem(MAPPING_QUEUE_NAME, 1, queued_id) # type: ignore
648+
logger.warning(msg="Removed un-queueable score set from the queue.", extra=logging_context)
649+
except Exception:
650+
pass
651+
649652
score_set_exc = db.scalars(select(ScoreSet).where(ScoreSet.id == queued_score_set.id)).one_or_none()
650653
if score_set_exc:
651654
score_set_exc.mapping_state = MappingState.failed

‎tests/conftest.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,8 @@ async def on_startup(ctx):
152152
async def on_job(ctx):
153153
ctx["db"] = session
154154
ctx["hdp"] = data_provider
155+
ctx["state"] = {}
156+
ctx["pool"] = futures.ProcessPoolExecutor()
155157

156158
worker_ = Worker(
157159
functions=[create_variants_for_score_set, map_variants_for_score_set, variant_mapper_manager],

‎tests/helpers/constants.py‎

Lines changed: 35 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from datetime import date
1+
from datetime import date, datetime
22
from humps import camelize
33
from mavedb.models.enums.processing_state import ProcessingState
44

@@ -56,7 +56,7 @@
5656
"is_first_login": True,
5757
}
5858

59-
TEST_DESCRIPTION = 'description'
59+
TEST_DESCRIPTION = "description"
6060

6161
ADMIN_USER_DECODED_JWT = {
6262
"sub": ADMIN_USER["username"],
@@ -77,50 +77,40 @@
7777
"key": "Variant Library Creation Method",
7878
"value": "Endogenous locus library method",
7979
"special": False,
80-
"description": "Description"
80+
"description": "Description",
8181
},
8282
{
8383
"key": "Variant Library Creation Method",
8484
"value": "In vitro construct library method",
8585
"special": False,
86-
"description": "Description"
87-
},
88-
{
89-
"key": "Variant Library Creation Method",
90-
"value": "Other",
91-
"special": False,
92-
"description": "Description"
86+
"description": "Description",
9387
},
88+
{"key": "Variant Library Creation Method", "value": "Other", "special": False, "description": "Description"},
9489
{
9590
"key": "Endogenous Locus Library Method System",
9691
"value": "SaCas9",
9792
"special": False,
98-
"description": "Description"
93+
"description": "Description",
9994
},
10095
{
10196
"key": "Endogenous Locus Library Method Mechanism",
10297
"value": "Base editor",
10398
"special": False,
104-
"description": "Description"
99+
"description": "Description",
105100
},
106101
{
107102
"key": "In Vitro Construct Library Method System",
108103
"value": "Oligo-directed mutagenic PCR",
109104
"special": False,
110-
"description": "Description"
105+
"description": "Description",
111106
},
112107
{
113108
"key": "In Vitro Construct Library Method Mechanism",
114109
"value": "Native locus replacement",
115110
"special": False,
116-
"description": "Description"
111+
"description": "Description",
117112
},
118-
{
119-
"key": "Delivery method",
120-
"value": "Other",
121-
"special": False,
122-
"description": "Description"
123-
}
113+
{"key": "Delivery method", "value": "Other", "special": False, "description": "Description"},
124114
]
125115

126116
TEST_KEYWORDS = [
@@ -129,33 +119,28 @@
129119
"key": "Variant Library Creation Method",
130120
"value": "Endogenous locus library method",
131121
"special": False,
132-
"description": "Description"
122+
"description": "Description",
133123
},
134124
},
135125
{
136126
"keyword": {
137127
"key": "Endogenous Locus Library Method System",
138128
"value": "SaCas9",
139129
"special": False,
140-
"description": "Description"
130+
"description": "Description",
141131
},
142132
},
143133
{
144134
"keyword": {
145135
"key": "Endogenous Locus Library Method Mechanism",
146136
"value": "Base editor",
147137
"special": False,
148-
"description": "Description"
138+
"description": "Description",
149139
},
150140
},
151141
{
152-
"keyword": {
153-
"key": "Delivery method",
154-
"value": "Other",
155-
"special": False,
156-
"description": "Description"
157-
},
158-
"description": "Details of delivery method"
142+
"keyword": {"key": "Delivery method", "value": "Other", "special": False, "description": "Description"},
143+
"description": "Details of delivery method",
159144
},
160145
]
161146

@@ -166,15 +151,10 @@
166151
"methodText": "Methods",
167152
"keywords": [
168153
{
169-
"keyword": {
170-
"key": "Delivery method",
171-
"value": "Other",
172-
"special": False,
173-
"description": "Description"
174-
},
175-
"description": "Details of delivery method"
154+
"keyword": {"key": "Delivery method", "value": "Other", "special": False, "description": "Description"},
155+
"description": "Details of delivery method",
176156
},
177-
]
157+
],
178158
}
179159

180160
TEST_MINIMAL_EXPERIMENT = {
@@ -232,15 +212,11 @@
232212
"modificationDate": date.today().isoformat(),
233213
"scoreSetUrns": [],
234214
"contributors": [],
235-
"keywords": [{
236-
"keyword": {
237-
"key": "Delivery method",
238-
"value": "Other",
239-
"special": False,
240-
"description": "Description"
241-
},
242-
"description": "Details of delivery method"
243-
},
215+
"keywords": [
216+
{
217+
"keyword": {"key": "Delivery method", "value": "Other", "special": False, "description": "Description"},
218+
"description": "Details of delivery method",
219+
},
244220
],
245221
"doiIdentifiers": [],
246222
"primaryPublicationIdentifiers": [],
@@ -276,18 +252,13 @@
276252
"key": "Variant Library Creation Method",
277253
"value": "Other",
278254
"special": False,
279-
"description": "Description"
255+
"description": "Description",
280256
},
281-
"description": "Description"
257+
"description": "Description",
282258
},
283259
{
284-
"keyword": {
285-
"key": "Delivery method",
286-
"value": "Other",
287-
"special": False,
288-
"description": "Description"
289-
},
290-
"description": "Description"
260+
"keyword": {"key": "Delivery method", "value": "Other", "special": False, "description": "Description"},
261+
"description": "Description",
291262
},
292263
],
293264
"doiIdentifiers": [],
@@ -527,13 +498,18 @@
527498

528499
TEST_VARIANT_MAPPING_SCAFFOLD = {
529500
"metadata": {},
530-
"computed_reference_sequence": {"sequence_type": "dna", "sequence_id": "ga4gh:SQ.ref_test", "sequence": "ACGTTT"},
531-
"mapped_reference_sequence": {
501+
"computed_genomic_reference_sequence": {
502+
"sequence_type": "dna",
503+
"sequence_id": "ga4gh:SQ.ref_test",
504+
"sequence": "ACGTTT",
505+
},
506+
"mapped_genomic_reference_sequence": {
532507
"sequence_type": "dna",
533508
"sequence_id": "ga4gh:SQ.map_test",
534509
"sequence_accessions": ["NC_000001.11"],
535510
},
536511
"mapped_scores": [],
537512
"vrs_version": "2.0",
538-
"api_version": "0.0.0",
513+
"dcd_mapping_version": "pytest.0.0",
514+
"mapped_date_utc": datetime.isoformat(datetime.now()),
539515
}

‎tests/helpers/util.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,3 +210,7 @@ def mark_user_inactive(session, username):
210210
session.refresh(user)
211211

212212
return user
213+
214+
215+
async def awaitable_exception():
216+
return Exception()

0 commit comments

Comments
 (0)