Skip to content

Commit 26800e8

Browse files
robertnovac1mfjarvis
robertnovac1
and
mfjarvis
authored
VED-241 and 243 SNOMED and terms refactoring JSON extraction (#412)
- Refactored ConversionLayout to include functionality to look up values using functions dedicated to each field. These functions are called in the expressionRule for each field. - Added unit tests for the SNOMED values and terms - Split up existing unit tests Co-authored-by: Matt Jarvis <[email protected]>
1 parent fb7972f commit 26800e8

20 files changed

+1797
-895
lines changed

‎delta_backend/src/ConversionChecker.py‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -91,15 +91,18 @@ def convertData(self, expressionType, expressionRule, fieldName, fieldValue):
9191
return self._convertToOnlyIfTo(
9292
expressionRule, fieldName, fieldValue, self.summarise, self.report_unexpected_exception
9393
)
94+
case "NORMAL":
95+
# TODO - check expression_rule is callable
96+
return expressionRule(fieldValue)
9497
case _:
95-
raise ValueError("Schema expression not found! Check your expression type : " + expressionType)
98+
raise ValueError("Schema expression not found! Check your expression type : " + expressionType)
9699

97100
# Utility function for logging errors
98101
def _log_error(self, fieldName, fieldValue, e, code=ExceptionMessages.RECORD_CHECK_FAILED):
99102
if isinstance(e, Exception):
100103
message = ExceptionMessages.MESSAGES[ExceptionMessages.UNEXPECTED_EXCEPTION] % (e.__class__.__name__, str(e))
101104
else:
102-
message = str(e) # if a simple string message was passed
105+
message = str(e) # if a simple string message was passed
103106

104107
self.errorRecords.append({
105108
"code": code,
@@ -110,7 +113,7 @@ def _log_error(self, fieldName, fieldValue, e, code=ExceptionMessages.RECORD_CHE
110113

111114
def _convertToDate(self, expressionRule, fieldName, fieldValue, summarise, report_unexpected_exception):
112115
"""
113-
Convert a date string according to match YYYYMMDD format.
116+
Convert a date string according to match YYYYMMDD format.
114117
"""
115118
if not fieldValue:
116119
return ""
@@ -156,7 +159,7 @@ def _convertToDateTime(self, expressionRule, fieldName, fieldValue, summarise, r
156159

157160
formatted = dt_format.strftime("%Y%m%dT%H%M%S%z")
158161
return formatted.replace("+0000", "00").replace("+0100", "01")
159-
162+
160163

161164
# Not Empty Validate - Returns exactly what is in the extracted fields no parsing or logic needed
162165
def _convertToNotEmpty(self, expressionRule, fieldName, fieldValue, summarise, report_unexpected_exception):
@@ -236,7 +239,7 @@ def _convertToChangeTo(self, expressionRule, fieldName, fieldValue, summarise, r
236239
def _convertToDose(self, expressionRule, fieldName, fieldValue, summarise, report_unexpected_exception):
237240
if isinstance(fieldValue, (int, float)) and 1 <= fieldValue <= 9:
238241
return fieldValue
239-
return ""
242+
return ""
240243

241244
# Change to Lookup (loads expected data as is but if empty use lookup extraction to populate value)
242245
def _convertToLookUp(self, expressionRule, fieldName, fieldValue, summarise, report_unexpected_exception):
@@ -310,11 +313,11 @@ def _convertToBoolean(self, expressionRule, fieldName, fieldValue, summarise, re
310313
return False
311314
elif report_unexpected_exception:
312315
self._log_error(fieldName, fieldValue, "Invalid String Data")
313-
return ""
316+
return ""
314317
except Exception as e:
315318
if report_unexpected_exception:
316319
self._log_error(fieldName, fieldValue, e)
317320
return ""
318321

319322
def get_error_records(self):
320-
return self.errorRecords
323+
return self.errorRecords

‎delta_backend/src/ConversionLayout.py‎

Lines changed: 126 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,108 @@
1-
21
# This file holds the schema/base layout that maps FHIR fields to flat JSON fields
32
# Each entry tells the converter how to extract and transform a specific value
3+
EXTENSION_URL_VACCINATION_PRODEDURE = "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-VaccinationProcedure"
4+
EXTENSION_URL_SCT_DESC_DISPLAY = "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-CodingSCTDescDisplay"
5+
6+
CODING_SYSTEM_URL_SNOMED = "http://snomed.info/sct"
7+
8+
9+
def _extract_vaccination_procedure_code(immunization) -> str:
10+
extensions = immunization.get("extension", [])
11+
for ext in extensions:
12+
if ext.get("url") == EXTENSION_URL_VACCINATION_PRODEDURE:
13+
value_cc = ext.get("valueCodeableConcept", {})
14+
return _get_first_snomed_code(value_cc)
15+
return ""
16+
17+
18+
def _extract_vaccine_product_code(immunization) -> str:
19+
vaccine_code = immunization.get("vaccineCode", {})
20+
return _get_first_snomed_code(vaccine_code)
21+
22+
23+
# Could be merged with smt
24+
def _extract_site_of_vaccination_code(immunization) -> str:
25+
site = immunization.get("site", {})
26+
return _get_first_snomed_code(site)
27+
28+
29+
def _extract_route_of_vaccination_code(immunization) -> str:
30+
route = immunization.get("route", {})
31+
return _get_first_snomed_code(route)
32+
33+
34+
def _extract_indication_code(immunization) -> str:
35+
for reason in immunization.get("reasonCode", []):
36+
codings = reason.get("coding", [])
37+
for coding in codings:
38+
if coding.get("system") == CODING_SYSTEM_URL_SNOMED:
39+
return coding.get("code", "")
40+
return ""
41+
42+
43+
def _extract_dose_unit_code(immunization) -> str:
44+
dose_quantity = immunization.get("doseQuantity", {})
45+
if dose_quantity.get("system") == CODING_SYSTEM_URL_SNOMED and dose_quantity.get("code"):
46+
return dose_quantity.get("code")
47+
return ""
48+
49+
def _extract_dose_unit_term(immunization) -> str:
50+
dose_quantity = immunization.get("doseQuantity", {})
51+
return dose_quantity.get("unit", "")
52+
53+
def _get_first_snomed_code(coding_container: dict) -> str:
54+
codings = coding_container.get("coding", [])
55+
for coding in codings:
56+
if coding.get("system") == CODING_SYSTEM_URL_SNOMED:
57+
return coding.get("code", "")
58+
return ""
59+
60+
def _get_term_from_codeable_concept(concept: dict) -> str:
61+
if concept.get("text"):
62+
return concept["text"]
63+
64+
codings = concept.get("coding", [])
65+
for coding in codings:
66+
if coding.get("system") == CODING_SYSTEM_URL_SNOMED:
67+
# Try SCTDescDisplay extension first
68+
for ext in coding.get("extension", []):
69+
if ext.get("url") == EXTENSION_URL_SCT_DESC_DISPLAY:
70+
value_string = ext.get("valueString")
71+
if value_string:
72+
return value_string
473

74+
# Fallback to display
75+
return coding.get("display", "")
76+
77+
return ""
78+
79+
def _extract_vaccination_procedure_term(immunization) -> str:
80+
extensions = immunization.get("extension", [])
81+
for ext in extensions:
82+
if ext.get("url") == EXTENSION_URL_VACCINATION_PRODEDURE:
83+
return _get_term_from_codeable_concept(ext.get("valueCodeableConcept", {}))
84+
return ""
85+
86+
def _extract_vaccine_product_term(immunization) -> str:
87+
return _get_term_from_codeable_concept(immunization.get("vaccineCode", {}))
88+
89+
def _extract_site_of_vaccination_term(immunization) -> str:
90+
return _get_term_from_codeable_concept(immunization.get("site", {}))
91+
92+
def _extract_route_of_vaccination_term(immunization) -> str:
93+
return _get_term_from_codeable_concept(immunization.get("route", {}))
94+
95+
# TBC
96+
# - requirements: If element doseNumberPositiveInt exists and is < 10 then populate as received, else null
97+
# - path : protocolApplied.doseNumber[x]
98+
def _extract_dose_sequence(immunization) -> str:
99+
protocol_applied = immunization.get("protocolApplied", [])
100+
101+
if protocol_applied:
102+
dose = protocol_applied[0].get("doseNumberPositiveInt", None)
103+
return str(dose) if dose else ""
104+
return ""
105+
5106
ConvertLayout = {
6107
"id": "7d78e9a6-d859-45d3-bb05-df9c405acbdb",
7108
"schemaName": "JSON Base",
@@ -157,44 +258,44 @@
157258
"fieldNameFlat": "VACCINATION_PROCEDURE_CODE",
158259
"expression": {
159260
"expressionName": "Not Empty",
160-
"expressionType": "SNOMED",
161-
"expressionRule": ""
261+
"expressionType": "NORMAL",
262+
"expressionRule": _extract_vaccination_procedure_code
162263
}
163264
},
164265
{
165266
"fieldNameFHIR": "extension|0|valueCodeableConcept|coding|0|display",
166267
"fieldNameFlat": "VACCINATION_PROCEDURE_TERM",
167268
"expression": {
168269
"expressionName": "Not Empty",
169-
"expressionType": "NOTEMPTY",
170-
"expressionRule": ""
270+
"expressionType": "NORMAL",
271+
"expressionRule": _extract_vaccination_procedure_term
171272
}
172273
},
173274
{
174275
"fieldNameFHIR": "protocolApplied|0|doseNumberPositiveInt",
175276
"fieldNameFlat": "DOSE_SEQUENCE",
176277
"expression": {
177278
"expressionName": "Not Empty",
178-
"expressionType": "DOSESEQUENCE",
179-
"expressionRule": ""
279+
"expressionType": "NORMAL",
280+
"expressionRule": _extract_dose_sequence
180281
}
181282
},
182283
{
183284
"fieldNameFHIR": "vaccineCode|coding|#:http://snomed.info/sct|code",
184285
"fieldNameFlat": "VACCINE_PRODUCT_CODE",
185286
"expression": {
186287
"expressionName": "Not Empty",
187-
"expressionType": "SNOMED",
188-
"expressionRule": ""
288+
"expressionType": "NORMAL",
289+
"expressionRule": _extract_vaccine_product_code
189290
}
190291
},
191292
{
192293
"fieldNameFHIR": "vaccineCode|coding|#:http://snomed.info/sct|display",
193294
"fieldNameFlat": "VACCINE_PRODUCT_TERM",
194295
"expression": {
195296
"expressionName": "Not Empty",
196-
"expressionType": "NOTEMPTY",
197-
"expressionRule": ""
297+
"expressionType": "NORMAL",
298+
"expressionRule": _extract_vaccine_product_term
198299
}
199300
},
200301
{
@@ -229,35 +330,35 @@
229330
"fieldNameFlat": "SITE_OF_VACCINATION_CODE",
230331
"expression": {
231332
"expressionName": "Not Empty",
232-
"expressionType": "SNOMED",
233-
"expressionRule": ""
333+
"expressionType": "NORMAL",
334+
"expressionRule": _extract_site_of_vaccination_code
234335
}
235336
},
236337
{
237338
"fieldNameFHIR": "site|coding|#:http://snomed.info/sct|display",
238339
"fieldNameFlat": "SITE_OF_VACCINATION_TERM",
239340
"expression": {
240341
"expressionName": "Look Up",
241-
"expressionType": "LOOKUP",
242-
"expressionRule": "site|coding|#:http://snomed.info/sct|code"
342+
"expressionType": "NORMAL",
343+
"expressionRule": _extract_site_of_vaccination_term
243344
}
244345
},
245346
{
246347
"fieldNameFHIR": "route|coding|#:http://snomed.info/sct|code",
247348
"fieldNameFlat": "ROUTE_OF_VACCINATION_CODE",
248349
"expression": {
249350
"expressionName": "Not Empty",
250-
"expressionType": "SNOMED",
251-
"expressionRule": ""
351+
"expressionType": "NORMAL",
352+
"expressionRule": _extract_route_of_vaccination_code
252353
}
253354
},
254355
{
255356
"fieldNameFHIR": "route|coding|#:http://snomed.info/sct|display",
256357
"fieldNameFlat": "ROUTE_OF_VACCINATION_TERM",
257358
"expression": {
258359
"expressionName": "Look Up",
259-
"expressionType": "LOOKUP",
260-
"expressionRule": "route|coding|#:http://snomed.info/sct|code"
360+
"expressionType": "NORMAL",
361+
"expressionRule": _extract_route_of_vaccination_term
261362
}
262363
},
263364
{
@@ -274,26 +375,26 @@
274375
"fieldNameFlat": "DOSE_UNIT_CODE",
275376
"expression": {
276377
"expressionName": "Only If",
277-
"expressionType": "ONLYIF",
278-
"expressionRule": "doseQuantity|system|http://snomed.info/sct"
378+
"expressionType": "NORMAL",
379+
"expressionRule": _extract_dose_unit_code
279380
}
280381
},
281382
{
282383
"fieldNameFHIR": "doseQuantity|unit",
283384
"fieldNameFlat": "DOSE_UNIT_TERM",
284385
"expression": {
285386
"expressionName": "Not Empty",
286-
"expressionType": "NOTEMPTY",
287-
"expressionRule": ""
387+
"expressionType": "NORMAL",
388+
"expressionRule": _extract_dose_unit_term
288389
}
289390
},
290391
{
291392
"fieldNameFHIR": "reasonCode|#:http://snomed.info/sct|coding|#:http://snomed.info/sct|code",
292393
"fieldNameFlat": "INDICATION_CODE",
293394
"expression": {
294395
"expressionName": "Not Empty",
295-
"expressionType": "SNOMED",
296-
"expressionRule": ""
396+
"expressionType": "NORMAL",
397+
"expressionRule": _extract_indication_code
297398
}
298399
},
299400
{

‎delta_backend/src/Converter.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ def _convertData(self, ConversionValidate: ConversionChecker, expression, dataPa
6060
expressionRule = expression["expression"]["expressionRule"]
6161

6262
try:
63-
conversionValues = dataParser.getKeyValue(FHIRFieldName, expressionType, expressionRule)
63+
conversionValues = dataParser.getKeyValue(FHIRFieldName, FlatFieldName, expressionType, expressionRule)
6464
except Exception as e:
6565
message = "Data get value Unexpected exception [%s]: %s" % (e.__class__.__name__, e)
6666
error = self._log_error(message, code=ExceptionMessages.PARSING_ERROR)

‎delta_backend/src/Extractor.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,6 @@ def extract_practitioner_names(json_data, occurrence_time):
104104

105105
return performing_professional_forename, performing_professional_surname
106106

107-
108107
def is_current_period(name, occurrence_time):
109108
period = name.get("period")
110109
if not isinstance(period, dict):

‎delta_backend/src/FHIRParser.py‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,21 +15,21 @@ def _validate_expression_rule(self, expression_type, expression_rule, key_value_
1515
"""
1616
Applies expression rules for filtering key-value pairs during searches.
1717
18-
This method provides a flexible foundation for implementing various filtering
19-
or validation rules, enabling more dynamic and configurable search behavior.
20-
While it currently supports only SNOMED code validation, the structure opens
18+
This method provides a flexible foundation for implementing various filtering
19+
or validation rules, enabling more dynamic and configurable search behavior.
20+
While it currently supports only SNOMED code validation, the structure opens
2121
the door to applying a wide range of expression rules in the future.
2222
23-
For example, when processing a list of items, this method helps determine
24-
which item(s) satisfy specific criteria based on the logic defined by the
23+
For example, when processing a list of items, this method helps determine
24+
which item(s) satisfy specific criteria based on the logic defined by the
2525
expression type and rule.
2626
"""
2727
if expression_type == "SNOMED" and expression_rule == "validate-code":
2828
if key_value_pair.get("code"):
2929
return is_valid_simple_snomed(key_value_pair["code"])
30-
30+
3131
return True
32-
32+
3333
# scan for a key name or a value
3434
def _scanValuesForMatch(self, parent, matchValue):
3535
try:
@@ -93,10 +93,14 @@ def _scanForValue(self, FHIRFields, expression_type, expression_rule: str = ""):
9393
return rootfield
9494

9595
# get the value for a key
96-
def getKeyValue(self, fieldName, expression_type: str = "", expression_rule: str = ""):
96+
def getKeyValue(self, fieldName, flatFieldName, expression_type: str = "", expression_rule = ""):
9797
value = []
9898
try:
99-
responseValue = self._scanForValue(fieldName, expression_type, expression_rule)
99+
# extract
100+
if expression_type == "NORMAL":
101+
responseValue = self.FHIRFile
102+
else:
103+
responseValue = self._scanForValue(fieldName, expression_type, expression_rule)
100104
except:
101105
responseValue = ""
102106

‎delta_backend/src/utils.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,4 @@ def is_valid_simple_snomed(simple_snomed: str) -> bool:
1717
and (simple_snomed[-3:-1] in ("00", "10"))
1818
)
1919
except:
20-
return False
20+
return False

0 commit comments

Comments
 (0)