Skip to content

Commit be10923

Browse files
nhsdevwsnhsdevws
authored
Merge pull request #392 from NHSDigital/VED-223-Delta-Review
VED-223 Review Resolve Delta Test & Associated Issues
2 parents f10088d + d5c01f3 commit be10923

File tree

12 files changed

+465
-113
lines changed

12 files changed

+465
-113
lines changed

‎.github/workflows/sonarcloud.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,9 +61,11 @@ jobs:
6161
6262
- name: Run unittest with coverage-delta
6363
id: delta
64+
env:
65+
PYTHONPATH: delta_backend/src:delta_backend/tests
6466
continue-on-error: true
6567
run: |
66-
pip install poetry==1.8.4 mypy-boto3-dynamodb==1.35.54 boto3==1.26.165 coverage botocore==1.29.165 jmespath==1.0.1 python-dateutil==2.9.0 urllib3==1.26.20 s3transfer==0.6.2 typing-extensions==4.12.2
68+
pip install poetry==1.8.4 moto==4.2.11 mypy-boto3-dynamodb==1.35.54 boto3==1.26.165 coverage botocore==1.29.165 jmespath==1.0.1 python-dateutil==2.9.0 urllib3==1.26.20 s3transfer==0.6.2 typing-extensions==4.12.2
6769
poetry run coverage run --source=delta_backend -m unittest discover -s delta_backend || echo "delta tests failed" >> failed_tests.txt
6870
poetry run coverage xml -o sonarcloud-coverage-delta.xml
6971

‎delta_backend/pyproject.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@ version = "0.1.0"
44
description = ""
55
authors = ["Your Name <[email protected]>"]
66
readme = "README.md"
7+
packages = [
8+
{include = "src"}
9+
]
710

811
[tool.poetry.dependencies]
912
python = "~3.10"

‎delta_backend/src/Converter.py‎

Lines changed: 16 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -14,21 +14,17 @@
1414
get_valid_address,
1515
)
1616

17-
# Converter variables
18-
FHIRData = ""
19-
SchemaFile = {}
20-
imms = []
21-
Converted = {}
22-
2317

2418
# Converter
2519
class Converter:
2620

2721
def __init__(self, fhir_data):
28-
self.FHIRData = fhir_data # Store JSON data directly
29-
self.SchemaFile = ConversionLayout.ConvertLayout
30-
self.ErrorRecords = []
31-
22+
self.imms = []
23+
self.converted = {}
24+
self.error_records = []
25+
self.fhir_data = fhir_data # Store JSON data directly
26+
self.schema_file = ConversionLayout.ConvertLayout
27+
3228
# Utility logs tailored to conveter class errors
3329
def _log_error(self,e,code=ExceptionMessages.UNEXPECTED_EXCEPTION):
3430
message = str(e) # if a simple string message was passed
@@ -38,8 +34,8 @@ def _log_error(self,e,code=ExceptionMessages.UNEXPECTED_EXCEPTION):
3834
"message": message
3935
}
4036

41-
# if not any(existing.get("message") == message for existing in self.ErrorRecords):
42-
self.ErrorRecords.append(error_obj)
37+
# if not any(existing.get("message") == message for existing in self.error_records):
38+
self.error_records.append(error_obj)
4339
return error_obj
4440

4541
# create a FHIR parser - uses fhir json data from delta
@@ -78,20 +74,20 @@ def _convertData(self, ConversionValidate, expression, dataParser, json_data):
7874
if "address" in FHIRFieldName or "performer" in FHIRFieldName or "name" in FHIRFieldName:
7975
convertedData = self.extract_patient_details(json_data, FlatFieldName)
8076
if convertedData is not None:
81-
Converted[FlatFieldName] = convertedData
77+
self.converted[FlatFieldName] = convertedData
8278

8379
# run the conversion against the data
8480
def runConversion(self, json_data, summarise=False, report_unexpected_exception=True):
8581
try:
86-
dataParser = self._getFHIRParser(self.FHIRData)
82+
dataParser = self._getFHIRParser(self.fhir_data)
8783
except Exception as e:
8884
if report_unexpected_exception:
8985
message = "FHIR Parser Unexpected exception [%s]: %s" % (e.__class__.__name__, e)
9086
error = self._log_error(message,code=ExceptionMessages.UNEXPECTED_EXCEPTION)
9187
return error
9288

9389
try:
94-
schemaParser = self._getSchemaParser(self.SchemaFile)
90+
schemaParser = self._getSchemaParser(self.schema_file)
9591
except Exception as e:
9692
if report_unexpected_exception:
9793
message = "Schema Parser Unexpected exception [%s]: %s" % (e.__class__.__name__, e)
@@ -120,21 +116,21 @@ def runConversion(self, json_data, summarise=False, report_unexpected_exception=
120116

121117
# Collect and store any errors from ConversionChecker
122118
all_errors = ConversionValidate.get_error_records()
123-
self.ErrorRecords.extend(all_errors)
119+
self.error_records.extend(all_errors)
124120

125121
# Add CONVERSION_ERRORS as the 35th field
126122
error_records = self.getErrorRecords()
127123
if error_records:
128124
error_summary = error_records
129125
else:
130126
error_summary = ""
131-
Converted["CONVERSION_ERRORS"] = error_summary
127+
self.converted["CONVERSION_ERRORS"] = error_summary
132128

133-
imms.append(Converted)
134-
return imms
129+
self.imms.append(self.converted)
130+
return self.imms
135131

136132
def getErrorRecords(self):
137-
return self.ErrorRecords
133+
return self.error_records
138134

139135
def extract_patient_details(self, json_data, FlatFieldName):
140136
if not hasattr(self, "_cached_values"):

‎delta_backend/src/SchemaParser.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@ class SchemaParser:
88
Conversions = {}
99

1010
def parseSchema(self, schemaFile): # changed to accept JSON better for cache
11-
self.SchemaFile = schemaFile
12-
self.Conversions = self.SchemaFile["conversions"]
11+
self.schema_file = schemaFile
12+
self.Conversions = self.schema_file["conversions"]
1313

1414
def conversionCount(self):
1515
count = 0

‎delta_backend/src/delta.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ def send_message(record):
2828
sqs_client.send_message(QueueUrl=failure_queue_url, MessageBody=json.dumps(message_body))
2929
logger.info("Record saved successfully to the DLQ")
3030
except ClientError as e:
31-
logger.info(f"Error sending record to DLQ: {e}")
31+
logger.error(f"Error sending record to DLQ: {e}")
3232

3333

3434
def get_vaccine_type(patientsk) -> str:
@@ -156,4 +156,7 @@ def handler(event, context):
156156
log_data["operation_outcome"] = operation_outcome
157157
firehose_log["event"] = log_data
158158
firehose_logger.send_log(firehose_log)
159-
raise Exception(f"Delta Lambda failure: {e}")
159+
return {
160+
"statusCode": 500,
161+
"body": "Records not processed",
162+
}

‎delta_backend/tests/sample_data/__init__.py‎

Whitespace-only changes.

‎delta_backend/tests/sample_data/test_resource_data.py‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
"""
2+
# This module provides a function to generate a sample FHIR Immunization resource
3+
"""
4+
def get_test_data_resource():
5+
"""
6+
The returned resource includes details about the practitioner, patient,
7+
vaccine code, location, and other relevant fields.
8+
"""
9+
return {
10+
"resourceType": "Immunization",
11+
"contained": [
12+
{
13+
"resourceType": "Practitioner",
14+
"id": "Pract1",
15+
"name": [
16+
{
17+
"family": "O'Reilly",
18+
"given": ["Ellena"]
19+
}
20+
]
21+
},
22+
{
23+
"resourceType": "Patient",
24+
"id": "Pat1",
25+
"identifier": [
26+
{
27+
"system": "https://fhir.nhs.uk/Id/nhs-number",
28+
"value": "9674963871"
29+
}
30+
],
31+
"name": [
32+
{
33+
"family": "GREIR",
34+
"given": ["SABINA"]
35+
}
36+
],
37+
"gender": "female",
38+
"birthDate": "2019-01-31",
39+
"address": [
40+
{
41+
"postalCode": "GU14 6TU"
42+
}
43+
]
44+
}
45+
],
46+
"extension": [
47+
{
48+
"url": "https://fhir.hl7.org.uk/StructureDefinition/Extension-UKCore-VaccinationProcedure",
49+
"valueCodeableConcept": {
50+
"coding": [
51+
{
52+
"system": "http://snomed.info/sct",
53+
"code": "1303503001",
54+
"display":
55+
"Administration of vaccine product containing only Human orthopneumovirus antigen (procedure)"
56+
}
57+
]
58+
}
59+
}
60+
],
61+
"identifier": [
62+
{
63+
"system": "https://www.ravs.england.nhs.uk/",
64+
"value": "0001_RSV_v5_RUN_2_CDFDPS-742_valid_dose_1"
65+
}
66+
],
67+
"status": "completed",
68+
"vaccineCode": {
69+
"coding": [
70+
{
71+
"system": "http://snomed.info/sct",
72+
"code": "42605811000001109",
73+
"display":
74+
"Abrysvo vaccine powder and solvent for solution for injection 0.5ml vials (Pfizer Ltd) (product)"
75+
}
76+
]
77+
},
78+
"patient": {
79+
"reference": "#Pat1"
80+
},
81+
"occurrenceDateTime": "2024-06-10T18:33:25+00:00",
82+
"recorded": "2024-06-10T18:33:25+00:00",
83+
"primarySource": True,
84+
"manufacturer": {
85+
"display": "Pfizer"
86+
},
87+
"location": {
88+
"type": "Location",
89+
"identifier": {
90+
"value": "J82067",
91+
"system": "https://fhir.nhs.uk/Id/ods-organization-code"
92+
}
93+
},
94+
"lotNumber": "RSVTEST",
95+
"expirationDate": "2024-12-31",
96+
"site": {
97+
"coding": [
98+
{
99+
"system": "http://snomed.info/sct",
100+
"code": "368208006",
101+
"display": "Left upper arm structure (body structure)"
102+
}
103+
]
104+
},
105+
"route": {
106+
"coding": [
107+
{
108+
"system": "http://snomed.info/sct",
109+
"code": "78421000",
110+
"display": "Intramuscular route (qualifier value)"
111+
}
112+
]
113+
},
114+
"doseQuantity": {
115+
"value": 0.5,
116+
"unit": "Milliliter (qualifier value)",
117+
"system": "http://unitsofmeasure.org",
118+
"code": "258773002"
119+
},
120+
"performer": [
121+
{
122+
"actor": {
123+
"reference": "#Pract1"
124+
}
125+
},
126+
{
127+
"actor": {
128+
"type": "Organization",
129+
"identifier": {
130+
"system": "https://fhir.nhs.uk/Id/ods-organization-code",
131+
"value": "X0X0X"
132+
}
133+
}
134+
}
135+
],
136+
"reasonCode": [
137+
{
138+
"coding": [
139+
{
140+
"code": "Test",
141+
"system": "http://snomed.info/sct"
142+
}
143+
]
144+
}
145+
],
146+
"protocolApplied": [
147+
{
148+
"targetDisease": [
149+
{
150+
"coding": [
151+
{
152+
"system": "http://snomed.info/sct",
153+
"code": "840539006",
154+
"display": "Disease caused by severe acute respiratory syndrome coronavirus 2"
155+
}
156+
]
157+
}
158+
],
159+
"doseNumberPositiveInt": 1
160+
}
161+
],
162+
"id": "ca8ba2c6-2383-4465-b456-c1174c21cf31"
163+
}
164+

0 commit comments

Comments
 (0)