Skip to content

Commit ca6ecf1

Browse files
Akol125Akol125
authored
Merge branch 'master' into VED-79-MNS-subscriptions-event
2 parents a5c8ac6 + 3761d09 commit ca6ecf1

File tree

8 files changed

+143
-108
lines changed

8 files changed

+143
-108
lines changed

‎config/common/disease_mapping.json‎

Lines changed: 15 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,18 @@
11
[
22
{
3-
"vacc_type": "COVID19",
3+
"vacc_type": "3IN1",
44
"diseases": [
55
{
6-
"code": "840539006",
7-
"term": "Disease caused by severe acute respiratory syndrome coronavirus 2"
6+
"code": "398102009",
7+
"term": "Acute poliomyelitis"
8+
},
9+
{
10+
"code": "397430003",
11+
"term": "Diphtheria caused by Corynebacterium diphtheriae"
12+
},
13+
{
14+
"code": "76902006",
15+
"term": "Tetanus (disorder)"
816
}
917
]
1018
},
@@ -27,24 +35,16 @@
2735
]
2836
},
2937
{
30-
"vacc_type": "MMR",
38+
"vacc_type": "MENACWY",
3139
"diseases": [
3240
{
33-
"code": "14189004",
34-
"term": "Measles (disorder)"
35-
},
36-
{
37-
"code": "36989005",
38-
"term": "Mumps (disorder)"
39-
},
40-
{
41-
"code": "36653000",
42-
"term": "Rubella (disorder)"
41+
"code": "23511006",
42+
"term": "Meningococcal infectious disease"
4343
}
4444
]
4545
},
4646
{
47-
"vacc_type": "MMRV",
47+
"vacc_type": "MMR",
4848
"diseases": [
4949
{
5050
"code": "14189004",
@@ -57,10 +57,6 @@
5757
{
5858
"code": "36653000",
5959
"term": "Rubella (disorder)"
60-
},
61-
{
62-
"code": "38907003",
63-
"term": "Varicella (disorder)"
6460
}
6561
]
6662
},
@@ -72,58 +68,5 @@
7268
"term": "Respiratory syncytial virus infection (disorder)"
7369
}
7470
]
75-
},
76-
{
77-
"vacc_type": "PERTUSSIS",
78-
"diseases": [
79-
{
80-
"code": "27836007",
81-
"term": "Pertussis (disorder)"
82-
}
83-
]
84-
},
85-
{
86-
"vacc_type": "SHINGLES",
87-
"diseases": [
88-
{
89-
"code": "4740000",
90-
"term": "Herpes zoster"
91-
}
92-
]
93-
},
94-
{
95-
"vacc_type": "PCV13",
96-
"diseases": [
97-
{
98-
"code": "16814004",
99-
"term": "Pneumococcal infectious disease"
100-
}
101-
]
102-
},
103-
{
104-
"vacc_type": "3in1",
105-
"diseases": [
106-
{
107-
"code": "398102009",
108-
"term": "Acute poliomyelitis"
109-
},
110-
{
111-
"code": "397430003",
112-
"term": "Diphtheria caused by Corynebacterium diphtheriae"
113-
},
114-
{
115-
"code": "76902006",
116-
"term": "Tetanus (disorder)"
117-
}
118-
]
119-
},
120-
{
121-
"vacc_type": "MENACWY",
122-
"diseases": [
123-
{
124-
"code": "23511006",
125-
"term": "Meningococcal infectious disease"
126-
}
127-
]
12871
}
12972
]

‎config/dev/permissions_config.json‎

Lines changed: 30 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -27,34 +27,12 @@
2727
"FLU.CRUDS"
2828
]
2929
},
30-
{
31-
"supplier": "EMIS",
32-
"permissions": [
33-
"RSV.U"
34-
]
35-
},
36-
{
37-
"supplier": "PINNACLE",
38-
"permissions": []
39-
},
4030
{
4131
"supplier": "SONAR",
4232
"permissions": [
4333
"FLU.CD"
4434
]
4535
},
46-
{
47-
"supplier": "TPP",
48-
"permissions": []
49-
},
50-
{
51-
"supplier": "AGEM-NIVS",
52-
"permissions": []
53-
},
54-
{
55-
"supplier": "NIMS",
56-
"permissions": []
57-
},
5836
{
5937
"supplier": "EVA",
6038
"permissions": [
@@ -64,22 +42,46 @@
6442
{
6543
"supplier": "RAVS",
6644
"permissions": [
67-
"RSV.CRUDS"
45+
"RSV.CRUDS",
46+
"MMR.CRUDS"
6847
]
6948
},
7049
{
7150
"supplier": "AGEM-RAVS-Integration",
7251
"permissions": [
73-
"RSV.CRUDS"
52+
"RSV.CRUDS",
53+
"MMR.CRUDS"
54+
]
55+
},
56+
{
57+
"supplier": "EMIS",
58+
"permissions": [
59+
"RSV.CRUDS",
60+
"MMR.CRUDS",
61+
"HPV.CRUDS",
62+
"3IN1.CRUDS",
63+
"MENACWY.CRUDS"
7464
]
7565
},
7666
{
77-
"supplier": "MEDICAL_DIRECTOR",
78-
"permissions": []
67+
"supplier": "TPP",
68+
"permissions": [
69+
"RSV.CRUDS",
70+
"MMR.CRUDS",
71+
"HPV.CRUDS",
72+
"3IN1.CRUDS",
73+
"MENACWY.CRUDS"
74+
]
7975
},
8076
{
81-
"supplier": "COVID19_VACCINE_RESOLUTION_SERVICEDESK",
82-
"permissions": []
77+
"supplier": "MEDICUS",
78+
"permissions": [
79+
"RSV.CRUDS",
80+
"MMR.CRUDS",
81+
"HPV.CRUDS",
82+
"3IN1.CRUDS",
83+
"MENACWY.CRUDS"
84+
]
8385
},
8486
{
8587
"supplier": "Test_App",

‎config/preprod/permissions_config.json‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,45 @@
3030
{
3131
"supplier": "RAVS",
3232
"permissions": [
33-
"RSV.CRUDS"
33+
"RSV.CRUDS",
34+
"MMR.CRUDS"
35+
]
36+
},
37+
{
38+
"supplier": "AGEM-RAVS-Integration",
39+
"permissions": [
40+
"RSV.CRUDS",
41+
"MMR.CRUDS"
42+
]
43+
},
44+
{
45+
"supplier": "EMIS",
46+
"permissions": [
47+
"RSV.CRUDS",
48+
"MMR.CRUDS",
49+
"HPV.CRUDS",
50+
"3IN1.CRUDS",
51+
"MENACWY.CRUDS"
52+
]
53+
},
54+
{
55+
"supplier": "TPP",
56+
"permissions": [
57+
"RSV.CRUDS",
58+
"MMR.CRUDS",
59+
"HPV.CRUDS",
60+
"3IN1.CRUDS",
61+
"MENACWY.CRUDS"
62+
]
63+
},
64+
{
65+
"supplier": "MEDICUS",
66+
"permissions": [
67+
"RSV.CRUDS",
68+
"MMR.CRUDS",
69+
"HPV.CRUDS",
70+
"3IN1.CRUDS",
71+
"MENACWY.CRUDS"
3472
]
3573
},
3674
{

‎filenameprocessor/src/constants.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ class Constants:
7272
"YA7": "SCOTLAND_DA",
7373
"N2N9I": "COVID19_VACCINE_RESOLUTION_SERVICEDESK",
7474
"YGJ": "EMIS",
75+
"YGMYW": "MEDICUS",
7576
"DPSREDUCED": "DPSREDUCED",
7677
"DPSFULL": "DPSFULL",
7778
}

‎mesh_processor/src/converter.py‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,11 +98,19 @@ def transfer_multipart_content(bucket_name: str, file_key: str, boundary: bytes,
9898
if content_disposition:
9999
_, content_disposition_params = parse_header_value(content_disposition)
100100
filename = content_disposition_params.get("filename") or filename
101+
content_type = headers.get("Content-Type") or "application/octet-stream"
101102

102103
with open(
103104
f"s3://{DESTINATION_BUCKET_NAME}/streaming/{filename}",
104105
"wb",
105-
transport_params={"client": s3_client}
106+
transport_params={
107+
"client": s3_client,
108+
"client_kwargs": {
109+
"S3.Client.create_multipart_upload": {
110+
"ContentType": content_type
111+
}
112+
}
113+
}
106114
) as output_file:
107115
stream_part_body(input_file, boundary, output_file)
108116

@@ -133,6 +141,10 @@ def process_record(record: dict) -> None:
133141

134142
logger.info(f"Transfer complete for {file_key}")
135143

144+
move_file(bucket_name, file_key, bucket_name, f"archive/{file_key}")
145+
146+
logger.info(f"Archived {file_key}")
147+
136148

137149
def lambda_handler(event: dict, _context: dict) -> dict:
138150
success = True

‎mesh_processor/tests/test_converter.py‎

Lines changed: 40 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,18 @@ def test_non_multipart_content_type(self):
4848
result = invoke_lambda("test-csv-file.csv")
4949
self.assertEqual(result["statusCode"], 200)
5050

51-
response = s3.get_object(Bucket="destination-bucket", Key="overridden-filename.csv")
52-
body = response["Body"].read().decode("utf-8")
51+
get_target_response = s3.get_object(Bucket="destination-bucket", Key="overridden-filename.csv")
52+
body = get_target_response["Body"].read().decode("utf-8")
5353
assert body == "some CSV content"
5454

55-
def test_non_multipart_content_type_no_mesh_metadata(self):
55+
with self.assertRaises(ClientError) as e:
56+
s3.head_object(Bucket="source-bucket", Key="test-csv-file.csv")
57+
self.assertEqual(e.exception.response["Error"]["Code"], "404")
58+
59+
head_archive_response = s3.head_object(Bucket="source-bucket", Key="archive/test-csv-file.csv")
60+
assert head_archive_response["ResponseMetadata"]["HTTPStatusCode"] == 200
61+
62+
def test_non_multipart_content_type_without_mesh_metadata(self):
5663
s3 = boto3.client("s3", region_name="eu-west-2")
5764
s3.put_object(
5865
Bucket="source-bucket",
@@ -142,8 +149,10 @@ def test_multipart_content_type(self):
142149
response = s3.get_object(Bucket="destination-bucket", Key="test-csv-file.csv")
143150
body = response["Body"].read().decode("utf-8")
144151
assert body == "some CSV content"
152+
content_type = response["ContentType"]
153+
assert content_type == "text/csv"
145154

146-
def test_multipart_content_type_without_filename_from_headers(self):
155+
def test_multipart_content_type_without_filename_in_headers(self):
147156
cases = [
148157
(
149158
"no filename in header",
@@ -187,6 +196,33 @@ def test_multipart_content_type_without_filename_from_headers(self):
187196
body = response["Body"].read().decode("utf-8")
188197
assert body == "some CSV content"
189198

199+
def test_multipart_content_type_without_content_type_in_headers(self):
200+
body = "\r\n".join([
201+
"",
202+
"--12345678",
203+
'Content-Disposition: form-data; name="File"; filename="test-csv-file.csv"',
204+
"",
205+
"some CSV content",
206+
"--12345678--",
207+
""
208+
])
209+
s3 = boto3.client("s3", region_name="eu-west-2")
210+
s3.put_object(
211+
Bucket="source-bucket",
212+
Key="test-dat-file.dat",
213+
Body=body.encode("utf-8"),
214+
ContentType="multipart/form-data; boundary=12345678",
215+
)
216+
217+
result = invoke_lambda("test-dat-file.dat")
218+
self.assertEqual(result["statusCode"], 200)
219+
220+
response = s3.get_object(Bucket="destination-bucket", Key="test-csv-file.csv")
221+
body = response["Body"].read().decode("utf-8")
222+
assert body == "some CSV content"
223+
content_type = response["ContentType"]
224+
assert content_type == "application/octet-stream"
225+
190226
def test_multipart_content_type_with_unix_line_endings(self):
191227
body = "\r\n".join([
192228
"",

‎recordprocessor/src/utils_for_recordprocessor.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ def get_csv_content_dict_reader(file_key: str) -> DictReader:
1919
"""Returns the requested file contents from the source bucket in the form of a DictReader"""
2020
response = s3_client.get_object(Bucket=os.getenv("SOURCE_BUCKET_NAME"), Key=file_key)
2121
binary_io = response["Body"]
22-
text_io = TextIOWrapper(binary_io, encoding="utf-8")
22+
text_io = TextIOWrapper(binary_io, encoding="utf-8", newline="")
2323
return DictReader(text_io, delimiter="|")
2424

2525

‎terraform/mesh_processor.tf‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,8 @@ resource "aws_iam_policy" "mesh_processor_lambda_exec_policy" {
137137
"s3:GetObject",
138138
"s3:ListBucket",
139139
"s3:PutObject",
140-
"s3:CopyObject"
140+
"s3:CopyObject",
141+
"s3:DeleteObject"
141142
]
142143
Resource = [
143144
aws_s3_bucket.batch_data_source_bucket.arn,
@@ -213,6 +214,7 @@ resource "aws_lambda_function" "mesh_file_converter_lambda" {
213214
image_uri = module.mesh_processor_docker_image[0].image_uri
214215
architectures = ["x86_64"]
215216
timeout = 900
217+
memory_size = 1024
216218

217219
environment {
218220
variables = {
@@ -240,6 +242,7 @@ resource "aws_s3_bucket_notification" "mesh_datasources_lambda_notification" {
240242
lambda_function {
241243
lambda_function_arn = aws_lambda_function.mesh_file_converter_lambda[0].arn
242244
events = ["s3:ObjectCreated:*"]
245+
filter_prefix = "inbound/"
243246
}
244247
}
245248

0 commit comments

Comments
 (0)