Skip to content

Commit 2e34355

Browse files
mfjarvismfjarvis
committed
VED-470: Consolidate tests.
1 parent 9044743 commit 2e34355

File tree

2 files changed

+158
-153
lines changed

2 files changed

+158
-153
lines changed

‎mesh_processor/src/converter.py‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,15 @@ def stream_part_body(input_file: BinaryIO, boundary: bytes, output_file: BinaryI
7676
raise ValueError("Unexpected EOF")
7777

7878

79+
def move_file(source_bucket: str, source_key: str, destination_bucket: str, destination_key: str) -> None:
80+
s3_client.copy_object(
81+
CopySource={"Bucket": source_bucket, "Key": source_key},
82+
Bucket=destination_bucket,
83+
Key=destination_key
84+
)
85+
s3_client.delete_object(Bucket=source_bucket, Key=source_key)
86+
87+
7988
def transfer_multipart_content(bucket_name: str, file_key: str, boundary: bytes, filename: str) -> None:
8089
with open(
8190
f"s3://{bucket_name}/{file_key}",
@@ -91,12 +100,14 @@ def transfer_multipart_content(bucket_name: str, file_key: str, boundary: bytes,
91100
filename = content_disposition_params.get("filename") or filename
92101

93102
with open(
94-
f"s3://{DESTINATION_BUCKET_NAME}/{filename}",
103+
f"s3://{DESTINATION_BUCKET_NAME}/streaming/{filename}",
95104
"wb",
96105
transport_params={"client": s3_client}
97106
) as output_file:
98107
stream_part_body(input_file, boundary, output_file)
99108

109+
move_file(DESTINATION_BUCKET_NAME, f"streaming/{filename}", DESTINATION_BUCKET_NAME, filename)
110+
100111

101112
def process_record(record: dict) -> None:
102113
bucket_name = record["s3"]["bucket"]["name"]

‎mesh_processor/tests/test_converter.py‎

Lines changed: 146 additions & 152 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,14 @@
33
from unittest.mock import patch
44

55
import boto3
6+
from botocore.exceptions import ClientError
67
from moto import mock_aws
78

89

910
def invoke_lambda(file_key: str):
1011
# Local import so that globals can be mocked
1112
from converter import lambda_handler
12-
lambda_handler(
13+
return lambda_handler(
1314
{
1415
"Records": [
1516
{
@@ -44,7 +45,8 @@ def test_non_multipart_content_type(self):
4445
}
4546
)
4647

47-
invoke_lambda("test-csv-file.csv")
48+
result = invoke_lambda("test-csv-file.csv")
49+
self.assertEqual(result["statusCode"], 200)
4850

4951
response = s3.get_object(Bucket="destination-bucket", Key="overridden-filename.csv")
5052
body = response["Body"].read().decode("utf-8")
@@ -59,114 +61,131 @@ def test_non_multipart_content_type_no_mesh_metadata(self):
5961
ContentType="text/csv",
6062
)
6163

62-
invoke_lambda("test-csv-file.csv")
64+
result = invoke_lambda("test-csv-file.csv")
65+
self.assertEqual(result["statusCode"], 200)
6366

6467
response = s3.get_object(Bucket="destination-bucket", Key="test-csv-file.csv")
6568
body = response["Body"].read().decode("utf-8")
6669
assert body == "some CSV content"
6770

6871
def test_multipart_content_type(self):
69-
body = "\r\n".join([
70-
"",
71-
"--12345678",
72-
'Content-Disposition: form-data; name="File"; filename="test-csv-file.csv"',
73-
"Content-Type: text/csv",
74-
"",
75-
"some CSV content",
76-
"--12345678--",
77-
""
78-
])
79-
s3 = boto3.client("s3", region_name="eu-west-2")
80-
s3.put_object(
81-
Bucket="source-bucket",
82-
Key="test-dat-file.dat",
83-
Body=body.encode("utf-8"),
84-
ContentType="multipart/form-data; boundary=12345678",
85-
)
86-
87-
invoke_lambda("test-dat-file.dat")
88-
89-
response = s3.get_object(Bucket="destination-bucket", Key="test-csv-file.csv")
90-
body = response["Body"].read().decode("utf-8")
91-
assert body == "some CSV content"
92-
93-
def test_multipart_content_type_multiple_parts(self):
94-
body = "\r\n".join([
95-
"",
96-
"--12345678",
97-
'Content-Disposition: form-data; name="File"; filename="test-csv-file.csv"',
98-
"Content-Type: text/csv",
99-
"",
100-
"some CSV content",
101-
"--12345678",
102-
'Content-Disposition: form-data; name="File"; filename="test-ignored-file"',
103-
"Content-Type: text/plain",
104-
"",
105-
"some ignored content",
106-
"--12345678--",
107-
""
108-
])
109-
s3 = boto3.client("s3", region_name="eu-west-2")
110-
s3.put_object(
111-
Bucket="source-bucket",
112-
Key="test-dat-file.dat",
113-
Body=body.encode("utf-8"),
114-
ContentType="multipart/form-data; boundary=12345678",
115-
)
116-
117-
invoke_lambda("test-dat-file.dat")
118-
119-
response = s3.get_object(Bucket="destination-bucket", Key="test-csv-file.csv")
120-
body = response["Body"].read().decode("utf-8")
121-
assert body == "some CSV content"
122-
123-
def test_multipart_content_type_without_filename(self):
124-
body = "\r\n".join([
125-
"",
126-
"--12345678",
127-
'Content-Disposition: form-data',
128-
"Content-Type: text/csv",
129-
"",
130-
"some CSV content",
131-
"--12345678--",
132-
""
133-
])
134-
s3 = boto3.client("s3", region_name="eu-west-2")
135-
s3.put_object(
136-
Bucket="source-bucket",
137-
Key="test-dat-file.dat",
138-
Body=body.encode("utf-8"),
139-
ContentType="multipart/form-data; boundary=12345678",
140-
)
141-
142-
invoke_lambda("test-dat-file.dat")
143-
144-
response = s3.get_object(Bucket="destination-bucket", Key="test-dat-file.dat")
145-
body = response["Body"].read().decode("utf-8")
146-
assert body == "some CSV content"
147-
148-
def test_multipart_content_type_without_headers(self):
149-
body = "\r\n".join([
150-
"",
151-
"--12345678",
152-
"",
153-
"some CSV content",
154-
"--12345678--",
155-
""
156-
])
157-
s3 = boto3.client("s3", region_name="eu-west-2")
158-
s3.put_object(
159-
Bucket="source-bucket",
160-
Key="test-dat-file.dat",
161-
Body=body.encode("utf-8"),
162-
ContentType="multipart/form-data; boundary=12345678",
163-
)
164-
165-
invoke_lambda("test-dat-file.dat")
166-
167-
response = s3.get_object(Bucket="destination-bucket", Key="test-dat-file.dat")
168-
body = response["Body"].read().decode("utf-8")
169-
assert body == "some CSV content"
72+
cases = [
73+
(
74+
"standard",
75+
"\r\n".join([
76+
"",
77+
"--12345678",
78+
'Content-Disposition: form-data; name="File"; filename="test-csv-file.csv"',
79+
"Content-Type: text/csv",
80+
"",
81+
"some CSV content",
82+
"--12345678--",
83+
""
84+
])
85+
),
86+
(
87+
"missing initial newline",
88+
"\r\n".join([
89+
"--12345678",
90+
'Content-Disposition: form-data; name="File"; filename="test-csv-file.csv"',
91+
"Content-Type: text/csv",
92+
"",
93+
"some CSV content",
94+
"--12345678--",
95+
""
96+
])
97+
),
98+
(
99+
"missing final newline",
100+
"\r\n".join([
101+
"",
102+
"--12345678",
103+
'Content-Disposition: form-data; name="File"; filename="test-csv-file.csv"',
104+
"Content-Type: text/csv",
105+
"",
106+
"some CSV content",
107+
"--12345678--",
108+
])
109+
),
110+
(
111+
"multiple parts",
112+
"\r\n".join([
113+
"",
114+
"--12345678",
115+
'Content-Disposition: form-data; name="File"; filename="test-csv-file.csv"',
116+
"Content-Type: text/csv",
117+
"",
118+
"some CSV content",
119+
"--12345678",
120+
'Content-Disposition: form-data; name="File"; filename="test-ignored-file"',
121+
"Content-Type: text/plain",
122+
"",
123+
"some ignored content",
124+
"--12345678--",
125+
""
126+
])
127+
)
128+
]
129+
for msg, body in cases:
130+
with self.subTest(msg=msg, body=body):
131+
s3 = boto3.client("s3", region_name="eu-west-2")
132+
s3.put_object(
133+
Bucket="source-bucket",
134+
Key="test-dat-file.dat",
135+
Body=body.encode("utf-8"),
136+
ContentType="multipart/form-data; boundary=12345678",
137+
)
138+
139+
result = invoke_lambda("test-dat-file.dat")
140+
self.assertEqual(result["statusCode"], 200)
141+
142+
response = s3.get_object(Bucket="destination-bucket", Key="test-csv-file.csv")
143+
body = response["Body"].read().decode("utf-8")
144+
assert body == "some CSV content"
145+
146+
def test_multipart_content_type_without_filename_from_headers(self):
147+
cases = [
148+
(
149+
"no filename in header",
150+
"\r\n".join([
151+
"",
152+
"--12345678",
153+
'Content-Disposition: form-data',
154+
"Content-Type: text/csv",
155+
"",
156+
"some CSV content",
157+
"--12345678--",
158+
""
159+
])
160+
),
161+
(
162+
"no header",
163+
"\r\n".join([
164+
"",
165+
"--12345678",
166+
"",
167+
"some CSV content",
168+
"--12345678--",
169+
""
170+
])
171+
)
172+
]
173+
for msg, body in cases:
174+
with self.subTest(msg=msg, body=body):
175+
s3 = boto3.client("s3", region_name="eu-west-2")
176+
s3.put_object(
177+
Bucket="source-bucket",
178+
Key="test-dat-file.dat",
179+
Body=body.encode("utf-8"),
180+
ContentType="multipart/form-data; boundary=12345678",
181+
)
182+
183+
result = invoke_lambda("test-dat-file.dat")
184+
self.assertEqual(result["statusCode"], 200)
185+
186+
response = s3.get_object(Bucket="destination-bucket", Key="test-dat-file.dat")
187+
body = response["Body"].read().decode("utf-8")
188+
assert body == "some CSV content"
170189

171190
def test_multipart_content_type_with_unix_line_endings(self):
172191
body = "\r\n".join([
@@ -187,56 +206,31 @@ def test_multipart_content_type_with_unix_line_endings(self):
187206
ContentType="multipart/form-data; boundary=12345678",
188207
)
189208

190-
invoke_lambda("test-dat-file.dat")
209+
result = invoke_lambda("test-dat-file.dat")
210+
self.assertEqual(result["statusCode"], 200)
191211

192212
response = s3.get_object(Bucket="destination-bucket", Key="test-csv-file.csv")
193213
body = response["Body"].read().decode("utf-8")
194214
assert body == "some CSV content\nsplit across\nmultiple lines"
195215

196-
def test_multipart_content_type_missing_first_newline(self):
197-
body = "\r\n".join([
198-
"--12345678",
199-
'Content-Disposition: form-data; name="File"; filename="test-csv-file.csv"',
200-
"Content-Type: text/csv",
201-
"",
202-
"some CSV content",
203-
"--12345678--",
204-
""
205-
])
206-
s3 = boto3.client("s3", region_name="eu-west-2")
207-
s3.put_object(
208-
Bucket="source-bucket",
209-
Key="test-dat-file.dat",
210-
Body=body.encode("utf-8"),
211-
ContentType="multipart/form-data; boundary=12345678",
212-
)
213-
214-
invoke_lambda("test-dat-file.dat")
215-
216-
response = s3.get_object(Bucket="destination-bucket", Key="test-csv-file.csv")
217-
body = response["Body"].read().decode("utf-8")
218-
assert body == "some CSV content"
219-
220-
def test_multipart_content_type_missing_final_newline(self):
221-
body = "\r\n".join([
222-
"",
223-
"--12345678",
224-
'Content-Disposition: form-data; name="File"; filename="test-csv-file.csv"',
225-
"Content-Type: text/csv",
226-
"",
227-
"some CSV content",
228-
"--12345678--",
229-
])
230-
s3 = boto3.client("s3", region_name="eu-west-2")
231-
s3.put_object(
232-
Bucket="source-bucket",
233-
Key="test-dat-file.dat",
234-
Body=body.encode("utf-8"),
235-
ContentType="multipart/form-data; boundary=12345678",
236-
)
237-
238-
invoke_lambda("test-dat-file.dat")
239-
240-
response = s3.get_object(Bucket="destination-bucket", Key="test-csv-file.csv")
241-
body = response["Body"].read().decode("utf-8")
242-
assert body == "some CSV content"
216+
def test_unexpected_end_of_file(self):
217+
for msg, body in [
218+
("before first part", ""),
219+
("in headers", "--12345678\r\n"),
220+
("in body", "--12345678\r\n\r\n"),
221+
]:
222+
with self.subTest(msg=msg, body=body):
223+
s3 = boto3.client("s3", region_name="eu-west-2")
224+
s3.put_object(
225+
Bucket="source-bucket",
226+
Key="test-dat-file.dat",
227+
Body=body.encode("utf-8"),
228+
ContentType="multipart/form-data; boundary=12345678",
229+
)
230+
231+
result = invoke_lambda("test-dat-file.dat")
232+
self.assertEqual(result["statusCode"], 500)
233+
234+
with self.assertRaises(ClientError) as e:
235+
s3.head_object(Bucket="destination-bucket", Key="test-csv-file.csv")
236+
self.assertEqual(e.exception.response["Error"]["Code"], "404")

0 commit comments

Comments
 (0)