VED-470: Handle some extra cases. Fix EOF check. Upgrade to Python 3.11.

Author: mfjarvis

.github/workflows/sonarcloud.yml

@@ -107,7 +107,7 @@ jobs:
         id: meshprocessor
         continue-on-error: true
         run: |
-          poetry env use 3.10
+          poetry env use 3.11
           poetry install
           poetry run coverage run -m unittest discover || echo "mesh_processor tests failed" >> ../failed_tests.txt
           poetry run coverage xml -o ../mesh_processor-coverage.xml

mesh_processor/Dockerfile

@@ -1,4 +1,4 @@
-FROM public.ecr.aws/lambda/python:3.10 AS base
+FROM public.ecr.aws/lambda/python:3.11 AS base
 
 # Create a non-root user with a specific UID and GID
 RUN mkdir -p /home/appuser && \

mesh_processor/poetry.lock

@@ -7,7 +7,7 @@ readme = "README.md"
 packages    = [{include = "src"}]
 
 [tool.poetry.dependencies]
-python = "~3.10"
+python = "~3.11"
 boto3 = "~1.38.42"
 mypy-boto3-dynamodb = "^1.38.4"
 moto = {extras = ["s3"], version = "^5.1.8"}

mesh_processor/pyproject.toml

@@ -57,24 +57,23 @@ def read_part_headers(input_file: BinaryIO) -> dict[str, str]:
 def stream_part_body(input_file: BinaryIO, boundary: bytes, output_file: BinaryIO) -> None:
     previous_line = None
     found_part_end = False
-    while not found_part_end:
-        if (line := input_file.readline()) is None:
-            raise ValueError("Unexpected EOF")
-
+    while line := input_file.readline():
         if line == b"--" + boundary + b"\r\n":
             logger.warning("Found additional part which will not be processed")
             found_part_end = True
-        if line == b"--" + boundary + b"--\r\n":
+        if line.startswith(b"--" + boundary + b"--"):
             found_part_end = True
 
         if previous_line is not None:
             if found_part_end:
                 # The final \r\n is part of the encapsulation boundary, so should not be included
                 output_file.write(previous_line.rstrip(b'\r\n'))
+                return
             else:
                 output_file.write(previous_line)
 
         previous_line = line
+    raise ValueError("Unexpected EOF")
 
 
 def transfer_multipart_content(bucket_name: str, file_key: str, boundary: bytes, filename: str) -> None:

mesh_processor/src/converter.py

@@ -67,6 +67,7 @@ def test_non_multipart_content_type_no_mesh_metadata(self):
 
     def test_multipart_content_type(self):
         body = "\r\n".join([
+            "",
             "--12345678",
             'Content-Disposition: form-data; name="File"; filename="test-csv-file.csv"',
             "Content-Type: text/csv",
@@ -91,6 +92,7 @@ def test_multipart_content_type(self):
 
     def test_multipart_content_type_multiple_parts(self):
         body = "\r\n".join([
+            "",
             "--12345678",
             'Content-Disposition: form-data; name="File"; filename="test-csv-file.csv"',
             "Content-Type: text/csv",
@@ -120,6 +122,7 @@ def test_multipart_content_type_multiple_parts(self):
 
     def test_multipart_content_type_without_filename(self):
         body = "\r\n".join([
+            "",
             "--12345678",
             'Content-Disposition: form-data',
             "Content-Type: text/csv",
@@ -144,6 +147,7 @@ def test_multipart_content_type_without_filename(self):
 
     def test_multipart_content_type_without_headers(self):
         body = "\r\n".join([
+            "",
             "--12345678",
             "",
             "some CSV content",
@@ -166,6 +170,7 @@ def test_multipart_content_type_without_headers(self):
 
     def test_multipart_content_type_with_unix_line_endings(self):
         body = "\r\n".join([
+            "",
             "--12345678",
             'Content-Disposition: form-data; name="File"; filename="test-csv-file.csv"',
             "Content-Type: text/csv",
@@ -187,3 +192,51 @@ def test_multipart_content_type_with_unix_line_endings(self):
         response = s3.get_object(Bucket="destination-bucket", Key="test-csv-file.csv")
         body = response["Body"].read().decode("utf-8")
         assert body == "some CSV content\nsplit across\nmultiple lines"
+
+    def test_multipart_content_type_missing_first_newline(self):
+        body = "\r\n".join([
+            "--12345678",
+            'Content-Disposition: form-data; name="File"; filename="test-csv-file.csv"',
+            "Content-Type: text/csv",
+            "",
+            "some CSV content",
+            "--12345678--",
+            ""
+        ])
+        s3 = boto3.client("s3", region_name="eu-west-2")
+        s3.put_object(
+            Bucket="source-bucket",
+            Key="test-dat-file.dat",
+            Body=body.encode("utf-8"),
+            ContentType="multipart/form-data; boundary=12345678",
+        )
+
+        invoke_lambda("test-dat-file.dat")
+
+        response = s3.get_object(Bucket="destination-bucket", Key="test-csv-file.csv")
+        body = response["Body"].read().decode("utf-8")
+        assert body == "some CSV content"
+
+    def test_multipart_content_type_missing_final_newline(self):
+        body = "\r\n".join([
+            "",
+            "--12345678",
+            'Content-Disposition: form-data; name="File"; filename="test-csv-file.csv"',
+            "Content-Type: text/csv",
+            "",
+            "some CSV content",
+            "--12345678--",
+        ])
+        s3 = boto3.client("s3", region_name="eu-west-2")
+        s3.put_object(
+            Bucket="source-bucket",
+            Key="test-dat-file.dat",
+            Body=body.encode("utf-8"),
+            ContentType="multipart/form-data; boundary=12345678",
+        )
+
+        invoke_lambda("test-dat-file.dat")
+
+        response = s3.get_object(Bucket="destination-bucket", Key="test-csv-file.csv")
+        body = response["Body"].read().decode("utf-8")
+        assert body == "some CSV content"