VED-470: Resolve Sonar warnings. Remove redundant multipart handling in record processor. Add missing TF changes.

Author: mfjarvis

mesh_processor/src/converter.py

@@ -36,8 +36,7 @@ def read_until_part_start(input_file: BinaryIO, boundary: bytes) -> None:
     while line := input_file.readline():
         if line == b"--" + boundary + b"\r\n":
             return
-    else:
-        raise ValueError(f"Unexpected EOF")
+    raise ValueError("Unexpected EOF")
 
 
 def read_headers_bytes(input_file: BinaryIO) -> bytes:
@@ -46,8 +45,7 @@ def read_headers_bytes(input_file: BinaryIO) -> bytes:
         if line == b"\r\n":
             return headers_bytes
         headers_bytes += line
-    else:
-        raise ValueError("Unexpected EOF")
+    raise ValueError("Unexpected EOF")
 
 
 def read_part_headers(input_file: BinaryIO) -> dict[str, str]:

recordprocessor/src/utils_for_recordprocessor.py

@@ -19,11 +19,8 @@ def get_environment() -> str:
 def get_csv_content_dict_reader(file_key: str) -> (DictReader, str):
     """Returns the requested file contents from the source bucket in the form of a DictReader"""
     response = s3_client.get_object(Bucket=os.getenv("SOURCE_BUCKET_NAME"), Key=file_key)
+    # TODO - this reads everything into memory! Look into streaming instead
     csv_data = response["Body"].read().decode("utf-8")
-
-    # Verify and process the DAT file content coming from MESH
-    if ".dat" in file_key:
-        csv_data = extract_content(csv_data)
     return DictReader(StringIO(csv_data), delimiter="|"), csv_data
 
 
@@ -46,21 +43,3 @@ def invoke_filename_lambda(file_key: str, message_id: str) -> None:
     except Exception as error:
         logger.error("Error invoking filename lambda: %s", error)
         raise
-
-
-def extract_content(dat_file_content):
-
-    boundary_pattern = re.compile(r"----------------------------\d+")
-
-    parts = boundary_pattern.split(dat_file_content)
-
-    # Extract the content between the boundaries
-    filecontent = None
-    for part in parts:
-        if "Content-Disposition" in part and "Content-Type" in part:
-
-            content_start = part.index("Content-Type") + len("Content-Type: text/csv") + 2
-            filecontent = part[content_start:].strip()
-            break
-
-    return filecontent

recordprocessor/tests/test_utils_for_recordprocessor.py

@@ -19,7 +19,6 @@
         get_environment,
         get_csv_content_dict_reader,
         create_diagnostics_dictionary,
-        extract_content,
     )
     from file_level_validation import move_file
 
@@ -81,19 +80,6 @@ def test_create_diagnostics_dictionary(self):
             },
         )
 
-    def test_extract_content_valid_input(self):
-        dat_file_content = (
-            "----------------------------1234567890\n"
-            'Content-Disposition: form-data; name="file";\n'
-            'filename="COVID19_Vaccinations_v5_YGM41_20250312T113455981.csv"\n'
-            "Content-Type: text/csv\n\n"
-            "NHS_NUMBER|PERSON_FORENAME|PERSON_SURNAME|PERSON_DOB|PERSON_GENDER_CODE|PERSON_POSTCODE\n"
-            "----------------------------1234567890\n"
-        )
-        expected_content = "NHS_NUMBER|PERSON_FORENAME|PERSON_SURNAME|PERSON_DOB|PERSON_GENDER_CODE|PERSON_POSTCODE"
-        result = extract_content(dat_file_content)
-        self.assertEqual(result, expected_content)
-
     def test_move_file(self):
         """Tests that move_file correctly moves a file from one location to another within a single S3 bucket"""
         source_file_key = "test_file_key"

terraform/mesh_processor.tf

@@ -212,11 +212,11 @@ resource "aws_lambda_function" "mesh_file_converter_lambda" {
   package_type  = "Image"
   image_uri     = module.mesh_processor_docker_image[0].image_uri
   architectures = ["x86_64"]
-  timeout       = 360
+  timeout       = 900
 
   environment {
     variables = {
-      Destination_BUCKET_NAME = aws_s3_bucket.batch_data_source_bucket.bucket
+      DESTINATION_BUCKET_NAME = aws_s3_bucket.batch_data_source_bucket.bucket
     }
   }
 }