VED-470: Increase MESH processor CPU & memory for faster processing. Only process files from the inbound prefix. Set the content type from headers. Override new line handling as recommended in the documentation.

Author: mfjarvis

mesh_processor/src/converter.py

@@ -98,11 +98,19 @@ def transfer_multipart_content(bucket_name: str, file_key: str, boundary: bytes,
         if content_disposition:
             _, content_disposition_params = parse_header_value(content_disposition)
             filename = content_disposition_params.get("filename") or filename
+        content_type = headers.get("Content-Type") or "application/octet-stream"
 
         with open(
             f"s3://{DESTINATION_BUCKET_NAME}/streaming/{filename}",
             "wb",
-            transport_params={"client": s3_client}
+            transport_params={
+                "client": s3_client,
+                "client_kwargs": {
+                    "S3.Client.create_multipart_upload": {
+                        "ContentType": content_type
+                    }
+                }
+            }
         ) as output_file:
             stream_part_body(input_file, boundary, output_file)
 

recordprocessor/src/utils_for_recordprocessor.py

@@ -19,7 +19,7 @@ def get_csv_content_dict_reader(file_key: str) -> DictReader:
     """Returns the requested file contents from the source bucket in the form of a DictReader"""
     response = s3_client.get_object(Bucket=os.getenv("SOURCE_BUCKET_NAME"), Key=file_key)
     binary_io = response["Body"]
-    text_io = TextIOWrapper(binary_io, encoding="utf-8")
+    text_io = TextIOWrapper(binary_io, encoding="utf-8", newline="")
     return DictReader(text_io, delimiter="|")
 
 

terraform/mesh_processor.tf

@@ -214,6 +214,7 @@ resource "aws_lambda_function" "mesh_file_converter_lambda" {
   image_uri     = module.mesh_processor_docker_image[0].image_uri
   architectures = ["x86_64"]
   timeout       = 900
+  memory_size   = 1024
 
   environment {
     variables = {
@@ -241,6 +242,7 @@ resource "aws_s3_bucket_notification" "mesh_datasources_lambda_notification" {
   lambda_function {
     lambda_function_arn = aws_lambda_function.mesh_file_converter_lambda[0].arn
     events              = ["s3:ObjectCreated:*"]
+    filter_prefix       = "inbound/"
   }
 }