Merge pull request #365 from NHSDigital/AMB-2144-mesh-integration

amitkr-22 · web-flow · commit e0b79b833045 · 2025-04-01T16:50:00.000+01:00
Amb 2144 mesh integration
diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml
@@ -78,6 +78,14 @@ jobs:
           pip install poetry==1.8.4 moto==4.2.11 coverage redis botocore==1.35.49 simplejson responses structlog fhir.resources jsonpath_ng pydantic==1.10.13 requests aws-lambda-typing cffi pyjwt boto3-stubs-lite[dynamodb]~=1.26.90 python-stdnum==1.20
           poetry run coverage run --source=backend -m unittest discover -s backend || echo "fhir-api tests failed" >> failed_tests.txt
           poetry run coverage xml -o sonarcloud-coverage.xml
+      
+      - name: Run unittest with coverage-mesh-processor
+        id: meshprocessor
+        continue-on-error: true
+        run: |
+          pip install poetry==1.8.4 moto==4.2.11 coverage redis botocore==1.35.49 simplejson responses structlog fhir.resources jsonpath_ng pydantic==1.10.13 requests aws-lambda-typing cffi pyjwt boto3-stubs-lite[dynamodb]~=1.26.90 python-stdnum==1.20
+          poetry run coverage run --source=meshfileprocessor -m unittest discover -s meshfileprocessor || echo "meshfileprocessor tests failed" >> failed_tests.txt
+          poetry run coverage xml -o sonarcloud-meshfileprocessor-coverage.xml
 
       - name: Run Test Failure Summary
         id: check_failure
diff --git a/filenameprocessor/src/file_key_validation.py b/filenameprocessor/src/file_key_validation.py
@@ -54,7 +54,7 @@ def validate_file_key(file_key: str) -> tuple[str, str]:
         and version in Constants.VALID_VERSIONS
         and supplier  # Note that if supplier could be identified, this also implies that ODS code is valid
         and is_valid_datetime(timestamp)
-        and extension == "CSV"
+        and ((extension == "CSV") or (extension == "DAT"))  # The DAT extension has been added for MESH file processing
     ):
         raise InvalidFileKeyError("Initial file validation failed: invalid file key")
 
diff --git a/filenameprocessor/tests/test_file_key_validation.py b/filenameprocessor/tests/test_file_key_validation.py
@@ -93,7 +93,7 @@ def test_validate_file_key(self):
             # File key with missing timestamp
             (VALID_FLU_EMIS_FILE_KEY.replace("20000101T00000001", ""), invalid_file_key_error_message),
             # File key with incorrect extension
-            (VALID_FLU_EMIS_FILE_KEY.replace(".csv", ".dat"), invalid_file_key_error_message),
+            (VALID_FLU_EMIS_FILE_KEY.replace(".csv", ".xlsx"), invalid_file_key_error_message),
         ]
 
         for file_key, expected_result in test_cases_for_failure_scenarios:
diff --git a/mesh-infra/main.tf b/mesh-infra/main.tf
@@ -0,0 +1,41 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5"
+    }    
+  }
+  backend "s3" {
+    region = "eu-west-2"
+    key    = "state"
+  }
+    required_version = ">= 1.5.0"
+}
+
+
+
+data "aws_vpc" "default" {
+    default = true
+}
+data "aws_subnets" "default" {
+    filter {
+        name   = "vpc-id"
+        values = [data.aws_vpc.default.id]
+    }
+}
+
+module "mesh" {
+  source = "git::https://github.com/nhsdigital/terraform-aws-mesh-client.git//module?ref=v2.1.5"
+
+  name_prefix                    = "local-immunisation"
+  mesh_env                       = "integration"
+  subnet_ids                     = data.aws_subnets.default.ids
+
+  mailbox_ids                    = ["X26OT303"]
+  verify_ssl                     = "true"
+  get_message_max_concurrency    = 10
+  compress_threshold             = 1 * 1024 * 1024
+  handshake_schedule             = "rate(24 hours)"
+
+  account_id                     = 345594581768
+}
diff --git a/meshfileprocessor/Dockerfile b/meshfileprocessor/Dockerfile
@@ -0,0 +1,27 @@
+FROM public.ecr.aws/lambda/python:3.10 AS base
+
+# Create a non-root user with a specific UID and GID
+RUN mkdir -p /home/appuser && \
+    echo 'appuser:x:1001:1001::/home/appuser:/sbin/nologin' >> /etc/passwd && \
+    echo 'appuser:x:1001:' >> /etc/group && \
+    chown -R 1001:1001 /home/appuser && pip install "poetry~=1.5.0"
+
+# Install Poetry as root
+COPY poetry.lock pyproject.toml README.md ./
+RUN poetry config virtualenvs.create false && poetry install --no-interaction --no-ansi --no-root --only main
+
+# -----------------------------
+# FROM base AS test
+# COPY src src
+# COPY tests tests
+# RUN poetry install --no-interaction --no-ansi --no-root && \
+#     pytest --disable-warnings tests
+
+# -----------------------------
+FROM base AS build
+
+COPY src .
+RUN chmod 644 $(find . -type f) && chmod 755 $(find . -type d)
+# Switch to the non-root user for running the container
+USER 1001:1001
+CMD ["converter.lambda_handler"]
diff --git a/meshfileprocessor/Makefile b/meshfileprocessor/Makefile
@@ -0,0 +1,19 @@
+build:
+	docker build -t mesh-lambda-build .
+
+package:build
+	mkdir -p build
+	docker run --rm -v $(shell pwd)/build:/build mesh-lambda-build
+# test:
+# 	python -m unittest	
+
+# coverage run:
+# 	coverage run -m unittest discover
+
+# coverage report:
+# 	coverage report -m
+
+# coverage html:
+# 	coverage html  
+
+.PHONY: build package
diff --git a/meshfileprocessor/README.md b/meshfileprocessor/README.md
diff --git a/meshfileprocessor/pyproject.toml b/meshfileprocessor/pyproject.toml
@@ -0,0 +1,31 @@
+[tool.poetry]
+name        = "meshconverter"
+version     = "0.1.0"
+description = ""
+authors     = ["Your Name <you@example.com>"]
+readme      = "README.md"
+packages    = [{include = "src"}]
+
+[tool.poetry.dependencies]
+python                  = "~3.10"
+"fhir.resources"        = "~7.0.2"
+boto3                   = "~1.26.90"
+boto3-stubs-lite        = {extras = ["dynamodb"], version = "~1.26.90"}
+aws-lambda-typing       = "~2.18.0"
+moto                    = "~4.2.11"
+requests                = "~2.31.0"
+responses               = "~0.24.1"
+pydantic                = "~1.10.13"
+pyjwt                   = "~2.8.0"
+cryptography            = "~42.0.4"
+cffi                    = "~1.16.0"
+jsonpath-ng             = "^1.6.0"
+simplejson              = "^3.19.2"
+structlog               = "^24.1.0"
+pandas                  = "^2.2.3"
+freezegun               = "^1.5.1"
+
+[build-system]
+requires      = ["poetry-core ~= 1.5.0"]
+
+build-backend = "poetry.core.masonry.api"
diff --git a/meshfileprocessor/src/__init__.py b/meshfileprocessor/src/__init__.py
diff --git a/meshfileprocessor/src/converter.py b/meshfileprocessor/src/converter.py
@@ -0,0 +1,44 @@
+import boto3
+import os
+
+
+def lambda_handler(event, context):
+    s3 = boto3.client('s3')
+
+    #  Destination bucket name
+    destination_bucket = os.getenv("Destination_BUCKET_NAME")
+
+    for record in event["Records"]:
+        bucket_name = record["s3"]["bucket"]["name"]
+        file_key = record["s3"]["object"]["key"]
+        copy_source = {
+            'Bucket': record["s3"]["bucket"]["name"],
+            'Key': record["s3"]["object"]["key"]
+        }
+
+    # Read the .dat file from S3
+    dat_obj = s3.get_object(Bucket=bucket_name, Key=file_key)
+
+    # Update the filename from Metadata
+    file_name = ensure_dat_extension(dat_obj['Metadata'].get('mex-filename', None))
+
+    s3.copy_object(CopySource=copy_source, Bucket=destination_bucket, Key=file_name)
+
+    return {
+        'statusCode': 200,
+        'body': 'Files converted and uploaded successfully!'
+    }
+
+
+def ensure_dat_extension(file_name):
+    if '.' in file_name:
+        # Split the filename and extension
+        base_name, extension = file_name.rsplit('.', 1)
+
+        # Check if the extension is not 'dat'
+        if extension != 'dat':
+            file_name = f"{base_name}.dat"
+    else:
+        file_name += '.dat'
+
+    return file_name
diff --git a/meshfileprocessor/tests/__init__.py b/meshfileprocessor/tests/__init__.py
@@ -0,0 +1,4 @@
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../src")))
diff --git a/meshfileprocessor/tests/test_converter.py b/meshfileprocessor/tests/test_converter.py
@@ -0,0 +1,60 @@
+from unittest import TestCase
+from unittest.mock import patch, MagicMock
+from src.converter import lambda_handler, ensure_dat_extension
+
+
+class TestLambdaHandler(TestCase):
+
+    @patch('boto3.client')
+    @patch('os.getenv')
+    def test_lambda_handler_success(self, mock_getenv, mock_boto_client):
+        # Mock environment variable
+        mock_getenv.return_value = "destination-bucket"
+
+        # Mock boto3 S3 client
+        mock_s3 = MagicMock()
+        mock_boto_client.return_value = mock_s3
+        mock_s3.get_object.return_value = {
+            'Metadata': {'mex-filename': '20250320121710483244_2DB240.txt'}
+        }
+
+        # Define the event
+        event = {
+            "Records": [
+                {
+                    "s3": {
+                        "bucket": {"name": "source-bucket"},
+                        "object": {"key": "20250320121710483244_2DB240.dat"}
+                    }
+                }
+            ]
+        }
+        context = {}
+
+        # Call the lambda_handler function
+        response = lambda_handler(event, context)
+
+        # Assertions
+        mock_s3.get_object.assert_called_with(Bucket="source-bucket", Key="20250320121710483244_2DB240.dat")
+        mock_s3.copy_object.assert_called_with(
+            CopySource={'Bucket': "source-bucket", 'Key': "20250320121710483244_2DB240.dat"},
+            Bucket="destination-bucket",
+            Key="20250320121710483244_2DB240.dat"
+        )
+        self.assertEqual(response['statusCode'], 200)
+        self.assertEqual(response['body'], 'Files converted and uploaded successfully!')
+
+    def test_ensure_dat_extension_with_other_extension(self):
+        # Test case where file has an extension other than 'dat'
+        result = ensure_dat_extension("COVID19_Vaccinations_v5_YGM41_20240927T13005921.txt")
+        self.assertEqual(result, "COVID19_Vaccinations_v5_YGM41_20240927T13005921.dat")
+
+    def test_ensure_dat_extension_with_dat_extension(self):
+        # Test case where file already has a 'dat' extension
+        result = ensure_dat_extension("COVID19_Vaccinations_v5_YGM41_20240927T13005921.dat")
+        self.assertEqual(result, "COVID19_Vaccinations_v5_YGM41_20240927T13005921.dat")
+
+    def test_ensure_dat_extension_without_extension(self):
+        # Test case where file has no extension
+        result = ensure_dat_extension("COVID19_Vaccinations_v5_YGM41_20240927T13005921")
+        self.assertEqual(result, "COVID19_Vaccinations_v5_YGM41_20240927T13005921.dat")
diff --git a/recordprocessor/src/utils_for_recordprocessor.py b/recordprocessor/src/utils_for_recordprocessor.py
@@ -2,6 +2,7 @@
 
 import os
 import json
+import re
 from csv import DictReader
 from io import StringIO
 from clients import s3_client, lambda_client, logger
@@ -19,6 +20,10 @@ def get_csv_content_dict_reader(file_key: str) -> DictReader:
     """Returns the requested file contents from the source bucket in the form of a DictReader"""
     response = s3_client.get_object(Bucket=os.getenv("SOURCE_BUCKET_NAME"), Key=file_key)
     csv_data = response["Body"].read().decode("utf-8")
+
+    # Verify and process the DAT file content coming from MESH
+    if ".dat" in file_key:
+        csv_data = extract_content(csv_data)
     return DictReader(StringIO(csv_data), delimiter="|"), csv_data
 
 
@@ -41,3 +46,21 @@ def invoke_filename_lambda(file_key: str, message_id: str) -> None:
     except Exception as error:
         logger.error("Error invoking filename lambda: %s", error)
         raise
+
+
+def extract_content(dat_file_content):
+
+    boundary_pattern = re.compile(r"----------------------------\d+")
+
+    parts = boundary_pattern.split(dat_file_content)
+
+    # Extract the content between the boundaries
+    filecontent = None
+    for part in parts:
+        if "Content-Disposition" in part and "Content-Type" in part:
+
+            content_start = part.index("Content-Type") + len("Content-Type: text/csv") + 2
+            filecontent = part[content_start:].strip()
+            break
+
+    return filecontent
diff --git a/recordprocessor/tests/test_utils_for_recordprocessor.py b/recordprocessor/tests/test_utils_for_recordprocessor.py
@@ -15,7 +15,12 @@
 from tests.utils_for_recordprocessor_tests.mock_environment_variables import MOCK_ENVIRONMENT_DICT, BucketNames
 
 with patch("os.environ", MOCK_ENVIRONMENT_DICT):
-    from utils_for_recordprocessor import get_environment, get_csv_content_dict_reader, create_diagnostics_dictionary
+    from utils_for_recordprocessor import (
+        get_environment,
+        get_csv_content_dict_reader,
+        create_diagnostics_dictionary,
+        extract_content,
+    )
     from file_level_validation import move_file
 
 s3_client = boto3.client("s3", region_name=REGION_NAME)
@@ -76,6 +81,18 @@ def test_create_diagnostics_dictionary(self):
             },
         )
 
+    def test_extract_content_valid_input(self):
+        dat_file_content = (
+            "----------------------------1234567890\n"
+            'Content-Disposition: form-data; name="file"; filename="COVID19_Vaccinations_v5_YGM41_20250312T113455981.csv"\n'
+            "Content-Type: text/csv\n\n"
+            "NHS_NUMBER|PERSON_FORENAME|PERSON_SURNAME|PERSON_DOB|PERSON_GENDER_CODE|PERSON_POSTCODE\n"
+            "----------------------------1234567890\n"
+        )
+        expected_content = "NHS_NUMBER|PERSON_FORENAME|PERSON_SURNAME|PERSON_DOB|PERSON_GENDER_CODE|PERSON_POSTCODE"
+        result = extract_content(dat_file_content)
+        self.assertEqual(result, expected_content)
+
     def test_move_file(self):
         """Tests that move_file correctly moves a file from one location to another within a single S3 bucket"""
         source_file_key = "test_file_key"
diff --git a/sonar-project.properties b/sonar-project.properties
@@ -4,7 +4,7 @@ sonar.organization=nhsdigital
 sonar.host.url=https://sonarcloud.io
 sonar.python.version=3.11
 sonar.exclusions=**/e2e/**,**/e2e_batch/**,**/temporary_sandbox/**,**/devtools/**,**/proxies/**,**/scripts/**,**/terraform/**,**/tests/**
-sonar.python.coverage.reportPaths=sonarcloud-coverage.xml,sonarcloud-coverage-delta.xml,sonarcloud-coverage-ack-lambda.xml,sonarcloud-coverage-filenameprocessor-coverage.xml,sonarcloud-coverage-recordforwarder-coverage.xml,sonarcloud-coverage-recordprocessor-coverage.xml
+sonar.python.coverage.reportPaths=sonarcloud-coverage.xml,sonarcloud-coverage-delta.xml,sonarcloud-coverage-ack-lambda.xml,sonarcloud-coverage-filenameprocessor-coverage.xml,sonarcloud-coverage-recordforwarder-coverage.xml,sonarcloud-coverage-recordprocessor-coverage.xml,sonarcloud-meshfileprocessor-coverage.xml
 sonar.issue.ignore.multicriteria=exclude_snomed_urls,exclude_hl7_urls
 sonar.issue.ignore.multicriteria.exclude_snomed_urls.ruleKey=python:S5332
 sonar.issue.ignore.multicriteria.exclude_snomed_urls.resourceKey=**http://snomed\.info/sct**
diff --git a/terraform/mesh_file_converter.tf b/terraform/mesh_file_converter.tf
diff --git a/terraform/variables.tf b/terraform/variables.tf

Original file line number	Diff line number	Diff line change
`@@ -93,7 +93,7 @@ def test_validate_file_key(self):`
`93`	`93`	`# File key with missing timestamp`
`94`	`94`	`(VALID_FLU_EMIS_FILE_KEY.replace("20000101T00000001", ""), invalid_file_key_error_message),`
`95`	`95`	`# File key with incorrect extension`
`96`		`- (VALID_FLU_EMIS_FILE_KEY.replace(".csv", ".dat"), invalid_file_key_error_message),`
	`96`	`+ (VALID_FLU_EMIS_FILE_KEY.replace(".csv", ".xlsx"), invalid_file_key_error_message),`
`97`	`97`	`]`
`98`	`98`
`99`	`99`	`for file_key, expected_result in test_cases_for_failure_scenarios:`