VED-355: Add MESH client and update mailbox details. (#643)

* initial code

* wip

* add s3 buckets to locals

* bucket names & dev id

* TODO Marker

* rename &  tidy

* remove mesh_processor from terraform folder

* tidy

* comments

* tidy & delete mesh file

* missing KMS

* VED-355: Move mesh processor back to terraform folder as it depends on the batch sources bucket.

---------

Co-authored-by: Matt Jarvis <[email protected]>

Author: nhsdevws

.github/dependabot.yml

@@ -71,7 +71,6 @@ updates:
     directories:
       - "/grafana/non-prod/terraform"
       - "/infra"
-      - "/mesh-infra"
       - "/terraform"
       - "/terraform_aws_backup/**"
     schedule:

README.md

@@ -45,7 +45,6 @@ See https://nhsd-confluence.digital.nhs.uk/display/APM/Glossary.
 | `terraform_old`        | Old tf code used to create INT to mimic prod. |
 | `terraform_sandbox`    | Sandbox environment for testing infrastructure changes. |
 | `terraform_aws_backup` | Streamlined backup processing with AWS. |
-| `mesh-infra`           | Infrastructure setup for Imms batch MESH integration. |
 | `proxies`              | Apigee API proxy definitions. |
 ---
 

infra/.terraform.lock.hcl

@@ -7,3 +7,5 @@ dspp_admin_role          = "root"
 environment              = "int"
 parent_route53_zone_name = "int.vds.platform.nhs.uk"
 child_route53_zone_name  = "imms.int.vds.platform.nhs.uk"
+mesh_mailbox_id          = "X26OT303"
+mesh_dlq_mailbox_id      = "X26OT304"

infra/environments/int/variables.tfvars

@@ -7,3 +7,8 @@ dspp_admin_role          = "root"
 environment              = "dev"
 parent_route53_zone_name = "dev.vds.platform.nhs.uk"
 child_route53_zone_name  = "imms.dev.vds.platform.nhs.uk"
+# TODO - null these out once we're using the int account
+# mesh_mailbox_id          = null
+# mesh_dlq_mailbox_id      = null
+mesh_mailbox_id          = "X26OT303"
+mesh_dlq_mailbox_id      = "X26OT304"

infra/environments/non-prod/variables.tfvars

@@ -7,3 +7,5 @@ dspp_admin_role          = "root"
 environment              = "prod"
 parent_route53_zone_name = "prod.vds.platform.nhs.uk"
 child_route53_zone_name  = "imms.prod.vds.platform.nhs.uk"
+mesh_mailbox_id          = "X26HC138"
+mesh_dlq_mailbox_id      = null

infra/environments/prod/variables.tfvars

@@ -0,0 +1,15 @@
+# MESH Client Module - conditionally created based on environment configuration
+module "mesh" {
+  count  = var.mesh_mailbox_id != null ? 1 : 0
+  source = "git::https://github.com/nhsdigital/terraform-aws-mesh-client.git//module?ref=v2.1.5"
+
+  name_prefix                    = "imms-${var.environment}"
+  account_id                     = var.imms_account_id
+  mesh_env                       = var.environment == "prod"? "production" : "integration"
+  subnet_ids                     = toset([])
+  mailbox_ids                    = [var.mesh_mailbox_id]
+
+  compress_threshold             = 1 * 1024 * 1024
+  get_message_max_concurrency    = 10
+  handshake_schedule             = "rate(24 hours)"
+}

infra/mesh.tf

@@ -16,3 +16,9 @@ variable "build_agent_account_id" {
 variable "environment" {
   default = "non-prod"
 }
+variable "mesh_mailbox_id" {
+  default = null
+}
+variable "mesh_dlq_mailbox_id" {
+  default = null
+}

infra/variables.tf

@@ -1,12 +1,28 @@
 # Define the directory containing the Docker image and calculate its SHA-256 hash for triggering redeployments
 locals {
+  create_mesh_processor         = local.environment == "int" || local.environment == "prod"
   mesh_processor_lambda_dir     = abspath("${path.root}/../mesh_processor")
   mesh_processor_lambda_files   = fileset(local.mesh_processor_lambda_dir, "**")
   mesh_processor_lambda_dir_sha = sha1(join("", [for f in local.mesh_processor_lambda_files : filesha1("${local.mesh_processor_lambda_dir}/${f}")]))
+  # This should match the prefix used in the infra Terraform
+  mesh_module_prefix = "imms-${local.config_env}"
 }
 
+data "aws_s3_bucket" "mesh" {
+  count = local.create_mesh_processor ? 1 : 0
+
+  bucket = "${local.mesh_module_prefix}-mesh"
+}
+
+data "aws_kms_key" "mesh" {
+  count = local.create_mesh_processor ? 1 : 0
+
+  key_id = "alias/${local.mesh_module_prefix}-mesh"
+}
 
 resource "aws_ecr_repository" "mesh_file_converter_lambda_repository" {
+  count = local.create_mesh_processor ? 1 : 0
+
   image_scanning_configuration {
     scan_on_push = true
   }
@@ -16,11 +32,13 @@ resource "aws_ecr_repository" "mesh_file_converter_lambda_repository" {
 
 # Module for building and pushing Docker image to ECR
 module "mesh_processor_docker_image" {
+  count = local.create_mesh_processor ? 1 : 0
+
   source  = "terraform-aws-modules/lambda/aws//modules/docker-build"
   version = "8.0.1"
 
   create_ecr_repo = false
-  ecr_repo        = aws_ecr_repository.mesh_file_converter_lambda_repository.name
+  ecr_repo        = aws_ecr_repository.mesh_file_converter_lambda_repository[0].name
   ecr_repo_lifecycle_policy = jsonencode({
     "rules" : [
       {
@@ -48,7 +66,9 @@ module "mesh_processor_docker_image" {
 
 # Define the lambdaECRImageRetreival policy
 resource "aws_ecr_repository_policy" "mesh_processor_lambda_ECRImageRetreival_policy" {
-  repository = aws_ecr_repository.mesh_file_converter_lambda_repository.name
+  count = local.create_mesh_processor ? 1 : 0
+
+  repository = aws_ecr_repository.mesh_file_converter_lambda_repository[0].name
 
   policy = jsonencode({
     Version = "2012-10-17"
@@ -78,6 +98,8 @@ resource "aws_ecr_repository_policy" "mesh_processor_lambda_ECRImageRetreival_po
 
 # IAM Role for Lambda
 resource "aws_iam_role" "mesh_processor_lambda_exec_role" {
+  count = local.create_mesh_processor ? 1 : 0
+
   name = "${local.short_prefix}-mesh_processor-lambda-exec-role"
   assume_role_policy = jsonencode({
     Version = "2012-10-17",
@@ -94,6 +116,8 @@ resource "aws_iam_role" "mesh_processor_lambda_exec_role" {
 
 # Policy for Lambda execution role
 resource "aws_iam_policy" "mesh_processor_lambda_exec_policy" {
+  count = local.create_mesh_processor ? 1 : 0
+
   name = "${local.short_prefix}-mesh_processor-lambda-exec-policy"
   policy = jsonencode({
     Version = "2012-10-17",
@@ -130,16 +154,17 @@ resource "aws_iam_policy" "mesh_processor_lambda_exec_policy" {
           "s3:DeleteObject"
         ]
         Resource = [
-          "arn:aws:s3:::local-immunisation-mesh",
-          "arn:aws:s3:::local-immunisation-mesh/*",
-          "arn:aws:s3:::local-immunisation-mesh-s3logs/*"
+          data.aws_s3_bucket.mesh[0].arn,
+          "${data.aws_s3_bucket.mesh[0].arn}/*"
         ]
       }
     ]
   })
 }
 
 resource "aws_iam_policy" "mesh_processor_lambda_kms_access_policy" {
+  count = local.create_mesh_processor ? 1 : 0
+
   name        = "${local.short_prefix}-mesh_processor-lambda-kms-policy"
   description = "Allow Lambda to decrypt environment variables"
 
@@ -154,8 +179,7 @@ resource "aws_iam_policy" "mesh_processor_lambda_kms_access_policy" {
           "kms:GenerateDataKey*"
         ]
         Resource = [
-          data.aws_kms_key.mesh_s3_encryption_key.arn
-          # "arn:aws:kms:eu-west-2:345594581768:key/9b756762-bc6f-42fb-ba56-2c0c00c15289"
+          data.aws_kms_key.mesh[0].arn
         ]
       }
     ]
@@ -164,59 +188,64 @@ resource "aws_iam_policy" "mesh_processor_lambda_kms_access_policy" {
 
 # Attach the execution policy to the Lambda role
 resource "aws_iam_role_policy_attachment" "mesh_processor_lambda_exec_policy_attachment" {
-  role       = aws_iam_role.mesh_processor_lambda_exec_role.name
-  policy_arn = aws_iam_policy.mesh_processor_lambda_exec_policy.arn
+  count = local.create_mesh_processor ? 1 : 0
+
+  role       = aws_iam_role.mesh_processor_lambda_exec_role[0].name
+  policy_arn = aws_iam_policy.mesh_processor_lambda_exec_policy[0].arn
 }
 
 
 # Attach the kms policy to the Lambda role
 resource "aws_iam_role_policy_attachment" "mesh_processor_lambda_kms_policy_attachment" {
-  role       = aws_iam_role.mesh_processor_lambda_exec_role.name
-  policy_arn = aws_iam_policy.mesh_processor_lambda_kms_access_policy.arn
+  count = local.create_mesh_processor ? 1 : 0
+
+  role       = aws_iam_role.mesh_processor_lambda_exec_role[0].name
+  policy_arn = aws_iam_policy.mesh_processor_lambda_kms_access_policy[0].arn
 }
 
 # Lambda Function with Security Group and VPC.
 resource "aws_lambda_function" "mesh_file_converter_lambda" {
+  count = local.create_mesh_processor ? 1 : 0
+
   function_name = "${local.short_prefix}-mesh_processor_lambda"
-  role          = aws_iam_role.mesh_processor_lambda_exec_role.arn
+  role          = aws_iam_role.mesh_processor_lambda_exec_role[0].arn
   package_type  = "Image"
-  image_uri     = module.mesh_processor_docker_image.image_uri
+  image_uri     = module.mesh_processor_docker_image[0].image_uri
   architectures = ["x86_64"]
   timeout       = 360
 
   environment {
     variables = {
-      Destination_BUCKET_NAME    = aws_s3_bucket.batch_data_source_bucket.bucket
-      MESH_FILE_PROC_LAMBDA_NAME = "imms-${local.env}-meshfileproc_lambda"
+      Destination_BUCKET_NAME = aws_s3_bucket.batch_data_source_bucket.bucket
     }
   }
-
 }
 
 # Permission for S3 to invoke Lambda function
 resource "aws_lambda_permission" "mesh_s3_invoke_permission" {
+  count = local.create_mesh_processor ? 1 : 0
+
   statement_id  = "AllowExecutionFromS3"
   action        = "lambda:InvokeFunction"
-  function_name = aws_lambda_function.mesh_file_converter_lambda.function_name
+  function_name = aws_lambda_function.mesh_file_converter_lambda[0].function_name
   principal     = "s3.amazonaws.com"
   source_arn    = "arn:aws:s3:::local-immunisation-mesh"
 }
 
-# TODO - This is scoped to the bucket, so is overwritten by each deployment
-# That might be intentional in prod, to switch between blue and green, but surely isn't in non-prod
-# S3 Bucket notification to trigger Lambda function
 resource "aws_s3_bucket_notification" "mesh_datasources_lambda_notification" {
-  # TODO - what is this bucket and why isn't it managed by Terraform?
-  bucket = "local-immunisation-mesh"
+  count = local.create_mesh_processor ? 1 : 0
+
+  bucket = data.aws_s3_bucket.mesh[0].bucket
 
   lambda_function {
-    lambda_function_arn = aws_lambda_function.mesh_file_converter_lambda.arn
+    lambda_function_arn = aws_lambda_function.mesh_file_converter_lambda[0].arn
     events              = ["s3:ObjectCreated:*"]
-    #filter_prefix      =""
   }
 }
 
 resource "aws_cloudwatch_log_group" "mesh_file_converter_log_group" {
+  count = local.create_mesh_processor ? 1 : 0
+
   name              = "/aws/lambda/${local.short_prefix}-mesh_processor_lambda"
   retention_in_days = 30
 }