Merge branch 'VED-480-Number-Update-branch' into VED-81-Number-Update

# Conflicts:
#	terraform/id_sync_lambda.tf

Author: nhsdevws

lambdas/id_sync/Dockerfile lambdas/id_sync.Dockerfilelambdas/id_sync/Dockerfile renamed to lambdas/id_sync.Dockerfile

@@ -12,10 +12,6 @@ COPY shared/poetry.lock shared/pyproject.toml shared/README.md ./shared/
 # Copy id_sync Poetry files
 COPY id_sync/poetry.lock id_sync/pyproject.toml id_sync/README.md ./
 
-# Install shared dependencies first
-WORKDIR /var/task/shared
-RUN poetry config virtualenvs.create false && poetry install --no-interaction --no-ansi --no-root --only main
-
 # Install id_sync dependencies
 WORKDIR /var/task
 RUN poetry config virtualenvs.create false && poetry install --no-interaction --no-ansi --no-root --only main

lambdas/shared/tests/test_common/test_pds_service.py

@@ -73,5 +73,3 @@ def test_env_mapping(self):
         env = "prod"
         service = PdsService(None, env)
         self.assertTrue(env not in service.base_url)
-
-

terraform/id_sync_lambda.tf

@@ -16,49 +16,6 @@ locals {
   combined_sha           = sha1("${local.shared_dir_sha}${local.id_sync_lambda_dir_sha}")
 }
 
-output "debug_build_paths" {
-  value = {
-    lambdas_dir            = local.lambdas_dir
-    shared_dir             = local.shared_dir
-    id_sync_lambda_dir     = local.id_sync_lambda_dir
-    shared_files_count     = length(local.shared_files)
-    id_sync_files_count    = length(local.id_sync_lambda_files)
-    combined_sha           = local.combined_sha
-    dockerfile_exists      = fileexists("${local.id_sync_lambda_dir}/Dockerfile")
-    shared_common_exists   = fileexists("${local.shared_dir}/src/common/__init__.py")
-  }
-}
-
-# Debug: List some files from each directory
-output "debug_file_listing" {
-  value = {
-    shared_files_sample    = slice(local.shared_files, 0, min(5, length(local.shared_files)))
-  }
-}
-
-resource "null_resource" "debug_build_context" {
-  provisioner "local-exec" {
-    command = <<-EOT
-      echo "SAW === HOST SYSTEM PATHS ==="
-      echo "Terraform execution directory: $(pwd)"
-      echo "Host build context: ${local.lambdas_dir}"
-      echo "Host Dockerfile location: ${local.id_sync_lambda_dir}/Dockerfile"
-      echo ""
-      echo "Docker build command that will be executed:"
-      echo "docker build -f id_sync/Dockerfile ${local.lambdas_dir}"
-      echo ""
-      echo "=== HOST BUILD CONTEXT CONTENTS ==="
-      echo "What Docker can see from host:"
-      ls -la "${local.lambdas_dir}/"
-    EOT
-  }
-}
-
-# Reference the existing SQS queue
-data "aws_sqs_queue" "existing_sqs_queue" {
-  name = "id_sync_test_queue"
-}
-
 resource "aws_ecr_repository" "id_sync_lambda_repository" {
   image_scanning_configuration {
     scan_on_push = true
@@ -74,6 +31,7 @@ module "id_sync_docker_image" {
 
   create_ecr_repo = false
   ecr_repo        = aws_ecr_repository.id_sync_lambda_repository.name
+  docker_file_path = "../id_sync.Dockerfile"
   ecr_repo_lifecycle_policy = jsonencode({
     "rules" : [
       {
@@ -93,40 +51,9 @@ module "id_sync_docker_image" {
 
   platform      = "linux/amd64"
   use_image_tag = false
-  source_path   = local.lambdas_dir    # parent lambdas directory
-  docker_file_path = "id_sync/Dockerfile"  # Add this line
-  triggers = {
-    dir_sha = local.combined_sha       # Changed to combined SHA
-  }
-}
-
-# Add a local provisioner to debug build context
-resource "null_resource" "debug_build_context2" {
+  source_path   = local.id_sync_lambda_dir
   triggers = {
-    dir_sha = local.combined_sha
-  }
-
-  provisioner "local-exec" {
-    command = <<-EOT
-      echo "SAW === BUILD CONTEXT DEBUG ==="
-      echo "Build context: ${local.lambdas_dir}"
-      echo "Dockerfile location: ${local.id_sync_lambda_dir}/Dockerfile"
-      echo ""
-      echo "Checking Dockerfile exists:"
-      ls -la "${local.id_sync_lambda_dir}/Dockerfile" || echo "Dockerfile NOT FOUND!"
-      echo ""
-      echo "Checking shared directory structure:"
-      ls -la "${local.shared_dir}/src/common/" || echo "Shared common directory NOT FOUND!"
-      echo ""
-      echo "Files in build context (lambdas dir):"
-      ls -la "${local.lambdas_dir}/"
-      echo ""
-      echo "Shared files structure:"
-      find "${local.shared_dir}" -type f -name "*.py" | head -10
-      echo ""
-      echo "ID Sync files structure:"
-      find "${local.id_sync_lambda_dir}" -type f -name "*.py" | head -10
-    EOT
+    dir_sha = local.id_sync_lambda_dir_sha
   }
 }
 
@@ -176,6 +103,7 @@ resource "aws_iam_role" "id_sync_lambda_exec_role" {
   })
 }
 
+# Policy for Lambda execution role
 resource "aws_iam_policy" "id_sync_lambda_exec_policy" {
   name = "${local.short_prefix}-id-sync-lambda-exec-policy"
   policy = jsonencode({
@@ -188,23 +116,21 @@ resource "aws_iam_policy" "id_sync_lambda_exec_policy" {
           "logs:CreateLogStream",
           "logs:PutLogEvents"
         ]
-        Resource = "arn:aws:logs:${var.aws_region}:${local.immunisation_account_id}:log-group:/aws/lambda/${local.short_prefix}-id_sync_lambda:*"
+        Resource = "arn:aws:logs:${var.aws_region}:${var.immunisation_account_id}:log-group:/aws/lambda/${local.short_prefix}-id_sync_lambda:*"
       },
       {
         Effect = "Allow"
         Action = [
-          "ecr:GetAuthorizationToken"
+          "s3:GetObject",
+          "s3:ListBucket",
+          "s3:PutObject",
+          "s3:CopyObject",
+          "s3:DeleteObject"
         ]
-        Resource = "*"
-      },
-      {
-        Effect = "Allow"
-        Action = [
-          "ecr:BatchCheckLayerAvailability",
-          "ecr:GetDownloadUrlForLayer",
-          "ecr:BatchGetImage"
+        Resource = [
+          aws_s3_bucket.batch_data_source_bucket.arn,
+          "${aws_s3_bucket.batch_data_source_bucket.arn}/*"
         ]
-        Resource = aws_ecr_repository.id_sync_lambda_repository.arn
       },
       {
         Effect = "Allow"
@@ -227,6 +153,18 @@ resource "aws_iam_policy" "id_sync_lambda_exec_policy" {
         ],
         Resource = "*"
       },
+      {
+        Effect = "Allow"
+        Action = [
+          "s3:GetObject",
+          "s3:PutObject",
+          "s3:ListBucket"
+        ]
+        Resource = [
+          local.config_bucket_arn,
+          "${local.config_bucket_arn}/*"
+        ]
+      },
       {
         Effect : "Allow",
         Action : [
@@ -239,17 +177,28 @@ resource "aws_iam_policy" "id_sync_lambda_exec_policy" {
         Effect = "Allow"
         Action = "lambda:InvokeFunction"
         Resource = [
-          "arn:aws:lambda:${var.aws_region}:${local.immunisation_account_id}:function:imms-${local.env}-id_sync_lambda",
+          "arn:aws:lambda:${var.aws_region}:${var.immunisation_account_id}:function:imms-${var.sub_environment}-id_sync_lambda",
         ]
       },
+      # NEW
+      # NB anomaly: do we want this in "id_sync_lambda_sqs_access_policy"?
       {
-        Effect = "Allow"
+        Effect = "Allow",
         Action = [
           "sqs:ReceiveMessage",
           "sqs:DeleteMessage",
           "sqs:GetQueueAttributes"
-        ]
-        Resource = data.aws_sqs_queue.existing_sqs_queue.arn
+        ],
+        Resource = "arn:aws:sqs:eu-west-2:${var.immunisation_account_id}:${local.short_prefix}-id-sync-queue"
+      },
+      # NB anomaly: in redis_sync this appears in "redis_sync_lambda_kms_access_policy"
+      {
+        Effect = "Allow",
+        Action = [
+          "kms:Decrypt",
+          "kms:GenerateDataKey"
+        ],
+        Resource = data.aws_kms_key.existing_id_sync_sqs_encryption_key.arn
       }
     ]
   })
@@ -268,6 +217,17 @@ resource "aws_iam_policy" "id_sync_lambda_kms_access_policy" {
           "kms:Decrypt"
         ]
         Resource = data.aws_kms_key.existing_lambda_encryption_key.arn
+      },
+      {
+        Effect = "Allow"
+        Action = [
+          "kms:Encrypt",
+          "kms:Decrypt",
+          "kms:GenerateDataKey*"
+        ]
+        Resource = [
+          data.aws_kms_key.existing_s3_encryption_key.arn,
+        ]
       }
     ]
   })
@@ -285,6 +245,29 @@ resource "aws_iam_role_policy_attachment" "id_sync_lambda_kms_policy_attachment"
   policy_arn = aws_iam_policy.id_sync_lambda_kms_access_policy.arn
 }
 
+data "aws_iam_policy_document" "id_sync_policy_document" {
+  source_policy_documents = [
+    templatefile("${local.policy_path}/dynamodb.json", {
+      "dynamodb_table_name" : aws_dynamodb_table.delta-dynamodb-table.name
+    }),
+    templatefile("${local.policy_path}/dynamodb_stream.json", {
+      "dynamodb_table_name" : aws_dynamodb_table.events-dynamodb-table.name
+    })
+  ]
+}
+
+resource "aws_iam_policy" "id_sync_lambda_dynamodb_access_policy" {
+  name        = "${local.short_prefix}-id-sync-lambda-dynamodb-access-policy"
+  description = "Allow Lambda to access DynamoDB"
+  policy     = data.aws_iam_policy_document.id_sync_policy_document.json
+}
+
+# Attach the dynamodb policy to the Lambda role
+resource "aws_iam_role_policy_attachment" "id_sync_lambda_dynamodb_policy_attachment" {
+  role       = aws_iam_role.id_sync_lambda_exec_role.name
+  policy_arn = aws_iam_policy.id_sync_lambda_dynamodb_access_policy.arn
+}
+
 # Lambda Function with Security Group and VPC.
 resource "aws_lambda_function" "id_sync_lambda" {
   function_name = "${local.short_prefix}-id_sync_lambda"
@@ -301,11 +284,15 @@ resource "aws_lambda_function" "id_sync_lambda" {
 
   environment {
     variables = {
-      ID_SYNC_PROC_LAMBDA_NAME = "imms-${local.env}-id_sync_lambda"
+      CONFIG_BUCKET_NAME          = local.config_bucket_name
+      REDIS_HOST                  = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].address
+      REDIS_PORT                  = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].port
+      ID_SYNC_PROC_LAMBDA_NAME    = "imms-${var.sub_environment}-id_sync_lambda"
+      # NEW
+      DELTA_TABLE_NAME            = aws_dynamodb_table.delta-dynamodb-table.name
+      IEDS_TABLE_NAME             = aws_dynamodb_table.events-dynamodb-table.name
+      PDS_ENV                     = var.pds_environment
       SPLUNK_FIREHOSE_NAME        = module.splunk.firehose_stream_name
-      PDS_ENV                     = local.environment == "prod" ? "prod" : local.environment == "ref" ? "ref" : "int"
-	  DELTA_TABLE_NAME 		= aws_dynamodb_table.delta-dynamodb-table.name
-	  IEDS_TABLE_NAME 		= aws_dynamodb_table.events-dynamodb-table.name
     }
   }
   kms_key_arn = data.aws_kms_key.existing_lambda_encryption_key.arn
@@ -321,15 +308,17 @@ resource "aws_cloudwatch_log_group" "id_sync_log_group" {
   retention_in_days = 30
 }
 
-# SQS Event Source Mapping for Lambda
-resource "aws_lambda_event_source_mapping" "id_sync_sqs_trigger" {
-  event_source_arn = data.aws_sqs_queue.existing_sqs_queue.arn
-  function_name    = aws_lambda_function.id_sync_lambda.arn
+# delete config_lambda_notification / new_s3_invoke_permission - not required; duplicate
 
+# NEW
+resource "aws_lambda_event_source_mapping" "id_sync_sqs_trigger" {
+  event_source_arn = "arn:aws:sqs:eu-west-2:${var.immunisation_account_id}:${local.short_prefix}-id-sync-queue"
+  function_name    = aws_lambda_function.id_sync_lambda.arn # TODO
+  
   # Optional: Configure batch size and other settings
   batch_size                         = 10
   maximum_batching_window_in_seconds = 5
-
+  
   # Optional: Configure error handling
   function_response_types = ["ReportBatchItemFailures"]
-}
+}

terraform/sqs_id_sync.tf

@@ -1,7 +1,8 @@
 resource "aws_sqs_queue" "id_sync_queue" {
   name                        = "${local.short_prefix}-id-sync-queue"
   kms_master_key_id           = data.aws_kms_key.existing_id_sync_sqs_encryption_key.arn
-  visibility_timeout_seconds  = 60
+  # TODO: visibility_timeout_seconds must not be less than aws_lambda_function.id_sync_lambda_timeout
+  visibility_timeout_seconds  = 360
   redrive_policy = jsonencode({
     deadLetterTargetArn = aws_sqs_queue.id_sync_dlq.arn
     maxReceiveCount     = 4