VED-887: Slack Api Error Alerting (#1066)

Akol125 · web-flow · commit 6650a6240b27 · 2025-12-18T14:44:37.000Z
* setting up account level config for slack and sns topic alerting
diff --git a/infrastructure/account/batch_processor_errors_sns_topic.tf b/infrastructure/account/batch_processor_errors_sns_topic.tf
@@ -1,6 +1,6 @@
 resource "aws_sns_topic" "batch_processor_errors" {
   name              = "${var.environment}-batch-processor-errors"
-  kms_master_key_id = aws_kms_key.batch_processor_errors_sns_encryption_key.arn
+  kms_master_key_id = aws_kms_key.error_alerts_sns_encryption_key.arn
 }
 
 resource "aws_sns_topic_policy" "batch_processor_errors_topic_policy" {
diff --git a/infrastructure/account/fhir_api_errors_slack_chatbot.tf b/infrastructure/account/fhir_api_errors_slack_chatbot.tf
@@ -0,0 +1,24 @@
+resource "aws_chatbot_slack_channel_configuration" "fhir_api_errors" {
+  configuration_name = "${var.environment}-fhir-api-errors-slack-channel-config"
+  iam_role_arn       = aws_iam_role.fhir_api_errors_chatbot.arn
+  slack_channel_id   = var.environment == "prod" ? "C0A3LPKNKEE" : "C0A4F3G8J0G"
+  slack_team_id      = "TJ00QR03U"
+  sns_topic_arns     = [aws_sns_topic.fhir_api_errors.arn]
+}
+
+resource "aws_iam_role" "fhir_api_errors_chatbot" {
+  name = "${var.environment}-fhir-api-errors-chatbot-channel-role"
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Sid    = "AssumeChatbotRole"
+        Principal = {
+          Service = "chatbot.amazonaws.com"
+        }
+      },
+    ]
+  })
+}
diff --git a/infrastructure/account/fhir_api_errors_sns_topic.tf b/infrastructure/account/fhir_api_errors_sns_topic.tf
@@ -0,0 +1,22 @@
+resource "aws_sns_topic" "fhir_api_errors" {
+  name              = "${var.environment}-fhir-api-errors"
+  kms_master_key_id = aws_kms_key.error_alerts_sns_encryption_key.arn
+}
+
+resource "aws_sns_topic_policy" "fhir_api_errors_topic_policy" {
+  arn = aws_sns_topic.fhir_api_errors.arn
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+        Sid    = "AllowCloudWatchToPublish",
+        Effect = "Allow",
+        Principal = {
+          Service = "cloudwatch.amazonaws.com"
+        },
+        Action   = "SNS:Publish",
+        Resource = aws_sns_topic.fhir_api_errors.arn
+      }
+    ]
+  })
+}
diff --git a/infrastructure/account/kms.tf b/infrastructure/account/kms.tf
@@ -179,7 +179,7 @@ resource "aws_kms_alias" "id_sync_sqs_encryption" {
   target_key_id = aws_kms_key.id_sync_sqs_encryption.key_id
 }
 
-resource "aws_kms_key" "batch_processor_errors_sns_encryption_key" {
+resource "aws_kms_key" "error_alerts_sns_encryption_key" {
   description             = "KMS key for encrypting the batch processor errors SNS Topic messages"
   deletion_window_in_days = 7
   enable_key_rotation     = true
@@ -218,5 +218,10 @@ resource "aws_kms_key" "batch_processor_errors_sns_encryption_key" {
 
 resource "aws_kms_alias" "batch_processor_errors_sns_encryption_key" {
   name          = "alias/${var.environment}-batch-processor-errors-imms-sns-encryption"
-  target_key_id = aws_kms_key.batch_processor_errors_sns_encryption_key.key_id
+  target_key_id = aws_kms_key.error_alerts_sns_encryption_key.key_id
+}
+
+resource "aws_kms_alias" "fhir_api_errors_sns_encryption_key" {
+  name          = "alias/${var.environment}-fhir-api-errors-imms-sns-encryption"
+  target_key_id = aws_kms_key.error_alerts_sns_encryption_key.key_id
 }
diff --git a/infrastructure/instance/batch_processor_filter_lambda.tf b/infrastructure/instance/batch_processor_filter_lambda.tf
@@ -305,7 +305,7 @@ resource "aws_lambda_event_source_mapping" "batch_file_created_sqs_to_lambda" {
 }
 
 resource "aws_cloudwatch_log_metric_filter" "batch_processor_filter_error_logs" {
-  count = var.batch_error_notifications_enabled ? 1 : 0
+  count = var.error_alarm_notifications_enabled ? 1 : 0
 
   name = "${local.short_prefix}-BatchProcessorFilterErrorLogsFilter"
   # Ignore errors with the below exception type. This is an expected error which returns items to the queue
@@ -320,7 +320,7 @@ resource "aws_cloudwatch_log_metric_filter" "batch_processor_filter_error_logs"
 }
 
 resource "aws_cloudwatch_metric_alarm" "batch_processor_filter_error_alarm" {
-  count = var.batch_error_notifications_enabled ? 1 : 0
+  count = var.error_alarm_notifications_enabled ? 1 : 0
 
   alarm_name          = "${local.short_prefix}-batch-processor-filter-lambda-error"
   comparison_operator = "GreaterThanOrEqualToThreshold"
diff --git a/infrastructure/instance/ecs_batch_processor_config.tf b/infrastructure/instance/ecs_batch_processor_config.tf
@@ -372,7 +372,7 @@ resource "aws_cloudwatch_log_group" "pipe_log_group" {
 }
 
 resource "aws_cloudwatch_log_metric_filter" "record_processor_task_error_logs" {
-  count = var.batch_error_notifications_enabled ? 1 : 0
+  count = var.error_alarm_notifications_enabled ? 1 : 0
 
   name           = "${local.short_prefix}-RecordProcessorTaskErrorLogsFilter"
   pattern        = "%ERROR:%"
@@ -386,7 +386,7 @@ resource "aws_cloudwatch_log_metric_filter" "record_processor_task_error_logs" {
 }
 
 resource "aws_cloudwatch_metric_alarm" "record_processor_task_error_alarm" {
-  count = var.batch_error_notifications_enabled ? 1 : 0
+  count = var.error_alarm_notifications_enabled ? 1 : 0
 
   alarm_name          = "${local.short_prefix}-record-processor-task-error"
   comparison_operator = "GreaterThanOrEqualToThreshold"
diff --git a/infrastructure/instance/endpoints.tf b/infrastructure/instance/endpoints.tf
@@ -9,12 +9,14 @@ data "aws_iam_policy_document" "logs_policy_document" {
   source_policy_documents = [templatefile("${local.policy_path}/log.json", {})]
 }
 module "get_status" {
-  source        = "./modules/lambda"
-  prefix        = local.prefix
-  short_prefix  = local.short_prefix
-  function_name = "get_status"
-  image_uri     = module.docker_image.image_uri
-  policy_json   = data.aws_iam_policy_document.logs_policy_document.json
+  source                            = "./modules/lambda"
+  prefix                            = local.prefix
+  short_prefix                      = local.short_prefix
+  function_name                     = "get_status"
+  image_uri                         = module.docker_image.image_uri
+  policy_json                       = data.aws_iam_policy_document.logs_policy_document.json
+  error_alarm_notifications_enabled = var.error_alarm_notifications_enabled
+  environment                       = var.environment
 }
 
 locals {
@@ -75,14 +77,16 @@ module "imms_event_endpoint_lambdas" {
   source = "./modules/lambda"
   count  = length(local.imms_endpoints)
 
-  prefix                 = local.prefix
-  short_prefix           = local.short_prefix
-  function_name          = local.imms_endpoints[count.index]
-  image_uri              = module.docker_image.image_uri
-  policy_json            = data.aws_iam_policy_document.imms_policy_document.json
-  environment_variables  = local.imms_lambda_env_vars
-  vpc_subnet_ids         = local.private_subnet_ids
-  vpc_security_group_ids = [data.aws_security_group.existing_securitygroup.id]
+  prefix                            = local.prefix
+  short_prefix                      = local.short_prefix
+  function_name                     = local.imms_endpoints[count.index]
+  image_uri                         = module.docker_image.image_uri
+  policy_json                       = data.aws_iam_policy_document.imms_policy_document.json
+  environment_variables             = local.imms_lambda_env_vars
+  vpc_subnet_ids                    = local.private_subnet_ids
+  vpc_security_group_ids            = [data.aws_security_group.existing_securitygroup.id]
+  error_alarm_notifications_enabled = var.error_alarm_notifications_enabled
+  environment                       = var.environment
 }
 
 
diff --git a/infrastructure/instance/environments/dev/int/variables.tfvars b/infrastructure/instance/environments/dev/int/variables.tfvars
@@ -3,6 +3,6 @@ environment                       = "dev"
 immunisation_account_id           = "345594581768"
 dspp_core_account_id              = "603871901111"
 pds_environment                   = "int"
-batch_error_notifications_enabled = true
+error_alarm_notifications_enabled = true
 create_mesh_processor             = true
 has_sub_environment_scope         = true
diff --git a/infrastructure/instance/environments/dev/internal-dev/variables.tfvars b/infrastructure/instance/environments/dev/internal-dev/variables.tfvars
@@ -2,6 +2,6 @@ environment                       = "dev"
 immunisation_account_id           = "345594581768"
 dspp_core_account_id              = "603871901111"
 pds_environment                   = "int"
-batch_error_notifications_enabled = true
+error_alarm_notifications_enabled = true
 create_mesh_processor             = false
 has_sub_environment_scope         = true
diff --git a/infrastructure/instance/environments/dev/internal-qa/variables.tfvars b/infrastructure/instance/environments/dev/internal-qa/variables.tfvars
@@ -2,6 +2,6 @@ environment                       = "dev"
 immunisation_account_id           = "345594581768"
 dspp_core_account_id              = "603871901111"
 pds_environment                   = "int"
-batch_error_notifications_enabled = false
+error_alarm_notifications_enabled = false
 create_mesh_processor             = false
 has_sub_environment_scope         = true
diff --git a/infrastructure/instance/environments/dev/pr/variables.tfvars b/infrastructure/instance/environments/dev/pr/variables.tfvars
@@ -2,6 +2,6 @@ environment                       = "dev"
 immunisation_account_id           = "345594581768"
 dspp_core_account_id              = "603871901111"
 pds_environment                   = "int"
-batch_error_notifications_enabled = false
+error_alarm_notifications_enabled = false
 create_mesh_processor             = false
 has_sub_environment_scope         = true
diff --git a/infrastructure/instance/environments/dev/ref/variables.tfvars b/infrastructure/instance/environments/dev/ref/variables.tfvars
@@ -2,6 +2,6 @@ environment                       = "dev"
 immunisation_account_id           = "345594581768"
 dspp_core_account_id              = "603871901111"
 pds_environment                   = "ref"
-batch_error_notifications_enabled = true
+error_alarm_notifications_enabled = true
 create_mesh_processor             = false
 has_sub_environment_scope         = true
diff --git a/infrastructure/instance/environments/preprod/int-blue/variables.tfvars b/infrastructure/instance/environments/preprod/int-blue/variables.tfvars
@@ -2,6 +2,6 @@ environment                       = "preprod"
 immunisation_account_id           = "084828561157"
 dspp_core_account_id              = "603871901111"
 pds_environment                   = "int"
-batch_error_notifications_enabled = true
+error_alarm_notifications_enabled = true
 create_mesh_processor             = true
 has_sub_environment_scope         = false
diff --git a/infrastructure/instance/environments/preprod/int-green/variables.tfvars b/infrastructure/instance/environments/preprod/int-green/variables.tfvars
@@ -2,6 +2,6 @@ environment                       = "preprod"
 immunisation_account_id           = "084828561157"
 dspp_core_account_id              = "603871901111"
 pds_environment                   = "int"
-batch_error_notifications_enabled = true
+error_alarm_notifications_enabled = true
 create_mesh_processor             = true
 has_sub_environment_scope         = false
diff --git a/infrastructure/instance/environments/prod/blue/variables.tfvars b/infrastructure/instance/environments/prod/blue/variables.tfvars
@@ -2,7 +2,7 @@ environment                       = "prod"
 immunisation_account_id           = "664418956997"
 dspp_core_account_id              = "232116723729"
 pds_environment                   = "prod"
-batch_error_notifications_enabled = true
+error_alarm_notifications_enabled = true
 create_mesh_processor             = true
 has_sub_environment_scope         = false
 dspp_kms_key_alias                = "nhsd-dspp-core-prod-extended-attributes-gdp-key"
diff --git a/infrastructure/instance/environments/prod/green/variables.tfvars b/infrastructure/instance/environments/prod/green/variables.tfvars
@@ -2,7 +2,7 @@ environment                       = "prod"
 immunisation_account_id           = "664418956997"
 dspp_core_account_id              = "232116723729"
 pds_environment                   = "prod"
-batch_error_notifications_enabled = true
+error_alarm_notifications_enabled = true
 create_mesh_processor             = true
 has_sub_environment_scope         = false
 dspp_kms_key_alias                = "nhsd-dspp-core-prod-extended-attributes-gdp-key"
diff --git a/infrastructure/instance/file_name_processor.tf b/infrastructure/instance/file_name_processor.tf
@@ -376,7 +376,7 @@ resource "aws_cloudwatch_log_group" "file_name_processor_log_group" {
 }
 
 resource "aws_cloudwatch_log_metric_filter" "file_name_processor_error_logs" {
-  count = var.batch_error_notifications_enabled ? 1 : 0
+  count = var.error_alarm_notifications_enabled ? 1 : 0
 
   name           = "${local.short_prefix}-FilenameProcessorErrorLogsFilter"
   pattern        = "%\\[ERROR\\]%"
@@ -390,7 +390,7 @@ resource "aws_cloudwatch_log_metric_filter" "file_name_processor_error_logs" {
 }
 
 resource "aws_cloudwatch_metric_alarm" "file_name_processor_error_alarm" {
-  count = var.batch_error_notifications_enabled ? 1 : 0
+  count = var.error_alarm_notifications_enabled ? 1 : 0
 
   alarm_name          = "${local.short_prefix}-file-name-processor-lambda-error"
   comparison_operator = "GreaterThanOrEqualToThreshold"
diff --git a/infrastructure/instance/modules/lambda/lambda.tf b/infrastructure/instance/modules/lambda/lambda.tf
@@ -50,3 +50,37 @@ resource "aws_cloudwatch_log_metric_filter" "max_memory_used_metric" {
     value     = "$18"
   }
 }
+
+resource "aws_cloudwatch_log_metric_filter" "fhir_api_error_logs" {
+  count = var.error_alarm_notifications_enabled ? 1 : 0
+
+  name           = "${var.short_prefix}_${var.function_name}-ErrorLogsFilter"
+  pattern        = "{ $.operation_outcome.status = \"500\" || $.operation_outcome.status = \"403\" }"
+  log_group_name = module.lambda_function_container_image.lambda_cloudwatch_log_group_name
+
+  metric_transformation {
+    name      = "${var.short_prefix}_${var.function_name}-ApiErrorLogs"
+    namespace = "${var.short_prefix}_${var.function_name}-Lambda"
+    value     = "1"
+  }
+}
+
+data "aws_sns_topic" "fhir_api_errors" {
+  name = "${var.environment}-fhir-api-errors"
+}
+
+resource "aws_cloudwatch_metric_alarm" "fhir_api_error_alarm" {
+  count = var.error_alarm_notifications_enabled ? 1 : 0
+
+  alarm_name          = "${var.short_prefix}_${var.function_name}-lambda-error"
+  comparison_operator = "GreaterThanOrEqualToThreshold"
+  evaluation_periods  = 1
+  metric_name         = "${var.short_prefix}_${var.function_name}-ApiErrorLogs"
+  namespace           = "${var.short_prefix}_${var.function_name}-Lambda"
+  period              = 120
+  statistic           = "Sum"
+  threshold           = 1
+  alarm_description   = "Triggers an alarm when 500 or 403 error responses are logged by the FHIR API Lambda function."
+  alarm_actions       = [data.aws_sns_topic.fhir_api_errors.arn]
+  treat_missing_data  = "notBreaching"
+}
diff --git a/infrastructure/instance/modules/lambda/variables.tf b/infrastructure/instance/modules/lambda/variables.tf
@@ -10,6 +10,11 @@ variable "function_name" {
   type = string
 }
 
+variable "error_alarm_notifications_enabled" {
+  description = "Switch to enable error alarm notifications to Slack"
+  type        = string
+}
+
 variable "image_uri" {
   type = string
 }
@@ -32,3 +37,8 @@ variable "vpc_subnet_ids" {
   type    = list(string)
   default = null
 }
+
+variable "environment" {
+  description = "The deployment environment (e.g., dev, int, internal-qa, prod)"
+  type        = string
+}
diff --git a/infrastructure/instance/variables.tf b/infrastructure/instance/variables.tf
@@ -41,9 +41,9 @@ variable "pds_environment" {
 }
 
 # Remember to switch off in PR envs after testing
-variable "batch_error_notifications_enabled" {
+variable "error_alarm_notifications_enabled" {
   default     = true
-  description = "Switch to enable batch processing error notifications to Slack"
+  description = "Switch to enable error alarm notifications to Slack"
   type        = bool
 }
 

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`resource "aws_sns_topic" "batch_processor_errors" {`
`2`	`2`	`name = "${var.environment}-batch-processor-errors"`
`3`		`- kms_master_key_id = aws_kms_key.batch_processor_errors_sns_encryption_key.arn`
	`3`	`+ kms_master_key_id = aws_kms_key.error_alerts_sns_encryption_key.arn`
`4`	`4`	`}`
`5`	`5`
`6`	`6`	`resource "aws_sns_topic_policy" "batch_processor_errors_topic_policy" {`
Original file line number	Diff line number	Diff line change
`@@ -305,7 +305,7 @@ resource "aws_lambda_event_source_mapping" "batch_file_created_sqs_to_lambda" {`
`305`	`305`	`}`
`306`	`306`
`307`	`307`	`resource "aws_cloudwatch_log_metric_filter" "batch_processor_filter_error_logs" {`
`308`		`- count = var.batch_error_notifications_enabled ? 1 : 0`
	`308`	`+ count = var.error_alarm_notifications_enabled ? 1 : 0`
`309`	`309`
`310`	`310`	`name = "${local.short_prefix}-BatchProcessorFilterErrorLogsFilter"`
`311`	`311`	`# Ignore errors with the below exception type. This is an expected error which returns items to the queue`
`@@ -320,7 +320,7 @@ resource "aws_cloudwatch_log_metric_filter" "batch_processor_filter_error_logs"`
`320`	`320`	`}`
`321`	`321`
`322`	`322`	`resource "aws_cloudwatch_metric_alarm" "batch_processor_filter_error_alarm" {`
`323`		`- count = var.batch_error_notifications_enabled ? 1 : 0`
	`323`	`+ count = var.error_alarm_notifications_enabled ? 1 : 0`
`324`	`324`
`325`	`325`	`alarm_name = "${local.short_prefix}-batch-processor-filter-lambda-error"`
`326`	`326`	`comparison_operator = "GreaterThanOrEqualToThreshold"`
Original file line number	Diff line number	Diff line change
`@@ -372,7 +372,7 @@ resource "aws_cloudwatch_log_group" "pipe_log_group" {`
`372`	`372`	`}`
`373`	`373`
`374`	`374`	`resource "aws_cloudwatch_log_metric_filter" "record_processor_task_error_logs" {`
`375`		`- count = var.batch_error_notifications_enabled ? 1 : 0`
	`375`	`+ count = var.error_alarm_notifications_enabled ? 1 : 0`
`376`	`376`
`377`	`377`	`name = "${local.short_prefix}-RecordProcessorTaskErrorLogsFilter"`
`378`	`378`	`pattern = "%ERROR:%"`
`@@ -386,7 +386,7 @@ resource "aws_cloudwatch_log_metric_filter" "record_processor_task_error_logs" {`
`386`	`386`	`}`
`387`	`387`
`388`	`388`	`resource "aws_cloudwatch_metric_alarm" "record_processor_task_error_alarm" {`
`389`		`- count = var.batch_error_notifications_enabled ? 1 : 0`
	`389`	`+ count = var.error_alarm_notifications_enabled ? 1 : 0`
`390`	`390`
`391`	`391`	`alarm_name = "${local.short_prefix}-record-processor-task-error"`
`392`	`392`	`comparison_operator = "GreaterThanOrEqualToThreshold"`