setup metrics and alarm for fhir api and change error notifications env

Akol125 · Akol125 · commit 749ac3b83075 · 2025-12-17T11:10:13.000Z
diff --git a/infrastructure/instance/batch_processor_filter_lambda.tf b/infrastructure/instance/batch_processor_filter_lambda.tf
@@ -305,7 +305,7 @@ resource "aws_lambda_event_source_mapping" "batch_file_created_sqs_to_lambda" {
 }
 
 resource "aws_cloudwatch_log_metric_filter" "batch_processor_filter_error_logs" {
-  count = var.batch_error_notifications_enabled ? 1 : 0
+  count = var.error_alarm_notifications_enabled ? 1 : 0
 
   name = "${local.short_prefix}-BatchProcessorFilterErrorLogsFilter"
   # Ignore errors with the below exception type. This is an expected error which returns items to the queue
@@ -320,7 +320,7 @@ resource "aws_cloudwatch_log_metric_filter" "batch_processor_filter_error_logs"
 }
 
 resource "aws_cloudwatch_metric_alarm" "batch_processor_filter_error_alarm" {
-  count = var.batch_error_notifications_enabled ? 1 : 0
+  count = var.error_alarm_notifications_enabled ? 1 : 0
 
   alarm_name          = "${local.short_prefix}-batch-processor-filter-lambda-error"
   comparison_operator = "GreaterThanOrEqualToThreshold"
diff --git a/infrastructure/instance/ecs_batch_processor_config.tf b/infrastructure/instance/ecs_batch_processor_config.tf
@@ -372,7 +372,7 @@ resource "aws_cloudwatch_log_group" "pipe_log_group" {
 }
 
 resource "aws_cloudwatch_log_metric_filter" "record_processor_task_error_logs" {
-  count = var.batch_error_notifications_enabled ? 1 : 0
+  count = var.error_alarm_notifications_enabled ? 1 : 0
 
   name           = "${local.short_prefix}-RecordProcessorTaskErrorLogsFilter"
   pattern        = "%ERROR:%"
@@ -386,7 +386,7 @@ resource "aws_cloudwatch_log_metric_filter" "record_processor_task_error_logs" {
 }
 
 resource "aws_cloudwatch_metric_alarm" "record_processor_task_error_alarm" {
-  count = var.batch_error_notifications_enabled ? 1 : 0
+  count = var.error_alarm_notifications_enabled ? 1 : 0
 
   alarm_name          = "${local.short_prefix}-record-processor-task-error"
   comparison_operator = "GreaterThanOrEqualToThreshold"
diff --git a/infrastructure/instance/environments/dev/int/variables.tfvars b/infrastructure/instance/environments/dev/int/variables.tfvars
@@ -3,6 +3,6 @@ environment                       = "dev"
 immunisation_account_id           = "345594581768"
 dspp_core_account_id              = "603871901111"
 pds_environment                   = "int"
-batch_error_notifications_enabled = true
+error_alarm_notifications_enabled = true
 create_mesh_processor             = true
 has_sub_environment_scope         = true
diff --git a/infrastructure/instance/environments/dev/internal-dev/variables.tfvars b/infrastructure/instance/environments/dev/internal-dev/variables.tfvars
@@ -2,6 +2,6 @@ environment                       = "dev"
 immunisation_account_id           = "345594581768"
 dspp_core_account_id              = "603871901111"
 pds_environment                   = "int"
-batch_error_notifications_enabled = true
+error_alarm_notifications_enabled = true
 create_mesh_processor             = false
 has_sub_environment_scope         = true
diff --git a/infrastructure/instance/environments/dev/internal-qa/variables.tfvars b/infrastructure/instance/environments/dev/internal-qa/variables.tfvars
@@ -2,6 +2,6 @@ environment                       = "dev"
 immunisation_account_id           = "345594581768"
 dspp_core_account_id              = "603871901111"
 pds_environment                   = "int"
-batch_error_notifications_enabled = false
+error_alarm_notifications_enabled = false
 create_mesh_processor             = false
 has_sub_environment_scope         = true
diff --git a/infrastructure/instance/environments/dev/pr/variables.tfvars b/infrastructure/instance/environments/dev/pr/variables.tfvars
@@ -2,6 +2,6 @@ environment                       = "dev"
 immunisation_account_id           = "345594581768"
 dspp_core_account_id              = "603871901111"
 pds_environment                   = "int"
-batch_error_notifications_enabled = false
+error_alarm_notifications_enabled = false
 create_mesh_processor             = false
 has_sub_environment_scope         = true
diff --git a/infrastructure/instance/environments/dev/ref/variables.tfvars b/infrastructure/instance/environments/dev/ref/variables.tfvars
@@ -2,6 +2,6 @@ environment                       = "dev"
 immunisation_account_id           = "345594581768"
 dspp_core_account_id              = "603871901111"
 pds_environment                   = "ref"
-batch_error_notifications_enabled = true
+error_alarm_notifications_enabled = true
 create_mesh_processor             = false
 has_sub_environment_scope         = true
diff --git a/infrastructure/instance/environments/preprod/int-blue/variables.tfvars b/infrastructure/instance/environments/preprod/int-blue/variables.tfvars
@@ -2,6 +2,6 @@ environment                       = "preprod"
 immunisation_account_id           = "084828561157"
 dspp_core_account_id              = "603871901111"
 pds_environment                   = "int"
-batch_error_notifications_enabled = true
+error_alarm_notifications_enabled = true
 create_mesh_processor             = true
 has_sub_environment_scope         = false
diff --git a/infrastructure/instance/environments/preprod/int-green/variables.tfvars b/infrastructure/instance/environments/preprod/int-green/variables.tfvars
@@ -2,6 +2,6 @@ environment                       = "preprod"
 immunisation_account_id           = "084828561157"
 dspp_core_account_id              = "603871901111"
 pds_environment                   = "int"
-batch_error_notifications_enabled = true
+error_alarm_notifications_enabled = true
 create_mesh_processor             = true
 has_sub_environment_scope         = false
diff --git a/infrastructure/instance/environments/prod/blue/variables.tfvars b/infrastructure/instance/environments/prod/blue/variables.tfvars
@@ -2,6 +2,6 @@ environment                       = "prod"
 immunisation_account_id           = "664418956997"
 dspp_core_account_id              = "232116723729"
 pds_environment                   = "prod"
-batch_error_notifications_enabled = true
+error_alarm_notifications_enabled = true
 create_mesh_processor             = true
 has_sub_environment_scope         = false
diff --git a/infrastructure/instance/environments/prod/green/variables.tfvars b/infrastructure/instance/environments/prod/green/variables.tfvars
@@ -2,6 +2,6 @@ environment                       = "prod"
 immunisation_account_id           = "664418956997"
 dspp_core_account_id              = "232116723729"
 pds_environment                   = "prod"
-batch_error_notifications_enabled = true
+error_alarm_notifications_enabled = true
 create_mesh_processor             = true
 has_sub_environment_scope         = false
diff --git a/infrastructure/instance/file_name_processor.tf b/infrastructure/instance/file_name_processor.tf
@@ -345,7 +345,7 @@ resource "aws_cloudwatch_log_group" "file_name_processor_log_group" {
 }
 
 resource "aws_cloudwatch_log_metric_filter" "file_name_processor_error_logs" {
-  count = var.batch_error_notifications_enabled ? 1 : 0
+  count = var.error_alarm_notifications_enabled ? 1 : 0
 
   name           = "${local.short_prefix}-FilenameProcessorErrorLogsFilter"
   pattern        = "%\\[ERROR\\]%"
@@ -359,7 +359,7 @@ resource "aws_cloudwatch_log_metric_filter" "file_name_processor_error_logs" {
 }
 
 resource "aws_cloudwatch_metric_alarm" "file_name_processor_error_alarm" {
-  count = var.batch_error_notifications_enabled ? 1 : 0
+  count = var.error_alarm_notifications_enabled ? 1 : 0
 
   alarm_name          = "${local.short_prefix}-file-name-processor-lambda-error"
   comparison_operator = "GreaterThanOrEqualToThreshold"
@@ -372,4 +372,4 @@ resource "aws_cloudwatch_metric_alarm" "file_name_processor_error_alarm" {
   alarm_description   = "This sets off an alarm for any error logs found in the file name processor Lambda function"
   alarm_actions       = [data.aws_sns_topic.batch_processor_errors.arn]
   treat_missing_data  = "notBreaching"
-}
+}
diff --git a/infrastructure/instance/modules/lambda/lambda.tf b/infrastructure/instance/modules/lambda/lambda.tf
@@ -46,7 +46,37 @@ resource "aws_cloudwatch_log_metric_filter" "max_memory_used_metric" {
 
   metric_transformation {
     name      = "max-memory-used"
-    namespace = "${var.short_prefix}_${var.function_name}"
+    namespace = var.short_prefix
     value     = "$18"
   }
 }
+
+resource "aws_cloudwatch_log_metric_filter" "fhir_api_error_logs" {
+  count = var.error_alarm_notifications_enabled ? 1 : 0
+
+  name           = "${local.short_prefix}_${var.function_name}-ErrorLogsFilter"
+  pattern        = "%\\[ERROR\\]%"
+  log_group_name = aws_cloudwatch_log_group.api_access_log.name
+
+  metric_transformation {
+    name      = "${local.short_prefix}_${var.function_name}-ApiErrorLogs"
+    namespace = "${local.short_prefix}-_${var.function_name}-Lambda"
+    value     = "1"
+  }
+}
+
+resource "aws_cloudwatch_metric_alarm" "fhir_api_error_alarm" {
+  count = var.error_alarm_notifications_enabled ? 1 : 0
+
+  alarm_name          = "${local.short_prefix}_${var.function_name}-lambda-error"
+  comparison_operator = "GreaterThanOrEqualToThreshold"
+  evaluation_periods  = 1
+  metric_name         = "${local.short_prefix}_${var.function_name}-ErrorLogs"
+  namespace           = "${local.short_prefix}_${var.function_name}-Lambda"
+  period              = 120
+  statistic           = "Sum"
+  threshold           = 1
+  alarm_description   = "This sets off an alarm for any error logs found in fhir api Lambda function"
+  alarm_actions       = [data.aws_sns_topic.fhir_api_errors.arn]
+  treat_missing_data  = "notBreaching"
+}
diff --git a/infrastructure/instance/variables.tf b/infrastructure/instance/variables.tf
@@ -35,9 +35,9 @@ variable "pds_environment" {
 }
 
 # Remember to switch off in PR envs after testing
-variable "batch_error_notifications_enabled" {
+variable "error_alarm_notifications_enabled" {
   default     = true
-  description = "Switch to enable batch processing error notifications to Slack"
+  description = "Switch to enable error alarm notifications to Slack"
   type        = bool
 }
 

Original file line number	Diff line number	Diff line change
`@@ -305,7 +305,7 @@ resource "aws_lambda_event_source_mapping" "batch_file_created_sqs_to_lambda" {`
`305`	`305`	`}`
`306`	`306`
`307`	`307`	`resource "aws_cloudwatch_log_metric_filter" "batch_processor_filter_error_logs" {`
`308`		`- count = var.batch_error_notifications_enabled ? 1 : 0`
	`308`	`+ count = var.error_alarm_notifications_enabled ? 1 : 0`
`309`	`309`
`310`	`310`	`name = "${local.short_prefix}-BatchProcessorFilterErrorLogsFilter"`
`311`	`311`	`# Ignore errors with the below exception type. This is an expected error which returns items to the queue`
`@@ -320,7 +320,7 @@ resource "aws_cloudwatch_log_metric_filter" "batch_processor_filter_error_logs"`
`320`	`320`	`}`
`321`	`321`
`322`	`322`	`resource "aws_cloudwatch_metric_alarm" "batch_processor_filter_error_alarm" {`
`323`		`- count = var.batch_error_notifications_enabled ? 1 : 0`
	`323`	`+ count = var.error_alarm_notifications_enabled ? 1 : 0`
`324`	`324`
`325`	`325`	`alarm_name = "${local.short_prefix}-batch-processor-filter-lambda-error"`
`326`	`326`	`comparison_operator = "GreaterThanOrEqualToThreshold"`
Original file line number	Diff line number	Diff line change
`@@ -372,7 +372,7 @@ resource "aws_cloudwatch_log_group" "pipe_log_group" {`
`372`	`372`	`}`
`373`	`373`
`374`	`374`	`resource "aws_cloudwatch_log_metric_filter" "record_processor_task_error_logs" {`
`375`		`- count = var.batch_error_notifications_enabled ? 1 : 0`
	`375`	`+ count = var.error_alarm_notifications_enabled ? 1 : 0`
`376`	`376`
`377`	`377`	`name = "${local.short_prefix}-RecordProcessorTaskErrorLogsFilter"`
`378`	`378`	`pattern = "%ERROR:%"`
`@@ -386,7 +386,7 @@ resource "aws_cloudwatch_log_metric_filter" "record_processor_task_error_logs" {`
`386`	`386`	`}`
`387`	`387`
`388`	`388`	`resource "aws_cloudwatch_metric_alarm" "record_processor_task_error_alarm" {`
`389`		`- count = var.batch_error_notifications_enabled ? 1 : 0`
	`389`	`+ count = var.error_alarm_notifications_enabled ? 1 : 0`
`390`	`390`
`391`	`391`	`alarm_name = "${local.short_prefix}-record-processor-task-error"`
`392`	`392`	`comparison_operator = "GreaterThanOrEqualToThreshold"`