VED-973: Slack Delta Alerts (#1075)

* setup delta error alarm and metrics

* change account level terraform

Author: Akol125

infrastructure/account/batch_processor_errors_slack_chatbot.tf

@@ -0,0 +1,24 @@
+resource "aws_chatbot_slack_channel_configuration" "imms_system_alert_errors" {
+  configuration_name = "${var.environment}-imms-system-alert-errors-slack-channel-config"
+  iam_role_arn       = aws_iam_role.imms_system_alert_errors_chatbot.arn
+  slack_channel_id   = var.environment == "prod" ? "C09EA0HE202" : "C09E48NDP18"
+  slack_team_id      = "TJ00QR03U"
+  sns_topic_arns     = [aws_sns_topic.imms_system_alert_errors.arn]
+}
+
+resource "aws_iam_role" "imms_system_alert_errors_chatbot" {
+  name = "${var.environment}-imms-system-alert-errors-chatbot-channel-role"
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Sid    = "AssumeChatbotRole"
+        Principal = {
+          Service = "chatbot.amazonaws.com"
+        }
+      },
+    ]
+  })
+}

infrastructure/account/imms_alerts_errors_slack_chatbot.tf

@@ -1,10 +1,10 @@
-resource "aws_sns_topic" "batch_processor_errors" {
-  name              = "${var.environment}-batch-processor-errors"
+resource "aws_sns_topic" "imms_system_alert_errors" {
+  name              = "${var.environment}-imms-system-alert-errors"
   kms_master_key_id = aws_kms_key.error_alerts_sns_encryption_key.arn
 }
 
-resource "aws_sns_topic_policy" "batch_processor_errors_topic_policy" {
-  arn = aws_sns_topic.batch_processor_errors.arn
+resource "aws_sns_topic_policy" "imms_system_alert_errors_topic_policy" {
+  arn = aws_sns_topic.imms_system_alert_errors.arn
   policy = jsonencode({
     Version = "2012-10-17",
     Statement = [
@@ -15,7 +15,7 @@ resource "aws_sns_topic_policy" "batch_processor_errors_topic_policy" {
           Service = "cloudwatch.amazonaws.com"
         },
         Action   = "SNS:Publish",
-        Resource = aws_sns_topic.batch_processor_errors.arn
+        Resource = aws_sns_topic.imms_system_alert_errors.arn
       }
     ]
   })

…ount/batch_processor_errors_sns_topic.tf …/account/imms_alerts_errors_sns_topic.tfinfrastructure/account/batch_processor_errors_sns_topic.tf renamed to infrastructure/account/imms_alerts_errors_sns_topic.tf infrastructure/account/batch_processor_errors_sns_topic.tf renamed to infrastructure/account/imms_alerts_errors_sns_topic.tf

@@ -216,8 +216,8 @@ resource "aws_kms_key" "error_alerts_sns_encryption_key" {
   })
 }
 
-resource "aws_kms_alias" "batch_processor_errors_sns_encryption_key" {
-  name          = "alias/${var.environment}-batch-processor-errors-imms-sns-encryption"
+resource "aws_kms_alias" "imms_system_alert_errors_sns_encryption_key" {
+  name          = "alias/${var.environment}-imms-alert-errors-sns-encryption"
   target_key_id = aws_kms_key.error_alerts_sns_encryption_key.key_id
 }
 

infrastructure/account/kms.tf

@@ -331,6 +331,6 @@ resource "aws_cloudwatch_metric_alarm" "batch_processor_filter_error_alarm" {
   statistic           = "Sum"
   threshold           = 1
   alarm_description   = "This sets off an alarm for any error logs found in the batch processor filter Lambda function"
-  alarm_actions       = [data.aws_sns_topic.batch_processor_errors.arn]
+  alarm_actions       = [data.aws_sns_topic.imms_system_alert_errors.arn]
   treat_missing_data  = "notBreaching"
 }

infrastructure/instance/batch_processor_filter_lambda.tf

@@ -178,3 +178,34 @@ resource "aws_cloudwatch_log_group" "delta_lambda" {
   name              = "/aws/lambda/${local.short_prefix}-${local.function_name}"
   retention_in_days = 30
 }
+
+
+resource "aws_cloudwatch_log_metric_filter" "delta_error_logs" {
+  count = var.error_alarm_notifications_enabled ? 1 : 0
+
+  name           = "${local.short_prefix}-DeltaErrorLogsFilter"
+  pattern        = "%\\[ERROR\\]%"
+  log_group_name = aws_cloudwatch_log_group.delta_lambda.name
+
+  metric_transformation {
+    name      = "${local.short_prefix}-DeltaErrorLogs"
+    namespace = "${local.short_prefix}-DeltaLambda"
+    value     = "1"
+  }
+}
+
+resource "aws_cloudwatch_metric_alarm" "delta_error_alarm" {
+  count = var.error_alarm_notifications_enabled ? 1 : 0
+
+  alarm_name          = "${local.short_prefix}-delta-lambda-error"
+  comparison_operator = "GreaterThanOrEqualToThreshold"
+  evaluation_periods  = 1
+  metric_name         = "${local.short_prefix}-DeltaErrorLogs"
+  namespace           = "${local.short_prefix}-DeltaLambda"
+  period              = 120
+  statistic           = "Sum"
+  threshold           = 1
+  alarm_description   = "This sets off an alarm for any error logs found in the delta Lambda function"
+  alarm_actions       = [data.aws_sns_topic.imms_system_alert_errors.arn]
+  treat_missing_data  = "notBreaching"
+}

infrastructure/instance/delta.tf

@@ -397,6 +397,6 @@ resource "aws_cloudwatch_metric_alarm" "record_processor_task_error_alarm" {
   statistic           = "Sum"
   threshold           = 1
   alarm_description   = "This sets off an alarm for any error logs found in the record processor ECS task"
-  alarm_actions       = [data.aws_sns_topic.batch_processor_errors.arn]
+  alarm_actions       = [data.aws_sns_topic.imms_system_alert_errors.arn]
   treat_missing_data  = "notBreaching"
 }

infrastructure/instance/ecs_batch_processor_config.tf

@@ -401,6 +401,6 @@ resource "aws_cloudwatch_metric_alarm" "file_name_processor_error_alarm" {
   statistic           = "Sum"
   threshold           = 1
   alarm_description   = "This sets off an alarm for any error logs found in the file name processor Lambda function"
-  alarm_actions       = [data.aws_sns_topic.batch_processor_errors.arn]
+  alarm_actions       = [data.aws_sns_topic.imms_system_alert_errors.arn]
   treat_missing_data  = "notBreaching"
 }

infrastructure/instance/file_name_processor.tf

@@ -108,6 +108,6 @@ data "aws_route53_zone" "project_zone" {
   name = local.project_domain_name
 }
 
-data "aws_sns_topic" "batch_processor_errors" {
-  name = "${var.environment}-batch-processor-errors"
+data "aws_sns_topic" "imms_system_alert_errors" {
+  name = "${var.environment}-imms-system-alert-errors"
 }