wip

Author: craigzour

aws/alarms/cloudwatch_app.tf

@@ -404,19 +404,55 @@ resource "aws_cloudwatch_metric_alarm" "vault_data_integrity_check_lambda_iterat
 // Cloudwatch log subscription filters
 
 locals {
-  map_of_lambda_log_group = {
-    audit_logs               = var.lambda_audit_logs_log_group_name,
-    audit_logs_archiver      = var.lambda_audit_logs_archiver_log_group_name,
-    form_archiver            = var.lambda_form_archiver_log_group_name,
-    nagware                  = var.lambda_nagware_log_group_name,
-    reliability              = var.lambda_reliability_log_group_name,
-    reliability_dlq_consumer = var.lambda_reliability_dlq_consumer_log_group_name,
-    response_archiver        = var.lambda_response_archiver_log_group_name,
-    submission               = var.lambda_submission_log_group_name,
-    vault_integrity          = var.lambda_vault_integrity_log_group_name,
-    api_end_to_end_test      = var.lambda_api_end_to_end_test_log_group_name,
-    file_upload_processor    = var.lambda_file_upload_processor_log_group_name,
-    file_upload_cleanup      = var.lambda_file_upload_cleanup_log_group_name
+  map_of_forms_app_lambda = {
+    audit_logs = {
+      function_name  = var.todo
+      log_group_name = var.lambda_audit_logs_log_group_name
+    }
+    audit_logs_archiver = {
+      function_name  = var.todo
+      log_group_name = var.lambda_audit_logs_archiver_log_group_name
+    }
+    form_archiver = {
+      function_name  = var.todo
+      log_group_name = var.lambda_form_archiver_log_group_name
+    }
+    nagware = {
+      function_name  = var.todo
+      log_group_name = var.lambda_nagware_log_group_name
+    }
+    reliability = {
+      function_name  = var.todo
+      log_group_name = var.lambda_reliability_log_group_name
+    }
+    reliability_dlq_consumer = {
+      function_name  = var.todo
+      log_group_name = var.lambda_reliability_dlq_consumer_log_group_name
+    }
+    response_archiver = {
+      function_name  = var.todo
+      log_group_name = var.lambda_response_archiver_log_group_name
+    }
+    submission = {
+      function_name  = var.todo
+      log_group_name = var.lambda_submission_log_group_name
+    }
+    vault_integrity = {
+      function_name  = var.todo
+      log_group_name = var.lambda_vault_integrity_log_group_name
+    }
+    api_end_to_end_test = {
+      function_name  = var.todo
+      log_group_name = var.lambda_api_end_to_end_test_log_group_name
+    }
+    file_upload_processor = {
+      function_name  = var.todo
+      log_group_name = var.lambda_file_upload_processor_log_group_name
+    }
+    file_upload_cleanup = {
+      function_name  = var.todo
+      log_group_name = var.lambda_file_upload_cleanup_log_group_name
+    }
   }
 }
 
@@ -441,25 +477,31 @@ resource "aws_cloudwatch_log_subscription_filter" "forms_app_log_stream" {
 }
 
 resource "aws_cloudwatch_log_subscription_filter" "lambda_error_detection" {
-  for_each        = local.map_of_lambda_log_group
+  for_each        = local.map_of_forms_app_lambda
   name            = "error_detection_in_${each.key}_lambda_logs"
-  log_group_name  = each.value
+  log_group_name  = each.value.log_group_name
   filter_pattern  = "{($.level = \"warn\") || ($.level = \"error\")}"
   destination_arn = aws_lambda_function.notify_slack.arn
 }
 
-/*
- * Lambda timeout detection
- * Note: We used the second and final lambda subscription filter to detect function time out.
- * If we ever need to create a new subscription filter we will have to rework the way we parse logs to extract errors and time out logs.
- */
+resource "aws_cloudwatch_metric_alarm" "forms_app_lambda_error_detection" {
+  for_each            = local.map_of_forms_app_lambda
+  alarm_name          = "${each.key}-lambda-error-detection"
+  alarm_description   = "Detects when the ${each.key} lambda function logs/throws an error or times out"
+  comparison_operator = "GreaterThanThreshold"
+  evaluation_periods  = "1"
+  metric_name         = "Errors"
+  namespace           = "AWS/Lambda"
+  period              = "60"
+  statistic           = "Sum"
+  threshold           = "0"
+  treat_missing_data  = "notBreaching"
 
-resource "aws_cloudwatch_log_subscription_filter" "lambda_timeout_detection" {
-  for_each        = local.map_of_lambda_log_group
-  name            = "timeout_detection_in_${each.key}_lambda_logs"
-  log_group_name  = each.value
-  filter_pattern  = "Task timed out"
-  destination_arn = aws_lambda_function.notify_slack.arn
+  dimensions = {
+    FunctionName = each.value.function_name
+  }
+
+  alarm_actions = [var.sns_topic_alert_critical_arn]
 }
 
 // Allow Cloudwatch filters to trigger Lambda

aws/alarms/cloudwatch_global.tf

@@ -11,4 +11,23 @@ resource "aws_cloudwatch_metric_alarm" "ip_added_to_block_list" {
   alarm_description   = "WAF - IP(s) Has been added to the dynamic block list."
 
   alarm_actions = [var.sns_topic_alert_warning_arn]
-}
+}
+
+resource "aws_cloudwatch_metric_alarm" "waf_ipv4_blocklist_lambda_error_detection" {
+  alarm_name          = "${var.waf_ipv4_blocklist_lambda_function_name}-lambda-error-detection"
+  alarm_description   = "Detects when the ${var.var.waf_ipv4_blocklist_lambda_function_name} lambda function logs/throws an error or times out"
+  comparison_operator = "GreaterThanThreshold"
+  evaluation_periods  = "1"
+  metric_name         = "Errors"
+  namespace           = "AWS/Lambda"
+  period              = "60"
+  statistic           = "Sum"
+  threshold           = "0"
+  treat_missing_data  = "notBreaching"
+
+  dimensions = {
+    FunctionName = var.waf_ipv4_blocklist_lambda_function_name
+  }
+
+  alarm_actions = [var.sns_topic_alert_critical_arn]
+}

aws/alarms/inputs.tf

@@ -316,6 +316,11 @@ variable "waf_ipv4_new_blocked_ip_metric_filter_namespace" {
   type        = string
 }
 
+variable "waf_ipv4_blocklist_lambda_function_name" {
+  description = "Name of WAF Blocklist associated Lambda"
+  type        = string
+}
+
 variable "unhealthy_host_count_for_target_group_1_alarm_arn" {
   description = "ARN of unhealthy host count alarm for target group 1"
   type        = string

aws/load_balancer/outputs.tf

@@ -68,6 +68,11 @@ output "waf_ipv4_new_blocked_ip_metric_filter_namespace" {
   value       = module.waf_ip_blocklist.ipv4_new_blocked_ip_metric_filter_namespace
 }
 
+output "waf_ipv4_blocklist_lambda_function_name" {
+  description = "Name of WAF Blocklist associated Lambda"
+  value       = module.waf_ip_blocklist.ipv4_lambda_function_name
+}
+
 output "unhealthy_host_count_for_target_group_1_alarm_arn" {
   description = "ARN of unhealthy host count alarm for target group 1"
   value       = aws_cloudwatch_metric_alarm.UnHealthyHostCount-TargetGroup1.arn

aws/load_balancer/waf.tf

@@ -793,7 +793,7 @@ resource "aws_wafv2_regex_pattern_set" "valid_maintenance_mode_uri_paths" {
 # that crosses a block threshold will be added to the blocklist.
 #
 module "waf_ip_blocklist" {
-  source = "github.com/cds-snc/terraform-modules//waf_ip_blocklist?ref=ae656dd178b8086ae4b422da6b310550fa1d86c0" # v10.8.2
+  source = "github.com/cds-snc/terraform-modules//waf_ip_blocklist?ref=5df29471c3868586b55149a72275a0e3b29fb3b0" # v10.11.1
 
   service_name                     = "forms_app"
   athena_database_name             = "access_logs"

env/cloud/alarms/terragrunt.hcl

@@ -45,6 +45,7 @@ dependency "load_balancer" {
     lb_target_group_api_arn_suffix                    = null
     waf_ipv4_new_blocked_ip_metric_filter_name        = "default"
     waf_ipv4_new_blocked_ip_metric_filter_namespace   = "default"
+    waf_ipv4_blocklist_lambda_function_name           = "ipv4_blocklist_forms_app"
     unhealthy_host_count_for_target_group_1_alarm_arn = ""
     unhealthy_host_count_for_target_group_2_alarm_arn = ""
   }
@@ -201,6 +202,7 @@ inputs = {
   lb_api_target_group_arn_suffix                    = dependency.load_balancer.outputs.lb_target_group_api_arn_suffix
   waf_ipv4_new_blocked_ip_metric_filter_name        = dependency.load_balancer.outputs.waf_ipv4_new_blocked_ip_metric_filter_name
   waf_ipv4_new_blocked_ip_metric_filter_namespace   = dependency.load_balancer.outputs.waf_ipv4_new_blocked_ip_metric_filter_namespace
+  waf_ipv4_blocklist_lambda_function_name           = dependency.load_balancer.outputs.waf_ipv4_blocklist_lambda_function_name
   unhealthy_host_count_for_target_group_1_alarm_arn = dependency.load_balancer.outputs.unhealthy_host_count_for_target_group_1_alarm_arn
   unhealthy_host_count_for_target_group_2_alarm_arn = dependency.load_balancer.outputs.unhealthy_host_count_for_target_group_2_alarm_arn