[PRMP-1579] Add alarm and DLQ to MNS (#256)

* [PRMP-1579] enable MNS queue and lambda for sandbox

* [PRMP-1579] add kms action permission to policy

* [PRMP-1579] add alarm to notify when items hit DLQ

* [PRMP-1579] pre-commit

* [PRMP-1579] adjust dimensions on mns dlq alarm

* [PRMP-1579] address PR comments

* [PRMP-1579] add indexing on resources

* [PRMP-1579] add var for dlq visibility timeout and a retry count

* [PRMP-1579] fix typo

* [PRMP-1579] fix variable

* [PRMP-1579] fix variable

* [PRMP-1579] change max visibility

* [PRMP-1579] adjust visibility timeout

Author: steph-torres-nhs

bootstrap/README.md

@@ -9,7 +9,7 @@
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | 5.70.0 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 4.0 |
 
 ## Modules
 

infrastructure/README.md

@@ -8,7 +8,7 @@
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | 5.86.1 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | 5.72.1 |
 
 ## Modules
 
@@ -95,6 +95,7 @@
 | <a name="module_manage-nrl-pointer-alarm"></a> [manage-nrl-pointer-alarm](#module\_manage-nrl-pointer-alarm) | ./modules/lambda_alarms | n/a |
 | <a name="module_manage-nrl-pointer-alarm-topic"></a> [manage-nrl-pointer-alarm-topic](#module\_manage-nrl-pointer-alarm-topic) | ./modules/sns | n/a |
 | <a name="module_manage-nrl-pointer-lambda"></a> [manage-nrl-pointer-lambda](#module\_manage-nrl-pointer-lambda) | ./modules/lambda | n/a |
+| <a name="module_mns-dlq-alarm-topic"></a> [mns-dlq-alarm-topic](#module\_mns-dlq-alarm-topic) | ./modules/sns | n/a |
 | <a name="module_mns-notification-alarm"></a> [mns-notification-alarm](#module\_mns-notification-alarm) | ./modules/lambda_alarms | n/a |
 | <a name="module_mns-notification-alarm-topic"></a> [mns-notification-alarm-topic](#module\_mns-notification-alarm-topic) | ./modules/sns | n/a |
 | <a name="module_mns-notification-lambda"></a> [mns-notification-lambda](#module\_mns-notification-lambda) | ./modules/lambda | n/a |
@@ -216,6 +217,7 @@
 | [aws_cloudwatch_metric_alarm.edge_presign_lambda_error](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_metric_alarm) | resource |
 | [aws_cloudwatch_metric_alarm.error_log_alarm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_metric_alarm) | resource |
 | [aws_cloudwatch_metric_alarm.inbox-messages-not-consumed](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_metric_alarm) | resource |
+| [aws_cloudwatch_metric_alarm.msn_dlq_new_message](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_metric_alarm) | resource |
 | [aws_cloudwatch_metric_alarm.nrl_dlq_new_messages](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_metric_alarm) | resource |
 | [aws_cloudwatch_metric_alarm.sns_topic_error_log_alarm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_metric_alarm) | resource |
 | [aws_ecs_cluster.mesh-forwarder-ecs-cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_cluster) | resource |

infrastructure/lambda-mns-notification.tf

@@ -1,5 +1,5 @@
 module "mns-notification-lambda" {
-  count   = local.is_sandbox ? 0 : 1
+  count   = 1
   source  = "./modules/lambda"
   name    = "MNSNotificationLambda"
   handler = "handlers.mns_notification_handler.lambda_handler"
@@ -29,13 +29,13 @@ module "mns-notification-lambda" {
 }
 
 resource "aws_lambda_event_source_mapping" "mns_notification_lambda" {
-  count            = local.is_sandbox ? 0 : 1
+  count            = 1
   event_source_arn = module.sqs-mns-notification-queue[0].endpoint
   function_name    = module.mns-notification-lambda[0].lambda_arn
 }
 
 module "mns-notification-alarm" {
-  count                = local.is_sandbox ? 0 : 1
+  count                = 1
   source               = "./modules/lambda_alarms"
   lambda_function_name = module.mns-notification-lambda[0].function_name
   lambda_timeout       = module.mns-notification-lambda[0].timeout
@@ -46,7 +46,7 @@ module "mns-notification-alarm" {
 }
 
 module "mns-notification-alarm-topic" {
-  count                 = local.is_sandbox ? 0 : 1
+  count                 = 1
   source                = "./modules/sns"
   sns_encryption_key_id = module.sns_encryption_key.id
   current_account_id    = data.aws_caller_identity.current.account_id
@@ -76,17 +76,18 @@ module "mns-notification-alarm-topic" {
 }
 
 resource "aws_iam_policy" "kms_mns_lambda_access" {
-  count = local.is_sandbox ? 0 : 1
+  count = 1
 
   name        = "${terraform.workspace}_mns_notification_lambda_access_policy"
-  description = "KMS policy to allow lambda to read MNS SQS messages"
+  description = "KMS policy to allow lambda to read and write MNS SQS messages"
 
   policy = jsonencode({
     Version = "2012-10-17"
     Statement = [
       {
         Action = [
           "kms:Decrypt",
+          "kms:GenerateDataKey"
         ]
         Effect   = "Allow"
         Resource = module.mns_encryption_key[0].kms_arn

infrastructure/mns.tf

@@ -4,7 +4,7 @@ data "aws_ssm_parameter" "mns_lambda_role" {
 
 
 module "mns_encryption_key" {
-  count                 = local.is_sandbox ? 0 : 1
+  count                 = 1
   source                = "./modules/kms"
   kms_key_name          = "alias/mns-notification-encryption-key-kms-${terraform.workspace}"
   kms_key_description   = "Custom KMS Key to enable server side encryption for mns subscriptions"
@@ -17,21 +17,24 @@ module "mns_encryption_key" {
 }
 
 module "sqs-mns-notification-queue" {
-  count             = local.is_sandbox ? 0 : 1
-  source            = "./modules/sqs"
-  name              = "mns-notification-queue"
-  max_size_message  = 256 * 1024        # allow message size up to 256 KB
-  message_retention = 60 * 60 * 24 * 14 # 14 days
-  environment       = var.environment
-  owner             = var.owner
-  max_visibility    = 1020
-  delay             = 60
-  enable_sse        = null
-  kms_master_key_id = module.mns_encryption_key[0].id
+  count                  = 1
+  source                 = "./modules/sqs"
+  name                   = "mns-notification-queue"
+  max_size_message       = 256 * 1024        # allow message size up to 256 KB
+  message_retention      = 60 * 60 * 24 * 14 # 14 days
+  environment            = var.environment
+  owner                  = var.owner
+  max_visibility         = 901
+  delay                  = 60
+  enable_sse             = null
+  kms_master_key_id      = module.mns_encryption_key[0].id
+  enable_dlq             = true
+  dlq_visibility_timeout = 0
+  max_receive_count      = 3
 }
 
 resource "aws_sqs_queue_policy" "mns_sqs_access" {
-  count = local.is_sandbox ? 0 : 1
+  count = 1
 
   queue_url = module.sqs-mns-notification-queue[0].sqs_url
 
@@ -49,3 +52,51 @@ resource "aws_sqs_queue_policy" "mns_sqs_access" {
     ]
   })
 }
+
+resource "aws_cloudwatch_metric_alarm" "msn_dlq_new_message" {
+  alarm_name          = "${terraform.workspace}_MNS_dlq_messages"
+  comparison_operator = "GreaterThanThreshold"
+  evaluation_periods  = 1
+  metric_name         = "ApproximateNumberOfMessagesVisible"
+  namespace           = "AWS/SQS"
+  period              = 60
+  statistic           = "Sum"
+  threshold           = 0
+  alarm_description   = "Alarm for when there are new messages in the MNS DLQ"
+  alarm_actions       = [module.mns-dlq-alarm-topic.arn]
+
+  dimensions = {
+    QueueName = module.sqs-mns-notification-queue[0].dlq_name
+  }
+}
+
+module "mns-dlq-alarm-topic" {
+  source                 = "./modules/sns"
+  sns_encryption_key_id  = module.sns_encryption_key.id
+  current_account_id     = data.aws_caller_identity.current.account_id
+  topic_name             = "mns-dlq-topic"
+  topic_protocol         = "email"
+  is_topic_endpoint_list = true
+  topic_endpoint_list    = nonsensitive(split(",", data.aws_ssm_parameter.cloud_security_notification_email_list.value))
+  delivery_policy = jsonencode({
+    "Version" : "2012-10-17",
+    "Statement" : [
+      {
+        "Effect" : "Allow",
+        "Principal" : {
+          "Service" : "cloudwatch.amazonaws.com"
+        },
+        "Action" : [
+          "SNS:Publish",
+        ],
+        "Condition" : {
+          "ArnLike" : {
+            "aws:SourceArn" : "arn:aws:cloudwatch:eu-west-2:${data.aws_caller_identity.current.account_id}:alarm:*"
+          }
+        }
+        "Resource" : "*"
+      }
+    ]
+  })
+  depends_on = [module.sqs-mns-notification-queue[0]]
+}

infrastructure/modules/sqs/README.md

@@ -1,55 +0,0 @@
-## Requirements
-
-No requirements.
-
-## Providers
-
-| Name | Version |
-|------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | n/a |
-
-## Modules
-
-No modules.
-
-## Resources
-
-| Name | Type |
-|------|------|
-| [aws_sqs_queue.queue_deadletter](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource |
-| [aws_sqs_queue.sqs_queue](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource |
-| [aws_sqs_queue_redrive_allow_policy.terraform_queue_redrive_allow_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_redrive_allow_policy) | resource |
-| [aws_sqs_queue_redrive_policy.dlq_redrive](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_redrive_policy) | resource |
-| [aws_iam_policy_document.sqs_read_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
-| [aws_iam_policy_document.sqs_write_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
-
-## Inputs
-
-| Name | Description | Type | Default | Required |
-|------|-------------|------|---------|:--------:|
-| <a name="input_delay"></a> [delay](#input\_delay) | The time in seconds that the delivery of all messages in the queue will be delayed | `number` | `0` | no |
-| <a name="input_enable_deduplication"></a> [enable\_deduplication](#input\_enable\_deduplication) | Prevent content based duplication in queue | `bool` | `false` | no |
-| <a name="input_enable_dlq"></a> [enable\_dlq](#input\_enable\_dlq) | n/a | `bool` | `false` | no |
-| <a name="input_enable_fifo"></a> [enable\_fifo](#input\_enable\_fifo) | Attach first in first out policy to sqs | `bool` | `false` | no |
-| <a name="input_enable_sse"></a> [enable\_sse](#input\_enable\_sse) | Enable server-side encryption (SSE) of message content with SQS-owned encryption keys, requires kms resource for queue | `bool` | `true` | no |
-| <a name="input_environment"></a> [environment](#input\_environment) | Tags | `string` | n/a | yes |
-| <a name="input_kms_master_key_id"></a> [kms\_master\_key\_id](#input\_kms\_master\_key\_id) | The ID of an AWS-managed customer master key (CMK) for Amazon SQS or a custom CMK | `string` | `null` | no |
-| <a name="input_max_receive_count"></a> [max\_receive\_count](#input\_max\_receive\_count) | n/a | `number` | `1` | no |
-| <a name="input_max_size_message"></a> [max\_size\_message](#input\_max\_size\_message) | Max message size in bytes before sqs rejects the message | `number` | `2048` | no |
-| <a name="input_max_visibility"></a> [max\_visibility](#input\_max\_visibility) | Time in seconds during which Amazon SQS prevents all consumers from receiving and processing the message | `number` | `30` | no |
-| <a name="input_message_retention"></a> [message\_retention](#input\_message\_retention) | Number of seconds sqs keeps a message | `number` | `86400` | no |
-| <a name="input_name"></a> [name](#input\_name) | n/a | `string` | n/a | yes |
-| <a name="input_owner"></a> [owner](#input\_owner) | n/a | `string` | n/a | yes |
-| <a name="input_receive_wait"></a> [receive\_wait](#input\_receive\_wait) | Number of seconds sqs will wait for a message when ReceiveMessage is received | `number` | `2` | no |
-
-## Outputs
-
-| Name | Description |
-|------|-------------|
-| <a name="output_dlq_name"></a> [dlq\_name](#output\_dlq\_name) | n/a |
-| <a name="output_endpoint"></a> [endpoint](#output\_endpoint) | Same as sqs queue arn. For use when setting the queue as endpoint of sns topic |
-| <a name="output_sqs_arn"></a> [sqs\_arn](#output\_sqs\_arn) | n/a |
-| <a name="output_sqs_id"></a> [sqs\_id](#output\_sqs\_id) | n/a |
-| <a name="output_sqs_read_policy_document"></a> [sqs\_read\_policy\_document](#output\_sqs\_read\_policy\_document) | n/a |
-| <a name="output_sqs_url"></a> [sqs\_url](#output\_sqs\_url) | n/a |
-| <a name="output_sqs_write_policy_document"></a> [sqs\_write\_policy\_document](#output\_sqs\_write\_policy\_document) | n/a |

infrastructure/modules/sqs/main.tf

@@ -22,7 +22,7 @@ resource "aws_sqs_queue" "queue_deadletter" {
   count                       = var.enable_dlq ? 1 : 0
   name                        = "${terraform.workspace}-deadletter-${var.name}"
   delay_seconds               = var.delay
-  visibility_timeout_seconds  = var.max_visibility
+  visibility_timeout_seconds  = var.dlq_visibility_timeout
   max_message_size            = var.max_size_message
   message_retention_seconds   = var.message_retention
   receive_wait_time_seconds   = var.receive_wait

infrastructure/modules/sqs/variable.tf

@@ -66,6 +66,11 @@ variable "enable_dlq" {
   default = false
 }
 
+variable "dlq_visibility_timeout" {
+  type    = number
+  default = 0
+}
+
 # Tags
 variable "environment" {
   type = string