CCM-7939: pipe dlq and retry settings

Author: harrim91

infrastructure/terraform/components/acct/module_sandbox_kms.tf

@@ -54,4 +54,35 @@ data "aws_iam_policy_document" "kms" {
       ]
     }
   }
+
+  statement {
+    sid    = "AllowLogDeliveryEncrypt"
+    effect = "Allow"
+
+    principals {
+      type = "Service"
+
+      identifiers = [
+        "delivery.logs.amazonaws.com"
+      ]
+    }
+
+    actions = [
+      "kms:Decrypt",
+      "kms:GenerateDataKey*",
+    ]
+
+    resources = [
+      "*",
+    ]
+
+    condition {
+      test     = "StringLike"
+      variable = "kms:EncryptionContext:SourceArn"
+
+      values = [
+        "arn:aws:logs:${var.region}:${var.aws_account_id}:*",
+      ]
+    }
+  }
 }

infrastructure/terraform/components/app/module_kms.tf

@@ -75,4 +75,35 @@ data "aws_iam_policy_document" "kms" {
       ]
     }
   }
+
+  statement {
+    sid    = "AllowLogDeliveryEncrypt"
+    effect = "Allow"
+
+    principals {
+      type = "Service"
+
+      identifiers = [
+        "delivery.logs.amazonaws.com"
+      ]
+    }
+
+    actions = [
+      "kms:Decrypt",
+      "kms:GenerateDataKey*",
+    ]
+
+    resources = [
+      "*",
+    ]
+
+    condition {
+      test     = "StringLike"
+      variable = "kms:EncryptionContext:SourceArn"
+
+      values = [
+        "arn:aws:logs:${var.region}:${var.aws_account_id}:*",
+      ]
+    }
+  }
 }

infrastructure/terraform/modules/backend-api/README.md

@@ -54,6 +54,7 @@ No requirements.
 | <a name="module_s3bucket_quarantine"></a> [s3bucket\_quarantine](#module\_s3bucket\_quarantine) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/s3bucket | v1.0.8 |
 | <a name="module_sqs_sftp_upload"></a> [sqs\_sftp\_upload](#module\_sqs\_sftp\_upload) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/sqs | v2.0.1 |
 | <a name="module_sqs_template_table_events"></a> [sqs\_template\_table\_events](#module\_sqs\_template\_table\_events) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/sqs | v2.0.8 |
+| <a name="module_sqs_template_table_events_pipe_dlq"></a> [sqs\_template\_table\_events\_pipe\_dlq](#module\_sqs\_template\_table\_events\_pipe\_dlq) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/sqs | v2.0.8 |
 | <a name="module_sqs_validate_letter_template_files"></a> [sqs\_validate\_letter\_template\_files](#module\_sqs\_validate\_letter\_template\_files) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/sqs | v2.0.1 |
 | <a name="module_submit_template_lambda"></a> [submit\_template\_lambda](#module\_submit\_template\_lambda) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/lambda | v2.0.4 |
 | <a name="module_update_template_lambda"></a> [update\_template\_lambda](#module\_update\_template\_lambda) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/lambda | v2.0.4 |

infrastructure/terraform/modules/backend-api/module_sqs_template_table_events.tf

@@ -1,14 +1,12 @@
 module "sqs_template_table_events" {
   source = "git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/sqs?ref=v2.0.8"
 
-  aws_account_id              = var.aws_account_id
-  component                   = var.component
-  environment                 = var.environment
-  project                     = var.project
-  region                      = var.region
-  name                        = "template-table-events"
-  fifo_queue                  = true
-  content_based_deduplication = true
-
+  aws_account_id  = var.aws_account_id
+  component       = var.component
+  environment     = var.environment
+  project         = var.project
+  region          = var.region
+  name            = "template-table-events"
+  fifo_queue      = true
   sqs_kms_key_arn = var.kms_key_arn
 }

infrastructure/terraform/modules/backend-api/module_sqs_template_table_events_pipe_dlq.tf

@@ -0,0 +1,11 @@
+module "sqs_template_table_events_pipe_dlq" {
+  source = "git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/sqs?ref=v2.0.8"
+
+  aws_account_id  = var.aws_account_id
+  component       = var.component
+  environment     = var.environment
+  project         = var.project
+  region          = var.region
+  name            = "template-table-events-pipe-dead-letter"
+  sqs_kms_key_arn = var.kms_key_arn
+}

infrastructure/terraform/modules/backend-api/pipes_pipe_template_table_events.tf

@@ -8,18 +8,28 @@ resource "aws_pipes_pipe" "template_table_events" {
 
   source_parameters {
     dynamodb_stream_parameters {
-      starting_position = "TRIM_HORIZON"
+      starting_position                  = "TRIM_HORIZON"
+      maximum_retry_attempts             = 3
+      maximum_record_age_in_seconds      = -1
+      maximum_batching_window_in_seconds = 5
+
+      dead_letter_config {
+        arn = module.sqs_template_table_events_pipe_dlq.sqs_queue_arn
+      }
     }
   }
 
   target_parameters {
     sqs_queue_parameters {
-      message_group_id = "$.dynamodb.Keys.id.S"
+      message_group_id         = "$.dynamodb.Keys.id.S"
+      message_deduplication_id = "$.eventID"
     }
   }
 
   log_configuration {
-    level = "ERROR"
+    level                  = "ERROR"
+    include_execution_data = ["ALL"]
+
     cloudwatch_logs_log_destination {
       log_group_arn = aws_cloudwatch_log_group.pipe_template_table_events.arn
     }
@@ -61,7 +71,7 @@ data "aws_iam_policy_document" "pipe_template_table_events" {
   version = "2012-10-17"
 
   statement {
-    sid    = "AllowDDBStreamRead"
+    sid    = "AllowDynamoStreamRead"
     effect = "Allow"
     actions = [
       "dynamodb:DescribeStream",
@@ -73,24 +83,28 @@ data "aws_iam_policy_document" "pipe_template_table_events" {
   }
 
   statement {
-    sid       = "AllowSQSSendMessage"
-    effect    = "Allow"
-    actions   = ["sqs:SendMessage"]
-    resources = [module.sqs_template_table_events.sqs_queue_arn]
+    sid     = "AllowSqsSendMessage"
+    effect  = "Allow"
+    actions = ["sqs:SendMessage"]
+    resources = [
+      module.sqs_template_table_events.sqs_queue_arn,
+      module.sqs_template_table_events_pipe_dlq.sqs_queue_arn,
+    ]
   }
 
   statement {
-    sid    = "AllowSqsKMS"
+    sid    = "AllowKmsUsage"
     effect = "Allow"
     actions = [
       "kms:Decrypt",
-      "kms:GenerateDataKey"
+      "kms:Encrypt",
+      "kms:GenerateDataKey*"
     ]
     resources = [var.kms_key_arn]
   }
 
   statement {
-    sid       = "AllowDynamoKMS"
+    sid       = "AllowDynamoKms"
     effect    = "Allow"
     actions   = ["kms:Decrypt"]
     resources = [local.dynamodb_kms_key_arn]