Skip to content

Commit 5ac636a

Browse files
harrim91harrim91
authored
CCM-7939: pipe template table stream to fifo queue (#506)
1 parent 2abb09c commit 5ac636a

File tree

10 files changed

+205
-0
lines changed

10 files changed

+205
-0
lines changed

‎infrastructure/terraform/components/acct/module_sandbox_kms.tf‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,4 +54,35 @@ data "aws_iam_policy_document" "kms" {
5454
]
5555
}
5656
}
57+
58+
statement {
59+
sid = "AllowLogDeliveryEncrypt"
60+
effect = "Allow"
61+
62+
principals {
63+
type = "Service"
64+
65+
identifiers = [
66+
"delivery.logs.amazonaws.com"
67+
]
68+
}
69+
70+
actions = [
71+
"kms:Decrypt",
72+
"kms:GenerateDataKey*",
73+
]
74+
75+
resources = [
76+
"*",
77+
]
78+
79+
condition {
80+
test = "StringLike"
81+
variable = "kms:EncryptionContext:SourceArn"
82+
83+
values = [
84+
"arn:aws:logs:${var.region}:${var.aws_account_id}:*",
85+
]
86+
}
87+
}
5788
}

‎infrastructure/terraform/components/app/module_kms.tf‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,4 +75,35 @@ data "aws_iam_policy_document" "kms" {
7575
]
7676
}
7777
}
78+
79+
statement {
80+
sid = "AllowLogDeliveryEncrypt"
81+
effect = "Allow"
82+
83+
principals {
84+
type = "Service"
85+
86+
identifiers = [
87+
"delivery.logs.amazonaws.com"
88+
]
89+
}
90+
91+
actions = [
92+
"kms:Decrypt",
93+
"kms:GenerateDataKey*",
94+
]
95+
96+
resources = [
97+
"*",
98+
]
99+
100+
condition {
101+
test = "StringLike"
102+
variable = "kms:EncryptionContext:SourceArn"
103+
104+
values = [
105+
"arn:aws:logs:${var.region}:${var.aws_account_id}:*",
106+
]
107+
}
108+
}
78109
}

‎infrastructure/terraform/components/sandbox/module_backend_api.tf‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,4 +23,6 @@ module "backend_api" {
2323
kms_key_arn = data.aws_kms_key.sandbox.arn
2424

2525
send_to_firehose = false
26+
27+
enable_event_stream = true
2628
}

‎infrastructure/terraform/modules/backend-api/README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ No requirements.
1515
| <a name="input_component"></a> [component](#input\_component) | The variable encapsulating the name of this component | `string` | n/a | yes |
1616
| <a name="input_csi"></a> [csi](#input\_csi) | CSI from the parent component | `string` | n/a | yes |
1717
| <a name="input_enable_backup"></a> [enable\_backup](#input\_enable\_backup) | Enable Backups for the DynamoDB table? | `bool` | `true` | no |
18+
| <a name="input_enable_event_stream"></a> [enable\_event\_stream](#input\_enable\_event\_stream) | Enable DynamoDB streaming to SQS? | `bool` | `false` | no |
1819
| <a name="input_enable_proofing"></a> [enable\_proofing](#input\_enable\_proofing) | Enable proofing feature flag | `bool` | n/a | yes |
1920
| <a name="input_environment"></a> [environment](#input\_environment) | The name of the tfscaffold environment | `string` | n/a | yes |
2021
| <a name="input_function_s3_bucket"></a> [function\_s3\_bucket](#input\_function\_s3\_bucket) | Name of S3 bucket to upload lambda artefacts to | `string` | n/a | yes |
@@ -51,6 +52,8 @@ No requirements.
5152
| <a name="module_s3bucket_internal"></a> [s3bucket\_internal](#module\_s3bucket\_internal) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/s3bucket | v1.0.8 |
5253
| <a name="module_s3bucket_quarantine"></a> [s3bucket\_quarantine](#module\_s3bucket\_quarantine) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/s3bucket | v1.0.8 |
5354
| <a name="module_sqs_sftp_upload"></a> [sqs\_sftp\_upload](#module\_sqs\_sftp\_upload) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/sqs | v2.0.1 |
55+
| <a name="module_sqs_template_mgmt_events"></a> [sqs\_template\_mgmt\_events](#module\_sqs\_template\_mgmt\_events) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/sqs | v2.0.8 |
56+
| <a name="module_sqs_template_table_events_pipe_dlq"></a> [sqs\_template\_table\_events\_pipe\_dlq](#module\_sqs\_template\_table\_events\_pipe\_dlq) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/sqs | v2.0.8 |
5457
| <a name="module_sqs_validate_letter_template_files"></a> [sqs\_validate\_letter\_template\_files](#module\_sqs\_validate\_letter\_template\_files) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/sqs | v2.0.1 |
5558
| <a name="module_submit_template_lambda"></a> [submit\_template\_lambda](#module\_submit\_template\_lambda) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/lambda | v2.0.4 |
5659
| <a name="module_update_template_lambda"></a> [update\_template\_lambda](#module\_update\_template\_lambda) | git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/lambda | v2.0.4 |

‎infrastructure/terraform/modules/backend-api/cloudwatch_log_group_pipe_template_table_events.tf‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
resource "aws_cloudwatch_log_group" "pipe_template_table_events" {
2+
name = "/aws/vendedlogs/pipes/${local.csi}-template-table-events"
3+
kms_key_id = var.kms_key_arn
4+
retention_in_days = var.log_retention_in_days
5+
}

‎infrastructure/terraform/modules/backend-api/dynamodb_table_templates.tf‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,4 +45,7 @@ resource "aws_dynamodb_table" "templates" {
4545
projection_type = "INCLUDE"
4646
non_key_attributes = ["owner"]
4747
}
48+
49+
stream_enabled = true
50+
stream_view_type = "NEW_AND_OLD_IMAGES"
4851
}

‎infrastructure/terraform/modules/backend-api/module_sqs_template_mgmt_events.tf‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
module "sqs_template_mgmt_events" {
2+
source = "git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/sqs?ref=v2.0.8"
3+
4+
aws_account_id = var.aws_account_id
5+
component = var.component
6+
environment = var.environment
7+
project = var.project
8+
region = var.region
9+
name = "template-mgmt-events"
10+
fifo_queue = true
11+
sqs_kms_key_arn = var.kms_key_arn
12+
}

‎infrastructure/terraform/modules/backend-api/module_sqs_template_table_events_pipe_dlq.tf‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
module "sqs_template_table_events_pipe_dlq" {
2+
source = "git::https://github.com/NHSDigital/nhs-notify-shared-modules.git//infrastructure/modules/sqs?ref=v2.0.8"
3+
4+
aws_account_id = var.aws_account_id
5+
component = var.component
6+
environment = var.environment
7+
project = var.project
8+
region = var.region
9+
name = "template-table-events-pipe-dead-letter"
10+
sqs_kms_key_arn = var.kms_key_arn
11+
}

‎infrastructure/terraform/modules/backend-api/pipes_pipe_template_table_events.tf‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
resource "aws_pipes_pipe" "template_table_events" {
2+
name = "${local.csi}-template-table-events"
3+
role_arn = aws_iam_role.pipe_template_table_events.arn
4+
source = aws_dynamodb_table.templates.stream_arn
5+
target = module.sqs_template_mgmt_events.sqs_queue_arn
6+
desired_state = var.enable_event_stream ? "RUNNING" : "STOPPED"
7+
kms_key_identifier = var.kms_key_arn
8+
9+
source_parameters {
10+
dynamodb_stream_parameters {
11+
starting_position = "TRIM_HORIZON"
12+
on_partial_batch_item_failure = "AUTOMATIC_BISECT"
13+
batch_size = 10
14+
maximum_batching_window_in_seconds = 5
15+
maximum_retry_attempts = 5
16+
maximum_record_age_in_seconds = -1
17+
18+
dead_letter_config {
19+
arn = module.sqs_template_table_events_pipe_dlq.sqs_queue_arn
20+
}
21+
}
22+
}
23+
24+
target_parameters {
25+
sqs_queue_parameters {
26+
message_group_id = "$.dynamodb.Keys.id.S"
27+
message_deduplication_id = "$.eventID"
28+
}
29+
}
30+
31+
log_configuration {
32+
level = "ERROR"
33+
include_execution_data = ["ALL"]
34+
35+
cloudwatch_logs_log_destination {
36+
log_group_arn = aws_cloudwatch_log_group.pipe_template_table_events.arn
37+
}
38+
}
39+
}
40+
41+
resource "aws_iam_role" "pipe_template_table_events" {
42+
name = "${local.csi}-pipe-template-table-events"
43+
description = "IAM Role for Pipe forward template table stream events to SQS"
44+
assume_role_policy = data.aws_iam_policy_document.pipes_trust_policy.json
45+
}
46+
47+
data "aws_iam_policy_document" "pipes_trust_policy" {
48+
statement {
49+
sid = "PipesAssumeRole"
50+
effect = "Allow"
51+
actions = ["sts:AssumeRole"]
52+
53+
principals {
54+
type = "Service"
55+
identifiers = ["pipes.amazonaws.com"]
56+
}
57+
}
58+
}
59+
60+
resource "aws_iam_role_policy" "pipe_template_table_events" {
61+
name = "${local.csi}-pipe-template-table-events"
62+
role = aws_iam_role.pipe_template_table_events.id
63+
policy = data.aws_iam_policy_document.pipe_template_table_events.json
64+
}
65+
66+
data "aws_iam_policy_document" "pipe_template_table_events" {
67+
version = "2012-10-17"
68+
69+
statement {
70+
sid = "AllowDynamoStreamRead"
71+
effect = "Allow"
72+
actions = [
73+
"dynamodb:DescribeStream",
74+
"dynamodb:GetRecords",
75+
"dynamodb:GetShardIterator",
76+
"dynamodb:ListStreams",
77+
]
78+
resources = [aws_dynamodb_table.templates.stream_arn]
79+
}
80+
81+
statement {
82+
sid = "AllowSqsSendMessage"
83+
effect = "Allow"
84+
actions = ["sqs:SendMessage"]
85+
resources = [
86+
module.sqs_template_mgmt_events.sqs_queue_arn,
87+
module.sqs_template_table_events_pipe_dlq.sqs_queue_arn,
88+
]
89+
}
90+
91+
statement {
92+
sid = "AllowKmsUsage"
93+
effect = "Allow"
94+
actions = [
95+
"kms:Decrypt",
96+
"kms:Encrypt",
97+
"kms:GenerateDataKey*"
98+
]
99+
resources = [var.kms_key_arn]
100+
}
101+
}

‎infrastructure/terraform/modules/backend-api/variables.tf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,12 @@ variable "enable_proofing" {
7676
description = "Enable proofing feature flag"
7777
}
7878

79+
variable "enable_event_stream" {
80+
type = bool
81+
description = "Enable DynamoDB streaming to SQS?"
82+
default = false
83+
}
84+
7985
variable "kms_key_arn" {
8086
type = string
8187
description = "KMS Key ARN"

0 commit comments

Comments
 (0)