Skip to content

Commit 8a677c9

Browse files
eddalmond1eddalmond1
authored
Merge pull request #276 from NHSDigital/feature/eja-eli-304-push-cloudwatch-alarms-to-itoc-splunk
Feature/eja eli 304 push cloudwatch alarms to itoc splunk
2 parents e3be5de + 68fa0f1 commit 8a677c9

File tree

17 files changed

+427
-4
lines changed

17 files changed

+427
-4
lines changed

‎.github/workflows/base-deploy.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,9 @@ jobs:
131131
TF_VAR_API_CA_CERT: ${{ secrets.API_CA_CERT }}
132132
TF_VAR_API_CLIENT_CERT: ${{ secrets.API_CLIENT_CERT }}
133133
TF_VAR_API_PRIVATE_KEY_CERT: ${{ secrets.API_PRIVATE_KEY_CERT }}
134+
TF_VAR_SPLUNK_HEC_TOKEN: ${{ secrets.SPLUNK_HEC_TOKEN }}
135+
TF_VAR_SPLUNK_HEC_ENDPOINT: ${{ secrets.SPLUNK_HEC_ENDPOINT }}
136+
134137
working-directory: ./infrastructure
135138
shell: bash
136139
run: |

‎.github/workflows/cicd-2-publish.yaml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,13 +93,15 @@ jobs:
9393
role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/service-roles/github-actions-api-deployment-role
9494
aws-region: eu-west-2
9595

96-
- name: "Terraform Plan Stacks"
96+
- name: "Terraform Apply"
9797
env:
9898
ENVIRONMENT: dev
9999
WORKSPACE: "default"
100100
TF_VAR_API_CA_CERT: ${{ secrets.API_CA_CERT }}
101101
TF_VAR_API_CLIENT_CERT: ${{ secrets.API_CLIENT_CERT }}
102102
TF_VAR_API_PRIVATE_KEY_CERT: ${{ secrets.API_PRIVATE_KEY_CERT }}
103+
TF_VAR_SPLUNK_HEC_TOKEN: ${{ secrets.SPLUNK_HEC_TOKEN }}
104+
TF_VAR_SPLUNK_HEC_ENDPOINT: ${{ secrets.SPLUNK_HEC_ENDPOINT }}
103105

104106
# just planning for now for safety and until review
105107
run: |

‎.github/workflows/cicd-3-test.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,8 @@ jobs:
119119
TF_VAR_API_CA_CERT: ${{ secrets.API_CA_CERT }}
120120
TF_VAR_API_CLIENT_CERT: ${{ secrets.API_CLIENT_CERT }}
121121
TF_VAR_API_PRIVATE_KEY_CERT: ${{ secrets.API_PRIVATE_KEY_CERT }}
122-
122+
TF_VAR_SPLUNK_HEC_TOKEN: ${{ secrets.SPLUNK_HEC_TOKEN }}
123+
TF_VAR_SPLUNK_HEC_ENDPOINT: ${{ secrets.SPLUNK_HEC_ENDPOINT }}
123124
run: |
124125
mkdir -p ./build
125126
echo "Running: make terraform env=$ENVIRONMENT workspace=$WORKSPACE stack=networking tf-command=apply"

‎.github/workflows/manual-terraform-apply.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,8 @@ jobs:
6363
TF_VAR_API_CA_CERT: ${{ secrets.API_CA_CERT }}
6464
TF_VAR_API_CLIENT_CERT: ${{ secrets.API_CLIENT_CERT }}
6565
TF_VAR_API_PRIVATE_KEY_CERT: ${{ secrets.API_PRIVATE_KEY_CERT }}
66-
66+
TF_VAR_SPLUNK_HEC_TOKEN: ${{ secrets.SPLUNK_HEC_TOKEN }}
67+
TF_VAR_SPLUNK_HEC_ENDPOINT: ${{ secrets.SPLUNK_HEC_ENDPOINT }}
6768
run: |
6869
mkdir -p ./build
6970
echo "Running: make terraform env=$ENVIRONMENT workspace=$WORKSPACE stack=networking tf-command=plan args=\"-auto-approve\""

‎infrastructure/modules/splunk_forwarder/data.tf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
data "aws_caller_identity" "current" {}

‎infrastructure/modules/splunk_forwarder/firehose.tf‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# KMS Key for Firehose encryption
2+
resource "aws_kms_key" "firehose_splunk_cmk" {
3+
description = "KMS key for encrypting Kinesis Firehose delivery stream data"
4+
deletion_window_in_days = 7
5+
enable_key_rotation = true
6+
tags = {
7+
Name = "firehose-splunk-cmk"
8+
Purpose = "Firehose encryption"
9+
ManagedBy = "terraform"
10+
}
11+
}
12+
13+
# KMS Key Alias for easier identification
14+
resource "aws_kms_alias" "firehose_splunk_cmk_alias" {
15+
name = "alias/firehose-splunk-cmk"
16+
target_key_id = aws_kms_key.firehose_splunk_cmk.key_id
17+
}
18+
19+
resource "aws_kinesis_firehose_delivery_stream" "splunk_delivery_stream" {
20+
name = "splunk-alarm-events"
21+
destination = "splunk"
22+
server_side_encryption {
23+
enabled = true
24+
key_type = "CUSTOMER_MANAGED_CMK"
25+
key_arn = aws_kms_key.firehose_splunk_cmk.arn
26+
}
27+
# VPC configuration is only supported for HTTP endpoint destinations in Kinesis Firehose
28+
# For Splunk destinations, the service runs in AWS-managed VPC but you can control network access
29+
# via the subnets where EventBridge (the source) runs and IAM policies
30+
31+
splunk_configuration {
32+
hec_endpoint = var.splunk_hec_endpoint
33+
hec_token = var.splunk_hec_token
34+
hec_endpoint_type = "Raw"
35+
s3_backup_mode = "FailedEventsOnly"
36+
37+
s3_configuration {
38+
role_arn = var.splunk_firehose_s3_role_arn
39+
bucket_arn = var.splunk_firehose_s3_backup_arn
40+
buffering_size = 10
41+
buffering_interval = 400
42+
compression_format = "GZIP"
43+
}
44+
}
45+
}

‎infrastructure/modules/splunk_forwarder/iam.tf‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
# EventBridge IAM roles now defined in api-layer stack for specific integration
2+
3+
resource "aws_kms_key_policy" "firehose_splunk_cmk_policy" {
4+
key_id = aws_kms_key.firehose_splunk_cmk.id
5+
policy = jsonencode({
6+
Version = "2012-10-17",
7+
Statement = [
8+
{
9+
Sid = "AllowRootAccountFullAccess"
10+
Effect = "Allow"
11+
Principal = { AWS = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:root" }
12+
Action = "kms:*"
13+
Resource = "*"
14+
},
15+
{
16+
Sid = "AllowFirehoseServiceUseOfKey"
17+
Effect = "Allow"
18+
Principal = { Service = "firehose.amazonaws.com" }
19+
Action = [
20+
"kms:Encrypt",
21+
"kms:Decrypt",
22+
"kms:ReEncrypt*",
23+
"kms:GenerateDataKey*",
24+
"kms:DescribeKey"
25+
]
26+
Resource = "*"
27+
},
28+
{
29+
Sid = "AllowEventBridgeUseOfKey"
30+
Effect = "Allow"
31+
Principal = { Service = "events.amazonaws.com" }
32+
Action = [
33+
"kms:Encrypt",
34+
"kms:Decrypt",
35+
"kms:ReEncrypt*",
36+
"kms:GenerateDataKey*",
37+
"kms:DescribeKey"
38+
]
39+
Resource = "*"
40+
},
41+
{
42+
Sid = "AllowCloudWatchUseOfKey"
43+
Effect = "Allow"
44+
Principal = { Service = "cloudwatch.amazonaws.com" }
45+
Action = [
46+
"kms:Encrypt",
47+
"kms:Decrypt",
48+
"kms:ReEncrypt*",
49+
"kms:GenerateDataKey*",
50+
"kms:DescribeKey"
51+
]
52+
Resource = "*"
53+
}
54+
]
55+
})
56+
}

‎infrastructure/modules/splunk_forwarder/outputs.tf‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# Output the Firehose delivery stream ARN for use by EventBridge
2+
output "firehose_delivery_stream_arn" {
3+
description = "ARN of the Kinesis Firehose delivery stream for Splunk"
4+
value = aws_kinesis_firehose_delivery_stream.splunk_delivery_stream.arn
5+
}
6+
7+
# Output the KMS key ARN for reference
8+
output "firehose_kms_key_arn" {
9+
description = "ARN of the KMS key used for Firehose encryption"
10+
value = aws_kms_key.firehose_splunk_cmk.arn
11+
}

‎infrastructure/modules/splunk_forwarder/variables.tf‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
variable "splunk_hec_endpoint" {
2+
description = "Splunk HEC endpoint URL"
3+
type = string
4+
}
5+
6+
variable "splunk_hec_token" {
7+
description = "Splunk HEC token"
8+
type = string
9+
}
10+
11+
variable "splunk_firehose_s3_backup_arn" {
12+
description = "s3 bucket ARN for Firehose backups"
13+
type = string
14+
}
15+
16+
variable "splunk_firehose_s3_role_arn" {
17+
description = "IAM role ARN for Firehose to access S3"
18+
type = string
19+
}

‎infrastructure/stacks/api-layer/eventbridge.tf‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
# IAM role for EventBridge to write to Firehose
2+
resource "aws_iam_role" "eventbridge_firehose_role" {
3+
name = "${var.environment}-eventbridge-to-firehose-role"
4+
5+
assume_role_policy = jsonencode({
6+
Version = "2012-10-17"
7+
Statement = [{
8+
Effect = "Allow"
9+
Principal = {
10+
Service = "events.amazonaws.com"
11+
}
12+
Action = "sts:AssumeRole"
13+
}]
14+
})
15+
16+
tags = {
17+
Environment = var.environment
18+
Purpose = "splunk-forwarding"
19+
ManagedBy = "terraform"
20+
}
21+
}
22+
23+
# IAM policy for EventBridge to access Firehose
24+
resource "aws_iam_role_policy" "eventbridge_to_firehose_policy" {
25+
name = "${var.environment}-eventbridge-to-firehose-policy"
26+
role = aws_iam_role.eventbridge_firehose_role.id
27+
28+
policy = jsonencode({
29+
Version = "2012-10-17"
30+
Statement = [{
31+
Effect = "Allow"
32+
Action = [
33+
"firehose:PutRecord",
34+
"firehose:PutRecordBatch"
35+
]
36+
Resource = module.splunk_forwarder.firehose_delivery_stream_arn
37+
}]
38+
})
39+
}
40+
41+
# EventBridge rule to capture CloudWatch alarm state changes
42+
resource "aws_cloudwatch_event_rule" "alarm_state_change" {
43+
name = "cloudwatch-alarm-state-change-to-splunk"
44+
description = "Forward CloudWatch alarm state changes to Splunk via Firehose"
45+
46+
event_pattern = jsonencode({
47+
source = ["aws.cloudwatch"]
48+
detail-type = ["CloudWatch Alarm State Change"]
49+
})
50+
51+
tags = {
52+
Environment = var.environment
53+
Purpose = "splunk-forwarding"
54+
ManagedBy = "terraform"
55+
}
56+
}
57+
58+
# EventBridge target to send events to Firehose
59+
resource "aws_cloudwatch_event_target" "firehose_target" {
60+
rule = aws_cloudwatch_event_rule.alarm_state_change.name
61+
arn = module.splunk_forwarder.firehose_delivery_stream_arn
62+
role_arn = aws_iam_role.eventbridge_firehose_role.arn
63+
64+
# Transform the CloudWatch alarm event into a format suitable for Splunk
65+
input_transformer {
66+
input_paths = {
67+
account = "$.account"
68+
region = "$.region"
69+
time = "$.time"
70+
alarm_name = "$.detail.alarmName"
71+
new_state = "$.detail.state.value"
72+
old_state = "$.detail.previousState.value"
73+
reason = "$.detail.state.reason"
74+
}
75+
76+
input_template = jsonencode({
77+
time = "<time>"
78+
source = "elid-${var.environment}:cloudwatch:alarm"
79+
sourcetype = "aws:cloudwatch:alarm"
80+
event = {
81+
alarm_name = "<alarm_name>"
82+
new_state = "<new_state>"
83+
old_state = "<old_state>"
84+
reason = "<reason>"
85+
region = "<region>"
86+
}
87+
})
88+
}
89+
}

0 commit comments

Comments
 (0)