Feature/eja eli 304 push cloudwatch alarms to itoc splunk

.github/workflows/base-deploy.yml

@@ -131,6 +131,9 @@ jobs:
           TF_VAR_API_CA_CERT: ${{ secrets.API_CA_CERT }}
           TF_VAR_API_CLIENT_CERT: ${{ secrets.API_CLIENT_CERT }}
           TF_VAR_API_PRIVATE_KEY_CERT: ${{ secrets.API_PRIVATE_KEY_CERT }}
+          TF_VAR_SPLUNK_HEC_TOKEN: ${{ secrets.SPLUNK_HEC_TOKEN }}
+          TF_VAR_SPLUNK_HEC_ENDPOINT: ${{ secrets.SPLUNK_HEC_ENDPOINT }}
+
         working-directory: ./infrastructure
         shell: bash
         run: |

.github/workflows/cicd-2-publish.yaml

@@ -93,13 +93,15 @@ jobs:
           role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/service-roles/github-actions-api-deployment-role
           aws-region: eu-west-2
 
-      - name: "Terraform Plan Stacks"
+      - name: "Terraform Apply"
         env:
           ENVIRONMENT: dev
           WORKSPACE: "default"
           TF_VAR_API_CA_CERT: ${{ secrets.API_CA_CERT }}
           TF_VAR_API_CLIENT_CERT: ${{ secrets.API_CLIENT_CERT }}
           TF_VAR_API_PRIVATE_KEY_CERT: ${{ secrets.API_PRIVATE_KEY_CERT }}
+          TF_VAR_SPLUNK_HEC_TOKEN: ${{ secrets.SPLUNK_HEC_TOKEN }}
+          TF_VAR_SPLUNK_HEC_ENDPOINT: ${{ secrets.SPLUNK_HEC_ENDPOINT }}
 
         # just planning for now for safety and until review
         run: |

.github/workflows/cicd-3-test.yaml

@@ -119,7 +119,8 @@ jobs:
           TF_VAR_API_CA_CERT: ${{ secrets.API_CA_CERT }}
           TF_VAR_API_CLIENT_CERT: ${{ secrets.API_CLIENT_CERT }}
           TF_VAR_API_PRIVATE_KEY_CERT: ${{ secrets.API_PRIVATE_KEY_CERT }}
-
+          TF_VAR_SPLUNK_HEC_TOKEN: ${{ secrets.SPLUNK_HEC_TOKEN }}
+          TF_VAR_SPLUNK_HEC_ENDPOINT: ${{ secrets.SPLUNK_HEC_ENDPOINT }}
         run: |
           mkdir -p ./build
           echo "Running: make terraform env=$ENVIRONMENT workspace=$WORKSPACE stack=networking tf-command=apply"

.github/workflows/manual-terraform-apply.yaml

@@ -63,7 +63,8 @@ jobs:
           TF_VAR_API_CA_CERT: ${{ secrets.API_CA_CERT }}
           TF_VAR_API_CLIENT_CERT: ${{ secrets.API_CLIENT_CERT }}
           TF_VAR_API_PRIVATE_KEY_CERT: ${{ secrets.API_PRIVATE_KEY_CERT }}
-
+          TF_VAR_SPLUNK_HEC_TOKEN: ${{ secrets.SPLUNK_HEC_TOKEN }}
+          TF_VAR_SPLUNK_HEC_ENDPOINT: ${{ secrets.SPLUNK_HEC_ENDPOINT }}
         run: |
           mkdir -p ./build
           echo "Running: make terraform env=$ENVIRONMENT workspace=$WORKSPACE stack=networking tf-command=plan args=\"-auto-approve\""

infrastructure/modules/lambda/kms.tf

@@ -7,7 +7,7 @@ resource "aws_kms_key" "lambda_cmk" {
 }
 
 resource "aws_kms_alias" "lambda_cmk" {
-  name          = "alias/${terraform.workspace == "default" ? "" : "${terraform.workspace}-"}${var.lambda_func_name}-cmk"
+  name          = "alias/${terraform.workspace == "default" ? "" : "${terraform.workspace}-"}${var.lambda_func_name}-key"
   target_key_id = aws_kms_key.lambda_cmk.key_id
 }
 

infrastructure/modules/lambda/lambda.tf

@@ -1,5 +1,5 @@
 resource "aws_lambda_function" "eligibility_signposting_lambda" {
-  #checkov:skip=CKV_AWS_116: No deadletter queue is configured for this Lambda function, yet
+  #checkov:skip=CKV_AWS_116: No deadletter queue is configured for this Lambda function, as the requests are synchronous
   #checkov:skip=CKV_AWS_115: Concurrent execution limit will be set at APIM level, not at Lambda level
   #checkov:skip=CKV_AWS_272: Skipping code signing but flagged to create ticket to investigate on ELI-238
   # If the file is not in the current working directory you will need to include a
@@ -11,7 +11,7 @@ resource "aws_lambda_function" "eligibility_signposting_lambda" {
 
   source_code_hash = filebase64sha256(var.file_name)
 
-  runtime     = "python3.13"
+  runtime     = var.runtime
   timeout     = 30
   memory_size = 2048
 
@@ -28,12 +28,35 @@ resource "aws_lambda_function" "eligibility_signposting_lambda" {
 
   kms_key_arn = aws_kms_key.lambda_cmk.arn
 
+  publish = true
+
   vpc_config {
     subnet_ids         = var.vpc_intra_subnets
     security_group_ids = var.security_group_ids
   }
 
+  layers = compact([
+  var.environment == "prod" ? "arn:aws:lambda:${var.region}:580247275435:layer:LambdaInsightsExtension:${var.lambda_insights_extension_version}" : null
+  ])
+
   tracing_config {
     mode = "Active"
   }
 }
+
+# lambda alias required for provisioning concurrency
+resource "aws_lambda_alias" "campaign_alias" {
+  count            = var.environment == "prod" ? 1 : 0
+  name             = "live"
+  function_name    = aws_lambda_function.eligibility_signposting_lambda.function_name
+  function_version = aws_lambda_function.eligibility_signposting_lambda.version
+}
+
+# provisioned concurrency - number of pre-warmed lambda containers
+resource "aws_lambda_provisioned_concurrency_config" "campaign_pc" {
+  count                             = var.environment == "prod" ? 1 : 0
+  function_name                     = var.lambda_func_name
+  qualifier                         = aws_lambda_alias.campaign_alias[0].name
+  provisioned_concurrent_executions = var.provisioned_concurrency_count
+}
+

infrastructure/modules/lambda/variables.tf

@@ -1,5 +1,10 @@
 variable "eligibility_lambda_role_arn" {
-  description = "lambda read role arn for dynamodb"
+  description = "lambda role arn"
+  type        = string
+}
+
+variable "eligibility_lambda_role_name" {
+  description = "lambda role name"
   type        = string
 }
 
@@ -8,6 +13,12 @@ variable "lambda_func_name" {
   type        = string
 }
 
+variable "runtime" {
+  description = "runtime of the Lambda function"
+  type        = string
+}
+
+
 variable "vpc_intra_subnets" {
   description = "vpc private subnets for lambda"
   type        = list(string)
@@ -52,3 +63,13 @@ variable "enable_xray_patching"{
   description = "flag to enable xray tracing, which puts an entry for dynamodb, s3 and firehose in trace map"
   type        = string
 }
+
+variable "provisioned_concurrency_count" {
+  description = "Number of prewarmed Lambda instances"
+  type        = number
+}
+
+variable "lambda_insights_extension_version" {
+  description = "version number of LambdaInsightsExtension"
+  type        = number
+}

infrastructure/modules/splunk_forwarder/data.tf

@@ -0,0 +1 @@
+data "aws_caller_identity" "current" {}

infrastructure/modules/splunk_forwarder/firehose.tf

@@ -0,0 +1,45 @@
+# KMS Key for Firehose encryption
+resource "aws_kms_key" "firehose_splunk_cmk" {
+  description             = "KMS key for encrypting Kinesis Firehose delivery stream data"
+  deletion_window_in_days = 7
+  enable_key_rotation     = true
+  tags = {
+    Name      = "firehose-splunk-cmk"
+    Purpose   = "Firehose encryption"
+    ManagedBy = "terraform"
+  }
+}
+
+# KMS Key Alias for easier identification
+resource "aws_kms_alias" "firehose_splunk_cmk_alias" {
+  name          = "alias/firehose-splunk-cmk"
+  target_key_id = aws_kms_key.firehose_splunk_cmk.key_id
+}
+
+resource "aws_kinesis_firehose_delivery_stream" "splunk_delivery_stream" {
+  name        = "splunk-alarm-events"
+  destination = "splunk"
+  server_side_encryption {
+    enabled  = true
+    key_type = "CUSTOMER_MANAGED_CMK"
+    key_arn  = aws_kms_key.firehose_splunk_cmk.arn
+  }
+  # VPC configuration is only supported for HTTP endpoint destinations in Kinesis Firehose
+  # For Splunk destinations, the service runs in AWS-managed VPC but you can control network access
+  # via the subnets where EventBridge (the source) runs and IAM policies
+
+  splunk_configuration {
+    hec_endpoint      = var.splunk_hec_endpoint
+    hec_token         = var.splunk_hec_token
+    hec_endpoint_type = "Raw"
+    s3_backup_mode    = "FailedEventsOnly"
+
+    s3_configuration {
+      role_arn           = var.splunk_firehose_s3_role_arn
+      bucket_arn         = var.splunk_firehose_s3_backup_arn
+      buffering_size     = 10
+      buffering_interval = 400
+      compression_format = "GZIP"
+    }
+  }
+}

infrastructure/modules/splunk_forwarder/iam.tf

@@ -0,0 +1,56 @@
+# EventBridge IAM roles now defined in api-layer stack for specific integration
+
+resource "aws_kms_key_policy" "firehose_splunk_cmk_policy" {
+  key_id = aws_kms_key.firehose_splunk_cmk.id
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+        Sid       = "AllowRootAccountFullAccess"
+        Effect    = "Allow"
+        Principal = { AWS = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:root" }
+        Action    = "kms:*"
+        Resource  = "*"
+      },
+      {
+        Sid       = "AllowFirehoseServiceUseOfKey"
+        Effect    = "Allow"
+        Principal = { Service = "firehose.amazonaws.com" }
+        Action = [
+          "kms:Encrypt",
+          "kms:Decrypt",
+          "kms:ReEncrypt*",
+          "kms:GenerateDataKey*",
+          "kms:DescribeKey"
+        ]
+        Resource = "*"
+      },
+      {
+        Sid       = "AllowEventBridgeUseOfKey"
+        Effect    = "Allow"
+        Principal = { Service = "events.amazonaws.com" }
+        Action = [
+          "kms:Encrypt",
+          "kms:Decrypt",
+          "kms:ReEncrypt*",
+          "kms:GenerateDataKey*",
+          "kms:DescribeKey"
+        ]
+        Resource = "*"
+      },
+      {
+        Sid       = "AllowCloudWatchUseOfKey"
+        Effect    = "Allow"
+        Principal = { Service = "cloudwatch.amazonaws.com" }
+        Action = [
+          "kms:Encrypt",
+          "kms:Decrypt",
+          "kms:ReEncrypt*",
+          "kms:GenerateDataKey*",
+          "kms:DescribeKey"
+        ]
+        Resource = "*"
+      }
+    ]
+  })
+}

infrastructure/modules/splunk_forwarder/outputs.tf

@@ -0,0 +1,11 @@
+# Output the Firehose delivery stream ARN for use by EventBridge
+output "firehose_delivery_stream_arn" {
+  description = "ARN of the Kinesis Firehose delivery stream for Splunk"
+  value       = aws_kinesis_firehose_delivery_stream.splunk_delivery_stream.arn
+}
+
+# Output the KMS key ARN for reference
+output "firehose_kms_key_arn" {
+  description = "ARN of the KMS key used for Firehose encryption"
+  value       = aws_kms_key.firehose_splunk_cmk.arn
+}

infrastructure/modules/splunk_forwarder/variables.tf

@@ -0,0 +1,19 @@
+variable "splunk_hec_endpoint" {
+  description = "Splunk HEC endpoint URL"
+  type        = string
+}
+
+variable "splunk_hec_token" {
+  description = "Splunk HEC token"
+  type        = string
+}
+
+variable "splunk_firehose_s3_backup_arn" {
+  description = "s3 bucket ARN for Firehose backups"
+  type        = string
+}
+
+variable "splunk_firehose_s3_role_arn" {
+  description = "IAM role ARN for Firehose to access S3"
+  type        = string
+}

infrastructure/stacks/api-layer/eventbridge.tf

@@ -0,0 +1,89 @@
+# IAM role for EventBridge to write to Firehose
+resource "aws_iam_role" "eventbridge_firehose_role" {
+  name = "${var.environment}-eventbridge-to-firehose-role"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [{
+      Effect = "Allow"
+      Principal = {
+        Service = "events.amazonaws.com"
+      }
+      Action = "sts:AssumeRole"
+    }]
+  })
+
+  tags = {
+    Environment = var.environment
+    Purpose     = "splunk-forwarding"
+    ManagedBy   = "terraform"
+  }
+}
+
+# IAM policy for EventBridge to access Firehose
+resource "aws_iam_role_policy" "eventbridge_to_firehose_policy" {
+  name = "${var.environment}-eventbridge-to-firehose-policy"
+  role = aws_iam_role.eventbridge_firehose_role.id
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [{
+      Effect = "Allow"
+      Action = [
+        "firehose:PutRecord",
+        "firehose:PutRecordBatch"
+      ]
+      Resource = module.splunk_forwarder.firehose_delivery_stream_arn
+    }]
+  })
+}
+
+# EventBridge rule to capture CloudWatch alarm state changes
+resource "aws_cloudwatch_event_rule" "alarm_state_change" {
+  name        = "cloudwatch-alarm-state-change-to-splunk"
+  description = "Forward CloudWatch alarm state changes to Splunk via Firehose"
+
+  event_pattern = jsonencode({
+    source      = ["aws.cloudwatch"]
+    detail-type = ["CloudWatch Alarm State Change"]
+  })
+
+  tags = {
+    Environment = var.environment
+    Purpose     = "splunk-forwarding"
+    ManagedBy   = "terraform"
+  }
+}
+
+# EventBridge target to send events to Firehose
+resource "aws_cloudwatch_event_target" "firehose_target" {
+  rule     = aws_cloudwatch_event_rule.alarm_state_change.name
+  arn      = module.splunk_forwarder.firehose_delivery_stream_arn
+  role_arn = aws_iam_role.eventbridge_firehose_role.arn
+
+  # Transform the CloudWatch alarm event into a format suitable for Splunk
+  input_transformer {
+    input_paths = {
+      account    = "$.account"
+      region     = "$.region"
+      time       = "$.time"
+      alarm_name = "$.detail.alarmName"
+      new_state  = "$.detail.state.value"
+      old_state  = "$.detail.previousState.value"
+      reason     = "$.detail.state.reason"
+    }
+
+    input_template = jsonencode({
+      time = "<time>"
+      source = "elid-${var.environment}:cloudwatch:alarm"
+      sourcetype = "aws:cloudwatch:alarm"
+      event = {
+        alarm_name  = "<alarm_name>"
+        new_state   = "<new_state>"
+        old_state   = "<old_state>"
+        reason      = "<reason>"
+        region      = "<region>"
+      }
+    })
+  }
+}