[NRL-1386] Add var to enable/disable all reporting services per env and account-wide

mattdean3-nhs · mattdean3-nhs · commit 950d036a2fe1 · 2025-06-18T09:23:09.000+01:00
diff --git a/terraform/account-wide-infrastructure/dev/athena.tf b/terraform/account-wide-infrastructure/dev/athena.tf
@@ -1,4 +1,5 @@
 module "dev-athena" {
+  count              = var.enable_reporting ? 1 : 0
   source             = "../modules/athena"
   name_prefix        = "nhsd-nrlf--dev"
   target_bucket_name = module.dev-glue.target_bucket_name
diff --git a/terraform/account-wide-infrastructure/dev/ec2.tf b/terraform/account-wide-infrastructure/dev/ec2.tf
@@ -1,5 +1,5 @@
 module "vpc" {
-  count                          = var.enable_powerbi_auto_push ? 1 : 0
+  count                          = var.enable_reporting && var.enable_powerbi_auto_push ? 1 : 0
   source                         = "../modules/vpc"
   vpc_cidr_block                 = var.vpc_cidr_block
   enable_dns_hostnames           = var.enable_dns_hostnames
@@ -10,15 +10,15 @@ module "vpc" {
 }
 
 module "powerbi_gw_instance" {
-  count              = var.enable_powerbi_auto_push ? 1 : 0
+  count              = var.enable_reporting && var.enable_powerbi_auto_push ? 1 : 0
   source             = "../modules/powerbi-gw-ec2"
   use_custom_ami     = var.use_powerbi_gw_custom_ami
   instance_type      = var.powerbi_gw_instance_type
   name_prefix        = "nhsd-nrlf--dev-powerbi-gw"
   target_bucket_arn  = module.dev-glue.target_bucket_arn
   glue_kms_key_arn   = module.dev-glue.aws_kms_key_arn
-  athena_kms_key_arn = module.dev-athena.kms_key_arn
-  athena_bucket_arn  = module.dev-athena.bucket_arn
+  athena_kms_key_arn = module.dev-athena[0].kms_key_arn
+  athena_bucket_arn  = module.dev-athena[0].bucket_arn
 
   subnet_id       = module.vpc[0].private_subnet_id
   security_groups = [module.vpc[0].powerbi_gw_security_group_id]
diff --git a/terraform/account-wide-infrastructure/dev/glue.tf b/terraform/account-wide-infrastructure/dev/glue.tf
@@ -1,4 +1,5 @@
 module "dev-glue" {
+  is_enabled     = var.enable_reporting
   source         = "../modules/glue"
   name_prefix    = "nhsd-nrlf--dev"
   python_version = 3
diff --git a/terraform/account-wide-infrastructure/dev/vars.tf b/terraform/account-wide-infrastructure/dev/vars.tf
@@ -14,6 +14,12 @@ variable "devsandbox_api_domain_name" {
   default     = "dev-sandbox.api.record-locator.dev.national.nhs.uk"
 }
 
+variable "enable_reporting" {
+  type        = bool
+  description = "Enable account-wide reporting services in the dev account"
+  default     = false
+}
+
 variable "aws_azs" {
   type        = string
   description = "AWS Availability Zones"
diff --git a/terraform/account-wide-infrastructure/modules/glue/glue.tf b/terraform/account-wide-infrastructure/modules/glue/glue.tf
@@ -1,13 +1,17 @@
 # Create Glue Data Catalog Database
 resource "aws_glue_catalog_database" "log_database" {
+  count = var.is_enabled ? 1 : 0
+
   name         = "${var.name_prefix}-reporting"
   location_uri = "${aws_s3_bucket.target-data-bucket.id}/"
 }
 
 # Create Glue Crawler
 resource "aws_glue_crawler" "log_crawler" {
+  count = var.is_enabled ? 1 : 0
+
   name          = "${var.name_prefix}-log-crawler"
-  database_name = aws_glue_catalog_database.log_database.name
+  database_name = aws_glue_catalog_database.log_database[0].name
   role          = aws_iam_role.glue_service_role.name
   s3_target {
     path = "s3://${aws_s3_bucket.target-data-bucket.id}/consumer_countDocumentReference/"
@@ -53,14 +57,18 @@ resource "aws_glue_crawler" "log_crawler" {
   })
 }
 resource "aws_glue_trigger" "log_trigger" {
+  count = var.is_enabled ? 1 : 0
+
   name = "${var.name_prefix}-org-report-trigger"
   type = "ON_DEMAND"
   actions {
-    crawler_name = aws_glue_crawler.log_crawler.name
+    crawler_name = aws_glue_crawler.log_crawler[0].name
   }
 }
 
 resource "aws_glue_job" "glue_job" {
+  count = var.is_enabled ? 1 : 0
+
   name              = "${var.name_prefix}-glue-job"
   role_arn          = aws_iam_role.glue_service_role.arn
   description       = "Transfer logs from source to bucket"
diff --git a/terraform/account-wide-infrastructure/modules/glue/outputs.tf b/terraform/account-wide-infrastructure/modules/glue/outputs.tf
@@ -23,5 +23,5 @@ output "glue_crawler_name" {
 }
 
 output "glue_database" {
-  value = aws_glue_catalog_database.log_database.name
+  value = var.is_enabled ? aws_glue_catalog_database.log_database[0].name : ""
 }
diff --git a/terraform/account-wide-infrastructure/modules/glue/s3.tf b/terraform/account-wide-infrastructure/modules/glue/s3.tf
@@ -62,17 +62,13 @@ resource "aws_s3_bucket_lifecycle_configuration" "source-data-bucket-lifecycle"
     expiration {
       days = local.s3.expiration.days
     }
-
-    noncurrent_version_expiration {
-      noncurrent_days = local.s3.expiration.days
-    }
   }
 }
 
 resource "aws_s3_bucket_versioning" "source-data-bucket-versioning" {
   bucket = aws_s3_bucket.source-data-bucket.id
   versioning_configuration {
-    status = "Enabled"
+    status = "Disabled"
   }
 }
 
diff --git a/terraform/account-wide-infrastructure/modules/glue/vars.tf b/terraform/account-wide-infrastructure/modules/glue/vars.tf
@@ -22,3 +22,9 @@ variable "code_bucket" {
   description = "S3 bucket for Glue job scripts"
   default     = "code-bucket"
 }
+
+variable "is_enabled" {
+  type        = bool
+  description = "Flag to enable or disable the Glue module"
+  default     = true
+}
diff --git a/terraform/account-wide-infrastructure/prod/athena.tf b/terraform/account-wide-infrastructure/prod/athena.tf
@@ -1,4 +1,5 @@
 module "prod-athena" {
+  count              = var.enable_reporting ? 1 : 0
   source             = "../modules/athena"
   name_prefix        = "nhsd-nrlf--prod"
   target_bucket_name = module.prod-glue.target_bucket_name
diff --git a/terraform/account-wide-infrastructure/prod/ec2.tf b/terraform/account-wide-infrastructure/prod/ec2.tf
@@ -1,5 +1,5 @@
 module "vpc" {
-  count                          = var.enable_powerbi_auto_push ? 1 : 0
+  count                          = var.enable_reporting && var.enable_powerbi_auto_push ? 1 : 0
   source                         = "../modules/vpc"
   vpc_cidr_block                 = var.vpc_cidr_block
   enable_dns_hostnames           = var.enable_dns_hostnames
@@ -10,15 +10,15 @@ module "vpc" {
 }
 
 module "powerbi_gw_instance" {
-  count              = var.enable_powerbi_auto_push ? 1 : 0
+  count              = var.enable_reporting && var.enable_powerbi_auto_push ? 1 : 0
   source             = "../modules/powerbi-gw-ec2"
   use_custom_ami     = false
   instance_type      = var.powerbi_gw_instance_type
   name_prefix        = "nhsd-nrlf--test-powerbi-gw"
   target_bucket_arn  = module.prod-glue.target_bucket_arn
   glue_kms_key_arn   = module.prod-glue.aws_kms_key_arn
-  athena_kms_key_arn = module.prod-athena.kms_key_arn
-  athena_bucket_arn  = module.prod-athena.bucket_arn
+  athena_kms_key_arn = module.prod-athena[0].kms_key_arn
+  athena_bucket_arn  = module.prod-athena[0].bucket_arn
 
   subnet_id       = module.vpc[0].private_subnet_id
   security_groups = [module.vpc[0].powerbi_gw_security_group_id]
diff --git a/terraform/account-wide-infrastructure/prod/glue.tf b/terraform/account-wide-infrastructure/prod/glue.tf
@@ -1,4 +1,5 @@
 module "prod-glue" {
+  is_enabled     = var.enable_reporting
   source         = "../modules/glue"
   name_prefix    = "nhsd-nrlf--prod"
   python_version = 3
diff --git a/terraform/account-wide-infrastructure/prod/vars.tf b/terraform/account-wide-infrastructure/prod/vars.tf
@@ -21,6 +21,12 @@ variable "enable_dns_hostnames" {
   default     = true
 }
 
+variable "enable_reporting" {
+  type        = bool
+  description = "Enable account-wide reporting services in the prod account"
+  default     = false
+}
+
 variable "vpc_cidr_block" {
   type        = string
   description = "Base CIDR Block for VPC"
diff --git a/terraform/account-wide-infrastructure/test/athena.tf b/terraform/account-wide-infrastructure/test/athena.tf
@@ -1,4 +1,4 @@
-module "qa-athena" {
+/*module "qa-athena" {
   source             = "../modules/athena"
   name_prefix        = "nhsd-nrlf--qa"
   target_bucket_name = module.qa-glue.target_bucket_name
@@ -24,4 +24,12 @@ module "ref-athena" {
   name_prefix        = "nhsd-nrlf--ref"
   target_bucket_name = module.ref-glue.target_bucket_name
   glue_database      = module.ref-glue.glue_database
+}*/
+
+module "test-athena" {
+  count              = var.enable_reporting ? 1 : 0
+  source             = "../modules/athena"
+  name_prefix        = "nhsd-nrlf--test"
+  target_bucket_name = module.test-glue.target_bucket_name
+  glue_database      = module.test-glue.glue_database
 }
diff --git a/terraform/account-wide-infrastructure/test/ec2.tf b/terraform/account-wide-infrastructure/test/ec2.tf
@@ -1,5 +1,5 @@
 module "vpc" {
-  count                          = var.enable_powerbi_auto_push ? 1 : 0
+  count                          = var.enable_reporting && var.enable_powerbi_auto_push ? 1 : 0
   source                         = "../modules/vpc"
   vpc_cidr_block                 = var.vpc_cidr_block
   enable_dns_hostnames           = var.enable_dns_hostnames
@@ -10,15 +10,15 @@ module "vpc" {
 }
 
 module "powerbi_gw_instance" {
-  count              = var.enable_powerbi_auto_push ? 1 : 0
+  count              = var.enable_reporting && var.enable_powerbi_auto_push ? 1 : 0
   source             = "../modules/powerbi-gw-ec2"
   use_custom_ami     = var.use_powerbi_gw_custom_ami
   instance_type      = var.powerbi_gw_instance_type
   name_prefix        = "nhsd-nrlf--test-powerbi-gw"
-  target_bucket_arn  = module.int-glue.target_bucket_arn
-  glue_kms_key_arn   = module.int-glue.aws_kms_key_arn
-  athena_kms_key_arn = module.int-athena.kms_key_arn
-  athena_bucket_arn  = module.int-athena.bucket_arn
+  target_bucket_arn  = module.test-glue.target_bucket_arn
+  glue_kms_key_arn   = module.test-glue.aws_kms_key_arn
+  athena_kms_key_arn = module.test-athena[0].kms_key_arn
+  athena_bucket_arn  = module.test-athena[0].bucket_arn
 
   subnet_id       = module.vpc[0].private_subnet_id
   security_groups = [module.vpc[0].powerbi_gw_security_group_id]
diff --git a/terraform/account-wide-infrastructure/test/glue.tf b/terraform/account-wide-infrastructure/test/glue.tf
@@ -1,4 +1,4 @@
-module "qa-glue" {
+/*module "qa-glue" {
   source         = "../modules/glue"
   name_prefix    = "nhsd-nrlf--qa"
   python_version = 3
@@ -20,4 +20,11 @@ module "ref-glue" {
   source         = "../modules/glue"
   name_prefix    = "nhsd-nrlf--ref"
   python_version = 3
+}*/
+
+module "test-glue" {
+  is_enabled     = var.enable_reporting
+  source         = "../modules/glue"
+  name_prefix    = "nhsd-nrlf--test"
+  python_version = 3
 }
diff --git a/terraform/account-wide-infrastructure/test/vars.tf b/terraform/account-wide-infrastructure/test/vars.tf
@@ -29,6 +29,12 @@ variable "ref_api_domain_name" {
   default     = "ref.api.record-locator.ref.national.nhs.uk"
 }
 
+variable "enable_reporting" {
+  type        = bool
+  description = "Enable account-wide reporting services in the test account"
+  default     = false
+}
+
 variable "aws_azs" {
   type        = string
   description = "AWS Availability Zones"
diff --git a/terraform/infrastructure/data.tf b/terraform/infrastructure/data.tf
@@ -43,9 +43,9 @@ data "external" "current-info" {
 }
 
 data "aws_s3_bucket" "source-data-bucket" {
-  bucket = "${local.shared_prefix}-source-data-bucket"
+  bucket = "${local.account_prefix}-source-data-bucket"
 }
 
 data "aws_kms_key" "glue" {
-  key_id = "alias/${local.shared_prefix}-glue"
+  key_id = "alias/${local.account_prefix}-glue"
 }
diff --git a/terraform/infrastructure/etc/dev.tfvars b/terraform/infrastructure/etc/dev.tfvars
@@ -1,6 +1,8 @@
-account_name = "dev"
+account_name     = "dev"
+aws_account_name = "dev"
 
 domain                = "api.record-locator.dev.national.nhs.uk"
 public_domain         = "internal-dev.api.service.nhs.uk"
 public_sandbox_domain = "internal-dev-sandbox.api.service.nhs.uk"
 log_retention_period  = 90
+enable_reporting      = true
diff --git a/terraform/infrastructure/etc/int.tfvars b/terraform/infrastructure/etc/int.tfvars
@@ -1,8 +1,10 @@
-// TODO-NOW - Change this file name to int and update all references in codebase (and github repo config)
-account_name        = "int"
+account_name     = "int"
+aws_account_name = "test"
+
 domain              = "api.record-locator.int.national.nhs.uk"
 deletion_protection = true
 
 public_domain         = "int.api.service.nhs.uk"
 public_sandbox_domain = "sandbox.api.service.nhs.uk"
 log_retention_period  = 90
+enable_reporting      = false
diff --git a/terraform/infrastructure/etc/prod.tfvars b/terraform/infrastructure/etc/prod.tfvars
@@ -1,5 +1,8 @@
-account_name         = "prod"
+account_name     = "prod"
+aws_account_name = "prod"
+
 domain               = "api.record-locator.national.nhs.uk"
 public_domain        = "api.service.nhs.uk"
 deletion_protection  = true
 log_retention_period = 2192
+enable_reporting     = false
diff --git a/terraform/infrastructure/etc/qa.tfvars b/terraform/infrastructure/etc/qa.tfvars
@@ -1,6 +1,8 @@
-account_name = "qa"
+account_name     = "qa"
+aws_account_name = "test"
 
 domain                = "qa.record-locator.national.nhs.uk"
 public_domain         = "internal-qa.api.service.nhs.uk"
 public_sandbox_domain = "internal-qa-sandbox.api.service.nhs.uk"
 log_retention_period  = 90
+enable_reporting      = false
diff --git a/terraform/infrastructure/etc/ref.tfvars b/terraform/infrastructure/etc/ref.tfvars
@@ -1,4 +1,7 @@
-account_name         = "ref"
+account_name     = "ref"
+aws_account_name = "test"
+
 domain               = "api.record-locator.ref.national.nhs.uk"
 public_domain        = "ref.api.service.nhs.uk"
 log_retention_period = 30
+enable_reporting     = false
diff --git a/terraform/infrastructure/locals.tf b/terraform/infrastructure/locals.tf
@@ -4,6 +4,7 @@ locals {
   stack_name          = terraform.workspace
   deletion_protection = var.deletion_protection
   prefix              = "${local.project}--${local.stack_name}"
+  account_prefix      = "${local.project}--${var.aws_account_name}"
 
   kms = {
     deletion_window_in_days = 7
@@ -28,8 +29,8 @@ locals {
   public_domain = local.is_sandbox_env ? var.public_sandbox_domain : var.public_domain
 
   # Logic / vars for reporting
-  reporting_bucket_arn = data.aws_s3_bucket.source-data-bucket[0].arn
-  reporting_kms_arn    = data.aws_kms_key.glue[0].arn
+  reporting_bucket_arn = data.aws_s3_bucket.source-data-bucket.arn
+  reporting_kms_arn    = data.aws_kms_key.glue.arn
   firehose_lambda_subscriptions = [
     module.firehose__processor.firehose_subscription,
     module.firehose__processor.firehose_reporting_subscription
diff --git a/terraform/infrastructure/modules/firehose/cloudwatch.tf b/terraform/infrastructure/modules/firehose/cloudwatch.tf
@@ -15,5 +15,5 @@ resource "aws_cloudwatch_log_group" "firehose_reporting" {
 
 resource "aws_cloudwatch_log_stream" "firehose_reporting" {
   name           = "${var.prefix}-firehose-reporting"
-  log_group_name = aws_cloudwatch_log_group.firehose_reporting[0].name
+  log_group_name = aws_cloudwatch_log_group.firehose_reporting.name
 }
diff --git a/terraform/infrastructure/modules/firehose/kinesis.tf b/terraform/infrastructure/modules/firehose/kinesis.tf
@@ -68,7 +68,7 @@ resource "aws_kinesis_firehose_delivery_stream" "reporting_stream" {
     buffering_interval = 600
 
     processing_configuration {
-      enabled = "true"
+      enabled = var.enable_reporting_stream
 
       processors {
         type = "Decompression"
@@ -90,9 +90,9 @@ resource "aws_kinesis_firehose_delivery_stream" "reporting_stream" {
     }
 
     cloudwatch_logging_options {
-      enabled         = true
-      log_group_name  = aws_cloudwatch_log_group.firehose_reporting[0].name
-      log_stream_name = aws_cloudwatch_log_stream.firehose_reporting[0].name
+      enabled         = var.enable_reporting_stream
+      log_group_name  = aws_cloudwatch_log_group.firehose_reporting.name
+      log_stream_name = aws_cloudwatch_log_stream.firehose_reporting.name
     }
   }
 }
diff --git a/terraform/infrastructure/modules/firehose/locals.tf b/terraform/infrastructure/modules/firehose/locals.tf
@@ -32,13 +32,13 @@ locals {
   }
 
   iam_firehose = {
-    cloudwatch_reporting_log_group_arn  = aws_cloudwatch_log_group.firehose_reporting[0].arn
-    cloudwatch_reporting_log_stream_arn = aws_cloudwatch_log_stream.firehose_reporting[0].arn
+    cloudwatch_reporting_log_group_arn  = aws_cloudwatch_log_group.firehose_reporting.arn
+    cloudwatch_reporting_log_stream_arn = aws_cloudwatch_log_stream.firehose_reporting.arn
     reporting_s3_arn                    = "${var.reporting_bucket_arn}/*"
   }
 
   iam_subscriptions = {
-    firehose_reporting_stream_arn = aws_kinesis_firehose_delivery_stream.reporting_stream[0].arn
+    firehose_reporting_stream_arn = aws_kinesis_firehose_delivery_stream.reporting_stream.arn
   }
 
   iam_kms_resources = compact([
diff --git a/terraform/infrastructure/modules/firehose/vars.tf b/terraform/infrastructure/modules/firehose/vars.tf
@@ -35,6 +35,13 @@ variable "error_prefix" {
   default = "errors"
 }
 
+
+variable "enable_reporting_stream" {
+  type        = bool
+  description = "Enable the reporting delivery stream"
+  default     = false
+
+}
 variable "reporting_bucket_arn" {
   type    = string
   default = null
diff --git a/terraform/infrastructure/vars.tf b/terraform/infrastructure/vars.tf

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`module "dev-athena" {`
	`2`	`+ count = var.enable_reporting ? 1 : 0`
`2`	`3`	`source = "../modules/athena"`
`3`	`4`	`name_prefix = "nhsd-nrlf--dev"`
`4`	`5`	`target_bucket_name = module.dev-glue.target_bucket_name`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`module "dev-glue" {`
	`2`	`+ is_enabled = var.enable_reporting`
`2`	`3`	`source = "../modules/glue"`
`3`	`4`	`name_prefix = "nhsd-nrlf--dev"`
`4`	`5`	`python_version = 3`
Original file line number	Diff line number	Diff line change
`@@ -23,5 +23,5 @@ output "glue_crawler_name" {`
`23`	`23`	`}`
`24`	`24`
`25`	`25`	`output "glue_database" {`
`26`		`- value = aws_glue_catalog_database.log_database.name`
	`26`	`+ value = var.is_enabled ? aws_glue_catalog_database.log_database[0].name : ""`
`27`	`27`	`}`
Original file line number	Diff line number	Diff line change
`@@ -62,17 +62,13 @@ resource "aws_s3_bucket_lifecycle_configuration" "source-data-bucket-lifecycle"`
`62`	`62`	`expiration {`
`63`	`63`	`days = local.s3.expiration.days`
`64`	`64`	`}`
`65`		`-`
`66`		`- noncurrent_version_expiration {`
`67`		`- noncurrent_days = local.s3.expiration.days`
`68`		`- }`
`69`	`65`	`}`
`70`	`66`	`}`
`71`	`67`
`72`	`68`	`resource "aws_s3_bucket_versioning" "source-data-bucket-versioning" {`
`73`	`69`	`bucket = aws_s3_bucket.source-data-bucket.id`
`74`	`70`	`versioning_configuration {`
`75`		`- status = "Enabled"`
	`71`	`+ status = "Disabled"`
`76`	`72`	`}`
`77`	`73`	`}`
`78`	`74`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`module "prod-athena" {`
	`2`	`+ count = var.enable_reporting ? 1 : 0`
`2`	`3`	`source = "../modules/athena"`
`3`	`4`	`name_prefix = "nhsd-nrlf--prod"`
`4`	`5`	`target_bucket_name = module.prod-glue.target_bucket_name`