[NRL-1386] Add missing glue_database to test+prod. Fix TF apply state issues

mattdean3-nhs · mattdean3-nhs · commit 0e8116a0bcab · 2025-06-16T16:24:30.000+01:00
diff --git a/terraform/account-wide-infrastructure/dev/ec2.tf b/terraform/account-wide-infrastructure/dev/ec2.tf
@@ -14,7 +14,7 @@ module "powerbi_gw_instance" {
   source             = "../modules/powerbi-gw-ec2"
   use_custom_ami     = var.use_powerbi_gw_custom_ami
   instance_type      = var.powerbi_gw_instance_type
-  name_prefix        = "nhsd-nrlf--dev-powerbi-gw-v2"
+  name_prefix        = "nhsd-nrlf--dev-powerbi-gw"
   target_bucket_arn  = module.dev-glue.target_bucket_arn
   glue_kms_key_arn   = module.dev-glue.aws_kms_key_arn
   athena_kms_key_arn = module.dev-athena.kms_key_arn
diff --git a/terraform/account-wide-infrastructure/modules/glue/glue.tf b/terraform/account-wide-infrastructure/modules/glue/glue.tf
@@ -10,37 +10,37 @@ resource "aws_glue_crawler" "log_crawler" {
   database_name = aws_glue_catalog_database.log_database.name
   role          = aws_iam_role.glue_service_role.name
   s3_target {
-    path = "${aws_s3_bucket.target-data-bucket.id}/consumer_countDocumentReference/"
+    path = "s3://${aws_s3_bucket.target-data-bucket.id}/consumer_countDocumentReference/"
   }
   s3_target {
-    path = "${aws_s3_bucket.target-data-bucket.id}/consumer_readDocumentReference/"
+    path = "s3://${aws_s3_bucket.target-data-bucket.id}/consumer_readDocumentReference/"
   }
   s3_target {
-    path = "${aws_s3_bucket.target-data-bucket.id}/consumer_searchDocumentReference/"
+    path = "s3://${aws_s3_bucket.target-data-bucket.id}/consumer_searchDocumentReference/"
   }
   s3_target {
-    path = "${aws_s3_bucket.target-data-bucket.id}/consumer_searchPostDocumentReference/"
+    path = "s3://${aws_s3_bucket.target-data-bucket.id}/consumer_searchPostDocumentReference/"
   }
   s3_target {
-    path = "${aws_s3_bucket.target-data-bucket.id}/producer_createDocumentReference/"
+    path = "s3://${aws_s3_bucket.target-data-bucket.id}/producer_createDocumentReference/"
   }
   s3_target {
-    path = "${aws_s3_bucket.target-data-bucket.id}/producer_deleteDocumentReference/"
+    path = "s3://${aws_s3_bucket.target-data-bucket.id}/producer_deleteDocumentReference/"
   }
   s3_target {
-    path = "${aws_s3_bucket.target-data-bucket.id}/producer_readDocumentReference/"
+    path = "s3://${aws_s3_bucket.target-data-bucket.id}/producer_readDocumentReference/"
   }
   s3_target {
-    path = "${aws_s3_bucket.target-data-bucket.id}/producer_searchDocumentReference/"
+    path = "s3://${aws_s3_bucket.target-data-bucket.id}/producer_searchDocumentReference/"
   }
   s3_target {
-    path = "${aws_s3_bucket.target-data-bucket.id}/producer_searchPostDocumentReference/"
+    path = "s3://${aws_s3_bucket.target-data-bucket.id}/producer_searchPostDocumentReference/"
   }
   s3_target {
-    path = "${aws_s3_bucket.target-data-bucket.id}/producer_updateDocumentReference/"
+    path = "s3://${aws_s3_bucket.target-data-bucket.id}/producer_updateDocumentReference/"
   }
   s3_target {
-    path = "${aws_s3_bucket.target-data-bucket.id}/producer_upsertDocumentReference/"
+    path = "s3://${aws_s3_bucket.target-data-bucket.id}/producer_upsertDocumentReference/"
   }
   schema_change_policy {
     delete_behavior = "LOG"
diff --git a/terraform/account-wide-infrastructure/modules/glue/s3.tf b/terraform/account-wide-infrastructure/modules/glue/s3.tf
@@ -180,10 +180,10 @@ resource "aws_s3_bucket_public_access_block" "code-bucket-public-access-block" {
 }
 
 resource "aws_s3_object" "script" {
-  bucket = aws_s3_bucket.code-bucket.bucket
-  key    = "main.py"
-  source = "${path.module}/src/main.py"
-  etag   = filemd5("${path.module}/src/main.py")
+  bucket      = aws_s3_bucket.code-bucket.bucket
+  key         = "main.py"
+  source      = "${path.module}/src/main.py"
+  source_hash = filemd5("${path.module}/src/main.py")
 }
 
 data "archive_file" "python" {
@@ -194,8 +194,8 @@ data "archive_file" "python" {
 }
 
 resource "aws_s3_object" "zip" {
-  bucket = aws_s3_bucket.code-bucket.bucket
-  key    = "src.zip"
-  source = data.archive_file.python.output_path
-  etag   = filemd5(data.archive_file.python.output_path)
+  bucket      = aws_s3_bucket.code-bucket.bucket
+  key         = "src.zip"
+  source      = data.archive_file.python.output_path
+  source_hash = filemd5(data.archive_file.python.output_path)
 }
diff --git a/terraform/account-wide-infrastructure/modules/powerbi-gw-ec2/ec2.tf b/terraform/account-wide-infrastructure/modules/powerbi-gw-ec2/ec2.tf
@@ -1,13 +1,11 @@
-resource "aws_instance" "web" {
+resource "aws_instance" "powerbi_gw" {
   associate_public_ip_address = false
   iam_instance_profile        = aws_iam_instance_profile.powerbi_profile.name
   ami                         = local.selected_ami_id
   instance_type               = var.instance_type
   key_name                    = aws_key_pair.ec2_key_pair.key_name
   subnet_id                   = var.subnet_id
-  # TODO WORK OUT WHY THIS IS HAVING SUCH AN ISSUE.....
-  #security_groups             = var.security_groups
-  vpc_security_group_ids = var.security_groups
+  vpc_security_group_ids      = var.security_groups
 
   user_data = file("${path.module}/scripts/user_data.tpl")
 
diff --git a/terraform/account-wide-infrastructure/modules/powerbi-gw-ec2/outputs.tf b/terraform/account-wide-infrastructure/modules/powerbi-gw-ec2/outputs.tf
@@ -1,7 +1,3 @@
 output "instance_id" {
-  value = aws_instance.web.id
-}
-
-output "public_ip" {
-  value = aws_instance.web.public_ip
+  value = aws_instance.powerbi_gw.id
 }
diff --git a/terraform/account-wide-infrastructure/prod/athena.tf b/terraform/account-wide-infrastructure/prod/athena.tf
@@ -2,4 +2,5 @@ module "prod-athena" {
   source             = "../modules/athena"
   name_prefix        = "nhsd-nrlf--prod"
   target_bucket_name = module.prod-glue.target_bucket_name
+  glue_database      = module.prod-glue.glue_database
 }
diff --git a/terraform/account-wide-infrastructure/prod/ec2.tf b/terraform/account-wide-infrastructure/prod/ec2.tf
@@ -14,7 +14,7 @@ module "powerbi_gw_instance" {
   source             = "../modules/powerbi-gw-ec2"
   use_custom_ami     = false
   instance_type      = var.powerbi_gw_instance_type
-  name_prefix        = "nhsd-nrlf--test-powerbi-gw-v2"
+  name_prefix        = "nhsd-nrlf--test-powerbi-gw"
   target_bucket_arn  = module.prod-glue.target_bucket_arn
   glue_kms_key_arn   = module.prod-glue.aws_kms_key_arn
   athena_kms_key_arn = module.prod-athena.kms_key_arn
diff --git a/terraform/account-wide-infrastructure/test/athena.tf b/terraform/account-wide-infrastructure/test/athena.tf
@@ -2,22 +2,26 @@ module "qa-athena" {
   source             = "../modules/athena"
   name_prefix        = "nhsd-nrlf--qa"
   target_bucket_name = module.qa-glue.target_bucket_name
+  glue_database      = module.qa-glue.glue_database
 }
 
 module "int-athena" {
   source             = "../modules/athena"
   name_prefix        = "nhsd-nrlf--int"
   target_bucket_name = module.int-glue.target_bucket_name
+  glue_database      = module.int-glue.glue_database
 }
 
 module "int-sandbox-athena" {
   source             = "../modules/athena"
   name_prefix        = "nhsd-nrlf--int-sandbox"
   target_bucket_name = module.int-sandbox-glue.target_bucket_name
+  glue_database      = module.int-sandbox-glue.glue_database
 }
 
 module "ref-athena" {
   source             = "../modules/athena"
   name_prefix        = "nhsd-nrlf--ref"
   target_bucket_name = module.ref-glue.target_bucket_name
+  glue_database      = module.ref-glue.glue_database
 }
diff --git a/terraform/account-wide-infrastructure/test/ec2.tf b/terraform/account-wide-infrastructure/test/ec2.tf
@@ -14,7 +14,7 @@ module "powerbi_gw_instance" {
   source             = "../modules/powerbi-gw-ec2"
   use_custom_ami     = var.use_powerbi_gw_custom_ami
   instance_type      = var.powerbi_gw_instance_type
-  name_prefix        = "nhsd-nrlf--test-powerbi-gw-v2"
+  name_prefix        = "nhsd-nrlf--test-powerbi-gw"
   target_bucket_arn  = module.int-glue.target_bucket_arn
   glue_kms_key_arn   = module.int-glue.aws_kms_key_arn
   athena_kms_key_arn = module.int-athena.kms_key_arn

Original file line number	Diff line number	Diff line change
`@@ -10,37 +10,37 @@ resource "aws_glue_crawler" "log_crawler" {`
`10`	`10`	`database_name = aws_glue_catalog_database.log_database.name`
`11`	`11`	`role = aws_iam_role.glue_service_role.name`
`12`	`12`	`s3_target {`
`13`		`- path = "${aws_s3_bucket.target-data-bucket.id}/consumer_countDocumentReference/"`
	`13`	`+ path = "s3://${aws_s3_bucket.target-data-bucket.id}/consumer_countDocumentReference/"`
`14`	`14`	`}`
`15`	`15`	`s3_target {`
`16`		`- path = "${aws_s3_bucket.target-data-bucket.id}/consumer_readDocumentReference/"`
	`16`	`+ path = "s3://${aws_s3_bucket.target-data-bucket.id}/consumer_readDocumentReference/"`
`17`	`17`	`}`
`18`	`18`	`s3_target {`
`19`		`- path = "${aws_s3_bucket.target-data-bucket.id}/consumer_searchDocumentReference/"`
	`19`	`+ path = "s3://${aws_s3_bucket.target-data-bucket.id}/consumer_searchDocumentReference/"`
`20`	`20`	`}`
`21`	`21`	`s3_target {`
`22`		`- path = "${aws_s3_bucket.target-data-bucket.id}/consumer_searchPostDocumentReference/"`
	`22`	`+ path = "s3://${aws_s3_bucket.target-data-bucket.id}/consumer_searchPostDocumentReference/"`
`23`	`23`	`}`
`24`	`24`	`s3_target {`
`25`		`- path = "${aws_s3_bucket.target-data-bucket.id}/producer_createDocumentReference/"`
	`25`	`+ path = "s3://${aws_s3_bucket.target-data-bucket.id}/producer_createDocumentReference/"`
`26`	`26`	`}`
`27`	`27`	`s3_target {`
`28`		`- path = "${aws_s3_bucket.target-data-bucket.id}/producer_deleteDocumentReference/"`
	`28`	`+ path = "s3://${aws_s3_bucket.target-data-bucket.id}/producer_deleteDocumentReference/"`
`29`	`29`	`}`
`30`	`30`	`s3_target {`
`31`		`- path = "${aws_s3_bucket.target-data-bucket.id}/producer_readDocumentReference/"`
	`31`	`+ path = "s3://${aws_s3_bucket.target-data-bucket.id}/producer_readDocumentReference/"`
`32`	`32`	`}`
`33`	`33`	`s3_target {`
`34`		`- path = "${aws_s3_bucket.target-data-bucket.id}/producer_searchDocumentReference/"`
	`34`	`+ path = "s3://${aws_s3_bucket.target-data-bucket.id}/producer_searchDocumentReference/"`
`35`	`35`	`}`
`36`	`36`	`s3_target {`
`37`		`- path = "${aws_s3_bucket.target-data-bucket.id}/producer_searchPostDocumentReference/"`
	`37`	`+ path = "s3://${aws_s3_bucket.target-data-bucket.id}/producer_searchPostDocumentReference/"`
`38`	`38`	`}`
`39`	`39`	`s3_target {`
`40`		`- path = "${aws_s3_bucket.target-data-bucket.id}/producer_updateDocumentReference/"`
	`40`	`+ path = "s3://${aws_s3_bucket.target-data-bucket.id}/producer_updateDocumentReference/"`
`41`	`41`	`}`
`42`	`42`	`s3_target {`
`43`		`- path = "${aws_s3_bucket.target-data-bucket.id}/producer_upsertDocumentReference/"`
	`43`	`+ path = "s3://${aws_s3_bucket.target-data-bucket.id}/producer_upsertDocumentReference/"`
`44`	`44`	`}`
`45`	`45`	`schema_change_policy {`
`46`	`46`	`delete_behavior = "LOG"`
Original file line number	Diff line number	Diff line change
`@@ -180,10 +180,10 @@ resource "aws_s3_bucket_public_access_block" "code-bucket-public-access-block" {`
`180`	`180`	`}`
`181`	`181`
`182`	`182`	`resource "aws_s3_object" "script" {`
`183`		`- bucket = aws_s3_bucket.code-bucket.bucket`
`184`		`- key = "main.py"`
`185`		`- source = "${path.module}/src/main.py"`
`186`		`- etag = filemd5("${path.module}/src/main.py")`
	`183`	`+ bucket = aws_s3_bucket.code-bucket.bucket`
	`184`	`+ key = "main.py"`
	`185`	`+ source = "${path.module}/src/main.py"`
	`186`	`+ source_hash = filemd5("${path.module}/src/main.py")`
`187`	`187`	`}`
`188`	`188`
`189`	`189`	`data "archive_file" "python" {`
`@@ -194,8 +194,8 @@ data "archive_file" "python" {`
`194`	`194`	`}`
`195`	`195`
`196`	`196`	`resource "aws_s3_object" "zip" {`
`197`		`- bucket = aws_s3_bucket.code-bucket.bucket`
`198`		`- key = "src.zip"`
`199`		`- source = data.archive_file.python.output_path`
`200`		`- etag = filemd5(data.archive_file.python.output_path)`
	`197`	`+ bucket = aws_s3_bucket.code-bucket.bucket`
	`198`	`+ key = "src.zip"`
	`199`	`+ source = data.archive_file.python.output_path`
	`200`	`+ source_hash = filemd5(data.archive_file.python.output_path)`
`201`	`201`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,3 @@`
`1`	`1`	`output "instance_id" {`
`2`		`- value = aws_instance.web.id`
`3`		`-}`
`4`		`-`
`5`		`-output "public_ip" {`
`6`		`- value = aws_instance.web.public_ip`
	`2`	`+ value = aws_instance.powerbi_gw.id`
`7`	`3`	`}`
Original file line number	Diff line number	Diff line change
`@@ -2,4 +2,5 @@ module "prod-athena" {`
`2`	`2`	`source = "../modules/athena"`
`3`	`3`	`name_prefix = "nhsd-nrlf--prod"`
`4`	`4`	`target_bucket_name = module.prod-glue.target_bucket_name`
	`5`	`+ glue_database = module.prod-glue.glue_database`
`5`	`6`	`}`
Original file line number	Diff line number	Diff line change
`@@ -2,22 +2,26 @@ module "qa-athena" {`
`2`	`2`	`source = "../modules/athena"`
`3`	`3`	`name_prefix = "nhsd-nrlf--qa"`
`4`	`4`	`target_bucket_name = module.qa-glue.target_bucket_name`
	`5`	`+ glue_database = module.qa-glue.glue_database`
`5`	`6`	`}`
`6`	`7`
`7`	`8`	`module "int-athena" {`
`8`	`9`	`source = "../modules/athena"`
`9`	`10`	`name_prefix = "nhsd-nrlf--int"`
`10`	`11`	`target_bucket_name = module.int-glue.target_bucket_name`
	`12`	`+ glue_database = module.int-glue.glue_database`
`11`	`13`	`}`
`12`	`14`
`13`	`15`	`module "int-sandbox-athena" {`
`14`	`16`	`source = "../modules/athena"`
`15`	`17`	`name_prefix = "nhsd-nrlf--int-sandbox"`
`16`	`18`	`target_bucket_name = module.int-sandbox-glue.target_bucket_name`
	`19`	`+ glue_database = module.int-sandbox-glue.glue_database`
`17`	`20`	`}`
`18`	`21`
`19`	`22`	`module "ref-athena" {`
`20`	`23`	`source = "../modules/athena"`
`21`	`24`	`name_prefix = "nhsd-nrlf--ref"`
`22`	`25`	`target_bucket_name = module.ref-glue.target_bucket_name`
	`26`	`+ glue_database = module.ref-glue.glue_database`
`23`	`27`	`}`