NRL-1188 Reference existing s3 bucket in firehose

Author: jackleary

terraform/account-wide-infrastructure/modules/glue/glue.tf

@@ -57,7 +57,3 @@ resource "aws_glue_job" "glue_job" {
     "--extra-py-files"                  = "s3://${aws_s3_bucket.code-bucket.id}/src.zip"
   }
 }
-
-output "glue_crawler_name" {
-  value = "s3//${aws_s3_bucket.source-data-bucket.id}/"
-}

terraform/account-wide-infrastructure/modules/glue/outputs.tf

@@ -2,3 +2,12 @@ output "target_bucket_name" {
   description = "Name of destination bucket"
   value       = aws_s3_bucket.target-data-bucket.id
 }
+
+output "source_bucket_name" {
+  description = "Name of source bucket"
+  value       = aws_s3_bucket.source-data-bucket.id
+}
+
+output "glue_crawler_name" {
+  value = "s3//${aws_s3_bucket.source-data-bucket.id}/"
+}

terraform/infrastructure/firehose.tf

@@ -8,4 +8,5 @@ module "firehose__processor" {
   splunk_environment = local.splunk_environment
   splunk_index       = local.splunk_index
   destination        = "splunk"
+  reporting_bucket   = module.dev-glue.source_bucket_name
 }

terraform/infrastructure/modules/firehose/kinesis.tf

@@ -63,7 +63,7 @@ resource "aws_kinesis_firehose_delivery_stream" "reporting_stream" {
 
   extended_s3_configuration {
     role_arn   = aws_iam_role.firehose.arn
-    bucket_arn = data.aws_s3_bucket.source-data-bucket.arn
+    bucket_arn = var.reporting_bucket
 
     processing_configuration {
       enabled = "true"

terraform/infrastructure/modules/firehose/s3.tf

@@ -106,7 +106,3 @@ resource "aws_iam_policy" "firehose-alert--s3-read" {
     ]
   })
 }
-
-data "aws_s3_bucket" "source-data-bucket" {
-  bucket = "source-data-bucket"
-}

terraform/infrastructure/modules/firehose/vars.tf

@@ -34,3 +34,7 @@ variable "error_prefix" {
   type    = string
   default = "errors"
 }
+
+variable "source_data_bucket" {
+  type = string
+}