Update/Create Crawlers, Secrets, and Temp Databases for Benefits Housing Needs and Revenues (#2299)

* add secretes and crawlers for benefits housing needs and revenues

* add crawlers

* add CombineCompatibleSchemas

* remove the unused creds

* create two databases

* update the comments

Author: Tian-2017

terraform/etl/06-aws-secrets-manager.tf

@@ -28,4 +28,40 @@ data "aws_secretsmanager_secret" "tascomi_api_private_key" {
 
 data "aws_ssm_parameter" "tascomi_api_private_key_arn" {
   name = "/${local.identifier_prefix}/secrets_manager/tascomi_api_private_key/arn"
-}
+}
+
+# Academy SQL Server credentials for Revenues department
+resource "aws_secretsmanager_secret" "revenues_academy_sql_server_creds" {
+  name        = "/revenues/academy_sql_server_creds"
+  description = "SQL Server credentials for Academy database ingestion - Revenues department"
+  tags        = module.tags.values
+}
+
+resource "aws_secretsmanager_secret_version" "revenues_academy_sql_server_creds" {
+  secret_id = aws_secretsmanager_secret.revenues_academy_sql_server_creds.id
+  secret_string = jsonencode({
+    value = "UPDATE_IN_CONSOLE"
+  })
+
+  lifecycle {
+    ignore_changes = [secret_string]
+  }
+}
+
+# Academy SQL Server credentials for Benefits and Housing Needs department
+resource "aws_secretsmanager_secret" "benefits_housing_needs_academy_sql_server_creds" {
+  name        = "/benefits-housing-needs/academy_sql_server_creds"
+  description = "SQL Server credentials for Academy database ingestion - Benefits and Housing Needs department"
+  tags        = module.tags.values
+}
+
+resource "aws_secretsmanager_secret_version" "benefits_housing_needs_academy_sql_server_creds" {
+  secret_id = aws_secretsmanager_secret.benefits_housing_needs_academy_sql_server_creds.id
+  secret_string = jsonencode({
+    value = "UPDATE_IN_CONSOLE"
+  })
+
+  lifecycle {
+    ignore_changes = [secret_string]
+  }
+}

terraform/etl/60-airflow-etl-used-crawlers.tf

@@ -173,3 +173,53 @@ resource "aws_glue_crawler" "google_sheet_ingestion_raw_zone" {
   })
   tags = module.tags.values
 }
+
+
+# Academy crawlers for raw zone
+resource "aws_glue_crawler" "benefits_housing_needs_academy_raw_zone" {
+  count         = local.is_live_environment ? 1 : 0
+  name          = "benefits-housing-needs-academy-raw-zone"
+  role          = module.department_benefits_and_housing_needs_data_source.glue_role_arn
+  database_name = aws_glue_catalog_database.temp_benefits_housing_needs_academy.name # This is a temp database will change to raw zone database when ready
+
+  s3_target {
+    path = "s3://${module.raw_zone_data_source.bucket_id}/benefits-housing-needs/academy"
+  }
+
+  configuration = jsonencode({
+    Version = 1.0
+    Grouping = {
+      TableLevelConfiguration = 4
+      TableGroupingPolicy     = "CombineCompatibleSchemas"
+    }
+    CrawlerOutput = {
+      Partitions = { AddOrUpdateBehavior = "InheritFromTable" }
+      Tables     = { AddOrUpdateBehavior = "MergeNewColumns" }
+    }
+  })
+  tags = module.tags.values
+}
+
+resource "aws_glue_crawler" "revenues_academy_raw_zone" {
+  count         = local.is_live_environment ? 1 : 0
+  name          = "revenues-academy-raw-zone"
+  role          = module.department_revenues_data_source.glue_role_arn
+  database_name = aws_glue_catalog_database.temp_revenues_academy.name # This is a temp database will change to raw zone database when ready
+
+  s3_target {
+    path = "s3://${module.raw_zone_data_source.bucket_id}/revenues/academy"
+  }
+
+  configuration = jsonencode({
+    Version = 1.0
+    Grouping = {
+      TableLevelConfiguration = 4
+      TableGroupingPolicy     = "CombineCompatibleSchemas"
+    }
+    CrawlerOutput = {
+      Partitions = { AddOrUpdateBehavior = "InheritFromTable" }
+      Tables     = { AddOrUpdateBehavior = "MergeNewColumns" }
+    }
+  })
+  tags = module.tags.values
+}

terraform/etl/60-airflow-variables-and-connnections.tf

@@ -92,22 +92,3 @@ resource "aws_secretsmanager_secret_version" "mtfh_secrets" {
     ignore_changes = [secret_string]
   }
 }
-
-
-# Academy SQL Server credentials
-resource "aws_secretsmanager_secret" "academy_sql_server_credentials" {
-  name        = "airflow/variables/academy_sql_server_credentials"
-  description = "SQL Server credentials for Academy database ingestion"
-  tags        = module.tags.values
-}
-
-resource "aws_secretsmanager_secret_version" "academy_sql_server_credentials" {
-  secret_id = aws_secretsmanager_secret.academy_sql_server_credentials.id
-  secret_string = jsonencode({
-    value = "UPDATE_IN_CONSOLE"
-  })
-
-  lifecycle {
-    ignore_changes = [secret_string]
-  }
-}

terraform/etl/61-aws-glue-catalog-database.tf

@@ -30,8 +30,17 @@ resource "aws_glue_catalog_database" "housing_nec_migration_database" {
   }
 }
 
-resource "aws_glue_catalog_database" "temp_academy_ingestion" {
-  name = "temp_academy_ingestion"
+
+resource "aws_glue_catalog_database" "temp_revenues_academy" {
+  name = "temp_revenues_academy"
+
+  lifecycle {
+    prevent_destroy = true
+  }
+}
+
+resource "aws_glue_catalog_database" "temp_benefits_housing_needs_academy" {
+  name = "temp_benefits_housing_needs_academy"
 
   lifecycle {
     prevent_destroy = true