DPF-226 remove crawlers from the source module and recreate it for each department (#2042)

* DPF-226 remove crawlers from the source module and recreate it for each department

* change the role arn of crawler

* add temporary triggers on crawlers

* add an index to trigger action

* add tags to make the terraform linter happy

Author: Tian-2017

terraform/etl/60-airflow-etl-used-crawlers.tf

@@ -90,3 +90,105 @@ resource "aws_glue_crawler" "parking_spatially_enriched_refined_zone" {
     }
   })
 }
+
+resource "aws_glue_crawler" "parking_google_sheet_ingestion_raw_zone" {
+  count         = local.is_live_environment ? 1 : 0
+  name          = "${local.short_identifier_prefix}${module.department_parking_data_source.identifier}-google-sheet-ingestion-raw-zone"
+  role          = data.aws_iam_role.glue_role.arn
+  database_name = module.department_parking_data_source.raw_zone_catalog_database_name
+  s3_target {
+    path = "s3://${module.raw_zone_data_source.bucket_id}/${module.department_parking_data_source.identifier}/google-sheets/"
+  }
+
+  configuration = jsonencode({
+    Version = 1.0
+    Grouping = {
+      TableLevelConfiguration = 4
+    }
+    CrawlerOutput = {
+      Partitions = { AddOrUpdateBehavior = "InheritFromTable" }
+      Tables     = { AddOrUpdateBehavior = "MergeNewColumns" }
+    }
+  })
+}
+
+resource "aws_glue_crawler" "housing_google_sheet_ingestion_raw_zone" {
+  count         = local.is_live_environment ? 1 : 0
+  name          = "${local.short_identifier_prefix}${module.department_housing_data_source.identifier}-google-sheet-ingestion-raw-zone"
+  role          = data.aws_iam_role.glue_role.arn
+  database_name = module.department_housing_data_source.raw_zone_catalog_database_name
+  s3_target {
+    path = "s3://${module.raw_zone_data_source.bucket_id}/${module.department_housing_data_source.identifier}/google-sheets/"
+  }
+
+  configuration = jsonencode({
+    Version = 1.0
+    Grouping = {
+      TableLevelConfiguration = 4
+    }
+    CrawlerOutput = {
+      Partitions = { AddOrUpdateBehavior = "InheritFromTable" }
+      Tables     = { AddOrUpdateBehavior = "MergeNewColumns" }
+    }
+  })
+}
+
+resource "aws_glue_crawler" "data_and_insight_google_sheet_ingestion_raw_zone" {
+  count         = local.is_live_environment ? 1 : 0
+  name          = "${local.short_identifier_prefix}${module.department_data_and_insight_data_source.identifier}-google-sheet-ingestion-raw-zone"
+  role          = data.aws_iam_role.glue_role.arn
+  database_name = module.department_data_and_insight_data_source.raw_zone_catalog_database_name
+  s3_target {
+    path = "s3://${module.raw_zone_data_source.bucket_id}/${module.department_data_and_insight_data_source.identifier}/google-sheets/"
+  }
+
+  configuration = jsonencode({
+    Version = 1.0
+    Grouping = {
+      TableLevelConfiguration = 4
+    }
+    CrawlerOutput = {
+      Partitions = { AddOrUpdateBehavior = "InheritFromTable" }
+      Tables     = { AddOrUpdateBehavior = "MergeNewColumns" }
+    }
+  })
+}
+
+# Below crawlers triggers are temporary and will be removed after enabling the airflow google sheet ingestion dag
+resource "aws_glue_trigger" "parking_google_sheet_ingestion_raw_zone_trigger" {
+  name                = "${local.short_identifier_prefix}${module.department_parking_data_source.identifier}-google-sheet-ingestion-raw-zone-trigger"
+  type                = "SCHEDULED"
+  schedule            = "cron(0 7 ? * * *)"
+  start_on_creation   = true
+
+  actions {
+    crawler_name = aws_glue_crawler.parking_google_sheet_ingestion_raw_zone[0].name
+  }
+  tags = module.tags.values
+}
+
+
+resource "aws_glue_trigger" "housing_google_sheet_ingestion_raw_zone_trigger" {
+  name                = "${local.short_identifier_prefix}${module.department_housing_data_source.identifier}-google-sheet-ingestion-raw-zone-trigger"
+  type                = "SCHEDULED"
+  schedule            = "cron(0 7 ? * * *)"
+  start_on_creation   = true
+
+  actions {
+    crawler_name = aws_glue_crawler.housing_google_sheet_ingestion_raw_zone[0].name
+  }
+  tags = module.tags.values
+}
+
+
+resource "aws_glue_trigger" "data_and_insight_google_sheet_ingestion_raw_zone_trigger" {
+  name                = "${local.short_identifier_prefix}${module.department_data_and_insight_data_source.identifier}-google-sheet-ingestion-raw-zone-trigger"
+  type                = "SCHEDULED"
+  schedule            = "cron(0 7 ? * * *)"
+  start_on_creation   = true
+
+  actions {
+    crawler_name = aws_glue_crawler.data_and_insight_google_sheet_ingestion_raw_zone[0].name
+  }
+  tags = module.tags.values
+}

terraform/modules/google-sheets-glue-job/03-input-derived.tf

@@ -1,6 +1,6 @@
 locals {
   dataset_name            = lower(replace(var.dataset_name, "_", "-"))
   import_name             = "${var.department.identifier}-${local.dataset_name}"
-  full_output_path        = "s3://${var.bucket_id}/${var.department.identifier}/${local.dataset_name}"
+  full_output_path        = "s3://${var.bucket_id}/${var.department.identifier}/google-sheets/${local.dataset_name}"
   sheets_credentials_name = var.sheets_credentials_name == null ? var.department.google_service_account.credentials_secret.name : var.sheets_credentials_name
 }

terraform/modules/google-sheets-glue-job/10-aws-glue-job.tf

@@ -24,17 +24,6 @@ module "google_sheet_import" {
   schedule        = var.google_sheet_import_schedule
   max_retries     = var.max_retries
   trigger_enabled = (var.is_live_environment && var.enable_glue_trigger)
-  crawler_details = {
-    database_name      = var.glue_catalog_database_name
-    s3_target_location = local.full_output_path
-    table_prefix       = "${var.department.identifier_snake_case}_"
-    configuration = jsonencode({
-      Version = 1.0
-      Grouping = {
-        TableGroupingPolicy = "CombineCompatibleSchemas"
-      }
-    })
-  }
 }
 
 resource "aws_glue_workflow" "workflow" {