add excluded paths to crawler (#2292)

* add excluded paths to crawler

* combine compatible schemas

Author: timburke-hackit

terraform/core/39-housing-interim-finance-db-ingestion.tf

@@ -70,6 +70,7 @@ module "ingest_housing_interim_finance_database_to_housing_raw_zone" {
     configuration = jsonencode({
       Version = 1.0
       Grouping = {
+        TableGroupingPolicy     = "CombineCompatibleSchemas"
         TableLevelConfiguration = 3
       }
       CrawlerOutput = {
@@ -78,6 +79,22 @@ module "ingest_housing_interim_finance_database_to_housing_raw_zone" {
     })
     table_prefix = null
   }
+  glue_crawler_excluded_blobs = [
+    "*/archive*",
+    "*/data-quality*",
+    "*/glue-*",
+    "*/google-sheets*",
+    "*/govnotify*",
+    "*/housingfinance*",
+    "*/ingestion-details*",
+    "*/mtfh*",
+    "*/temp_backup*",
+    "*.json",
+    "*.txt",
+    "*.zip",
+    "*.xlsx",
+    "*.html",
+  ]
 }
 
 resource "aws_ssm_parameter" "ingest_housing_interim_finance_database_to_housing_raw_zone_crawler_name" {