Merge pull request #2072 from LBHackney-IT/llpg-jdbc-ingestion

Llpg jdbc ingestion

Author: LBHSBALLEY

terraform/config/prod.tfvars

@@ -10,7 +10,7 @@ stack                               = "Core"
 transit_gateway_availability_zones  = ["eu-west-2a", "eu-west-2b", "eu-west-2c"]
 transit_gateway_cidr                = "10.120.32.0/24"
 transit_gateway_private_subnets     = ["10.120.32.0/26", "10.120.32.64/26", "10.120.32.128/26"]
-rds_instance_ids                    = ["addresses-api-db-production-emergency-temp"]
+rds_instance_ids                    = []
 qlik_server_instance_type           = "m5.2xlarge"
 qlik_ssl_certificate_domain         = "qliksense.hackney.gov.uk"
 redshift_public_ips                 = ["18.168.126.173"]

terraform/core/15-unrestricted-geolive-database-ingestion.tf

@@ -247,3 +247,53 @@ module "housing_boundaries_geolive_ingestion_job" {
     table_prefix = null
   }
 }
+
+module "llpg_geolive_database_ingestion" {
+  count = local.is_live_environment ? 1 : 0
+  tags  = module.tags.values
+
+  source = "../modules/database-ingestion-via-jdbc-connection"
+
+  name                        = "geolive-llpg-schema"
+  jdbc_connection_url         = "jdbc:postgresql://geolive-db-prod.cjgyygrtgrhl.eu-west-2.rds.amazonaws.com:5432/geolive"
+  jdbc_connection_description = "JDBC connection to Geolive PostgreSQL database, to access the llpg schema only"
+  jdbc_connection_subnet      = data.aws_subnet.network[local.instance_subnet_id]
+  identifier_prefix           = local.short_identifier_prefix
+  database_secret_name        = "database-credentials/geolive-llpg"
+  schema_name                 = "llpg"
+  job_schedule                = "cron(1 1 ? * * *)"
+}
+
+module "llpg_geolive_ingestion_job" {
+  count                     = local.is_production_environment ? 1 : 0
+  source                    = "../modules/aws-glue-job"
+  is_live_environment       = local.is_live_environment
+  is_production_environment = local.is_production_environment
+
+  department                 = module.department_unrestricted
+  job_name                   = "${local.short_identifier_prefix}geolive llpg tables ingestion"
+  glue_version               = "4.0"
+  script_s3_object_key       = aws_s3_object.ingest_database_tables_via_jdbc_connection.key
+  spark_ui_output_storage_id = module.spark_ui_output_storage.bucket_id
+  helper_module_key          = aws_s3_object.helpers.key
+  pydeequ_zip_key            = aws_s3_object.pydeequ.key
+  jdbc_connections           = [module.llpg_geolive_database_ingestion[0].jdbc_connection_name]
+  triggered_by_crawler       = module.llpg_geolive_database_ingestion[0].crawler_name
+  workflow_name              = module.llpg_geolive_database_ingestion[0].workflow_name
+  job_parameters = {
+    "--s3_ingestion_bucket_target"  = "s3://${module.raw_zone.bucket_id}/unrestricted/geolive/llpg/"
+    "--s3_ingestion_details_target" = "s3://${module.raw_zone.bucket_id}/unrestricted/geolive/llpg/ingestion-details/"
+    "--source_data_database"        = module.llpg_geolive_database_ingestion[0].ingestion_database_name
+  }
+  crawler_details = {
+    database_name      = module.department_unrestricted.raw_zone_catalog_database_name
+    s3_target_location = "s3://${module.raw_zone.bucket_id}/unrestricted/geolive/llpg/"
+    configuration = jsonencode({
+      Version = 1.0
+      Grouping = {
+        TableLevelConfiguration = 5
+      }
+    })
+    table_prefix = null
+  }
+}