delete data migration (#2578)

* delete data migration

* remove the scripts

* delete the python script too

Author: Tian-2017

scripts/jobs/academy_data/load_all_academy_data_into_redshift.py

@@ -93,12 +93,3 @@ resource "aws_s3_object" "parking_copy_ringgo_sftp_data_to_raw" {
   source      = "../../scripts/jobs/parking/parking_copy_ringgo_sftp_data_to_raw.py"
   source_hash = filemd5("../../scripts/jobs/parking/parking_copy_ringgo_sftp_data_to_raw.py")
 }
-
-resource "aws_s3_object" "load_all_academy_data_into_redshift" {
-  bucket      = module.glue_scripts.bucket_id # this is glue_scripts_data_source in etl folder
-  key         = "scripts/load_all_academy_data_into_redshift.py"
-  acl         = "private"
-  source      = "../../scripts/jobs/academy_data/load_all_academy_data_into_redshift.py"
-  source_hash = filemd5("../../scripts/jobs/academy_data/load_all_academy_data_into_redshift.py")
-}
-

terraform/core/12-aws-s3-scripts.tf

@@ -26,19 +26,19 @@ locals {
 
 # option 2: tailored for this module
 resource "aws_glue_connection" "database_ingestion_via_jdbc_connection" {
-  count                        = local.is_live_environment && !local.is_production_environment ? 1 : 0
-  name                         = "${local.short_identifier_prefix}redshift-serverless-connection-${data.aws_subnet.network[local.instance_subnet_id].availability_zone}"
+  count       = local.is_live_environment && !local.is_production_environment ? 1 : 0
+  name        = "${local.short_identifier_prefix}redshift-serverless-connection-${data.aws_subnet.network[local.instance_subnet_id].availability_zone}"
   description = "JDBC connection for Redshift Serverless"
   connection_properties = {
-    JDBC_CONNECTION_URL        = "jdbc:redshift://${local.redshift_serverless_credentials["host"]}:${local.redshift_serverless_credentials["port"]}/${local.redshift_serverless_credentials["database_name"]}"
-    PASSWORD                   = local.redshift_serverless_credentials["password"]
-    USERNAME                   = local.redshift_serverless_credentials["username"]
+    JDBC_CONNECTION_URL = "jdbc:redshift://${local.redshift_serverless_credentials["host"]}:${local.redshift_serverless_credentials["port"]}/${local.redshift_serverless_credentials["database_name"]}"
+    PASSWORD            = local.redshift_serverless_credentials["password"]
+    USERNAME            = local.redshift_serverless_credentials["username"]
   }
 
   physical_connection_requirements {
-    availability_zone          = data.aws_subnet.network[local.instance_subnet_id].availability_zone
-    security_group_id_list     = [aws_security_group.ingestion_database_connection.id]
-    subnet_id                  = data.aws_subnet.network[local.instance_subnet_id].id
+    availability_zone      = data.aws_subnet.network[local.instance_subnet_id].availability_zone
+    security_group_id_list = [aws_security_group.ingestion_database_connection.id]
+    subnet_id              = data.aws_subnet.network[local.instance_subnet_id].id
   }
 
 }
@@ -69,37 +69,3 @@ resource "aws_security_group_rule" "ingestion_database_connection_allow_tcp_egre
   ipv6_cidr_blocks  = ["::/0"]
   security_group_id = aws_security_group.ingestion_database_connection.id
 }
-
-
-module "load_all_academy_data_into_redshift" {
-  count                           = local.is_live_environment && !local.is_production_environment ? 1 : 0
-  tags  = module.tags.values
-  source                          = "../modules/aws-glue-job"
-  is_live_environment             = local.is_live_environment
-  is_production_environment       = local.is_production_environment
-  job_name                        = "${local.short_identifier_prefix}load_all_academy_data_into_redshift"
-  script_s3_object_key            = aws_s3_object.load_all_academy_data_into_redshift.key
-  pydeequ_zip_key                 = aws_s3_object.pydeequ.key
-  helper_module_key               = aws_s3_object.helpers.key
-  glue_role_arn                   = aws_iam_role.glue_role.arn
-  glue_temp_bucket_id             = module.glue_temp_storage.bucket_id
-  glue_scripts_bucket_id          = module.glue_scripts.bucket_id
-  spark_ui_output_storage_id      = module.spark_ui_output_storage.bucket_id
-  glue_version                    = "4.0"
-  glue_job_worker_type            = "G.1X"
-  number_of_workers_for_glue_job  = 2
-  glue_job_timeout                = 220
-  schedule                        = "cron(15 7 ? * MON-FRI *)"
-  # jdbc_connections                = [module.database_ingestion_via_jdbc_connection[0].name]
-  jdbc_connections                = [aws_glue_connection.database_ingestion_via_jdbc_connection[0].name]
-  job_parameters = {
-    "--additional-python-modules"        = "botocore==1.27.59, redshift_connector==2.1.0"
-    "--environment"                      = var.environment
-    # This is the ARN of the IAM role used by Redshift Serverless. We have count in redshift-serverless module so index 0 is to get the ARN.
-    "--role_arn"                         = try(module.redshift_serverless[0].redshift_serverless_role_arn, "") 
-    "--enable-auto-scaling"              = "false"
-    "--job-bookmark-option"              = "job-bookmark-disable"
-    "--base_s3_url"                      = "${module.raw_zone.bucket_url}/revenues/"
-    "--conf"                             = "spark.sql.legacy.timeParserPolicy=LEGACY --conf spark.sql.legacy.parquet.int96RebaseModeInRead=LEGACY --conf spark.sql.legacy.parquet.int96RebaseModeInWrite=LEGACY --conf spark.sql.legacy.parquet.datetimeRebaseModeInRead=LEGACY --conf spark.sql.legacy.parquet.datetimeRebaseModeInWrite=LEGACY"
-  }
-}