Merge branch 'main' into di-447-amend-dq-tests-for-housing

Author: annajgibson

docker/sql-to-parquet/entrypoint.sh

@@ -19,23 +19,18 @@ echo "Snapshot Id - $SNAPSHOT_ID"
 FILENAME="liberator_dump_${DATE}"
 DBNAME="liberator"
 
-MYSQL_CONN_PARAMS="--user=${MYSQL_USER} --password=${MYSQL_PASS} --host=${MYSQL_HOST}"
+echo "Retrieving pem file..."
+wget -O /tmp/rds-combined-ca-bundle.pem https://truststore.pki.rds.amazonaws.com/eu-west-2/eu-west-2-bundle.pem
 
-SSL_MODE=${MYSQL_SSL_MODE:-""}  
-OTHER_FLAGS=${MYSQL_OTHER_FLAGS:-""}  
+MYSQL_CONN_PARAMS="--user=${MYSQL_USER} --password=${MYSQL_PASS} --host=${MYSQL_HOST} --ssl-ca=/tmp/rds-combined-ca-bundle.pem"
 
-if [ -n "$SSL_MODE" ]; then
-  MYSQL_CONN_PARAMS="$MYSQL_CONN_PARAMS --ssl-mode=$SSL_MODE"
-  echo "SSL Mode set to $SSL_MODE"
-fi
+OTHER_FLAGS=${MYSQL_OTHER_FLAGS:-""}  
 
 if [ -n "$OTHER_FLAGS" ]; then
   MYSQL_CONN_PARAMS="$MYSQL_CONN_PARAMS $OTHER_FLAGS"
   echo "Additional MySQL flags: $OTHER_FLAGS"
 fi
 
-echo "MYSQL connection params: $MYSQL_CONN_PARAMS"
-
 echo "Deleting old snapshots in database..."
 python3 delete_db_snapshots_in_db.py
 

lambdas/lambda_layers/requirements3.txt

@@ -1 +1 @@
-notifications-python-client==10.0.0
+notifications-python-client==10.0.1

lambdas/lambda_layers/requirements4.txt

@@ -1 +1 @@
-numpy==2.1.3
+numpy==2.2.1

lambdas/lambda_layers/requirements5.txt

@@ -1,4 +1,4 @@
 gspread==6.1.4
 oauth2client==4.1.3
-google-api-python-client==2.154.0
+google-api-python-client==2.156.0
 yagmail==0.15.293

lambdas/lambda_layers/requirements6.txt

@@ -1 +1 @@
-urllib3==2.2.3
+urllib3==2.3.0

lambdas/lambda_layers/requirements7.txt

@@ -1 +1 @@
-s3fs==2024.10.0
+s3fs==2024.12.0

lambdas/vonage_api_ingestion/Pipfile.lock

@@ -62,7 +62,10 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra
 SELECT
    cc, subj,analysis, trandate,
    CASE
-      When trandate like '%/%'Then CAST(substr(trandate, 7, 4)||'-'||substr(trandate, 4,2)||'-'||'01' as date)
+      When trandate like '%/%'                  Then 
+                CAST(substr(trandate, 7, 4)||'-'||substr(trandate, 4,2)||'-'||'01' as date)
+      When substr(trandate, 1, 4) like '%-%'    Then
+                CAST(substr(trandate, 7, 4)||'-'||substr(trandate, 4,2)||'-'||'01' as date)
       ELSE cast(concat(substr(Cast(trandate as varchar(10)),1, 7), '-01') as date)
    END as PayMonthYear,
    description, o_description, cast(financialvalue as decimal(10,2)) as financialvalue

scripts/jobs/parking/parking_cedar_payments.py

@@ -3,5 +3,5 @@ freezegun
 pytest-mock
 pydeequ
 # Explicitly set urllib3 version compatible with botocore to pass CI tests
-urllib3>=1.25.4,<1.27 # glue 4 is using urllib3==1.25.11 so don't need add to requirements.build.txt
+urllib3>=1.25.4,<2.4 # glue 4 is using urllib3==1.25.11 so don't need add to requirements.build.txt
 redshift-connector==2.1.4 # used to connect with redshift

scripts/requirements.test.txt

@@ -970,3 +970,25 @@ module "diplomatic_country_vrm_prefix_data_load" {
   google_sheet_import_schedule    = "cron(0 4 ? * MON *)"
   spark_ui_output_storage_id      = module.spark_ui_output_storage_data_source.bucket_id
 }
+
+module "parking_cycle_hangar_unsubscribed_emails" {
+  count                           = local.is_live_environment ? 1 : 0
+  source                          = "../modules/google-sheets-glue-job"
+  is_production_environment       = local.is_production_environment
+  identifier_prefix               = local.short_identifier_prefix
+  is_live_environment             = local.is_live_environment
+  glue_scripts_bucket_id          = module.glue_scripts_data_source.bucket_id
+  helper_module_key               = data.aws_s3_object.helpers.key
+  pydeequ_zip_key                 = data.aws_s3_object.pydeequ.key
+  glue_catalog_database_name      = module.department_parking_data_source.raw_zone_catalog_database_name
+  glue_temp_storage_bucket_url    = module.glue_temp_storage_data_source.bucket_url
+  glue_crawler_excluded_blobs     = local.glue_crawler_excluded_blobs
+  google_sheets_import_script_key = aws_s3_object.google_sheets_import_script.key
+  bucket_id                       = module.raw_zone_data_source.bucket_id
+  google_sheets_document_id       = "1I9OPrlWIpidOM1pnwcc8G6Xq86IJz_UcIia1u7zX6_A"
+  google_sheets_worksheet_name    = "Unsubscribed emails"
+  department                      = module.department_parking_data_source
+  dataset_name                    = "parking_cycle_hangar_unsubscribed_emails"
+  google_sheet_import_schedule    = "cron(0 4 ? * * *)"
+  spark_ui_output_storage_id      = module.spark_ui_output_storage_data_source.bucket_id
+}