Merge branch 'main' into di-447-amend-dq-tests-for-housing

Author: annajgibson

docker/sql-to-parquet/entrypoint.sh

@@ -19,23 +19,18 @@ echo "Snapshot Id - $SNAPSHOT_ID"
 FILENAME="liberator_dump_${DATE}"
 DBNAME="liberator"
 
-MYSQL_CONN_PARAMS="--user=${MYSQL_USER} --password=${MYSQL_PASS} --host=${MYSQL_HOST}"
+echo "Retrieving pem file..."
+wget -O /tmp/rds-combined-ca-bundle.pem https://truststore.pki.rds.amazonaws.com/eu-west-2/eu-west-2-bundle.pem
 
-SSL_MODE=${MYSQL_SSL_MODE:-""}  
-OTHER_FLAGS=${MYSQL_OTHER_FLAGS:-""}  
+MYSQL_CONN_PARAMS="--user=${MYSQL_USER} --password=${MYSQL_PASS} --host=${MYSQL_HOST} --ssl-ca=/tmp/rds-combined-ca-bundle.pem"
 
-if [ -n "$SSL_MODE" ]; then
-  MYSQL_CONN_PARAMS="$MYSQL_CONN_PARAMS --ssl-mode=$SSL_MODE"
-  echo "SSL Mode set to $SSL_MODE"
-fi
+OTHER_FLAGS=${MYSQL_OTHER_FLAGS:-""}  
 
 if [ -n "$OTHER_FLAGS" ]; then
   MYSQL_CONN_PARAMS="$MYSQL_CONN_PARAMS $OTHER_FLAGS"
   echo "Additional MySQL flags: $OTHER_FLAGS"
 fi
 
-echo "MYSQL connection params: $MYSQL_CONN_PARAMS"
-
 echo "Deleting old snapshots in database..."
 python3 delete_db_snapshots_in_db.py
 

lambdas/lambda_layers/requirements4.txt

@@ -1 +1 @@
-numpy==2.1.3
+numpy==2.2.0

lambdas/lambda_layers/requirements5.txt

@@ -1,4 +1,4 @@
 gspread==6.1.4
 oauth2client==4.1.3
-google-api-python-client==2.154.0
+google-api-python-client==2.155.0
 yagmail==0.15.293

lambdas/vonage_api_ingestion/Pipfile.lock

@@ -62,7 +62,10 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra
 SELECT
    cc, subj,analysis, trandate,
    CASE
-      When trandate like '%/%'Then CAST(substr(trandate, 7, 4)||'-'||substr(trandate, 4,2)||'-'||'01' as date)
+      When trandate like '%/%'                  Then 
+                CAST(substr(trandate, 7, 4)||'-'||substr(trandate, 4,2)||'-'||'01' as date)
+      When substr(trandate, 1, 4) like '%-%'    Then
+                CAST(substr(trandate, 7, 4)||'-'||substr(trandate, 4,2)||'-'||'01' as date)
       ELSE cast(concat(substr(Cast(trandate as varchar(10)),1, 7), '-01') as date)
    END as PayMonthYear,
    description, o_description, cast(financialvalue as decimal(10,2)) as financialvalue

scripts/jobs/parking/parking_cedar_payments.py

@@ -287,11 +287,11 @@ module "Cedar_Parking_Payments" {
   landing_zone_bucket_id         = module.landing_zone_data_source.bucket_id
   landing_zone_kms_key_arn       = module.landing_zone_data_source.kms_key_arn
   landing_zone_bucket_arn        = module.landing_zone_data_source.bucket_arn
-  google_drive_document_id       = "1er6XED0byp3Gu3yPfOI9Ao3_Gp8ZnQ9s"
+  google_drive_document_id       = "10Rw1t_0Yvrhq7yocwRiDnjAELeFMOwx6"
   glue_job_name                  = "Cedar Parking Payments"
   output_folder_name             = "g-drive"
   raw_zone_bucket_id             = module.raw_zone_data_source.bucket_id
-  input_file_name                = "Cedar_Parking_Payments/Cedar_Parking_Payments_nov_2024.csv"
+  input_file_name                = "Cedar_Parking_Payments/Cedar_Parking_Payments_nov_update_2024.csv"
   ingestion_schedule             = "cron(0 21 * * ? *)"
   enable_bookmarking             = true
   worksheets = {
@@ -992,11 +992,11 @@ module "interim_cycle_wait_list" {
   landing_zone_bucket_id         = module.landing_zone_data_source.bucket_id
   landing_zone_kms_key_arn       = module.landing_zone_data_source.kms_key_arn
   landing_zone_bucket_arn        = module.landing_zone_data_source.bucket_arn
-  google_drive_document_id       = "1ObXnGJmYS04j1-lBQiz3-kSwuP5BvXFz"
+  google_drive_document_id       = "1EFk08b0_eN0BJ_eAYUWKlhTLnhGeU6Oa"
   glue_job_name                  = "interim_cycle_wait_list"
   output_folder_name             = "g-drive"
   raw_zone_bucket_id             = module.raw_zone_data_source.bucket_id
-  input_file_name                = "Interim_Cycle_Hangar_Wait_List/interim_cycle_wait_list @ 19-11-2024.csv"
+  input_file_name                = "Interim_Cycle_Hangar_Wait_List/interim_cycle_wait_list @ 17-12-2024.csv"
   ingestion_schedule             = "cron(0 21 * * ? *)"
   enable_bookmarking             = true
   worksheets = {

terraform/etl/09-spreadsheet-imports-from-g-drive.tf

@@ -168,8 +168,8 @@ module "addressbasepremium_load_files" {
   job_parameters = {
     "--job-bookmark-option"  = "job-bookmark-disable"    
     "--raw_bucket"           = module.raw_zone_data_source.bucket_id
-    "--raw_prefix"           = "unrestricted/os-addressbase-premium/full-supply/epoch-105/raw/"
-    "--processed_data_path"  = "s3://${module.raw_zone_data_source.bucket_id}/unrestricted/os-addressbase-premium/full-supply/epoch-105/processed/"
+    "--raw_prefix"           = "unrestricted/os-addressbase-premium/full-supply/epoch-114/raw/"
+    "--processed_data_path"  = "s3://${module.raw_zone_data_source.bucket_id}/unrestricted/os-addressbase-premium/full-supply/epoch-114/processed/"
   }
   script_name                = "addressbasepremium_load_files"
 }
@@ -192,7 +192,7 @@ module "addressbasepremium_create_address_table" {
     "--job-bookmark-option"          = "job-bookmark-disable"    
     "--blpu_class_lookup_path"       = "s3://${module.glue_scripts_data_source.bucket_id}/${aws_s3_object.blpu_class_lookup.key}"
     "--ward_lookup_path"             = "s3://${module.glue_scripts_data_source.bucket_id}/${aws_s3_object.ons_ward_lookup.key}"
-    "--processed_source_data_path"   = "s3://${module.raw_zone_data_source.bucket_id}/unrestricted/os-addressbase-premium/full-supply/epoch-105/processed/"
+    "--processed_source_data_path"   = "s3://${module.raw_zone_data_source.bucket_id}/unrestricted/os-addressbase-premium/full-supply/epoch-114/processed/"
     "--target_path"                  = "s3://${module.refined_zone_data_source.bucket_id}/unrestricted/national_address"
   }
   script_name                    = "addressbasepremium_create_address_table"