Merge branch 'main' into di-447-amend-dq-tests-for-housing

Author: annajgibson

docker/sql-to-parquet/entrypoint.sh

@@ -19,23 +19,18 @@ echo "Snapshot Id - $SNAPSHOT_ID"
 FILENAME="liberator_dump_${DATE}"
 DBNAME="liberator"
 
-MYSQL_CONN_PARAMS="--user=${MYSQL_USER} --password=${MYSQL_PASS} --host=${MYSQL_HOST}"
+echo "Retrieving pem file..."
+wget -O /tmp/rds-combined-ca-bundle.pem https://truststore.pki.rds.amazonaws.com/eu-west-2/eu-west-2-bundle.pem
 
-SSL_MODE=${MYSQL_SSL_MODE:-""}  
-OTHER_FLAGS=${MYSQL_OTHER_FLAGS:-""}  
+MYSQL_CONN_PARAMS="--user=${MYSQL_USER} --password=${MYSQL_PASS} --host=${MYSQL_HOST} --ssl-ca=/tmp/rds-combined-ca-bundle.pem"
 
-if [ -n "$SSL_MODE" ]; then
-  MYSQL_CONN_PARAMS="$MYSQL_CONN_PARAMS --ssl-mode=$SSL_MODE"
-  echo "SSL Mode set to $SSL_MODE"
-fi
+OTHER_FLAGS=${MYSQL_OTHER_FLAGS:-""}  
 
 if [ -n "$OTHER_FLAGS" ]; then
   MYSQL_CONN_PARAMS="$MYSQL_CONN_PARAMS $OTHER_FLAGS"
   echo "Additional MySQL flags: $OTHER_FLAGS"
 fi
 
-echo "MYSQL connection params: $MYSQL_CONN_PARAMS"
-
 echo "Deleting old snapshots in database..."
 python3 delete_db_snapshots_in_db.py
 

lambdas/vonage_api_ingestion/Pipfile.lock

@@ -62,7 +62,10 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra
 SELECT
    cc, subj,analysis, trandate,
    CASE
-      When trandate like '%/%'Then CAST(substr(trandate, 7, 4)||'-'||substr(trandate, 4,2)||'-'||'01' as date)
+      When trandate like '%/%'                  Then 
+                CAST(substr(trandate, 7, 4)||'-'||substr(trandate, 4,2)||'-'||'01' as date)
+      When substr(trandate, 1, 4) like '%-%'    Then
+                CAST(substr(trandate, 7, 4)||'-'||substr(trandate, 4,2)||'-'||'01' as date)
       ELSE cast(concat(substr(Cast(trandate as varchar(10)),1, 7), '-01') as date)
    END as PayMonthYear,
    description, o_description, cast(financialvalue as decimal(10,2)) as financialvalue