Merge branch 'main' into di-447-update-data-quality-tests

Author: annajgibson

.github/CODEOWNERS

@@ -0,0 +1,4 @@
+# This file specifies owners for pull request approval
+# See https://help.github.com/articles/about-code-owners/
+
+* @LBHackney-IT/data-analytics-platform @LBHackney-IT/data-insight

.github/workflows/deploy_terraform.yml

@@ -77,7 +77,7 @@ on:
 jobs:
   deploy:
     name: Terraform Apply
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     environment: ${{ inputs.environment }}
     steps:
       - name: Checkout Source
@@ -86,7 +86,7 @@ jobs:
       - name: Set Github Auth
         run: git config --global url."https://oauth2:${{ secrets.TERRAFORM_SECRET_TOKEN}}@github.com".insteadOf https://github.com
         shell: bash
-      
+
       - name: Install Terraform
         uses: hashicorp/[email protected]
         with:

.github/workflows/deploy_terraform_networking.yml

@@ -73,15 +73,15 @@ on:
 jobs:
   deploy:
     name: Terraform Apply
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     environment: ${{ inputs.environment }}
     steps:
       - name: Checkout Source
         uses: actions/checkout@v3
 
       - name: Set Github Auth
         run: git config --global url."https://oauth2:${{ secrets.TERRAFORM_SECRET_TOKEN}}@github.com".insteadOf https://github.com
-        shell: bash   
+        shell: bash
 
       - name: Install Terraform
         uses: hashicorp/[email protected]

.github/workflows/lint-terraform.yml

@@ -66,15 +66,15 @@ on:
 jobs:
   lint:
     name: Terraform Lint
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v3
 
       - name: Set Github Auth
         run: git config --global url."https://oauth2:${{ secrets.TERRAFORM_SECRET_TOKEN}}@github.com".insteadOf https://github.com
         shell: bash
-      
+
       - name: Install Terraform
         uses: hashicorp/[email protected]
         with:

.github/workflows/plan-terraform.yml

@@ -66,17 +66,17 @@ on:
 jobs:
   plan:
     name: Terraform Plan
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
       - uses: terraform-compliance/github_action@main
 
       - name: Checkout
         uses: actions/checkout@v3
-      
+
       - name: Set Github Auth
         run: git config --global url."https://oauth2:${{ secrets.TERRAFORM_SECRET_TOKEN}}@github.com".insteadOf https://github.com
         shell: bash
-      
+
       - name: Install Terraform
         uses: hashicorp/[email protected]
         with:

.github/workflows/test-python-and-lambda.yml

@@ -6,7 +6,7 @@ on:
 jobs:
   tests:
     name: Test Python Jobs & Lambda Functions
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v3

.github/workflows/unlock_terraform_state.yml

@@ -76,7 +76,7 @@ on:
 jobs:
   deploy:
     name: Terraform State Unlock
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout Source
         uses: actions/checkout@v3

.github/workflows/validate-and-lint-terraform.yml

@@ -66,7 +66,7 @@ on:
 jobs:
   validate:
     name: Terraform Validate
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -108,15 +108,15 @@ jobs:
 
   lint:
     name: Terraform Lint
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v3
 
       - name: Set Github Auth
         run: git config --global url."https://oauth2:${{ secrets.TERRAFORM_SECRET_TOKEN}}@github.com".insteadOf https://github.com
         shell: bash
-      
+
       - name: Install Terraform
         uses: hashicorp/[email protected]
         with:

docker/sql-to-parquet/entrypoint.sh

@@ -19,7 +19,17 @@ echo "Snapshot Id - $SNAPSHOT_ID"
 FILENAME="liberator_dump_${DATE}"
 DBNAME="liberator"
 
-MYSQL_CONN_PARAMS="--user=${MYSQL_USER} --password=${MYSQL_PASS} --host=${MYSQL_HOST}"
+echo "Retrieving pem file..."
+wget -O /tmp/rds-combined-ca-bundle.pem https://truststore.pki.rds.amazonaws.com/eu-west-2/eu-west-2-bundle.pem
+
+MYSQL_CONN_PARAMS="--user=${MYSQL_USER} --password=${MYSQL_PASS} --host=${MYSQL_HOST} --ssl-ca=/tmp/rds-combined-ca-bundle.pem"
+
+OTHER_FLAGS=${MYSQL_OTHER_FLAGS:-""}  
+
+if [ -n "$OTHER_FLAGS" ]; then
+  MYSQL_CONN_PARAMS="$MYSQL_CONN_PARAMS $OTHER_FLAGS"
+  echo "Additional MySQL flags: $OTHER_FLAGS"
+fi
 
 echo "Deleting old snapshots in database..."
 python3 delete_db_snapshots_in_db.py
@@ -50,3 +60,4 @@ echo "Taking snapshot of RDS database..."
 aws rds create-db-snapshot --db-instance-identifier "${RDS_INSTANCE_ID}" --db-snapshot-identifier "${SNAPSHOT_ID}"
 
 echo "Done"
+

external-lib/Makefile

@@ -3,7 +3,7 @@
 all: target/java-lib-1.0-SNAPSHOT-jar-with-dependencies.jar target/deequ-1.0.3.jar target/pydeequ-1.0.1.zip
 
 target/java-lib-1.0-SNAPSHOT-jar-with-dependencies.jar: pom.xml
-	mvn assembly:assembly -DdescriptorId=jar-with-dependencies
+	mvn clean package
 
 target/deequ-1.0.3.jar:
 	wget https://repo1.maven.org/maven2/com/amazon/deequ/deequ/1.0.3/deequ-1.0.3.jar -O target/deequ-1.0.3.jar