Add workaround for working with dates in GX. Add expectation to person_reshape suite. Amend sql query for person_reshape.

Author: annajgibson

scripts/helpers/housing_gx_dq_inputs.py

@@ -1,5 +1,8 @@
 sql_config = {'person_reshape': {
-    'sql': """SELECT *, cast(date_parse(substr(startdate, 1, 10), '%Y-%m-%d') as date) as startdate_parsed, cast(date_parse(substr(enddate, 1, 10), '%Y-%m-%d') as date) as enddate_parsed,  cast(date_parse(substr(dateofbirth, 1, 10), '%Y-%m-%d') as date) as dateofbirth_parsed FROM "housing-refined-zone"."person_reshape" where import_date=(select max(import_date) from "housing-refined-zone"."person_reshape") and enddate is NULL and type in ('Secure', 'Introductory')""",
+    'sql': """SELECT *, substr(startdate, 1, 10) as startdate_parsed, substr(enddate, 1, 10) as enddate_parsed,
+    substr(dateofbirth, 1, 10) as dateofbirth_parsed
+    FROM "housing-refined-zone"."person_reshape" WHERE import_date = (SELECT max(import_date) FROM "housing-refined-zone"."person_reshape"
+	) AND enddate IS NULL AND type IN ('Secure', 'Introductory')	and substr(dateofbirth, 1, 10) between '1850-01-01' and '2100-01-01' 	and substr(startdate, 1, 10) between '1900-01-01' and '2100-01-01'""",
     'id_field': 'person_id'},
     'tenure_reshape': {
         'sql': """SELECT * FROM "housing-refined-zone"."tenure_reshape" where import_date>'20240412' and import_date=(select max(import_date) from "housing-refined-zone"."tenure_reshape" where import_date>'20240412') and isterminated=False and description in ('Secure', 'Introductory')""",

scripts/jobs/housing/housing_apply_gx_dq_tests.py

@@ -1,7 +1,7 @@
 # flake8: noqa: F821
 
 import awswrangler as wr
-from datetime import datetime
+from datetime import datetime, date
 import json
 import logging
 import sys
@@ -27,6 +27,13 @@
 locals().update(args)
 
 
+def json_serial(obj):
+    """JSON serializer for objects not serializable by default."""
+    if isinstance(obj, (datetime, date)):
+        return obj.isoformat()
+    raise TypeError(f"Type {type(obj)} not serializable")
+
+
 def main():
     # add GX context
     context = gx.get_context(mode="file", project_root_dir=s3_target_location)
@@ -96,7 +103,11 @@ def main():
         )
 
         checkpoint_result = checkpoint.run(batch_parameters=batch_parameters)
-        results = json.loads(checkpoint_result.describe())
+        results_dict = checkpoint_result.describe_dict()
+        # Serialize the result to handle any datetime objects
+        json_results = json.dumps(results_dict, default=json_serial)
+        logger.info(f"json_results: {json_results}")
+        results = json.loads(json_results)
         table_results_df = pd.json_normalize(results['validation_results'][0]['expectations'])
         table_results_df_list.append(table_results_df)
 

scripts/jobs/housing/housing_person_reshape_gx_suite.py

@@ -1,5 +1,5 @@
 # flake8: noqa: F821
-from datetime import datetime, date
+from datetime import datetime
 import sys
 
 from awsglue.utils import getResolvedOptions
@@ -85,7 +85,14 @@
 )
 suite.add_expectation(
     gxe.ExpectColumnValuesToNotBeNull(
-        column='dateofbirth')
+        column='dateofbirth_parsed')
+)
+suite.add_expectation(
+    gxe.ExpectColumnValuesToBeBetween(
+        column='dateofbirth_parsed',
+        min_value=datetime(1900, 1, 1, 0, 0, 0).isoformat(),
+        max_value=datetime.today().isoformat()
+    )
 )
 
 suite = context.suites.add(suite)