Add GX DQ tests for tenancies data load

Author: annajgibson

scripts/helpers/housing_nec_migration_gx_dq_inputs.py

@@ -1,6 +1,9 @@
-sql_config = {"properties": {"id_field": "LPRO_PROPREF"}}
+sql_config = {
+    "properties": {"id_field": "LPRO_PROPREF"},
+    "tenancies": {"id_field": "LTCY_ALT_REF"},
+}
 
-data_load_list = ["properties"]
+data_load_list = ["properties", "tenancies"]
 
 table_list = {
     "properties": [
@@ -11,8 +14,10 @@
         "properties_1e",
         "properties_2a",
         "properties_4a",
-        "properties_4c",
-    ]
+        "properties_4b",
+        "properties_4c"
+    ],
+    "tenancies": ["tenancies_1a"],
 }
 
 partition_keys = ["import_date"]

scripts/jobs/housing/housing_nec_migration_apply_gx_dq_tests.py

@@ -16,6 +16,8 @@
     table_list,
 )
 import scripts.jobs.housing.housing_nec_migration_properties_data_load_gx_suite
+import scripts.jobs.housing.housing_nec_migration_properties_data_load_gx_suite
+
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)

scripts/jobs/housing/housing_nec_migration_tenancies_data_load_gx_suite.py

@@ -0,0 +1,91 @@
+# flake8: noqa: F821
+
+import sys
+
+from awsglue.utils import getResolvedOptions
+import great_expectations as gx
+import great_expectations.expectations as gxe
+
+
+class ExpectTagRefColumnValuesToBeUnique(gxe.ExpectColumnValuesToBeUnique):
+    column: str = "LTCY_ALT_REF"
+    description: str = "Expect LTCY_ALT_REF (tenancy ref) values to be unique"
+
+
+class ExpectTenancyTypeCodeToBeInSet(gxe.ExpectColumnValuesToBeInSet):
+    column: str = "LTCY_TTY_CODE"
+    value_set: list = [
+        "ASH",
+        "ASY",
+        "Demoted",
+        "FRS",
+        "HAL",
+        "INT",
+        "LEA",
+        "LTA",
+        "LHS",
+        "MPA",
+        "PVG",
+        "SPS",
+        "RTM",
+        "SEC",
+        "SSE",
+        "SHO",
+        "SLL",
+        "TLA",
+        "TBB",
+        "TBBFam",
+        "DEC",
+        "THGF",
+        "THO",
+        "THL",
+        "TPL",
+        "TRA",
+        "TACCFLAT",
+        "TGA",
+        "UNDER18",
+        "NONSECTA",
+        "NONSECHR",
+        "OFFICESE",
+        "LIVINGRT",
+        "FRE",
+    ]
+    description: str = "Expect tenancy type code to contain one of the set"
+
+
+class ExpectTenureTypeCodeToBeInSet(gxe.ExpectColumnValuesToBeInSet):
+    column: str = "LTCY_HRV_TTYP_CODE"
+    value_set: list = [
+        "Secure",
+        "NonSec",
+        "NonRes",
+        "Leasehold",
+        "Temporary",
+        "Freehold",
+        "Commercial",
+        "LivingRent"
+    ]
+    description: str = "Expect tenure type code to be one of the set"
+
+
+class ExpectTenancyStatusCodeToBeInSet(gxe.ExpectColumnValuesToBeInSet):
+    column: str = "LTCY_HRV_TST_CODE"
+    value_set: list = ["Notice", "Decant", "UnautOcc"]
+    description: str = "Expect tenancy status code to be one of the set"
+
+
+
+arg_key = ["s3_target_location"]
+args = getResolvedOptions(sys.argv, arg_key)
+locals().update(args)
+
+# add to GX context
+context = gx.get_context(mode="file", project_root_dir=s3_target_location)
+
+suite = gx.ExpectationSuite(name="tenancies_data_load_suite")
+
+suite.add_expectation(ExpectTagRefColumnValuesToBeUnique())
+suite.add_expectation(ExpectTenancyTypeCodeToBeInSet())
+suite.add_expectation(ExpectTenureTypeCodeToBeInSet())
+suite.add_expectation(ExpectTenancyStatusCodeToBeInSet())
+suite = context.suites.add(suite)