More test updates (#2575)

* Update test values; pause housing GX job

* Update test values; pause housing GX job

* Add more property type value codes

Author: annajgibson

scripts/jobs/housing/housing_nec_migration_properties_data_load_gx_suite.py

@@ -17,28 +17,6 @@ class PropertiesExpectPropRefColumnValuesToNotBeNull(gxe.ExpectColumnValuesToNot
     description: str = "Expect LPRO_PROPREF (prop ref) values to not be Null"
 
 
-class PropertiesExpectPropTypeCodeToBeInSet(gxe.ExpectColumnValuesToBeInSet):
-    column: str = "LPRO_HOU_PTV_CODE"
-    value_set: list = [
-        "BUN",
-        "CMC",
-        "CMF",
-        "COM",
-        "CYC",
-        "DUP",
-        "FLT",
-        "GAR",
-        "HOU",
-        "MAI",
-        "PRA",
-        "PSP",
-        "ROM",
-        "STD",
-        "TRV",
-    ]
-    description: str = "Expect property type codes to contain one of the set"
-
-
 class PropertiesExpectOccStatusCodeToBeInSet(gxe.ExpectColumnValuesToBeInSet):
     column: str = "LPRO_SCO_CODE"
     value_set: list = ["OCC", "VOI", "CLO"]
@@ -88,7 +66,7 @@ class PropertiesExpectPropTypeValuesToBeInSet(gxe.ExpectColumnValuesToBeInSet):
     column: str = "LPRO_HOU_PTV_CODE"
     value_set: list = [
         "CMC",
-        "CMC",
+        "CMF",
         "GAR",
         "FLT",
         "HOU",
@@ -102,11 +80,18 @@ class PropertiesExpectPropTypeValuesToBeInSet(gxe.ExpectColumnValuesToBeInSet):
         "PRA",
         "CYC",
         "DUP",
+        "BOI",
+        "LFT",
+        "CON",
+        "PLY",
+
     ]
     description: str = "Expect property type values to be one of the set"
 
 
-class PropertiesExpectPropColumnsToMatchOrderedList(gxe.ExpectTableColumnsToMatchOrderedList):
+class PropertiesExpectPropColumnsToMatchOrderedList(
+    gxe.ExpectTableColumnsToMatchOrderedList
+):
     column_list = [
         "LPRO_PROPREF",
         "LPRO_HOU_FRB",
@@ -155,7 +140,6 @@ class PropertiesExpectPropColumnsToMatchOrderedList(gxe.ExpectTableColumnsToMatc
 suite = gx.ExpectationSuite(name="properties_data_load_suite")
 
 suite.add_expectation(PropertiesExpectPropRefColumnValuesToBeUnique())
-suite.add_expectation(PropertiesExpectPropTypeCodeToBeInSet())
 suite.add_expectation(PropertiesExpectOccStatusCodeToBeInSet())
 suite.add_expectation(PropertiesExpectOrgIndicatorToBeInSet())
 suite.add_expectation(PropertiesExpectOwnTypeToBeInSet())

scripts/jobs/housing/housing_nec_migration_tenancies_data_load_gx_suite.py

@@ -33,7 +33,7 @@ class TenanciesExpectTenancyTypeCodeToBeInSet(gxe.ExpectColumnValuesToBeInSet):
         "LHS",
         "LTA",
         "MPA",
-        "NON",
+        "NONSEC",
         "PVG",
         "RTM",
         "SEC",

terraform/etl/54-aws-glue-housing-apply-gx-dq-tests.tf

@@ -13,10 +13,10 @@ module "housing_apply_gx_dq_tests" {
   helper_module_key              = data.aws_s3_object.helpers.key
   pydeequ_zip_key                = data.aws_s3_object.pydeequ.key
   spark_ui_output_storage_id     = module.spark_ui_output_storage_data_source.bucket_id
-  trigger_enabled                = local.is_production_environment
+  trigger_enabled                = false
   number_of_workers_for_glue_job = 2
   schedule                       = "cron(0 10 ? * MON-FRI *)"
-  job_parameters                 = {
+  job_parameters = {
     "--job-bookmark-option"              = "job-bookmark-enable"
     "--enable-glue-datacatalog"          = "true"
     "--enable-continuous-cloudwatch-log" = "true"
@@ -51,7 +51,7 @@ module "housing_gx_dq_metadata" {
   spark_ui_output_storage_id     = module.spark_ui_output_storage_data_source.bucket_id
   trigger_enabled                = local.is_production_environment
   number_of_workers_for_glue_job = 2
-  job_parameters                 = {
+  job_parameters = {
     "--job-bookmark-option"              = "job-bookmark-enable"
     "--enable-glue-datacatalog"          = "true"
     "--enable-continuous-cloudwatch-log" = "true"
@@ -68,11 +68,11 @@ module "housing_gx_dq_metadata" {
 }
 
 resource "aws_glue_trigger" "housing_gx_dq_metadata" {
-  name = "${local.short_identifier_prefix}Housing GX Data Quality Metadata Trigger"
-  type = "CONDITIONAL"
-  tags     = module.department_housing_data_source.tags
-  enabled  = local.is_production_environment
-  count    = local.is_live_environment ? 1 : 0
+  name    = "${local.short_identifier_prefix}Housing GX Data Quality Metadata Trigger"
+  type    = "CONDITIONAL"
+  tags    = module.department_housing_data_source.tags
+  enabled = local.is_production_environment
+  count   = local.is_live_environment ? 1 : 0
 
   actions {
     job_name = "${local.short_identifier_prefix}Housing GX Data Quality Metadata"