- change S3 file writing from overwrite to overwrote_partitions

- remove uprn dq tests in tables where uprns are not critical

Author: annajgibson

scripts/jobs/housing/housing_gx_dq_metadata.py

@@ -73,7 +73,7 @@ def main():
         dataset=True,
         database=target_database,
         table=target_table,
-        mode="overwrite",
+        mode="overwrite_partitions",
         partition_cols=partition_keys,
         dtype=dtype_dict
     )

scripts/jobs/housing/housing_person_reshape_gx_suite.py

@@ -23,24 +23,6 @@ class ExpectSurnameColumnValueLength(gxe.ExpectColumnValueLengthsToBeBetween):
     description: str = "Expect surname to be at least 1 character length"
 
 
-class ExpectUPRNColumnValueLengthsBetween(gxe.ExpectColumnValueLengthsToBeBetween):
-    column: str = "uprn"
-    min_value: int = 11
-    max_value: int = 12
-    description: str = "Expect UPRN to be between 11 and 12 characters length inclusive"
-
-
-class ExpectUPRNColumnValuesToMatchRegex(gxe.ExpectColumnValuesToMatchRegex):
-    column: str = "uprn"
-    regex: str = r"^[1-9]\d{10,11}"
-    description: str = "Expect UPRN to match regex ^[1-9]\d{10,11} (starting with digit 1-9, followed by 10 or 11 digits"
-
-
-class ExpectUPRNNotToBeNull(gxe.ExpectColumnValuesToNotBeNull):
-    column: str = "uprn"
-    description: str = "Expect UPRN column to be complete with no missing values"
-
-
 class ExpectPersonTypeValuesToBeInSet(gxe.ExpectColumnValuesToBeInSet):
     column: str = 'person_type'
     value_set: list = ['Tenant', 'HouseholdMember', 'Leaseholder', 'Freeholder', 'Occupant', 'HousingOfficer',
@@ -81,11 +63,6 @@ class ExpectPersonIDAndPaymentReferenceColumnValuesToBeUniqueWithinRecord(
     description: str = "Expect Person ID and Payment Reference to be unique within dataset"
 
 
-class ExpectUPRNColumnValuesToNotBeNull(gxe.ExpectColumnValuesToNotBeNull):
-    column: str = 'uprn'
-    description: str = "Expect UPRN be complete with no missing values"
-
-
 class ExpectDateOfBirthColumnValuesToNotBeNull(gxe.ExpectColumnValuesToNotBeNull):
     column: str = 'dateofbirth_parsed'
     description: str = "Expect dateofbirth_parsed be complete with no missing values"
@@ -104,17 +81,13 @@ class ExpectDateOfBirthToBeBetween(gxe.ExpectColumnValuesToBeBetween):
 suite = gx.ExpectationSuite(name='person_reshape_suite')
 suite.add_expectation(ExpectFirstNameColumnValueLength())
 suite.add_expectation(ExpectSurnameColumnValueLength())
-suite.add_expectation(ExpectUPRNColumnValueLengthsBetween())
-suite.add_expectation(ExpectUPRNColumnValuesToMatchRegex())
-suite.add_expectation(ExpectUPRNNotToBeNull())
 suite.add_expectation(ExpectPersonTypeValuesToBeInSet())
 suite.add_expectation(ExpectPreferredTitleValuesToBeInSet())
 suite.add_expectation(ExpectPersonIDColumnValuesToBeUnique())
 suite.add_expectation(ExpectPersonIDColumnValuesToNotBeNull())
 suite.add_expectation(ExpectPersonIDAndPropertyReferenceColumnValuesToBeUniqueWithinRecord())
 suite.add_expectation(ExpectPropertyRefColumnValuesToNotBeNull())
 suite.add_expectation(ExpectPersonIDAndPaymentReferenceColumnValuesToBeUniqueWithinRecord())
-suite.add_expectation(ExpectUPRNColumnValuesToNotBeNull())
 suite.add_expectation(ExpectDateOfBirthColumnValuesToNotBeNull())
 suite.add_expectation(ExpectDateOfBirthToBeBetween())
 

scripts/jobs/housing/housing_tenure_reshape_gx_suite.py

@@ -17,19 +17,6 @@ class ExpectMemberFullNameColumnValueLengthsBetween(gxe.ExpectColumnValueLengths
     description: str = "Expect Member Fullname to be at least 1 character length"
 
 
-class ExpectUPRNColumnValueLengthsBetween(gxe.ExpectColumnValueLengthsToBeBetween):
-    column: str = "uprn"
-    min_value: int = 11
-    max_value: int = 12
-    description: str = "Expect UPRN to be between 11 and 12 characters length inclusive"
-
-
-class ExpectUPRNColumnValuesToMatchRegex(gxe.ExpectColumnValuesToMatchRegex):
-    column: str = "uprn"
-    regex: str = r"^[1-9]\d{10,11}"
-    description: str = "Expect UPRN to match regex ^[1-9]\d{10,11} (starting with digit 1-9, followed by 10 or 11 digits"
-
-
 class ExpectDescriptionValuesToBeInSet(gxe.ExpectColumnValuesToBeInSet):
     column: str = 'description'
     value_set: list = ['Asylum Seeker', 'Commercial Let', 'Temp Decant', 'Freehold', 'Freehold (Serv)', 'Introductory',
@@ -76,11 +63,6 @@ class ExpectTenancyIDColumnNotToBeNull(gxe.ExpectColumnValuesToNotBeNull):
     description: str = "Expect Tenancy ID column to be complete with no missing values"
 
 
-class ExpectUPRNNotToBeNull(gxe.ExpectColumnValuesToNotBeNull):
-    column: str = "uprn"
-    description: str = "Expect UPRN column to be complete with no missing values"
-
-
 class ExpectStartOfTenureDateColumnNotToBeNull(gxe.ExpectColumnValuesToNotBeNull):
     column: str = "startoftenuredate"
     description: str = "Expect Start of Tenure Date column to be complete with no missing values"
@@ -97,14 +79,11 @@ class ExpectTenureCodeColumnNotToBeNull(gxe.ExpectColumnValuesToNotBeNull):
 suite = gx.ExpectationSuite(name='tenure_reshape_suite')
 suite.add_expectation(ExpectTenancyIDAndPaymentReferenceColumnValuesToBeUniqueWithinRecord())
 suite.add_expectation(ExpectMemberFullNameColumnValueLengthsBetween())
-suite.add_expectation(ExpectUPRNColumnValueLengthsBetween())
-suite.add_expectation(ExpectUPRNColumnValuesToMatchRegex())
 suite.add_expectation(ExpectDescriptionValuesToBeInSet())
 suite.add_expectation(ExpectAssetTypeValuesToBeInSet())
 suite.add_expectation(ExpectTenancyIDAndPropertyReferenceColumnValuesToBeUniqueWithinRecord())
 suite.add_expectation(ExpectPaymentReferenceColumnNotToBeNull())
 suite.add_expectation(ExpectTenancyIDColumnNotToBeNull())
-suite.add_expectation(ExpectUPRNNotToBeNull())
 suite.add_expectation(ExpectStartOfTenureDateColumnNotToBeNull())
 suite.add_expectation(ExpectTenureCodeColumnNotToBeNull())