Skip to content

Commit 21d8432

Browse files
annajgibsonannajgibson
committed
Amend sql queries to only include ke columns and rows
1 parent 5e1fbb9 commit 21d8432

File tree

1 file changed

+97
-97
lines changed

1 file changed

+97
-97
lines changed

‎scripts/helpers/housing_gx_dq_inputs.py‎

Lines changed: 97 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
sql_config = {'person_reshape': {
2-
'sql': """SELECT *, substr(startdate, 1, 10) as startdate_parsed, substr(enddate, 1, 10) as enddate_parsed,
2+
'sql': """SELECT *, substr(startdate, 1, 10) as startdate_parsed, substr(enddate, 1, 10) as enddate_parsed,
33
substr(dateofbirth, 1, 10) as dateofbirth_parsed FROM "housing-refined-zone"."person_reshape" WHERE import_date = (SELECT max(import_date) FROM "housing-refined-zone"."person_reshape") AND enddate IS NULL AND type IN ('Secure', 'Introductory')""",
4-
'id_field': 'person_id'},
5-
'tenure_reshape': {
6-
'sql': """SELECT * FROM "housing-refined-zone"."tenure_reshape" where import_date=(select max(import_date) from "housing-refined-zone"."tenure_reshape") and description in ('Secure', 'Introductory', 'Mesne Profit Ac', 'Non-Secure') and (endoftenuredate is null or substr(endoftenuredate, 1, 11) = '1900-01-01')""",
7-
'id_field': 'tenancy_id'},
8-
'contacts_reshape': {
9-
'sql': """SELECT
4+
'id_field': 'person_id'},
5+
'tenure_reshape': {
6+
'sql': """SELECT * FROM "housing-refined-zone"."tenure_reshape" where import_date=(select max(import_date) from "housing-refined-zone"."tenure_reshape") and description in ('Secure', 'Introductory', 'Mesne Profit Ac', 'Non-Secure') and (endoftenuredate is null or substr(endoftenuredate, 1, 11) = '1900-01-01')""",
7+
'id_field': 'tenancy_id'},
8+
'contacts_reshape': {
9+
'sql': """SELECT
1010
DATE_FORMAT(from_iso8601_timestamp(createdat), '%Y-%b-%d') AS createdat,
1111
contacttype,
1212
subtype,
@@ -21,12 +21,12 @@
2121
FROM "housing-refined-zone"."contacts_reshape"
2222
where targettype = 'person'
2323
and import_date = (select max(import_date) from "housing-refined-zone"."contacts_reshape")""",
24-
'id_field': 'id'},
25-
'housing_homeowner_record_sheet': {
26-
'sql': """SELECT * FROM "housing-raw-zone"."housing_homeowner_record_sheet" where import_date=(select max(import_date) from "housing-raw-zone"."housing_homeowner_record_sheet")""",
27-
'id_field': 'property_no'},
28-
'housing_dwellings_list': {
29-
'sql': """SELECT
24+
'id_field': 'id'},
25+
'housing_homeowner_record_sheet': {
26+
'sql': """SELECT * FROM "housing-raw-zone"."housing_homeowner_record_sheet" where import_date=(select max(import_date) from "housing-raw-zone"."housing_homeowner_record_sheet")""",
27+
'id_field': 'property_no'},
28+
'housing_dwellings_list': {
29+
'sql': """SELECT
3030
cast(property_dwelling_reference_number as varchar) as property_dwelling_reference_number,
3131
cast(llpg as varchar) as llpg,
3232
cast(estate_reference_number as varchar) as estate_reference_number,
@@ -46,9 +46,9 @@
4646
registered_building,
4747
year_built_block
4848
FROM "housing-raw-zone"."housing_dwellings_list" where import_date = (select max(import_date) from "housing-raw-zone"."housing_dwellings_list")""",
49-
'id_field': 'property_dwelling_reference_number'},
50-
'assets_reshape': {
51-
'sql': """select
49+
'id_field': 'property_dwelling_reference_number'},
50+
'assets_reshape': {
51+
'sql': """select
5252
cast(assetid as varchar) as assetid,
5353
assettype,
5454
cast(uprn as varchar) as uprn,
@@ -88,91 +88,91 @@
8888
import_date
8989
FROM "housing-refined-zone"."assets_reshape" where import_date = (select max(import_date) from "housing-refined-zone"."assets_reshape")
9090
and assettype in ('BoilerHouse','BoosterPump','CleanersFacilities','CombinedHeatAndPowerUnit','CommunityHall','Concierge','Dwelling','LettableNonDwelling','Lift','NA','NBD','OutBuilding','TravellerSite')""",
91-
'id_field': 'asset_id'},
92-
'matenancyagreement': {
93-
'sql': """SELECT *, substr(cast(eot as varchar), 1, 10) as eot_parsed, substr(cast(cot as varchar), 1, 10) as cot_parsed FROM "housing-raw-zone"."sow2b_dbo_matenancyagreement" where import_date=(select max(import_date) FROM "housing-raw-zone"."sow2b_dbo_matenancyagreement")""",
94-
'id_field': 'tag_ref'},
95-
'maproperty': {
96-
'sql': """SELECT * FROM "housing-raw-zone"."sow2b_dbo_maproperty" where import_date=(select max(import_date) FROM "housing-raw-zone"."sow2b_dbo_maproperty")""",
97-
'id_field': 'prop_ref'}
91+
'id_field': 'asset_id'},
92+
'matenancyagreement': {
93+
'sql': """SELECT *, substr(cast(eot as varchar), 1, 10) as eot_parsed, substr(cast(cot as varchar), 1, 10) as cot_parsed FROM "housing-raw-zone"."sow2b_dbo_matenancyagreement" where import_date=(select max(import_date) FROM "housing-raw-zone"."sow2b_dbo_matenancyagreement")""",
94+
'id_field': 'tag_ref'},
95+
'maproperty': {
96+
'sql': """SELECT * FROM "housing-raw-zone"."sow2b_dbo_maproperty" where import_date=(select max(import_date) FROM "housing-raw-zone"."sow2b_dbo_maproperty")""",
97+
'id_field': 'prop_ref'}
9898
}
9999

100100
table_list = ['person_reshape', 'tenure_reshape', 'contacts_reshape', 'housing_homeowner_record_sheet',
101-
'housing_dwellings_list', 'assets_reshape', 'matenancyagreement', 'maproperty']
101+
'housing_dwellings_list', 'assets_reshape', 'matenancyagreement', 'maproperty']
102102

103103
partition_keys = ['import_year', 'import_month', 'import_day', 'import_date']
104104

105105
dq_dimensions_map = {
106-
'expect_arr_patch_not_to_be_null': 'COMPLETENESS',
107-
'expect_asset_id_not_to_be_null': 'COMPLETENESS',
108-
'expect_asset_type_not_to_be_null': 'COMPLETENESS',
109-
'expect_asset_type_values_to_be_in_set': 'CONSISTENCY',
110-
'expect_block_ref_no_column_values_to_match_regex': 'ACCURACY',
111-
'expect_co_t_not_to_be_null': 'COMPLETENESS',
112-
'expect_co_t_to_be_between': 'VALIDITY',
113-
'expect_column_value_lengths_to_be_between': 'VALIDITY',
114-
'expect_column_values_to_be_in_set': 'CONSISTENCY',
115-
'expect_column_values_to_be_unique': 'UNIQUENESS',
116-
'expect_column_values_to_match_regex': 'ACCURACY',
117-
'expect_column_values_to_not_be_null': 'COMPLETENESS',
118-
'expect_contact_type_column_values_to_be_in_set': 'CONSISTENCY',
119-
'expect_contact_type_column_values_to_not_be_null': 'COMPLETENESS',
120-
'expect_contact_value_column_values_to_be_unique': 'UNIQUENESS',
121-
'expect_contact_value_column_values_to_not_be_null': 'COMPLETENESS',
122-
'expect_date_of_birth_column_values_to_not_be_null': 'COMPLETENESS',
123-
'expect_date_of_birth_to_be_between': 'VALIDITY',
124-
'expect_description_values_to_be_in_set': 'CONSISTENCY',
125-
'expect_end_of_tenure_date_column_to_be_null': 'COMPLETENESS',
126-
'expect_eo_t_to_be_between': 'VALIDITY',
127-
'expect_estate_ref_no_column_values_to_match_regex': 'ACCURACY',
128-
'expect_first_name_column_value_length': 'ACCURACY',
129-
'expect_firstname_column_value_length': 'ACCURACY',
130-
'expect_is_organisation_column_values_to_not_be_null': 'COMPLETENESS',
131-
'expect_is_organisation_values_to_be_in_set': 'CONSISTENCY',
132-
'expect_llpg_and_prop_ref_column_values_to_be_unique_within_record': 'UNIQUENESS',
133-
'expect_llpg_column_value_lengths_between': 'VALIDITY',
134-
'expect_llpg_column_values_to_be_unique': 'UNIQUENESS',
135-
'expect_llpg_column_values_to_match_regex': 'ACCURACY',
136-
'expect_llpg_column_values_to_not_be_null': 'COMPLETENESS',
137-
'expect_member_full_name_column_value_lengths_between': 'VALIDITY',
138-
'expect_member_is_responsible_values_to_be_in_set': 'CONSISTENCY',
139-
'expect_payment_ref_no_column_values_to_match_regex': 'ACCURACY',
140-
'expect_payment_reference_column_not_to_be_null': 'COMPLETENESS',
141-
'expect_person_id_and_payment_reference_column_values_to_be_unique_within_record': 'UNIQUENESS',
142-
'expect_person_id_and_property_reference_column_values_to_be_unique_within_record': 'UNIQUENESS',
143-
'expect_person_id_column_values_to_be_unique': 'UNIQUENESS',
144-
'expect_person_id_column_values_to_not_be_null': 'COMPLETENESS',
145-
'expect_person_type_values_to_be_in_set': 'CONSISTENCY',
146-
'expect_preferred_title_values_to_be_in_set': 'CONSISTENCY',
147-
'expect_prop_no_and_payment_ref_column_values_to_be_unique_within_record': 'UNIQUENESS',
148-
'expect_prop_no_column_values_to_be_unique': 'UNIQUENESS',
149-
'expect_prop_no_column_values_to_not_be_null': 'COMPLETENESS',
150-
'expect_prop_no_no_column_values_to_match_regex': 'ACCURACY',
151-
'expect_prop_ref_column_values_to_be_unique': 'UNIQUENESS',
152-
'expect_prop_ref_not_to_be_null': 'COMPLETENESS',
153-
'expect_property_ref_column_values_to_not_be_null': 'COMPLETENESS',
154-
'expect_rent_group_ref_not_to_be_null': 'COMPLETENESS',
155-
'expect_saff_rent_acc_not_to_be_null': 'COMPLETENESS',
156-
'expect_select_column_values_to_be_unique_within_record': 'UNIQUENESS',
157-
'expect_start_of_tenure_date_column_not_to_be_null': 'COMPLETENESS',
158-
'expect_sub_type_column_values_to_be_in_set': 'CONSISTENCY',
159-
'expect_sub_type_column_values_to_not_be_null': 'COMPLETENESS',
160-
'expect_surname_column_value_length': 'ACCURACY',
161-
'expect_tag_ref_column_not_to_be_null': 'COMPLETENESS',
162-
'expect_tag_ref_column_values_to_be_unique': 'UNIQUENESS',
163-
'expect_tag_ref_not_to_be_null': 'COMPLETENESS',
164-
'expect_target_id_and_value_column_values_to_be_unique_within_record': 'UNIQUENESS',
165-
'expect_target_id_column_values_to_not_be_null': 'COMPLETENESS',
166-
'expect_target_type_column_values_to_be_in_set': 'CONSISTENCY',
167-
'expect_tenancy_id_and_payment_reference_column_values_to_be_unique_within_record': 'UNIQUENESS',
168-
'expect_tenancy_id_and_property_reference_column_values_to_be_unique_within_record': 'UNIQUENESS',
169-
'expect_tenancy_id_column_not_to_be_null': 'COMPLETENESS',
170-
'expect_tenure_code_column_not_to_be_null': 'COMPLETENESS',
171-
'expect_tenure_code_values_to_be_in_set': 'CONSISTENCY',
172-
'expect_tenure_not_to_be_null': 'COMPLETENESS',
173-
'expect_tenure_type_column_values_to_be_in_set': 'CONSISTENCY',
174-
'expect_tenure_values_to_be_in_set': 'CONSISTENCY',
175-
'expect_uprn_column_value_lengths_between': 'VALIDITY',
176-
'expect_uprn_column_values_to_match_regex': 'ACCURACY',
177-
'expect_uprn_column_values_to_not_be_null': 'COMPLETENESS',
178-
'expect_uprn_not_to_be_null': 'COMPLETENESS'}
106+
'expect_arr_patch_not_to_be_null': 'COMPLETENESS',
107+
'expect_asset_id_not_to_be_null': 'COMPLETENESS',
108+
'expect_asset_type_not_to_be_null': 'COMPLETENESS',
109+
'expect_asset_type_values_to_be_in_set': 'CONSISTENCY',
110+
'expect_block_ref_no_column_values_to_match_regex': 'ACCURACY',
111+
'expect_co_t_not_to_be_null': 'COMPLETENESS',
112+
'expect_co_t_to_be_between': 'VALIDITY',
113+
'expect_column_value_lengths_to_be_between': 'VALIDITY',
114+
'expect_column_values_to_be_in_set': 'CONSISTENCY',
115+
'expect_column_values_to_be_unique': 'UNIQUENESS',
116+
'expect_column_values_to_match_regex': 'ACCURACY',
117+
'expect_column_values_to_not_be_null': 'COMPLETENESS',
118+
'expect_contact_type_column_values_to_be_in_set': 'CONSISTENCY',
119+
'expect_contact_type_column_values_to_not_be_null': 'COMPLETENESS',
120+
'expect_contact_value_column_values_to_be_unique': 'UNIQUENESS',
121+
'expect_contact_value_column_values_to_not_be_null': 'COMPLETENESS',
122+
'expect_date_of_birth_column_values_to_not_be_null': 'COMPLETENESS',
123+
'expect_date_of_birth_to_be_between': 'VALIDITY',
124+
'expect_description_values_to_be_in_set': 'CONSISTENCY',
125+
'expect_end_of_tenure_date_column_to_be_null': 'COMPLETENESS',
126+
'expect_eo_t_to_be_between': 'VALIDITY',
127+
'expect_estate_ref_no_column_values_to_match_regex': 'ACCURACY',
128+
'expect_first_name_column_value_length': 'ACCURACY',
129+
'expect_firstname_column_value_length': 'ACCURACY',
130+
'expect_is_organisation_column_values_to_not_be_null': 'COMPLETENESS',
131+
'expect_is_organisation_values_to_be_in_set': 'CONSISTENCY',
132+
'expect_llpg_and_prop_ref_column_values_to_be_unique_within_record': 'UNIQUENESS',
133+
'expect_llpg_column_value_lengths_between': 'VALIDITY',
134+
'expect_llpg_column_values_to_be_unique': 'UNIQUENESS',
135+
'expect_llpg_column_values_to_match_regex': 'ACCURACY',
136+
'expect_llpg_column_values_to_not_be_null': 'COMPLETENESS',
137+
'expect_member_full_name_column_value_lengths_between': 'VALIDITY',
138+
'expect_member_is_responsible_values_to_be_in_set': 'CONSISTENCY',
139+
'expect_payment_ref_no_column_values_to_match_regex': 'ACCURACY',
140+
'expect_payment_reference_column_not_to_be_null': 'COMPLETENESS',
141+
'expect_person_id_and_payment_reference_column_values_to_be_unique_within_record': 'UNIQUENESS',
142+
'expect_person_id_and_property_reference_column_values_to_be_unique_within_record': 'UNIQUENESS',
143+
'expect_person_id_column_values_to_be_unique': 'UNIQUENESS',
144+
'expect_person_id_column_values_to_not_be_null': 'COMPLETENESS',
145+
'expect_person_type_values_to_be_in_set': 'CONSISTENCY',
146+
'expect_preferred_title_values_to_be_in_set': 'CONSISTENCY',
147+
'expect_prop_no_and_payment_ref_column_values_to_be_unique_within_record': 'UNIQUENESS',
148+
'expect_prop_no_column_values_to_be_unique': 'UNIQUENESS',
149+
'expect_prop_no_column_values_to_not_be_null': 'COMPLETENESS',
150+
'expect_prop_no_no_column_values_to_match_regex': 'ACCURACY',
151+
'expect_prop_ref_column_values_to_be_unique': 'UNIQUENESS',
152+
'expect_prop_ref_not_to_be_null': 'COMPLETENESS',
153+
'expect_property_ref_column_values_to_not_be_null': 'COMPLETENESS',
154+
'expect_rent_group_ref_not_to_be_null': 'COMPLETENESS',
155+
'expect_saff_rent_acc_not_to_be_null': 'COMPLETENESS',
156+
'expect_select_column_values_to_be_unique_within_record': 'UNIQUENESS',
157+
'expect_start_of_tenure_date_column_not_to_be_null': 'COMPLETENESS',
158+
'expect_sub_type_column_values_to_be_in_set': 'CONSISTENCY',
159+
'expect_sub_type_column_values_to_not_be_null': 'COMPLETENESS',
160+
'expect_surname_column_value_length': 'ACCURACY',
161+
'expect_tag_ref_column_not_to_be_null': 'COMPLETENESS',
162+
'expect_tag_ref_column_values_to_be_unique': 'UNIQUENESS',
163+
'expect_tag_ref_not_to_be_null': 'COMPLETENESS',
164+
'expect_target_id_and_value_column_values_to_be_unique_within_record': 'UNIQUENESS',
165+
'expect_target_id_column_values_to_not_be_null': 'COMPLETENESS',
166+
'expect_target_type_column_values_to_be_in_set': 'CONSISTENCY',
167+
'expect_tenancy_id_and_payment_reference_column_values_to_be_unique_within_record': 'UNIQUENESS',
168+
'expect_tenancy_id_and_property_reference_column_values_to_be_unique_within_record': 'UNIQUENESS',
169+
'expect_tenancy_id_column_not_to_be_null': 'COMPLETENESS',
170+
'expect_tenure_code_column_not_to_be_null': 'COMPLETENESS',
171+
'expect_tenure_code_values_to_be_in_set': 'CONSISTENCY',
172+
'expect_tenure_not_to_be_null': 'COMPLETENESS',
173+
'expect_tenure_type_column_values_to_be_in_set': 'CONSISTENCY',
174+
'expect_tenure_values_to_be_in_set': 'CONSISTENCY',
175+
'expect_uprn_column_value_lengths_between': 'VALIDITY',
176+
'expect_uprn_column_values_to_match_regex': 'ACCURACY',
177+
'expect_uprn_column_values_to_not_be_null': 'COMPLETENESS',
178+
'expect_uprn_not_to_be_null': 'COMPLETENESS'}

0 commit comments

Comments
 (0)