Skip to content

Commit 812f791

Browse files
trslatertrslater
authored
Merge pull request #2835 from bcgov/2517-etl-property-data
2517: Make property ETL
2 parents cc03d4b + 32b3e72 commit 812f791

File tree

9 files changed

+242
-2
lines changed

9 files changed

+242
-2
lines changed

‎bin/migrate-nris-data/migrate.py‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
from common.constants import BATCH_UPLOAD_SIZE
55
import ce_files
6+
import properties
67
import responsible_parties
78
import submitters
89

@@ -14,7 +15,11 @@ def cli():
1415
pass
1516

1617

17-
@cli.group(chain=True, invoke_without_command=True, help="Runs all imports in sequence if no subcommand is specified.")
18+
@cli.group(
19+
chain=True,
20+
invoke_without_command=True,
21+
help="Runs all imports in sequence if no subcommand is specified.",
22+
)
1823
@click.option(
1924
"--batch-size",
2025
default=BATCH_UPLOAD_SIZE,
@@ -30,6 +35,7 @@ def import_data(ctx, batch_size):
3035
ctx.invoke(import_ce_files)
3136
ctx.invoke(import_submitters)
3237
ctx.invoke(import_responsible_parties)
38+
ctx.invoke(import_properties)
3339

3440

3541
@import_data.command("ce-files")
@@ -56,14 +62,26 @@ def import_responsible_parties(ctx):
5662
console.log("Responsible party import complete.")
5763

5864

65+
@import_data.command("properties")
66+
@click.pass_context
67+
def import_properties(ctx):
68+
console.log("Start importing properties...")
69+
properties.etl(batch_size=ctx.parent.params["batch_size"])
70+
console.log("Property import complete.")
71+
72+
5973
@import_data.result_callback()
6074
def import_cleanup(results, batch_size):
6175
console.log("All imports complete. Cleaning up...")
6276
# Any import cleanup can go here.
6377
console.log("Cleanup complete.")
6478

6579

66-
@cli.group(chain=True, invoke_without_command=True, help="Obfuscates all data in sequence if no subcommand is specified.")
80+
@cli.group(
81+
chain=True,
82+
invoke_without_command=True,
83+
help="Obfuscates all data in sequence if no subcommand is specified.",
84+
)
6785
@click.option(
6886
"--batch-size",
6987
default=BATCH_UPLOAD_SIZE,
@@ -79,6 +97,7 @@ def obfuscate(ctx, batch_size):
7997
ctx.invoke(obfuscate_ce_files)
8098
ctx.invoke(obfuscate_submitters)
8199
ctx.invoke(obfuscate_responsible_parties)
100+
ctx.invoke(obfuscate_properties)
82101

83102

84103
@obfuscate.command("ce-files")
@@ -105,6 +124,14 @@ def obfuscate_responsible_parties(ctx):
105124
console.log("Responsible party obfuscation complete.")
106125

107126

127+
@obfuscate.command("properties")
128+
@click.pass_context
129+
def obfuscate_properties(ctx):
130+
console.log("Start obfuscating properties...")
131+
properties.obfuscate(batch_size=ctx.parent.params["batch_size"])
132+
console.log("Property obfuscation complete.")
133+
134+
108135
@obfuscate.result_callback()
109136
def obfuscation_cleanup(results, batch_size):
110137
console.log("All obfuscation complete. Cleaning up...")

‎bin/migrate-nris-data/properties/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
from .properties import *

‎bin/migrate-nris-data/properties/properties.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
from common.etl_logger import setup_and_get_logger
2+
from config import ABS_PATH
3+
from db import batch_read_write
4+
from faker import Faker
5+
6+
7+
def etl(batch_size):
8+
logger = setup_and_get_logger("property-etl")
9+
10+
batch_read_write(
11+
logger,
12+
batch_size,
13+
ABS_PATH / "properties/sql/count.sql",
14+
ABS_PATH / "properties/sql/et.sql",
15+
ABS_PATH / "properties/sql/l.sql",
16+
)
17+
18+
19+
def obfuscate(batch_size):
20+
logger = setup_and_get_logger("property-obfuscation")
21+
22+
batch_read_write(
23+
logger,
24+
batch_size, # Obfuscation can be done in smaller batches to reduce transaction size.
25+
ABS_PATH / "properties/sql/obfuscate_count.sql",
26+
ABS_PATH / "properties/sql/obfuscate_get_rows.sql",
27+
ABS_PATH / "properties/sql/obfuscate_update.sql",
28+
row_processor=row_obfuscator(),
29+
)
30+
31+
32+
def row_obfuscator():
33+
faker = Faker("la")
34+
35+
def obfuscate_row(row):
36+
row["alc_history"] = "\n\n".join(faker.paragraphs())
37+
38+
return row
39+
40+
return obfuscate_row

‎bin/migrate-nris-data/properties/sql/count.sql‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
select
2+
count(*)
3+
from
4+
nris.complaint nc;

‎bin/migrate-nris-data/properties/sql/et.sql‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
select
2+
distinct coalesce(nc.street_address, '')::text as civic_address,
3+
case
4+
when nc.region = 'Okanagan' then 'OKAR'
5+
when nc.region = 'South Coast' then 'SOUR'
6+
when nc.region = 'Kootenay' then 'KOOR'
7+
when nc.region = 'Interior' then 'INTR'
8+
when nc.region = 'Island' then 'ISLR'
9+
when nc.region = 'North' then 'NORR'
10+
end::text as region_code,
11+
nc.latitude::numeric(10, 7) as latitude,
12+
nc.longitude::numeric(10, 7) as longitude,
13+
case
14+
when nc.ownership_type = 'Fee Simple' then 'SMPL'
15+
when nc.ownership_type = 'Crown' then 'CRWN'
16+
end as ownership_type_code,
17+
nc.primary_location_pid::text as pid,
18+
nc.area::numeric(10, 2) as area_hectares,
19+
nc.percent_of_alr::numeric(5, 2) as alr_percentage,
20+
coalesce(nc.application_history_prior_c_e_issues, 'Migrated from NRIS, please verify using LAN folder')::text as alc_history,
21+
ace.uuid as file_uuid,
22+
case
23+
when nc.local_government = 'Alberni-Clayoquot' then 'e4b0f43c-7c7d-47cd-af05-24b6313d80b0'
24+
when nc.local_government = 'Bowen Island (Island Municipality)' then '6e5da7c6-9a72-44c5-8b8d-690439fb7cd1'
25+
when nc.local_government = 'Bulkley-Nechako' then '0a0c9cad-ef11-4b85-a3d9-1d0d055d7e0f'
26+
when nc.local_government = 'Capital' then 'beb93bee-6c60-404b-979b-2567a1a34492'
27+
when nc.local_government = 'Cariboo' then '773a4edc-7af3-4dcb-b9ae-28bea98099a1'
28+
when nc.local_government = 'Central Coast' then '51b0522b-b6e3-4d8f-904b-3413b33f936e'
29+
when nc.local_government = 'Central Kootenay' then 'eb9a3858-b065-46cf-8bf7-73cdb5b6c2f7'
30+
when nc.local_government = 'Central Okanagan' then 'c180bc79-c78b-4c58-91a6-aa3e92408d15'
31+
when nc.local_government = 'Chilliwack' then 'cf727428-c2c7-4478-8924-6b8d41570a17'
32+
when nc.local_government = 'City of Chilliwack' then 'cf727428-c2c7-4478-8924-6b8d41570a17'
33+
when nc.local_government = 'City of Abbotsford' then 'b30fcb96-d5c5-498c-933c-22563f25b606'
34+
when nc.local_government = 'City of Armstrong' then 'f9364f76-10bb-4036-9f0f-9e497e71c06f'
35+
when nc.local_government = 'City of Burnaby' then '72a9c8c8-28cb-4b08-b79a-8afebabdd327'
36+
when nc.local_government = 'City of Coquitlam' then '4754ed2c-100d-4bba-ac60-6a97232a9a31'
37+
when nc.local_government = 'City of Courtenay' then 'b9195d80-81b4-42c5-9047-e1a1a00a04f0'
38+
when nc.local_government = 'City of Dawson Creek' then '8d2378a8-edef-4b04-8b13-8aa67603f4f5'
39+
when nc.local_government = 'City of Delta' then 'f1db2dca-61f5-46ac-819c-2c5575b007c9'
40+
when nc.local_government = 'City of Kamloops' then 'ac411bf2-66b3-401d-9155-27f3026bbed3'
41+
when nc.local_government = 'City of Kelowna' then 'cc7cebbb-d126-46bb-8a7a-6936f20ddce9'
42+
when nc.local_government = 'City of Langford' then 'c438ec9e-8446-4b15-a794-629da2330bb8'
43+
when nc.local_government = 'City of Langley' then '825f2ecf-fac1-467c-a96c-7b29473073b3'
44+
when nc.local_government = 'City of Maple Ridge' then '8c834efa-9c33-4f46-a2c7-b0cfdedece04'
45+
when nc.local_government = 'City of Merritt' then 'd57ff5bc-5715-4017-ab5c-fa8d00334bec'
46+
when nc.local_government = 'City of Nanaimo' then 'afaa920f-7cd3-43c3-a7a1-bcd77b4a2af0'
47+
when nc.local_government = 'City of Penticton' then '9fda6054-e4e5-4fcb-9752-174abc8ee9ea'
48+
when nc.local_government = 'City of Pitt Meadows' then '9da89a5c-9a27-4371-8e60-d93a4c85e4a9'
49+
when nc.local_government = 'City of Port Coquitlam' then '3115b2e5-b038-4292-b238-eadc032de2e7'
50+
when nc.local_government = 'City of Powell River' then '7d2a7a18-2e2e-4a45-a3d3-0543652810e4'
51+
when nc.local_government = 'City of Prince George' then '692b797a-d4a3-4c02-b60d-6f7c559665d8'
52+
when nc.local_government = 'City of Richmond' then '961e2cf5-1508-4775-80c4-164f1199585d'
53+
when nc.local_government = 'City of Salmon Arm' then 'b2883559-a22c-49dc-b6f7-20fe9117d085'
54+
when nc.local_government = 'City of Surrey' then 'dd81e07c-499d-4416-bdfa-2a70f08302ce'
55+
when nc.local_government = 'City of Vancouver' then 'de39716c-a867-4403-86da-bb6b05a34908'
56+
when nc.local_government = 'City of Vernon' then 'f625fb10-c89e-49ac-afac-41b50e7ad3c3'
57+
when nc.local_government = 'City of West Kelowna' then 'd5636832-75e1-4f64-ad25-046a3cf594a2'
58+
when nc.local_government = 'Columbia Shuswap' then '614c63a6-599f-4c3e-bd51-43a0be5eaecd'
59+
when nc.local_government = 'Comox Valley' then '693e3cfa-ce7f-43b4-8952-a4b02bb273f1'
60+
when nc.local_government = 'Comox-Strathcona (Historical)' then 'ed2cfbb2-2139-4542-8063-212fd2654c9e'
61+
when nc.local_government = 'Cowichan Valley' then 'abfe022f-64f2-472b-b312-afc0c9ae7e8f'
62+
when nc.local_government = 'District of 100 Mile House' then '8eaa02e6-fd7d-4f51-89f7-b881b8137fa2'
63+
when nc.local_government = 'District of Central Saanich' then 'c6109982-ec0c-4624-91d7-77c4405bbf71'
64+
when nc.local_government = 'District of Coldstream' then '35906d62-8e9a-4170-9666-c21d2e16a506'
65+
when nc.local_government = 'District of Delta' then 'f1db2dca-61f5-46ac-819c-2c5575b007c9'
66+
when nc.local_government = 'District of Hope' then 'b17f2d87-2952-409d-a0d7-5cbbb76a11f5'
67+
when nc.local_government = 'District of Hudson''s Hope' then 'df19457c-4511-4104-bc85-9675412a6359'
68+
when nc.local_government = 'District of Kent' then 'bf6277e3-ad02-418f-bc0b-c61db4a3e469'
69+
when nc.local_government = 'District of Lake Country' then '311704d0-8db8-4789-90f2-ac15c6437b3f'
70+
when nc.local_government = 'District of Lantzville' then 'df1dea47-66e3-48ef-925f-d6f7ac40f688'
71+
when nc.local_government = 'District of Lillooet' then '5ca6cd49-989a-430c-8805-a45a7c95bc71'
72+
when nc.local_government = 'District of Metchosin' then 'b0d9e0fe-742e-4158-a9c5-3e06e9a42145'
73+
when nc.local_government = 'District of Mission' then 'd9d9545b-748a-46da-85d4-fee20c0fb124'
74+
when nc.local_government = 'District of North Cowichan' then '8db386a1-93a6-4848-8b49-68af04464bc8'
75+
when nc.local_government = 'District of North Saanich' then 'a1457f58-7566-4222-acb1-8b7fdeb27810'
76+
when nc.local_government = 'District of Peachland' then 'a1bdd9cd-fd63-46a8-a925-f38f81a3b5a1'
77+
when nc.local_government = 'District of Saanich' then '8cc35f6b-bd3f-4985-8bca-7607889de964'
78+
when nc.local_government = 'District of Sechelt' then '71dfa376-1bfc-4e0f-a753-b65e594be1e5'
79+
when nc.local_government = 'District of Sooke' then '991b9a44-01bb-459b-8b92-115ca49af99f'
80+
when nc.local_government = 'District of Summerland' then '919716db-c316-4ae5-92ab-74b0a4e5e3b6'
81+
when nc.local_government = 'East Kootenay' then 'a3e3dd07-9efa-4a01-ac38-5dafa43911c3'
82+
when nc.local_government = 'Fraser Fort George' then '5f144a91-b3f7-458e-872b-30eb8ddc06a8'
83+
when nc.local_government = 'Fraser Valley' then '6b0d5e6b-69f7-4287-a941-a1abc99cdd04'
84+
when nc.local_government = 'Islands Trust' then '597a183d-e958-468c-ac4d-b62affe1594a'
85+
when nc.local_government = 'Islands Trust Gabriola Island' then '927ece9d-b4e3-4862-aa85-67539d9abb0d'
86+
when nc.local_government = 'Islands Trust Hornby Island' then 'd325a7a4-fe68-49b0-9df2-599beb06766e'
87+
when nc.local_government = 'Islands Trust Pender Island' then '13de89f7-1c6b-487c-87c8-d5089195f8c4'
88+
when nc.local_government = 'Islands Trust Salt Spring Island' then 'dc0357b1-aacd-45ba-8fa9-0e62cbe88e9f'
89+
when nc.local_government = 'Islands Trust- Comox Strathcona' then '88532d18-3059-4720-986a-363d9bce6be0'
90+
when nc.local_government = 'Islands Trust- Nanaimo' then '4e721d39-09b3-4f9b-ad1f-601a69315808'
91+
when nc.local_government = 'Islands Trust-Capital' then '9355bf78-280a-4695-adad-fafa06969c99'
92+
when nc.local_government = 'Kitimat Stikine' then '35be0556-d281-4f7c-8b0a-8f6cdc202425'
93+
when nc.local_government = 'Kootenay Boundary' then 'c4ddce75-e3bb-4b8d-87a2-a199fe31d665'
94+
when nc.local_government = 'Metro Vancouver' then 'e30d6777-1345-4f63-9e63-bc7bbad5067a'
95+
when nc.local_government = 'Multiple Jurisdictions for Capital' then 'a1ba516d-3637-42f6-9727-12b6e0d04d8a'
96+
when nc.local_government = 'Multiple Jurisdictions for East Kootenay' then '3ef6f25c-8b5d-49f4-b6d1-5e167e9252ba'
97+
when nc.local_government = 'Multiple Jurisdictions for Fraser Valley' then '8fde1050-463c-4226-acfb-b58263cb2387'
98+
when nc.local_government = 'Multiple Jurisdictions for Fraser-Fort George' then '1777e5b5-5520-406d-8b05-05872478f112'
99+
when nc.local_government = 'Multiple Jurisdictions for Kitimat-Stikine' then '2e485e96-81c6-4f4d-8b25-bb848da1a994'
100+
when nc.local_government = 'Multiple Jurisdictions for Nanaimo' then '8a0002dd-5783-4822-8c29-6b60e421e5eb'
101+
when nc.local_government = 'Multiple Jurisdictions for Okanagan-Similkameen' then '70960e85-252d-41ef-a1d5-6f4026d73fff'
102+
when nc.local_government = 'Multiple Jurisdictions for Squamish-Lillooet' then 'fb2972f5-b5b7-435a-b667-492028f8beb6'
103+
when nc.local_government = 'Multiple Jurisdictions for Thompson-Nicola' then 'f0bd3817-5825-4656-b685-d306ddd1fa70'
104+
when nc.local_government = 'Nanaimo' then 'ba5bd974-ad17-48cf-ab1c-55abfa91c291'
105+
when nc.local_government = 'North Okanagan' then '29a7e1ae-ba19-4dda-a7b8-7ae5a0a272d8'
106+
when nc.local_government = 'Northern Rockies' then '33aa1f7d-3b65-4ed5-badf-11dafb0b2789'
107+
when nc.local_government = 'Okanagan Similkameen' then '1aa6ba16-51e8-4789-8624-67ac30f25636'
108+
when nc.local_government = 'Okanagan-Similkameen' then '1aa6ba16-51e8-4789-8624-67ac30f25636'
109+
when nc.local_government = 'Peace River' then '001cfdad-bc6e-4d25-9294-1550603da980'
110+
when nc.local_government = 'Peace River Regional District' then '001cfdad-bc6e-4d25-9294-1550603da980'
111+
when nc.local_government = 'Powell River' then '437aa538-47ac-4b54-a512-a99ea976ced1'
112+
when nc.local_government = 'qathet Regional District' then '437aa538-47ac-4b54-a512-a99ea976ced1'
113+
when nc.local_government = 'Squamish Lillooet' then 'ba21a867-8a74-4364-95b9-4c3648591ba4'
114+
when nc.local_government = 'Strathcona' then '2d778f4d-27a2-4f92-a561-f4b626f6b5d8'
115+
when nc.local_government = 'Sunshine Coast' then '036e3d84-6ea4-4293-a44d-85a44bbad7c2'
116+
when nc.local_government = 'Thompson Nicola' then '1549f188-b6a6-4401-92d5-d3a480c74a2d'
117+
when nc.local_government = 'Town of Gibsons' then 'a2978d25-77e9-481b-9f84-726894b4570f'
118+
when nc.local_government = 'Town of Lake Cowichan' then 'afb8feab-5604-4257-95cc-bd036251e387'
119+
when nc.local_government = 'Town of Oliver' then 'a6ae4789-3fa5-463f-90a3-68295d4a83ab'
120+
when nc.local_government = 'Town of Osoyoos' then 'd4a9a573-749b-45df-bef9-9e5435807144'
121+
when nc.local_government = 'Town of Qualicum Beach' then '5a5b3fff-f341-40aa-9377-08c52d5e1def'
122+
when nc.local_government = 'Township of Langley' then 'cea7033d-f36d-42c1-bc2a-77e25c189436'
123+
when nc.local_government = 'Township of Spallumcheen' then '516876da-c350-4c4a-9379-224633972938'
124+
when nc.local_government = 'Village of Midway' then 'c7e91940-0d7a-4cc8-8beb-c5471cc257c5'
125+
when nc.local_government = 'Village of Nakusp' then '884bc103-5191-4c67-988b-5ebb074cf5d5'
126+
when nc.local_government = 'Village of Pemberton' then '87b7e1f6-4946-4714-a7c9-c4777058d2a5'
127+
end::uuid as local_government_uuid
128+
from
129+
nris.complaint nc
130+
join alcs.compliance_and_enforcement ace on ace.file_number = nc.record_id
131+
join alcs.compliance_and_enforcement_responsible_party acerp on acerp.file_uuid = ace.uuid;

‎bin/migrate-nris-data/properties/sql/l.sql‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
insert into
2+
alcs.compliance_and_enforcement_property (
3+
civic_address,
4+
region_code,
5+
latitude,
6+
longitude,
7+
ownership_type_code,
8+
pid,
9+
area_hectares,
10+
alr_percentage,
11+
alc_history,
12+
file_uuid,
13+
local_government_uuid
14+
)
15+
values %s;

‎bin/migrate-nris-data/properties/sql/obfuscate_count.sql‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
select
2+
count(*)
3+
from
4+
alcs.compliance_and_enforcement_property aces;

‎bin/migrate-nris-data/properties/sql/obfuscate_get_rows.sql‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
select
2+
aces.uuid
3+
from
4+
alcs.compliance_and_enforcement_property aces;

‎bin/migrate-nris-data/properties/sql/obfuscate_update.sql‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
update
2+
alcs.compliance_and_enforcement_property as acep
3+
set
4+
alc_history = case
5+
when acep.alc_history <> '' then v.alc_history
6+
else acep.alc_history
7+
end
8+
from
9+
(values %s) as v(
10+
uuid,
11+
alc_history
12+
)
13+
where
14+
acep.uuid::text = v.uuid::text;

0 commit comments

Comments
 (0)