Skip to content

Commit 33750fb

Browse files
Tian-2017Tian-2017
committed
correct the file
1 parent fd7dd11 commit 33750fb

File tree

1 file changed

+159
-65
lines changed

1 file changed

+159
-65
lines changed

‎scripts/jobs/parking/parking_pcn_dvla_response_no_address.py‎

Lines changed: 159 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -10,76 +10,170 @@
1010
environment = get_glue_env_var("environment")
1111

1212
# The target table in liberator refined zone
13-
table_name = "Parking_PCN_LTN_Report_Summary"
13+
table_name = "parking_pcn_dvla_response_no_address"
1414

1515
# The exact same query prototyped in pre-prod(stg) or prod Athena
1616
query_on_athena = """
17-
/*************************************************************************************************************************
18-
Parking_PCN_LTN_Report_Summary
17+
/*
18+
All VRMs with PCNs response from DVLA has no address still open and not due to be written off
1919
20-
The SQL builds the PCN Low Traffic Network figures, for use with the answering of the LTN FOI's
20+
Criteria:
21+
All PCNs issued
22+
and is not void
23+
and does not have a warning flag
24+
and is not vda and pcn issue date before '2021-06-01' or can be vda after pcn issue date '2021-05-31'
25+
and has had a dvla_request
26+
and has had a dvla_response
27+
and registered_keeper_address is not null
28+
and has not had a PCN cancelled date
29+
and has not had a pcn_casecloseddate
30+
and next progression stage not like 'WRITEOFF'
2131
22-
12/07/2021 - Create SQL.
23-
*************************************************************************************************************************/
24-
SELECT concat(
25-
substr(Cast(pcnissuedate as varchar(10)), 1, 7),
26-
'-01'
27-
) AS IssueMonthYear,
28-
street_location,
29-
COUNT(distinct pcn) AS PCNs_Issued,
30-
CAST(
31-
SUM(cast(lib_payment_received as double)) as decimal(11, 2)
32-
) AS Total_Amount_Paid,
33-
cast(current_timestamp as timestamp) as ImportDateTime,
34-
format_datetime(current_date, 'yyyy') AS import_year,
35-
format_datetime(current_date, 'MM') AS import_month,
36-
format_datetime(current_date, 'dd') AS import_day,
37-
format_datetime(current_date, 'yyyyMMdd') AS import_date
38-
FROM "dataplatform-prod-liberator-refined-zone".pcnfoidetails_pcn_foi_full as A
39-
WHERE warningflag = 0
40-
and isvda = 0
41-
and isvoid = 0
42-
AND import_date =(
43-
select max(import_date)
44-
from "dataplatform-prod-liberator-refined-zone".pcnfoidetails_pcn_foi_full
45-
)
46-
AND vrm != 'T123EST'
47-
AND contraventioncode = '52'
48-
AND street_location IN (
49-
'Allen Road',
50-
'Ashenden Road junction of Glyn Road',
51-
'Barnabas Road JCT Berger Road',
52-
'Barnabas Road JCT Oriel Road',
53-
'Brooke Road (E)',
54-
'Brooke Road junction of Evering Road',
55-
'Dove Row',
56-
'Gore Road junction of Lauriston Road.',
57-
'Hyde Road',
58-
'Hyde Road JCT Northport Street',
59-
'Lee Street junction of Stean Street',
60-
'Maury Road junction of Evering Road',
61-
'Meeson Street junction of Kingsmead Way',
62-
'Nevill Road junction of Osterley Road',
63-
'Neville Road junction of Osterley Road',
64-
'Pitfield Street (F)',
65-
'Pitfield Street JCT Hemsworth Street',
66-
'Powell Road junction of Kenninghall Road',
67-
'Pritchard`s Road',
68-
'Pritchards Road',
69-
'Richmond Road junction of Greenwood Road',
70-
'Shepherdess Walk',
71-
'Ufton Road junction of Downham Road',
72-
'Wilton Way junction of Greenwood Road'
73-
)
74-
GROUP BY concat(
75-
substr(Cast(pcnissuedate as varchar(10)), 1, 7),
76-
'-01'
77-
),
78-
street_location,
79-
import_date,
80-
import_day,
81-
import_month,
82-
import_year;
32+
11/07/2024 - Created
33+
34+
*/
35+
with no_resp_ceo as (
36+
Select distinct vrm as vrm_ceo,
37+
count(distinct pcn) as num_pcns_ceo --, LISTAGG(pcn, ' ') WITHIN GROUP (ORDER BY vrm) as vl_multi_pcns_ceo
38+
,
39+
array_join(
40+
array_distinct(array_agg(pcnfoidetails_pcn_foi_full.pcn)),
41+
-- aggregate that collects distinct values into an array
42+
' | ' -- delimiter
43+
) as vl_multi_pcns_ceo
44+
FROM "dataplatform-prod-liberator-refined-zone".pcnfoidetails_pcn_foi_full
45+
WHERE pcnfoidetails_pcn_foi_full.import_date = (
46+
SELECT max(pcnfoidetails_pcn_foi_full.import_date)
47+
from "dataplatform-prod-liberator-refined-zone".pcnfoidetails_pcn_foi_full
48+
)
49+
and isvoid = 0
50+
and warningflag = 0
51+
and (
52+
(
53+
isvda = 0
54+
and cast(pcnissuedate as date) < cast('2021-06-01' as date)
55+
)
56+
or (
57+
cast(pcnissuedate as date) > cast('2021-05-31' as date)
58+
)
59+
)
60+
and dvla_request is not null
61+
and dvla_response is not null
62+
and registered_keeper_address = ''
63+
and pcn_canx_date is null
64+
and pcn_casecloseddate is null
65+
and upper(debttype) like 'CEO'
66+
and upper(nextprogressionstage) not like 'WRITEOFF' --and cast(pcnissuedate as date) > cast('2019-12-31' as date) and cast(pcnissuedate as date) < cast('2024-01-01' as date)--Parking Tickets issued between 01/01/2020 and 31/12/2023.
67+
group by vrm
68+
order by vrm_ceo
69+
),
70+
no_resp_cctv as (
71+
Select distinct vrm as vrm_cctv,
72+
count(distinct pcn) as num_pcns_cctv --, LISTAGG(pcn, ' ') WITHIN GROUP (ORDER BY vrm) as vl_multi_pcns_cctv
73+
,
74+
array_join(
75+
array_distinct(array_agg(pcnfoidetails_pcn_foi_full.pcn)),
76+
-- aggregate that collects distinct values into an array
77+
' | ' -- delimiter
78+
) as vl_multi_pcns_cctv
79+
FROM "dataplatform-prod-liberator-refined-zone".pcnfoidetails_pcn_foi_full
80+
WHERE pcnfoidetails_pcn_foi_full.import_date = (
81+
SELECT max(pcnfoidetails_pcn_foi_full.import_date)
82+
from "dataplatform-prod-liberator-refined-zone".pcnfoidetails_pcn_foi_full
83+
)
84+
and isvoid = 0
85+
and warningflag = 0
86+
and (
87+
(
88+
isvda = 0
89+
and cast(pcnissuedate as date) < cast('2021-06-01' as date)
90+
)
91+
or (
92+
cast(pcnissuedate as date) > cast('2021-05-31' as date)
93+
)
94+
)
95+
and dvla_request is not null
96+
and dvla_response is not null
97+
and registered_keeper_address = ''
98+
and pcn_canx_date is null
99+
and pcn_casecloseddate is null
100+
and upper(debttype) in (
101+
'CCTV MOVING TRAFFIC',
102+
'CCTV STATIC',
103+
'CCTV BUS LANE'
104+
)
105+
and upper(nextprogressionstage) not like 'WRITEOFF' --and cast(pcnissuedate as date) > cast('2019-12-31' as date) and cast(pcnissuedate as date) < cast('2024-01-01' as date)--Parking Tickets issued between 01/01/2020 and 31/12/2023.
106+
group by vrm
107+
order by vrm
108+
) -- summary sent to DRT
109+
Select distinct pcnfoidetails_pcn_foi_full.vrm,
110+
min(
111+
cast(
112+
substr(Cast(pcnissuedate as varchar(10)), 1, 10) as date
113+
)
114+
) as min_pcnissuedate,
115+
max(
116+
cast(
117+
substr(Cast(pcnissuedate as varchar(10)), 1, 10) as date
118+
)
119+
) as max_pcnissuedate,
120+
count(distinct pcn) as num_pcns_all --, LISTAGG(pcn, ' ') WITHIN GROUP (ORDER BY vrm) as vl_multi_pcns_all
121+
,
122+
array_join(
123+
array_distinct(array_agg(pcnfoidetails_pcn_foi_full.pcn)),
124+
-- aggregate that collects distinct values into an array
125+
' | ' -- delimiter
126+
) as vl_multi_pcns_all --CEO PCNs
127+
,
128+
num_pcns_ceo,
129+
vl_multi_pcns_ceo --CCTV PCNs
130+
,
131+
num_pcns_cctv,
132+
vl_multi_pcns_cctv
133+
/*Partitions*/
134+
,
135+
pcnfoidetails_pcn_foi_full.import_year,
136+
pcnfoidetails_pcn_foi_full.import_month,
137+
pcnfoidetails_pcn_foi_full.import_day,
138+
pcnfoidetails_pcn_foi_full.import_date
139+
FROM "dataplatform-prod-liberator-refined-zone".pcnfoidetails_pcn_foi_full
140+
left join no_resp_cctv on no_resp_cctv.vrm_cctv = pcnfoidetails_pcn_foi_full.vrm
141+
left join no_resp_ceo on no_resp_ceo.vrm_ceo = pcnfoidetails_pcn_foi_full.vrm
142+
WHERE pcnfoidetails_pcn_foi_full.import_date = (
143+
SELECT max(pcnfoidetails_pcn_foi_full.import_date)
144+
from "dataplatform-prod-liberator-refined-zone".pcnfoidetails_pcn_foi_full
145+
)
146+
and isvoid = 0
147+
and warningflag = 0
148+
and (
149+
(
150+
isvda = 0
151+
and cast(pcnissuedate as date) < cast('2021-06-01' as date)
152+
)
153+
or (
154+
cast(pcnissuedate as date) > cast('2021-05-31' as date)
155+
)
156+
)
157+
and dvla_request is not null
158+
and dvla_response is not null
159+
and registered_keeper_address = ''
160+
and pcn_canx_date is null
161+
and pcn_casecloseddate is null
162+
and upper(nextprogressionstage) not like 'WRITEOFF' --and cast(pcnissuedate as date) > cast('2019-12-31' as date) and cast(pcnissuedate as date) < cast('2024-01-01' as date)--Parking Tickets issued between 01/01/2020 and 31/12/2023.
163+
group by pcnfoidetails_pcn_foi_full.vrm --CEO PCNs
164+
,
165+
num_pcns_ceo,
166+
vl_multi_pcns_ceo --CCTV PCNs
167+
,
168+
num_pcns_cctv,
169+
vl_multi_pcns_cctv
170+
/*Partitions*/
171+
,
172+
pcnfoidetails_pcn_foi_full.import_year,
173+
pcnfoidetails_pcn_foi_full.import_month,
174+
pcnfoidetails_pcn_foi_full.import_day,
175+
pcnfoidetails_pcn_foi_full.import_date
176+
order by pcnfoidetails_pcn_foi_full.vrm
83177
"""
84178

85179
create_update_table_with_partition(

0 commit comments

Comments
 (0)