|
| 1 | +""" |
| 2 | +Only need to change the table name and the query prototyped on the Athena UI |
| 3 | +by replacing table_name and query_on_athena |
| 4 | +Note: python file name should be the same as the table name |
| 5 | +""" |
| 6 | + |
| 7 | +from scripts.helpers.athena_helpers import create_update_table_with_partition |
| 8 | +from scripts.helpers.helpers import get_glue_env_var |
| 9 | + |
| 10 | +environment = get_glue_env_var("environment") |
| 11 | + |
| 12 | +# The target table in liberator refined zone |
| 13 | +table_name = "Parking_interim_cycle_hangar_waiting_list" |
| 14 | + |
| 15 | +# The exact same query prototyped in pre-prod(stg) or prod Athena |
| 16 | +query_on_athena = """ |
| 17 | +/********************************************************************************* |
| 18 | +Parking_interim_cycle_hangar_waiting_list |
| 19 | +
|
| 20 | +Process the interim cycle hangar waiting list (supplied by Michael W) to add |
| 21 | +additional fields (telephone Number, etc). |
| 22 | +
|
| 23 | +23/12/2024 - Create SQL |
| 24 | +*********************************************************************************/ |
| 25 | +With Interim_Wait as ( |
| 26 | + SELECT |
| 27 | + distinct forename |
| 28 | + ,surname |
| 29 | + ,email |
| 30 | + ,party_id_to |
| 31 | + ,party_id |
| 32 | + ,cast(uprn as decimal) as uprn |
| 33 | + ,address1 |
| 34 | + ,address2 |
| 35 | + ,post_code |
| 36 | + ,x |
| 37 | + ,y |
| 38 | + ,lat |
| 39 | + ,long |
| 40 | + FROM "parking-raw-zone".interim_cycle_wait_list |
| 41 | + WHERE import_date = (select max(import_date) |
| 42 | + from "parking-raw-zone".interim_cycle_wait_list)), |
| 43 | + |
| 44 | +/*** Obtain the llpg data ***/ |
| 45 | +FULL_LLPG as ( |
| 46 | + SELECT * From "parking-refined-zone".spatially_enriched_liberator_permit_llpg |
| 47 | + WHERE import_Date = format_datetime(current_date, 'yyyyMMdd') |
| 48 | + AND address1 not like '%STREET RECORD%'), |
| 49 | +
|
| 50 | +STREET_LLPG as ( |
| 51 | + SELECT * From "parking-refined-zone".spatially_enriched_liberator_permit_llpg |
| 52 | + WHERE import_Date = format_datetime(current_date, 'yyyyMMdd') |
| 53 | + AND address1 like '%STREET RECORD%'), |
| 54 | +
|
| 55 | +/*** Obtain the Party details, where available ***/ |
| 56 | +Party as ( |
| 57 | + SELECT |
| 58 | + * |
| 59 | + From "dataplatform-stg-liberator-raw-zone".liberator_licence_party |
| 60 | + WHERE import_Date = format_datetime(current_date, 'yyyyMMdd')), |
| 61 | +
|
| 62 | +/*** obtain the emails (from Tom) of those parties that are NOT interested in a Hangar***/ |
| 63 | +unsubscribed_emails as ( |
| 64 | + SELECT *, |
| 65 | + ROW_NUMBER() OVER ( PARTITION BY email_address ORDER BY email_address DESC) row1 |
| 66 | + FROM "parking-raw-zone".parking_parking_cycle_hangar_unsubscribed_emails |
| 67 | + WHERE import_Date = format_datetime(current_date, 'yyyyMMdd')) |
| 68 | + |
| 69 | +SELECT |
| 70 | + A.*, cast(D.telephone_number as varchar) as Telephone_Number, C.address2 as Street, B.housing_estate, E.email_address, |
| 71 | +
|
| 72 | + format_datetime(CAST(CURRENT_TIMESTAMP AS timestamp), |
| 73 | + 'yyyy-MM-dd HH:mm:ss') AS import_date_timestamp, |
| 74 | +
|
| 75 | + format_datetime(current_date, 'yyyy') AS import_year, |
| 76 | + format_datetime(current_date, 'MM') AS import_month, |
| 77 | + format_datetime(current_date, 'dd') AS import_day, |
| 78 | + format_datetime(current_date, 'yyyyMMdd') AS import_date |
| 79 | +
|
| 80 | +FROM Interim_Wait as A |
| 81 | +LEFT JOIN FULL_LLPG as B ON A.uprn = B.UPRN |
| 82 | +LEFT JOIN STREET_LLPG as C ON B.USRN = C.USRN |
| 83 | +LEFT JOIN Party as D ON A.party_id = D.business_party_id |
| 84 | +LEFT JOIN unsubscribed_emails as E ON upper(ltrim(rtrim(A.email))) = upper(ltrim(rtrim(E.email_address))) |
| 85 | + AND row1 = 1 |
| 86 | +""" |
| 87 | + |
| 88 | +create_update_table_with_partition( |
| 89 | + environment=environment, query_on_athena=query_on_athena, table_name=table_name |
| 90 | +) |
0 commit comments