Skip to content

Commit f643802

Browse files
lizihao-anulizihao-anu
committed
update run_sdl for mh and maternity
1 parent be34502 commit f643802

File tree

1 file changed

+20
-164
lines changed

1 file changed

+20
-164
lines changed

‎SDL_process/run_sdl.r‎

Lines changed: 20 additions & 164 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
1-
# run_sdl.R for testing
1+
# run_sdl.R
22

3-
# First step rename current dataset so it is not overwritten.####
4-
# will need to change/remove renaming step once switched to seer
5-
6-
# 00 setup logger ----
3+
# Setup ----
74
logger::log_info("Run SDL starts.")
85

96
library(createslf)
@@ -45,20 +42,10 @@ library(targets)
4542
library(testthat)
4643
library(crew)
4744

48-
# Include reporting of last run date of ACADME
49-
dplyr::tbl(
50-
denodo_connect,
51-
dbplyr::in_schema("sdl", "sdl_byoc_acadme_load_detail")
52-
) %>%
53-
collect() %>%
54-
# Optional: Format the date to look clean first
55-
mutate(load_str = format(load_date, "%Y-%m-%d %H:%M:%S")) %>%
56-
pwalk(function(data_mart, load_str, ...) {
57-
logger::log_info("{data_mart} loaded at {load_str}")
58-
})
5945

46+
## Detect BYOC_MODE ----
6047
BYOC_MODE <- Sys.getenv("BYOC_MODE")
61-
# set up logger and system environment variable BYOC_MODE
48+
# Set up logger and system environment variable BYOC_MODE
6249
if (tolower(BYOC_MODE) %in% c("true", "t")) {
6350
logger::log_info("Detect run_sdl.r run on Denodo")
6451
BYOC_MODE <- TRUE
@@ -67,18 +54,19 @@ if (tolower(BYOC_MODE) %in% c("true", "t")) {
6754
BYOC_MODE <- FALSE
6855
}
6956

57+
## Set up targets store path ----
7058
store_path <- dplyr::if_else(
7159
BYOC_MODE,
7260
"/sdl_byoc/_targets",
7361
"/conf/sourcedev/Source_Linkage_File_Updates/_targets"
7462
)
7563

64+
## Set up run_id and run_date_time ----
7665
# run_id <- Sys.getenv("run_id")
7766
# run_date_time <- Sys.getenv("run_date_time")
7867
run_date_time <- script_run_time
7968

80-
# Include reporting of last run date of ACADME
81-
69+
## Include reporting of last run date of ACADME ----
8270
if (isFALSE(BYOC_MODE)) {
8371
denodo_connect <- createslf::get_denodo_connection(BYOC_MODE = BYOC_MODE)
8472
}
@@ -96,160 +84,28 @@ if (isFALSE(BYOC_MODE)) {
9684
odbc::dbDisconnect(denodo_connect)
9785
}
9886

99-
write_to_disk <- TRUE
100-
101-
## create dummy data for testing -----
102-
# hard coded dummy data
103-
# TODO: remove this
104-
la_code_lookup <- data.frame(
105-
CA = c(
106-
"S12000005", "S12000006", "S12000008", "S12000010", "S12000011",
107-
"S12000013", "S12000014", "S12000015", "S12000017", "S12000018",
108-
"S12000019", "S12000020", "S12000021", "S12000023", "S12000024",
109-
"S12000026", "S12000027", "S12000028", "S12000029", "S12000030",
110-
"S12000033", "S12000034", "S12000035", "S12000036", "S12000038",
111-
"S12000039", "S12000040", "S12000041", "S12000042", "S12000044",
112-
"S12000045", "S12000046", "S12000047", "S12000048", "S12000049",
113-
"S12000050"
114-
),
115-
CAName = c(
116-
"Clackmannanshire", "Dumfries and Galloway", "East Ayrshire",
117-
"East Lothian", "East Renfrewshire", "Na h-Eileanan Siar",
118-
"Falkirk", "Fife", "Highland", "Inverclyde", "Midlothian", "Moray",
119-
"North Ayrshire", "Orkney Islands", "Perth and Kinross",
120-
"Scottish Borders", "Shetland Islands", "South Ayrshire",
121-
"South Lanarkshire", "Stirling", "Aberdeen City", "Aberdeenshire",
122-
"Argyll and Bute", "City of Edinburgh", "Renfrewshire",
123-
"West Dunbartonshire", "West Lothian", "Angus", "Dundee City",
124-
"North Lanarkshire", "East Dunbartonshire", "Glasgow City",
125-
"Fife", "Perth and Kinross", "Glasgow City", "North Lanarkshire"
126-
),
127-
sending_local_authority_name = c(
128-
"Clackmannanshire", "Dumfries & Galloway", "East Ayrshire",
129-
"East Lothian", "East Renfrewshire", "Eilean Siar", "Falkirk",
130-
"Fife", "Highland", "Inverclyde", "Midlothian", "Moray",
131-
"North Ayrshire", "Orkney Islands", "Perth & Kinross",
132-
"Scottish Borders", "Shetland Islands", "South Ayrshire",
133-
"South Lanarkshire", "Stirling", "Aberdeen City", "Aberdeenshire",
134-
"Argyll & Bute", "Edinburgh", "Renfrewshire", "West Dunbartonshire",
135-
"West Lothian", "Angus", "Dundee City", "North Lanarkshire",
136-
"East Dunbartonshire", "Glasgow City", "Fife", "Perth & Kinross",
137-
"Glasgow City", "North Lanarkshire"
138-
),
139-
stringsAsFactors = FALSE
140-
)
87+
## Set up years to run ----
88+
years <- createslf::years_to_run()
14189

142-
set.seed(1)
143-
sg_pub_data <- data.frame(
144-
CAName = c(
145-
"Aberdeen City", "Aberdeen City", "Aberdeen City", "Aberdeen City", "Aberdeen City", "Aberdeen City", "Aberdeen City", "Aberdeen City", "Aberdeen City",
146-
"Aberdeenshire", "Aberdeenshire", "Aberdeenshire", "Aberdeenshire", "Aberdeenshire", "Aberdeenshire", "Aberdeenshire", "Aberdeenshire", "Aberdeenshire",
147-
"Angus", "Angus", "Angus", "Angus", "Angus", "Angus", "Angus", "Angus", "Angus",
148-
"Argyll & Bute", "Argyll & Bute", "Argyll & Bute", "Argyll & Bute", "Argyll & Bute", "Argyll & Bute", "Argyll & Bute", "Argyll & Bute", "Argyll & Bute",
149-
"Clackmannanshire", "Clackmannanshire", "Clackmannanshire", "Clackmannanshire", "Clackmannanshire", "Clackmannanshire", "Clackmannanshire", "Clackmannanshire", "Clackmannanshire",
150-
"Dumfries & Galloway", "Dumfries & Galloway", "Dumfries & Galloway", "Dumfries & Galloway", "Dumfries & Galloway", "Dumfries & Galloway", "Dumfries & Galloway", "Dumfries & Galloway", "Dumfries & Galloway",
151-
"Dundee City", "Dundee City", "Dundee City", "Dundee City", "Dundee City", "Dundee City", "Dundee City", "Dundee City", "Dundee City",
152-
"East Ayrshire", "East Ayrshire", "East Ayrshire", "East Ayrshire", "East Ayrshire", "East Ayrshire", "East Ayrshire", "East Ayrshire", "East Ayrshire",
153-
"East Dunbartonshire", "East Dunbartonshire", "East Dunbartonshire", "East Dunbartonshire", "East Dunbartonshire", "East Dunbartonshire", "East Dunbartonshire", "East Dunbartonshire", "East Dunbartonshire",
154-
"East Lothian", "East Lothian", "East Lothian", "East Lothian", "East Lothian", "East Lothian", "East Lothian", "East Lothian", "East Lothian",
155-
"East Renfrewshire", "East Renfrewshire", "East Renfrewshire", "East Renfrewshire", "East Renfrewshire", "East Renfrewshire", "East Renfrewshire", "East Renfrewshire", "East Renfrewshire",
156-
"Edinburgh", "Edinburgh", "Edinburgh", "Edinburgh", "Edinburgh", "Edinburgh", "Edinburgh", "Edinburgh", "Edinburgh",
157-
"Eilean Siar", "Eilean Siar", "Eilean Siar", "Eilean Siar", "Eilean Siar", "Eilean Siar", "Eilean Siar", "Eilean Siar", "Eilean Siar",
158-
"Falkirk", "Falkirk", "Falkirk", "Falkirk", "Falkirk", "Falkirk", "Falkirk", "Falkirk", "Falkirk",
159-
"Fife", "Fife", "Fife", "Fife", "Fife", "Fife", "Fife", "Fife", "Fife",
160-
"Glasgow City", "Glasgow City", "Glasgow City", "Glasgow City", "Glasgow City", "Glasgow City", "Glasgow City", "Glasgow City", "Glasgow City",
161-
"Highland", "Highland", "Highland", "Highland", "Highland", "Highland", "Highland", "Highland", "Highland",
162-
"Inverclyde", "Inverclyde", "Inverclyde", "Inverclyde", "Inverclyde", "Inverclyde", "Inverclyde", "Inverclyde", "Inverclyde",
163-
"Midlothian", "Midlothian", "Midlothian", "Midlothian", "Midlothian", "Midlothian", "Midlothian", "Midlothian", "Midlothian",
164-
"Moray", "Moray", "Moray", "Moray", "Moray", "Moray", "Moray", "Moray", "Moray",
165-
"North Ayrshire", "North Ayrshire", "North Ayrshire", "North Ayrshire", "North Ayrshire", "North Ayrshire", "North Ayrshire", "North Ayrshire", "North Ayrshire",
166-
"North Lanarkshire", "North Lanarkshire", "North Lanarkshire", "North Lanarkshire", "North Lanarkshire", "North Lanarkshire", "North Lanarkshire", "North Lanarkshire", "North Lanarkshire",
167-
"Orkney", "Orkney", "Orkney", "Orkney", "Orkney", "Orkney", "Orkney", "Orkney", "Orkney",
168-
"Perth & Kinross", "Perth & Kinross", "Perth & Kinross", "Perth & Kinross", "Perth & Kinross", "Perth & Kinross", "Perth & Kinross", "Perth & Kinross", "Perth & Kinross",
169-
"Renfrewshire", "Renfrewshire", "Renfrewshire", "Renfrewshire", "Renfrewshire", "Renfrewshire", "Renfrewshire", "Renfrewshire", "Renfrewshire",
170-
"Scottish Borders", "Scottish Borders", "Scottish Borders", "Scottish Borders", "Scottish Borders", "Scottish Borders", "Scottish Borders", "Scottish Borders", "Scottish Borders",
171-
"Shetland", "Shetland", "Shetland", "Shetland", "Shetland", "Shetland", "Shetland", "Shetland", "Shetland",
172-
"South Ayrshire", "South Ayrshire", "South Ayrshire", "South Ayrshire", "South Ayrshire", "South Ayrshire", "South Ayrshire", "South Ayrshire", "South Ayrshire",
173-
"South Lanarkshire", "South Lanarkshire", "South Lanarkshire", "South Lanarkshire", "South Lanarkshire", "South Lanarkshire", "South Lanarkshire", "South Lanarkshire", "South Lanarkshire",
174-
"Stirling", "Stirling", "Stirling", "Stirling", "Stirling", "Stirling", "Stirling", "Stirling", "Stirling",
175-
"West Dunbartonshire", "West Dunbartonshire", "West Dunbartonshire", "West Dunbartonshire", "West Dunbartonshire", "West Dunbartonshire", "West Dunbartonshire", "West Dunbartonshire", "West Dunbartonshire",
176-
"West Lothian", "West Lothian", "West Lothian", "West Lothian", "West Lothian", "West Lothian", "West Lothian", "West Lothian", "West Lothian"
177-
),
178-
sg_year = rep(c("1617", "1718", "1819", "1920", "2021", "2122", "2223", "2324", "2425"), times = 32),
179-
sg_all_assessments = sample(120:9000, size = 32 * 9, replace = TRUE),
180-
stringsAsFactors = FALSE
90+
## Build BYOC Output File Paths ----
91+
byoc_output_files <- get_byoc_output_files(
92+
year = "1920",
93+
types = c("maternity", "mh")
18194
)
95+
# using homelessness for test purpose. When development is complete,
96+
# we change to "types = "byoc_input_files""
97+
# can always use any other type for testing also
18298

183-
# just test one year
184-
year <- "1920"
185-
186-
# # Build BYOC Output File Paths
187-
# byoc_output_files <- get_byoc_output_files(
188-
# year = year,
189-
# types = c("homelessness", "maternity", "mh") # using homelessness for test purpose. When development is complete, we change to "types = "byoc_input_files""
190-
# ) # can always use any other type for testing also
191-
192-
## targets ----
99+
# targets ----
100+
logger::log_info("Targets started.")
193101
targets::tar_make(
194102
script = "SDL_process/dummy_targets.R",
195103
store = store_path
196104
)
197105
logger::log_info("Targets finished.")
198-
# targets::tar_make()
199106

200-
# test homelessness data only
201-
## create homelessness data ----
202-
# logger::log_info("Read and process homelessness data")
203-
# hl1 <- read_extract_homelessness(
204-
# year,
205-
# denodo_connect = get_denodo_connection(BYOC_MODE = BYOC_MODE),
206-
# file_path = get_boxi_extract_path(
207-
# year = year,
208-
# type = "homelessness",
209-
# BYOC_MODE = BYOC_MODE
210-
# ),
211-
# BYOC_MODE = BYOC_MODE
212-
# ) %>% process_extract_homelessness(
213-
# year = year,
214-
# write_to_disk = write_to_disk,
215-
# la_code_lookup = la_code_lookup,
216-
# sg_pub_data = sg_pub_data,
217-
# BYOC_MODE = BYOC_MODE,
218-
# run_id = run_id,
219-
# run_date_time = run_date_time
220-
# )
221-
#
222-
# logger::log_info("Read and process maternity data")
223-
# maternity <- read_extract_maternity(
224-
# year,
225-
# denodo_connect = get_denodo_connection(BYOC_MODE = BYOC_MODE),
226-
# file_path = get_boxi_extract_path(year, type = "maternity", BYOC_MODE),
227-
# BYOC_MODE = BYOC_MODE
228-
# ) %>%
229-
# process_extract_maternity(
230-
# year = year,
231-
# write_to_disk = TRUE,
232-
# BYOC_MODE = BYOC_MODE,
233-
# run_id = run_id,
234-
# run_date_time = run_date_time
235-
# )
107+
# Episode file ----
236108

237-
logger::log_info("Read and process mental health data")
238-
mental_health <- read_extract_mental_health(
239-
year,
240-
denodo_connect = get_denodo_connection(BYOC_MODE = BYOC_MODE),
241-
file_path = get_boxi_extract_path(
242-
year = year,
243-
type = "mh",
244-
BYOC_MODE = BYOC_MODE
245-
),
246-
BYOC_MODE = BYOC_MODE
247-
) %>%
248-
process_extract_mental_health(
249-
year = year,
250-
BYOC_MODE = BYOC_MODE,
251-
run_id = run_id,
252-
run_date_time = run_date_time
253-
)
109+
# Individual file ----
254110

255111
logger::log_info("Run SDL ended.")

0 commit comments

Comments
 (0)