From 323e36bd08e129a90dbbac25ea46f2f23610aae1 Mon Sep 17 00:00:00 2001 From: Oluwatobi Date: Mon, 2 Mar 2026 14:42:16 +0000 Subject: [PATCH 01/16] first draft - read_extract_ae refactor --- R/read_extract_ae.R | 136 ++++++++++++++++++-------------------------- 1 file changed, 54 insertions(+), 82 deletions(-) diff --git a/R/read_extract_ae.R b/R/read_extract_ae.R index c0fd9ed6e..0dc229be8 100644 --- a/R/read_extract_ae.R +++ b/R/read_extract_ae.R @@ -4,89 +4,61 @@ #' #' @export #' -read_extract_ae <- function( - year, - file_path = get_boxi_extract_path(year = year, type = "ae") -) { - extract_ae <- read_file(file_path, - col_type = readr::cols( - "Arrival Date" = readr::col_date(format = "%Y/%m/%d %T"), - "DAT Date" = readr::col_date(format = "%Y/%m/%d %T"), - "anon_chi" = readr::col_character(), - "Pat Date Of Birth [C]" = readr::col_date(format = "%Y/%m/%d %T"), - "Pat Gender Code" = readr::col_double(), - "NHS Board of Residence Code - current" = readr::col_character(), - "Treatment NHS Board Code - current" = readr::col_character(), - "Treatment Location Code" = readr::col_character(), - "GP Practice Code" = readr::col_character(), - "Council Area Code" = readr::col_character(), - "Postcode (epi) [C]" = readr::col_character(), - "Postcode (CHI) [C]" = readr::col_character(), - "HSCP of Residence Code - current" = readr::col_character(), - "Arrival Time" = readr::col_time(""), - "DAT Time" = readr::col_time(""), - "Arrival Mode Code" = readr::col_character(), - "Referral Source Code" = readr::col_character(), - "Attendance Category Code" = readr::col_character(), - "Discharge Destination Code" = readr::col_character(), - "Patient Flow Code" = readr::col_double(), - "Place of Incident Code" = readr::col_character(), - "Reason for Wait Code" = readr::col_character(), - "Disease 1 Code" = readr::col_character(), - "Disease 2 Code" = readr::col_character(), - "Disease 3 Code" = readr::col_character(), - "Bodily Location Of Injury Code" = readr::col_character(), - "Alcohol Involved Code" = readr::col_character(), - "Alcohol Related Admission" = readr::col_character(), - "Substance Misuse Related Admission" = readr::col_character(), - "Falls Related Admission" = readr::col_character(), - "Self Harm Related Admission" = readr::col_character(), - "Total Net Costs" = readr::col_double(), - "Age at Midpoint of Financial Year" = readr::col_double(), - "Case Reference Number" = readr::col_character(), - "Significant Facility Code" = readr::col_character(), - "Community Hospital Flag" = readr::col_character(), - ) +read_extract_ae <- function(year, + denodo_connect, + BYOC_MODE = FALSE) { + + year <- check_year_format(year, format = "fyyear") + c_year <- convert_fyyear_to_year(year) + + extract_ae <- dplyr::tbl( + denodo_connect, + dbplyr::in_schema("sdl", "sdl_ae2_episode_level_source") ) %>% - # rename variables - dplyr::rename( - record_keydate1 = "Arrival Date", - record_keydate2 = "DAT Date", - dob = "Pat Date Of Birth [C]", - postcode_epi = "Postcode (epi) [C]", - postcode_chi = "Postcode (CHI) [C]", - age = "Age at Midpoint of Financial Year", - ae_alcohol = "Alcohol Involved Code", - alcohol_adm = "Alcohol Related Admission", - ae_arrivalmode = "Arrival Mode Code", - keytime1 = "Arrival Time", - ae_attendcat = "Attendance Category Code", - ae_bodyloc = "Bodily Location Of Injury Code", - lca = "Council Area Code", - ae_disdest = "Discharge Destination Code", - keytime2 = "DAT Time", - diag1 = "Disease 1 Code", - diag2 = "Disease 2 Code", - diag3 = "Disease 3 Code", - falls_adm = "Falls Related Admission", - gpprac = "GP Practice Code", - hscp = "HSCP of Residence Code - current", - hbrescode = "NHS Board of Residence Code - current", - hbtreatcode = "Treatment NHS Board Code - current", - anon_chi = "anon_chi", - gender = "Pat Gender Code", - ae_patflow = "Patient Flow Code", - ae_placeinc = "Place of Incident Code", - ae_reasonwait = "Reason for Wait Code", - refsource = "Referral Source Code", - selfharm_adm = "Self Harm Related Admission", - submis_adm = "Substance Misuse Related Admission", - sigfac = "Significant Facility Code", - cost_total_net = "Total Net Costs", - location = "Treatment Location Code", - case_ref_number = "Case Reference Number", - commhosp = "Community Hospital Flag" - ) + dplyr::filter(costs_financial_year == !!c_year) %>% # (Assuming the column name in Denodo is 'costs_financial_year') + + dplyr::select( + record_keydate1 = "arrival_date", + record_keydate2 = "dat_date", + keytime1 = "arrival_time", + keytime2 = "dat_time", + chi = "patient_chi", # following the logic from Zihao's refactor-mat + gender = "patient_sex", + dob = "patient_dob", + gpprac = "gp_practice_code", + lca = "council_area_code", + hscp = "hscp_of_residence_code_curr", + location = "treatment_location_code", + hbrescode = "nhs_board_of_residence_code_curr", + hbtreatcode = "treatment_nhs_board_code_curr", + diag1 = "disease_1_code", + diag2 = "disease_2_code", + diag3 = "disease_3_code", + ae_arrivalmode = "arrival_mode_code", + refsource = "referral_source_code", + sigfac = "significant_facility_code", + ae_attendcat = "attendance_category_code", + ae_disdest = "discharge_destination_code", + ae_patflow = "patient_flow_code", + ae_placeinc = "place_of_incident_code", + ae_reasonwait = "reason_for_wait_code", + ae_bodyloc = "bodily_location_of_injury_code", + ae_alcohol = "alcohol_involved_code", + alcohol_adm = "alcohol_related_admission", + submis_adm = "substance_misuse_related_admission", + falls_adm = "falls_related_admission", + selfharm_adm = "self_harm_related_admission", + cost_total_net = "total_net_cost", + age = "age_at_midpoint_of_financial_year" + # case_ref_number = "case_reference_number", + # postcode_epi = "postcode_epi", + # postcode_chi = "postcode_chi", + # commhosp = "community_hospital_flag" + ) %>% + dplyr::collect() %>% + slfhelper::get_anon_chi("chi") return(extract_ae) } + +# Note arrival financial year From 1b828798ab95058c71c867392ab97727233418f3 Mon Sep 17 00:00:00 2001 From: Oluwatobi Date: Mon, 9 Mar 2026 18:45:55 +0000 Subject: [PATCH 02/16] refactor ae --- R/process_extract_ae.R | 40 ++++++++++++++++++++++++++++++++++++++-- R/read_extract_ae.R | 32 +++++++++++++++++++------------- 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/R/process_extract_ae.R b/R/process_extract_ae.R index cf71578bd..782e7cb63 100644 --- a/R/process_extract_ae.R +++ b/R/process_extract_ae.R @@ -12,7 +12,13 @@ #' @return the final data as a [tibble][tibble::tibble-package]. #' @export #' @family process extracts -process_extract_ae <- function(data, year, write_to_disk = TRUE) { +process_extract_ae <- function(data, + year, + write_to_disk = TRUE, + BYOC_MODE = FALSE, + run_id = NA, + run_date_time = NA + ) { # Only run for a single year stopifnot(length(year) == 1L) @@ -221,6 +227,29 @@ process_extract_ae <- function(data, year, write_to_disk = TRUE) { cup_pathway = "CUP Pathway Name" ) + # ---------------------------------------------------------------------------- + + # denodo_connect_cup <- get_denodo_connection() + # + # c_year_cup <- convert_fyyear_to_year(check_year_format(year)) + # + # ae_cup_file <- dplyr::tbl( + # denodo_connect_cup, + # dbplyr::in_schema("sdl", "sdl_ae_cup_source_placeholder") # TO-DO: Placeholder for data path in denodo + # ) %>% + # dplyr::filter(year == !!c_year_cup) %>% # TO-DO: Placeholder for the variable to filter by year + # dplyr::select( # TO-DO: Placeholder variables + # record_keydate1 = "ED Arrival Date", + # keytime1 = "ED Arrival Time", + # record_keydate2 = "ED Discharge Date", + # keytime2 = "ED Discharge Time", + # case_ref_number = "ED Case Reference Number [C]", + # cup_marker = "CUP Marker", + # cup_pathway = "CUP Pathway Name" + # ) %>% + # dplyr::collect() + # ---------------------------------------------------------------------------- + # Data Cleaning--------------------------------------- @@ -254,7 +283,13 @@ process_extract_ae <- function(data, year, write_to_disk = TRUE) { ) ae_processed <- matched_ae_data %>% + dplyr::mutate( + run_id = run_id, + run_date_time = run_date_time + ) %>% dplyr::select( + "run_id", + "run_date_time", "year", "recid", "smrtype", @@ -310,7 +345,8 @@ process_extract_ae <- function(data, year, write_to_disk = TRUE) { if (write_to_disk) { write_file( ae_processed, - get_source_extract_path(year, "ae", check_mode = "write"), + get_source_extract_path(year, "ae", check_mode = "write", BYOC_MODE = BYOC_MODE), + BYOC_MODE = BYOC_MODE, group_id = 3356 # sourcedev owner ) } diff --git a/R/read_extract_ae.R b/R/read_extract_ae.R index 0dc229be8..d360af0ba 100644 --- a/R/read_extract_ae.R +++ b/R/read_extract_ae.R @@ -5,24 +5,32 @@ #' @export #' read_extract_ae <- function(year, - denodo_connect, - BYOC_MODE = FALSE) { - + denodo_connect, # TO-DO: will be hardcoded to denodo_connect = get_denodo_connection() + file_path = get_boxi_extract_path(year = year, type = "ae", BYOC_MODE), + BYOC_MODE) { year <- check_year_format(year, format = "fyyear") c_year <- convert_fyyear_to_year(year) + # Specify years available for running + if (file_path == get_dummy_boxi_extract_path(BYOC_MODE = BYOC_MODE)) { + return(tibble::tibble()) + } + + # Read Extract extract_ae <- dplyr::tbl( denodo_connect, dbplyr::in_schema("sdl", "sdl_ae2_episode_level_source") ) %>% - dplyr::filter(costs_financial_year == !!c_year) %>% # (Assuming the column name in Denodo is 'costs_financial_year') - + dplyr::filter( + financial_year == !!c_year & # TO-DO: check assumption that arrival_financial_year == financial_year + (significant_facility_code == "32" | is.na(significant_facility_code)) + ) %>% dplyr::select( record_keydate1 = "arrival_date", record_keydate2 = "dat_date", keytime1 = "arrival_time", keytime2 = "dat_time", - chi = "patient_chi", # following the logic from Zihao's refactor-mat + chi = "patient_chi", gender = "patient_sex", dob = "patient_dob", gpprac = "gp_practice_code", @@ -49,16 +57,14 @@ read_extract_ae <- function(year, falls_adm = "falls_related_admission", selfharm_adm = "self_harm_related_admission", cost_total_net = "total_net_cost", - age = "age_at_midpoint_of_financial_year" - # case_ref_number = "case_reference_number", - # postcode_epi = "postcode_epi", - # postcode_chi = "postcode_chi", - # commhosp = "community_hospital_flag" + age = "age_at_midpoint_of_financial_year", + case_ref_number = "care_reference_number", # TO-DO: needs to be renamed by NSS from care to case? + postcode_epi = "postcode_epi", + postcode_chi = "postcode_chi", + commhosp = "community_hospital_flag" ) %>% dplyr::collect() %>% slfhelper::get_anon_chi("chi") return(extract_ae) } - -# Note arrival financial year From f30f6b7d503ca6190999165ec4902bb290afa750 Mon Sep 17 00:00:00 2001 From: OluwatobiOni <108776112+OluwatobiOni@users.noreply.github.com> Date: Mon, 9 Mar 2026 18:48:26 +0000 Subject: [PATCH 03/16] Update documentation --- man/process_extract_ae.Rd | 9 ++++++++- man/read_extract_ae.Rd | 4 +++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/man/process_extract_ae.Rd b/man/process_extract_ae.Rd index 36d2bb4d3..4f72dd724 100644 --- a/man/process_extract_ae.Rd +++ b/man/process_extract_ae.Rd @@ -4,7 +4,14 @@ \alias{process_extract_ae} \title{Process the A&E extract} \usage{ -process_extract_ae(data, year, write_to_disk = TRUE) +process_extract_ae( + data, + year, + write_to_disk = TRUE, + BYOC_MODE = FALSE, + run_id = NA, + run_date_time = NA +) } \arguments{ \item{data}{The extract to process} diff --git a/man/read_extract_ae.Rd b/man/read_extract_ae.Rd index 1a15efbc1..d0151da8d 100644 --- a/man/read_extract_ae.Rd +++ b/man/read_extract_ae.Rd @@ -6,7 +6,9 @@ \usage{ read_extract_ae( year, - file_path = get_boxi_extract_path(year = year, type = "ae") + denodo_connect, + file_path = get_boxi_extract_path(year = year, type = "ae", BYOC_MODE), + BYOC_MODE ) } \arguments{ From 50d4db33a9db8a17b03f704cdb4f83327662d85f Mon Sep 17 00:00:00 2001 From: OluwatobiOni <108776112+OluwatobiOni@users.noreply.github.com> Date: Mon, 9 Mar 2026 18:53:27 +0000 Subject: [PATCH 04/16] Style code --- R/process_extract_ae.R | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/R/process_extract_ae.R b/R/process_extract_ae.R index 782e7cb63..ed8203c4a 100644 --- a/R/process_extract_ae.R +++ b/R/process_extract_ae.R @@ -17,8 +17,7 @@ process_extract_ae <- function(data, write_to_disk = TRUE, BYOC_MODE = FALSE, run_id = NA, - run_date_time = NA - ) { + run_date_time = NA) { # Only run for a single year stopifnot(length(year) == 1L) From ef1bfed015baceb92d057cfe4ca0ae8f69880c5d Mon Sep 17 00:00:00 2001 From: Oluwatobi Date: Fri, 13 Mar 2026 11:23:48 +0000 Subject: [PATCH 05/16] add denodo_connect as a function parameter --- R/process_extract_ae.R | 7 ++++--- R/read_extract_ae.R | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/R/process_extract_ae.R b/R/process_extract_ae.R index ed8203c4a..e3af4c59a 100644 --- a/R/process_extract_ae.R +++ b/R/process_extract_ae.R @@ -14,6 +14,7 @@ #' @family process extracts process_extract_ae <- function(data, year, + denodo_connect = NULL, # TO-DO: will be hardcoded to denodo_connect = get_denodo_connection() write_to_disk = TRUE, BYOC_MODE = FALSE, run_id = NA, @@ -227,13 +228,11 @@ process_extract_ae <- function(data, ) # ---------------------------------------------------------------------------- - - # denodo_connect_cup <- get_denodo_connection() # # c_year_cup <- convert_fyyear_to_year(check_year_format(year)) # # ae_cup_file <- dplyr::tbl( - # denodo_connect_cup, + # denodo_connect, # dbplyr::in_schema("sdl", "sdl_ae_cup_source_placeholder") # TO-DO: Placeholder for data path in denodo # ) %>% # dplyr::filter(year == !!c_year_cup) %>% # TO-DO: Placeholder for the variable to filter by year @@ -350,5 +349,7 @@ process_extract_ae <- function(data, ) } + DBI::dbDisconnect(denodo_connect) + return(ae_processed) } diff --git a/R/read_extract_ae.R b/R/read_extract_ae.R index d360af0ba..70b8bd044 100644 --- a/R/read_extract_ae.R +++ b/R/read_extract_ae.R @@ -66,5 +66,7 @@ read_extract_ae <- function(year, dplyr::collect() %>% slfhelper::get_anon_chi("chi") + DBI::dbDisconnect(denodo_connect) + return(extract_ae) } From 419a7439e1f08b3c073464ed253ea2ab8e62b14e Mon Sep 17 00:00:00 2001 From: OluwatobiOni <108776112+OluwatobiOni@users.noreply.github.com> Date: Fri, 13 Mar 2026 11:27:07 +0000 Subject: [PATCH 06/16] Update documentation --- man/process_extract_ae.Rd | 1 + 1 file changed, 1 insertion(+) diff --git a/man/process_extract_ae.Rd b/man/process_extract_ae.Rd index 4f72dd724..74750b86b 100644 --- a/man/process_extract_ae.Rd +++ b/man/process_extract_ae.Rd @@ -7,6 +7,7 @@ process_extract_ae( data, year, + denodo_connect = NULL, write_to_disk = TRUE, BYOC_MODE = FALSE, run_id = NA, From bd9862ebea5f582b9e6a6b146c8015bdd0244e77 Mon Sep 17 00:00:00 2001 From: LucyEmma22 Date: Fri, 20 Mar 2026 18:23:44 +0000 Subject: [PATCH 07/16] Add log_info() and get_denodo_connection(), and move line to disconnect from Denodo --- R/read_extract_ae.R | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/R/read_extract_ae.R b/R/read_extract_ae.R index 70b8bd044..69dd1cf5e 100644 --- a/R/read_extract_ae.R +++ b/R/read_extract_ae.R @@ -5,7 +5,7 @@ #' @export #' read_extract_ae <- function(year, - denodo_connect, # TO-DO: will be hardcoded to denodo_connect = get_denodo_connection() + denodo_connect = get_denodo_connection(BYOC_MODE = BYOC_MODE), file_path = get_boxi_extract_path(year = year, type = "ae", BYOC_MODE), BYOC_MODE) { year <- check_year_format(year, format = "fyyear") @@ -16,13 +16,17 @@ read_extract_ae <- function(year, return(tibble::tibble()) } + on.exit(try(DBI::dbDisconnect(denodo_connect), silent = TRUE), add = TRUE) + + logger::log_info("Read A&E data from Denodo") + # Read Extract extract_ae <- dplyr::tbl( denodo_connect, dbplyr::in_schema("sdl", "sdl_ae2_episode_level_source") ) %>% dplyr::filter( - financial_year == !!c_year & # TO-DO: check assumption that arrival_financial_year == financial_year + financial_year == c_year & # TO-DO: check assumption that arrival_financial_year == financial_year (significant_facility_code == "32" | is.na(significant_facility_code)) ) %>% dplyr::select( @@ -66,7 +70,5 @@ read_extract_ae <- function(year, dplyr::collect() %>% slfhelper::get_anon_chi("chi") - DBI::dbDisconnect(denodo_connect) - return(extract_ae) } From bc7bfcfe9edb5d62aa69627d40f2dfbb7184c07f Mon Sep 17 00:00:00 2001 From: LucyEmma22 Date: Mon, 23 Mar 2026 11:34:44 +0000 Subject: [PATCH 08/16] Temporarily correct data types after data is read in from Denodo. --- R/read_extract_ae.R | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/R/read_extract_ae.R b/R/read_extract_ae.R index 69dd1cf5e..71f6d73e4 100644 --- a/R/read_extract_ae.R +++ b/R/read_extract_ae.R @@ -70,5 +70,17 @@ read_extract_ae <- function(year, dplyr::collect() %>% slfhelper::get_anon_chi("chi") + # ---------------------------------------------------------------------------- + # TO-DO: Remove this when data types are fixed + extract_ae <- extract_ae %>% + mutate(keytime1 = hms::parse_hms(keytime1), + keytime2 = hms::parse_hms(keytime2), + gender = as.numeric(gender), + ae_patflow = as.numeric(ae_patflow), + age = as.numeric(age), + commhosp = as.character(commhosp) + ) + # ---------------------------------------------------------------------------- + return(extract_ae) } From 4d7c4c22c3330af257d473efde595987555ff625 Mon Sep 17 00:00:00 2001 From: LucyEmma22 Date: Mon, 23 Mar 2026 11:48:12 +0000 Subject: [PATCH 09/16] Add logger, denodo connect/disconnect and new ae_cup column names to process_extract_ae --- R/process_extract_ae.R | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/R/process_extract_ae.R b/R/process_extract_ae.R index e3af4c59a..d3ed4ebf8 100644 --- a/R/process_extract_ae.R +++ b/R/process_extract_ae.R @@ -14,7 +14,7 @@ #' @family process extracts process_extract_ae <- function(data, year, - denodo_connect = NULL, # TO-DO: will be hardcoded to denodo_connect = get_denodo_connection() + denodo_connect = get_denodo_connection(BYOC_MODE = BYOC_MODE), write_to_disk = TRUE, BYOC_MODE = FALSE, run_id = NA, @@ -25,6 +25,8 @@ process_extract_ae <- function(data, # Check that the supplied year is in the correct format year <- check_year_format(year) + logger::log_info("Process A&E data") + # Data Cleaning --------------------------------------- ae_clean <- data %>% @@ -228,27 +230,29 @@ process_extract_ae <- function(data, ) # ---------------------------------------------------------------------------- - # # c_year_cup <- convert_fyyear_to_year(check_year_format(year)) # + # on.exit(try(DBI::dbDisconnect(denodo_connect), silent = TRUE), add = TRUE) + # + # logger::log_info("Read A&E CUP data from Denodo") + # # ae_cup_file <- dplyr::tbl( # denodo_connect, # dbplyr::in_schema("sdl", "sdl_ae_cup_source_placeholder") # TO-DO: Placeholder for data path in denodo # ) %>% - # dplyr::filter(year == !!c_year_cup) %>% # TO-DO: Placeholder for the variable to filter by year - # dplyr::select( # TO-DO: Placeholder variables - # record_keydate1 = "ED Arrival Date", - # keytime1 = "ED Arrival Time", - # record_keydate2 = "ED Discharge Date", - # keytime2 = "ED Discharge Time", - # case_ref_number = "ED Case Reference Number [C]", - # cup_marker = "CUP Marker", - # cup_pathway = "CUP Pathway Name" + # dplyr::filter(year == c_year_cup) %>% # TO-DO: Placeholder for the variable to filter by year + # dplyr::select( + # record_keydate1 = "ed_arrival_date", + # keytime1 = "ed_arrival_time", + # record_keydate2 = "ed_discharge_date", + # keytime2 = "ed_discharge_time", + # case_ref_number = "ed_case_reference_number", + # cup_marker = "cup_marker", + # cup_pathway = "cup_pathway_name" # ) %>% # dplyr::collect() # ---------------------------------------------------------------------------- - # Data Cleaning--------------------------------------- ae_cup_clean <- ae_cup_file %>% @@ -340,6 +344,8 @@ process_extract_ae <- function(data, "cup_pathway" ) + logger::log_info("Write processed A&E data to Denodo intermediate drive") + if (write_to_disk) { write_file( ae_processed, @@ -349,7 +355,5 @@ process_extract_ae <- function(data, ) } - DBI::dbDisconnect(denodo_connect) - return(ae_processed) } From 0af724a8d77f74462401b1bf498b246266cbbb67 Mon Sep 17 00:00:00 2001 From: LucyEmma22 Date: Mon, 23 Mar 2026 11:49:12 +0000 Subject: [PATCH 10/16] Add new parameters to targets for refactored A&E --- _targets.R | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/_targets.R b/_targets.R index 2b0c964b0..5445135b6 100644 --- a/_targets.R +++ b/_targets.R @@ -476,12 +476,16 @@ list( ), # Accident & Emergency (AE2) activity -------------------------------------- # READ - A&E - tar_file_read( + tar_target( # Target name ae_data, - get_boxi_extract_path(year, type = "ae"), # Function - read_extract_ae(year, !!.x) + read_extract_ae( + year = year, + denodo_connect = get_denodo_connection(BYOC_MODE = BYOC_MODE), + file_path = get_boxi_extract_path(year = year, type = "ae", BYOC_MODE), + BYOC_MODE = BYOC_MODE + ) ), # PROCESS - A&E tar_target( @@ -489,9 +493,13 @@ list( source_ae_extract, # Function process_extract_ae( - ae_data, - year, - write_to_disk = write_to_disk + data = ae_data, + year = year, + denodo_connect = get_denodo_connection(BYOC_MODE = BYOC_MODE), + write_to_disk = write_to_disk, + BYOC_MODE = BYOC_MODE, + run_id = run_id, + run_date_time = run_date_time ) ), # TESTS - A&E From 3185a6000a599aa8e26c434f3725f7d2cea50f1d Mon Sep 17 00:00:00 2001 From: LucyEmma22 Date: Mon, 23 Mar 2026 11:50:53 +0000 Subject: [PATCH 11/16] Add A&E to run_sdl --- SDL_process/run_sdl.r | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/SDL_process/run_sdl.r b/SDL_process/run_sdl.r index 5cbfa47a5..9666adffc 100644 --- a/SDL_process/run_sdl.r +++ b/SDL_process/run_sdl.r @@ -151,12 +151,12 @@ year <- "1920" # Build BYOC Output File Paths byoc_output_files <- get_byoc_output_files( year = year, - types = c("homelessness", "maternity") # using homelessness for test purpose. When development is complete, we change to "types = "byoc_input_files"" + types = c("homelessness", "maternity", "ae") # using homelessness for test purpose. When development is complete, we change to "types = "byoc_input_files"" ) # can always use any other type for testing also # targets::tar_make() -# test homelessness data only +# Test homelessness data ## create homelessness data ---- logger::log_info("Read and process homelessness data") hl1 <- read_extract_homelessness( @@ -178,6 +178,7 @@ hl1 <- read_extract_homelessness( run_date_time = run_date_time ) +# Test maternity data logger::log_info("Read and process maternity data") maternity <- read_extract_maternity( year, @@ -193,4 +194,21 @@ maternity <- read_extract_maternity( run_date_time = run_date_time ) +# Test ae data +logger::log_info("Read and process A&E data") +ae <- read_extract_ae( + year, + denodo_connect = get_denodo_connection(BYOC_MODE = BYOC_MODE), + file_path = get_boxi_extract_path(year, type = "ae", BYOC_MODE), + BYOC_MODE = BYOC_MODE + ) %>% + process_extract_ae( + year = year, + denodo_connect = get_denodo_connection(BYOC_MODE = BYOC_MODE), + write_to_disk = TRUE, + BYOC_MODE = BYOC_MODE, + run_id = run_id, + run_date_time = run_date_time + ) + logger::log_info("Run SDL ended.") From 9756bcb0f44a5b566b5481402c2c58214a86af6a Mon Sep 17 00:00:00 2001 From: LucyEmma22 <78858900+LucyEmma22@users.noreply.github.com> Date: Mon, 23 Mar 2026 11:53:15 +0000 Subject: [PATCH 12/16] Style code --- R/read_extract_ae.R | 13 +++++++------ SDL_process/run_sdl.r | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/R/read_extract_ae.R b/R/read_extract_ae.R index 71f6d73e4..2f46c573b 100644 --- a/R/read_extract_ae.R +++ b/R/read_extract_ae.R @@ -73,12 +73,13 @@ read_extract_ae <- function(year, # ---------------------------------------------------------------------------- # TO-DO: Remove this when data types are fixed extract_ae <- extract_ae %>% - mutate(keytime1 = hms::parse_hms(keytime1), - keytime2 = hms::parse_hms(keytime2), - gender = as.numeric(gender), - ae_patflow = as.numeric(ae_patflow), - age = as.numeric(age), - commhosp = as.character(commhosp) + mutate( + keytime1 = hms::parse_hms(keytime1), + keytime2 = hms::parse_hms(keytime2), + gender = as.numeric(gender), + ae_patflow = as.numeric(ae_patflow), + age = as.numeric(age), + commhosp = as.character(commhosp) ) # ---------------------------------------------------------------------------- diff --git a/SDL_process/run_sdl.r b/SDL_process/run_sdl.r index 9666adffc..9616d2c3b 100644 --- a/SDL_process/run_sdl.r +++ b/SDL_process/run_sdl.r @@ -201,7 +201,7 @@ ae <- read_extract_ae( denodo_connect = get_denodo_connection(BYOC_MODE = BYOC_MODE), file_path = get_boxi_extract_path(year, type = "ae", BYOC_MODE), BYOC_MODE = BYOC_MODE - ) %>% +) %>% process_extract_ae( year = year, denodo_connect = get_denodo_connection(BYOC_MODE = BYOC_MODE), @@ -209,6 +209,6 @@ ae <- read_extract_ae( BYOC_MODE = BYOC_MODE, run_id = run_id, run_date_time = run_date_time - ) + ) logger::log_info("Run SDL ended.") From 3b79705598ae9cb13f8405fa4734720adcdab996 Mon Sep 17 00:00:00 2001 From: LucyEmma22 <78858900+LucyEmma22@users.noreply.github.com> Date: Mon, 23 Mar 2026 11:56:24 +0000 Subject: [PATCH 13/16] Update documentation --- man/process_extract_ae.Rd | 2 +- man/read_extract_ae.Rd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/man/process_extract_ae.Rd b/man/process_extract_ae.Rd index 74750b86b..a44cecb3a 100644 --- a/man/process_extract_ae.Rd +++ b/man/process_extract_ae.Rd @@ -7,7 +7,7 @@ process_extract_ae( data, year, - denodo_connect = NULL, + denodo_connect = get_denodo_connection(BYOC_MODE = BYOC_MODE), write_to_disk = TRUE, BYOC_MODE = FALSE, run_id = NA, diff --git a/man/read_extract_ae.Rd b/man/read_extract_ae.Rd index d0151da8d..390842e54 100644 --- a/man/read_extract_ae.Rd +++ b/man/read_extract_ae.Rd @@ -6,7 +6,7 @@ \usage{ read_extract_ae( year, - denodo_connect, + denodo_connect = get_denodo_connection(BYOC_MODE = BYOC_MODE), file_path = get_boxi_extract_path(year = year, type = "ae", BYOC_MODE), BYOC_MODE ) From 7939364a90fcf4f126e5787af492f1ec3fe8c7b7 Mon Sep 17 00:00:00 2001 From: LucyEmma22 Date: Tue, 24 Mar 2026 11:01:09 +0000 Subject: [PATCH 14/16] Change manual logger messages to log_slf_event() function. --- R/process_extract_ae.R | 11 +++++------ R/read_extract_ae.R | 6 ++++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/R/process_extract_ae.R b/R/process_extract_ae.R index d3ed4ebf8..ddd9ee073 100644 --- a/R/process_extract_ae.R +++ b/R/process_extract_ae.R @@ -19,14 +19,15 @@ process_extract_ae <- function(data, BYOC_MODE = FALSE, run_id = NA, run_date_time = NA) { + + log_slf_event(stage = "process", status = "start", type = "ae", year = year) + # Only run for a single year stopifnot(length(year) == 1L) # Check that the supplied year is in the correct format year <- check_year_format(year) - logger::log_info("Process A&E data") - # Data Cleaning --------------------------------------- ae_clean <- data %>% @@ -234,8 +235,6 @@ process_extract_ae <- function(data, # # on.exit(try(DBI::dbDisconnect(denodo_connect), silent = TRUE), add = TRUE) # - # logger::log_info("Read A&E CUP data from Denodo") - # # ae_cup_file <- dplyr::tbl( # denodo_connect, # dbplyr::in_schema("sdl", "sdl_ae_cup_source_placeholder") # TO-DO: Placeholder for data path in denodo @@ -344,8 +343,6 @@ process_extract_ae <- function(data, "cup_pathway" ) - logger::log_info("Write processed A&E data to Denodo intermediate drive") - if (write_to_disk) { write_file( ae_processed, @@ -355,5 +352,7 @@ process_extract_ae <- function(data, ) } + log_slf_event(stage = "process", status = "complete", type = "ae", year = year) + return(ae_processed) } diff --git a/R/read_extract_ae.R b/R/read_extract_ae.R index 71f6d73e4..5cc5d0e17 100644 --- a/R/read_extract_ae.R +++ b/R/read_extract_ae.R @@ -8,6 +8,9 @@ read_extract_ae <- function(year, denodo_connect = get_denodo_connection(BYOC_MODE = BYOC_MODE), file_path = get_boxi_extract_path(year = year, type = "ae", BYOC_MODE), BYOC_MODE) { + + log_slf_event(stage = "read", status = "start", type = "ae", year = year) + year <- check_year_format(year, format = "fyyear") c_year <- convert_fyyear_to_year(year) @@ -18,8 +21,6 @@ read_extract_ae <- function(year, on.exit(try(DBI::dbDisconnect(denodo_connect), silent = TRUE), add = TRUE) - logger::log_info("Read A&E data from Denodo") - # Read Extract extract_ae <- dplyr::tbl( denodo_connect, @@ -81,6 +82,7 @@ read_extract_ae <- function(year, commhosp = as.character(commhosp) ) # ---------------------------------------------------------------------------- + log_slf_event(stage = "read", status = "complete", type = "ae", year = year) return(extract_ae) } From 835aa4dd7c38f3634373ce2e3793914464d58835 Mon Sep 17 00:00:00 2001 From: LucyEmma22 <78858900+LucyEmma22@users.noreply.github.com> Date: Tue, 24 Mar 2026 11:23:05 +0000 Subject: [PATCH 15/16] Style code --- R/process_extract_ae.R | 1 - R/read_extract_ae.R | 1 - 2 files changed, 2 deletions(-) diff --git a/R/process_extract_ae.R b/R/process_extract_ae.R index ddd9ee073..316e14e99 100644 --- a/R/process_extract_ae.R +++ b/R/process_extract_ae.R @@ -19,7 +19,6 @@ process_extract_ae <- function(data, BYOC_MODE = FALSE, run_id = NA, run_date_time = NA) { - log_slf_event(stage = "process", status = "start", type = "ae", year = year) # Only run for a single year diff --git a/R/read_extract_ae.R b/R/read_extract_ae.R index 563a81485..7b84265e7 100644 --- a/R/read_extract_ae.R +++ b/R/read_extract_ae.R @@ -8,7 +8,6 @@ read_extract_ae <- function(year, denodo_connect = get_denodo_connection(BYOC_MODE = BYOC_MODE), file_path = get_boxi_extract_path(year = year, type = "ae", BYOC_MODE), BYOC_MODE) { - log_slf_event(stage = "read", status = "start", type = "ae", year = year) year <- check_year_format(year, format = "fyyear") From 8435f54f0dce96ec5611f2f4b7e9fba3e43c065c Mon Sep 17 00:00:00 2001 From: Zihao Li Date: Fri, 27 Mar 2026 15:33:41 +0000 Subject: [PATCH 16/16] tiny updates --- R/process_extract_ae.R | 3 ++- R/read_extract_ae.R | 16 ++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/R/process_extract_ae.R b/R/process_extract_ae.R index 316e14e99..329254136 100644 --- a/R/process_extract_ae.R +++ b/R/process_extract_ae.R @@ -255,7 +255,8 @@ process_extract_ae <- function(data, ae_cup_clean <- ae_cup_file %>% # Remove any duplicates - dplyr::distinct(.data$record_keydate1, + dplyr::distinct( + .data$record_keydate1, .data$keytime1, .data$record_keydate2, .data$keytime2, diff --git a/R/read_extract_ae.R b/R/read_extract_ae.R index 0970ebb17..6067b017f 100644 --- a/R/read_extract_ae.R +++ b/R/read_extract_ae.R @@ -4,10 +4,12 @@ #' #' @export #' -read_extract_ae <- function(year, - denodo_connect = get_denodo_connection(BYOC_MODE = BYOC_MODE), - file_path = get_boxi_extract_path(year = year, type = "ae", BYOC_MODE), - BYOC_MODE) { +read_extract_ae <- function( + year, + denodo_connect = get_denodo_connection(BYOC_MODE = BYOC_MODE), + file_path = get_boxi_extract_path(year = year, type = "ae", BYOC_MODE), + BYOC_MODE +) { log_slf_event(stage = "read", status = "start", type = "ae", year = year) year <- check_year_format(year, format = "fyyear") @@ -26,8 +28,8 @@ read_extract_ae <- function(year, dbplyr::in_schema("sdl", "sdl_ae2_episode_level_source") ) %>% dplyr::filter( - financial_year == c_year & # TO-DO: check assumption that arrival_financial_year == financial_year - (significant_facility_code == "32" | is.na(significant_facility_code)) + financial_year == c_year, # TO-DO: check assumption that arrival_financial_year == financial_year + significant_facility_code == "32" | is.na(significant_facility_code) ) %>% dplyr::select( record_keydate1 = "arrival_date", @@ -84,7 +86,5 @@ read_extract_ae <- function(year, # ---------------------------------------------------------------------------- log_slf_event(stage = "read", status = "complete", type = "ae", year = year) - log_slf_event(stage = "read", status = "complete", type = "ae", year = year) - return(extract_ae) }