diff --git a/DESCRIPTION b/DESCRIPTION index c3f0c21a2..295c630af 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -25,6 +25,7 @@ Imports: hms (>= 1.1.0), janitor (>= 2.2.0), keyring (>= 1.3.0), + logger (>= 0.4.1), lubridate (>= 1.9.2), magrittr (>= 2.0.3), odbc (>= 1.3.1), diff --git a/NAMESPACE b/NAMESPACE index 0d294c226..cade26547 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -95,6 +95,12 @@ export(is_missing) export(last_date_month) export(latest_cost_year) export(latest_update) +export(log_ep_message) +export(log_ep_substage) +export(log_ind_message) +export(log_ind_substage) +export(log_slf_event) +export(log_tar_message) export(make_lowercase_ext) export(midpoint_fy) export(next_fy) @@ -198,6 +204,7 @@ export(setup_keyring) export(start_fy) export(start_fy_quarter) export(start_next_fy_quarter) +export(write_console_output) export(write_file) export(write_temp_data) export(years_to_run) diff --git a/R/create_episode_file.R b/R/create_episode_file.R index 25e35fee1..3d6d7802e 100644 --- a/R/create_episode_file.R +++ b/R/create_episode_file.R @@ -33,7 +33,7 @@ create_episode_file <- function( write_to_disk = TRUE, write_temp_to_disk = FALSE ) { - cli::cli_alert_info("Create episode file function started at {Sys.time()}") + log_ep_substage("Create episode file", "started", year) processed_data_list <- purrr::discard(processed_data_list, ~ is.null(.x) | identical(.x, tibble::tibble())) @@ -317,7 +317,7 @@ store_ep_file_vars <- function(data, year, vars_to_keep) { group_id = 3356 # sourcedev owner ) - cli::cli_alert_info("Store episode file variables function finished at {Sys.time()}") + log_ep_substage("Store episode file variables", "finished", year) return( dplyr::select( @@ -352,7 +352,7 @@ load_ep_file_vars <- function(data, year) { fs::file_delete(tempfile_path) - cli::cli_alert_info("Load episode file variable function finished at {Sys.time()}") + log_ep_substage("Load episode file variables", "finished", year) return(full_data) } @@ -409,7 +409,7 @@ fill_missing_cij_markers <- function(data) { return_data <- dplyr::bind_rows(non_fixable_data, fixed_data) - cli::cli_alert_info("Fill missing cij markers function finished at {Sys.time()}") + log_ep_substage("Fill missing cij markers", "finished", year) return(return_data) } @@ -455,7 +455,7 @@ correct_cij_vars <- function(data) { ) ) - cli::cli_alert_info("Correct cij variables function finished at {Sys.time()}") + log_ep_substage("Correct cij variables", "finished", year) return(data) } @@ -483,7 +483,7 @@ create_cost_inc_dna <- function(data) { ) ) - cli::cli_alert_info("Create cost inc dna function finished at {Sys.time()}") + log_ep_substage("Create cost inc dna", "finished", year) return(data) } @@ -510,7 +510,7 @@ create_cohort_lookups <- function(data, year, update = latest_update()) { write_to_disk = TRUE ) - cli::cli_alert_info("Create cohort lookups function finished at {Sys.time()}") + log_ep_substage("Create cohort lookups", "finished", year) return(data) } @@ -545,7 +545,7 @@ join_cohort_lookups <- function( by = "anon_chi" ) - cli::cli_alert_info("Join cohort lookups function finished at {Sys.time()}") + log_ep_substage("Join cohort lookups", "finished", year) return(join_cohort_lookups) } @@ -612,7 +612,9 @@ join_sc_client <- function(data, ) } - cli::cli_alert_info("Join social care client function finished at {Sys.time()}") + log_ep_substage("Join social care client", "finished", year) + + log_ep_substage("Create episode file", "finished", year) return(data_file) } diff --git a/R/create_individual_file.R b/R/create_individual_file.R index 6282c031d..81701f612 100644 --- a/R/create_individual_file.R +++ b/R/create_individual_file.R @@ -17,7 +17,7 @@ create_individual_file <- function( write_to_disk = TRUE, write_temp_to_disk ) { - cli::cli_alert_info("Create individual file function started at {Sys.time()}") + log_ind_substage("Create individual file", "started", year) individual_file <- episode_file %>% dplyr::select(dplyr::any_of(c( @@ -177,7 +177,7 @@ remove_blank_chi <- function(episode_file) { dplyr::mutate(anon_chi = dplyr::na_if(.data$anon_chi, "")) %>% dplyr::filter(!is.na(.data$anon_chi)) - cli::cli_alert_info("Remove blank CHI function finished at {Sys.time()}") + log_ind_substage("Remove blank CHI function", "finished", year) return(episode_file) } @@ -218,7 +218,7 @@ add_cij_columns <- function(episode_file) { ) ) - cli::cli_alert_info("Add cij columns function finished at {Sys.time()}") + log_ind_substage("Add cij columns function", "finished", year) return(episode_file) } @@ -276,7 +276,7 @@ add_all_columns <- function(episode_file, year) { ) ) - cli::cli_alert_info("Add all columns function finished at {Sys.time()}") + log_ind_substage("Add all columns function", "finished", year) return(episode_file) } @@ -293,7 +293,7 @@ add_acute_columns <- function(episode_file, prefix, condition) { add_standard_cols(prefix, condition, episode = TRUE, cost = TRUE) %>% add_ipdc_cols(prefix, condition) - cli::cli_alert_info("Add acute columns function finished at {Sys.time()}") + log_ind_substage("Add acute columns function", "finished", year) return(episode_file) } @@ -308,7 +308,7 @@ add_mat_columns <- function(episode_file, prefix, condition) { add_standard_cols(prefix, condition, episode = TRUE, cost = TRUE) %>% add_ipdc_cols(prefix, condition, elective = FALSE) - cli::cli_alert_info("Add maternity columns function finished at {Sys.time()}") + log_ind_substage("Add maternity columns function", "finished", year) return(episode_file) } @@ -323,7 +323,7 @@ add_mh_columns <- function(episode_file, prefix, condition) { add_standard_cols(prefix, condition, episode = TRUE, cost = TRUE) %>% add_ipdc_cols(prefix, condition, ipdc_d = FALSE) - cli::cli_alert_info("Add mental health columns function finished at {Sys.time()}") + log_ind_substage("Add mental health columns function", "finished", year) return(episode_file) } @@ -338,7 +338,7 @@ add_gls_columns <- function(episode_file, prefix, condition) { add_standard_cols(prefix, condition, episode = TRUE, cost = TRUE) %>% add_ipdc_cols(prefix, condition, ipdc_d = FALSE) - cli::cli_alert_info("Add geriatric long stay columns function finished at {Sys.time()}") + log_ind_substage("Add geriatric long stay columns function", "finished", year) return(episode_file) } @@ -364,7 +364,7 @@ add_op_columns <- function(episode_file, prefix, condition) { "{prefix}_cost_dnas" := dplyr::if_else(eval(condition_5_8), .data$cost_total_net_inc_dnas, NA_real_) ) - cli::cli_alert_info("Add outpatient columns function finished at {Sys.time()}") + log_ind_substage("Add outpatient columns function", "finished", year) return(episode_file) } @@ -379,7 +379,7 @@ add_ae_columns <- function(episode_file, prefix, condition) { add_standard_cols(prefix, condition, cost = TRUE) %>% dplyr::mutate("{prefix}_attendances" := dplyr::if_else(eval(condition), 1L, NA_integer_)) - cli::cli_alert_info("Add A&E columns function finished at {Sys.time()}") + log_ind_substage("Add A&E columns function", "finished", year) return(episode_file) } @@ -393,7 +393,7 @@ add_pis_columns <- function(episode_file, prefix, condition) { episode_file <- episode_file %>% add_standard_cols(prefix, condition, cost = TRUE) %>% dplyr::mutate("{prefix}_paid_items" := dplyr::if_else(eval(condition), .data$no_paid_items, NA_integer_)) - cli::cli_alert_info("Add prescribing columns function finished at {Sys.time()}") + log_ind_substage("Add prescribing columns function", "finished", year) return(episode_file) } @@ -430,7 +430,7 @@ add_ooh_columns <- function(episode_file, prefix, condition) { ) ) - cli::cli_alert_info("Add out of hours columns function finished at {Sys.time()}") + log_ind_substage("Add out of hours columns function", "finished", year) return(episode_file) } @@ -457,7 +457,7 @@ add_dn_columns <- function(episode_file, prefix, condition) { dplyr::mutate("{prefix}_contacts" := NA_integer_) } - cli::cli_alert_info("Add district nursing columns function finished at {Sys.time()}") + log_ind_substage("Add district nursing columns function", "finished", year) return(episode_file) } @@ -472,7 +472,7 @@ add_cmh_columns <- function(episode_file, prefix, condition) { add_standard_cols(prefix, condition) %>% dplyr::mutate("{prefix}_contacts" := dplyr::if_else(eval(condition), 1L, NA_integer_)) - cli::cli_alert_info("Add communicty mental health columns function finished at {Sys.time()}") + log_ind_substage("Add community mental health columns function", "finished", year) return(episode_file) } @@ -496,7 +496,7 @@ add_dd_columns <- function(episode_file, prefix, condition) { "{prefix}_Code9_beddays" := dplyr::if_else(eval(condition_delay_9), .data$yearstay, NA_real_) ) - cli::cli_alert_info("Add delayed discharges columns function finished at {Sys.time()}") + log_ind_substage("Add delayed discharges columns function", "finished", year) return(episode_file) } @@ -511,7 +511,7 @@ add_nsu_columns <- function(episode_file, prefix, condition) { add_standard_cols(prefix, condition) %>% dplyr::mutate("{prefix}" := dplyr::if_else(eval(condition), 1L, NA_integer_)) - cli::cli_alert_info("Add non service users columns function finished at {Sys.time()}") + log_ind_substage("Add non service users columns function", "finished", year) return(episode_file) } @@ -526,7 +526,7 @@ add_nrs_columns <- function(episode_file, prefix, condition) { add_standard_cols(prefix, condition) %>% dplyr::mutate("{prefix}" := dplyr::if_else(eval(condition), 1L, NA_integer_)) - cli::cli_alert_info("Add nrs columns function finished at {Sys.time()}") + log_ind_substage("Add nrs columns function", "finished", year) return(episode_file) } @@ -540,7 +540,7 @@ add_hl1_columns <- function(episode_file, prefix, condition) { episode_file <- episode_file %>% add_standard_cols(prefix, condition) - cli::cli_alert_info("Add homelessness columns function finished at {Sys.time()}") + log_ind_substage("Add homelessness columns function", "finished", year) return(episode_file) } @@ -574,7 +574,7 @@ add_ch_columns <- function(episode_file, prefix, condition) { ) ) - cli::cli_alert_info("Add care home columns function finished at {Sys.time()}") + log_ind_substage("Add care home columns function", "finished", year) return(episode_file) } @@ -622,7 +622,7 @@ add_hc_columns <- function(episode_file, prefix, condition) { "{prefix}_reablement_hours_cost" := dplyr::if_else(eval(condition_reabl), .data$cost_total_net, NA_real_) ) - cli::cli_alert_info("Add home care columns function finished at {Sys.time()}") + log_ind_substage("Add home care columns function", "finished", year) return(episode_file) } @@ -640,7 +640,7 @@ add_at_columns <- function(episode_file, prefix, condition) { "{prefix}_telecare" := dplyr::if_else(eval(condition) & .data$smrtype == "AT-Tele", 1L, NA_integer_) ) - cli::cli_alert_info("Add alarms telecare columns function finished at {Sys.time()}") + log_ind_substage("Add alarms telecare columns function", "finished", year) return(episode_file) } @@ -660,7 +660,7 @@ add_sds_columns <- function(episode_file, prefix, condition) { "{prefix}_option_4" := dplyr::if_else(eval(condition) & .data$smrtype == "SDS-4", 1L, NA_integer_) ) - cli::cli_alert_info("Add SDS columns function finished at {Sys.time()}") + log_ind_substage("Add SDS columns function", "finished", year) return(episode_file) } @@ -708,7 +708,7 @@ add_ipdc_cols <- function(episode_file, prefix, condition, ipdc_d = TRUE, electi ) } - cli::cli_alert_info("Add ipdc columns function finished at {Sys.time()}") + log_ind_substage("Add ipdc columns function", "finished", year) return(episode_file) } @@ -730,7 +730,7 @@ add_standard_cols <- function(episode_file, prefix, condition, episode = FALSE, episode_file <- dplyr::mutate(episode_file, "{prefix}_cost" := dplyr::if_else(eval(condition), .data$cost_total_net, NA_real_)) } - cli::cli_alert_info("Add standard columns function finished at {Sys.time()}") + log_ind_substage("Add standard columns function", "finished", year) return(episode_file) } @@ -775,7 +775,7 @@ clean_up_ch <- function(episode_file, year) { ) %>% dplyr::select(-c("fy_end", "fy_start", "term_1", "term_2")) - cli::cli_alert_info("Clean up CH function finished at {Sys.time()}") + log_ind_substage("Clean up CH function", "finished", year) return(episode_file) } @@ -796,7 +796,7 @@ recode_gender <- function(episode_file) { ) ) - cli::cli_alert_info("Recode Gender function finished at {Sys.time()}") + log_ind_substage("Recode Gender function", "finished", year) return(episode_file) } @@ -812,7 +812,7 @@ condition_cols <- function() { date_cols <- paste0(conditions, "_date") all_cols <- c(conditions, date_cols) - cli::cli_alert_info("Return condition columns function finished at {Sys.time()}") + log_ind_substage("Return condition columns function", "finished", year) return(all_cols) } @@ -858,7 +858,7 @@ clean_individual_file <- function(individual_file, year) { clean_up_gender() %>% dplyr::mutate(age = compute_mid_year_age(year, .data$dob)) - cli::cli_alert_info("Clean individual file function finished at {Sys.time()}") + log_ind_substage("Clean individual file function", "finished", year) return(individual_file) } @@ -881,7 +881,7 @@ clean_up_gender <- function(individual_file) { # change back to anon_chi slfhelper::get_anon_chi() - cli::cli_alert_info("Clean up gender column function finished at {Sys.time()}") + log_ind_substage("Clean up gender column function", "finished", year) return(individual_file) } @@ -913,7 +913,9 @@ join_slf_lookup_vars <- function(individual_file, ) %>% dplyr::rename(hbrescode = hbrescode_var) - cli::cli_alert_info("Join slf lookup variables function finished at {Sys.time()}") + log_ind_substage("Join slf lookup variables function", "finished", year) + + log_ind_substage("Create individual file", "finished", year) return(individual_file) } diff --git a/R/logger_utils.R b/R/logger_utils.R new file mode 100644 index 000000000..ac4d3c157 --- /dev/null +++ b/R/logger_utils.R @@ -0,0 +1,215 @@ +#' Write Console Output to File +#' +#' @description Initialises logger for run_episode_file_fy.R and run_individual_file_fy.R. +#' +#' @param console_outputs If TRUE, capture console output and messages to file. +#' @param file_type Type of file being processed: "episode", "individual", or "targets". +#' @param year Financial year. +#' +#' @export +write_console_output <- function(console_outputs = TRUE, + file_type = c("episode", "individual", "targets"), + year = NULL) { + if (!console_outputs) { + return(invisible(NULL)) + } + + file_type <- match.arg(file_type) + + # Update information + update <- latest_update() + update_year <- as.integer(substr(phsmethods::extract_fin_year(end_date()), 1, 4)) + update_quarter <- qtr() + + # Output directory path + con_output_dir <- file.path( + "/conf/sourcedev/Source_Linkage_File_Updates/_console_output", + update_year, + update_quarter + ) + + if (!dir.exists(con_output_dir)) { + dir.create(con_output_dir, recursive = TRUE) + } + + base_filename <- switch(file_type, + "episode" = stringr::str_glue("ep_{year}_{update}_update"), + "individual" = stringr::str_glue("ind_{year}_{update}_update"), + "targets" = stringr::str_glue("targets_console_{update}_update") + ) + + existing_files <- list.files( + path = con_output_dir, + pattern = stringr::str_glue("^{base_filename}_[0-9]+\\.txt$"), + full.names = FALSE + ) + + if (length(existing_files) == 0) { + increment <- 1 + } else { + numbers <- stringr::str_extract(existing_files, "[0-9]+(?=\\.txt$)") %>% + as.integer() + increment <- max(numbers, na.rm = TRUE) + 1 + } + + file_name <- stringr::str_glue("{base_filename}_{increment}.txt") + file_path <- file.path(con_output_dir, file_name) + + # Add logger appender to write to file + logger::log_appender(logger::appender_tee(file_path), index = 2) + logger::log_info("Console output will be saved to: {file_path}") + + invisible(file_path) +} + +#' Log Episode File Messages +#' +#' @param stage Stage of episode file (run_episode_file_fy.R) creation. +#' @param year Financial year. +#' +#' @export +log_ep_message <- function(stage = c("start", "read_data", "creating", "complete"), + year) { + stage <- match.arg(stage) + + messages <- list( + start = "Starting episode file creation for ep_{year}", + read_data = "Reading processed data from source extracts for year {year}", + creating = "Creating episode file and running tests for year {year}", + complete = "Episode file creation complete for year {year}" + ) + + logger::log_info(stringr::str_glue(messages[[stage]])) +} + +#' Log Individual File Messages +#' +#' @param stage Stage of individual file (run_individual_file_fy.R) creation. +#' @param year Financial year. +#' +#' @export +log_ind_message <- function(stage = c("start", "read_data", "creating", "complete"), + year) { + stage <- match.arg(stage) + + messages <- list( + start = "Starting individual file creation for ind_{year}", + read_data = "Reading episode file for year {year}", + creating = "Creating individual file and running tests for year {year}", + complete = "Individual file creation complete for year {year}" + ) + + logger::log_info(stringr::str_glue(messages[[stage]])) +} + +#' Log Targets Messages +#' +#' @param stage Stage of targets (run_targets_fy.R) creation. +#' +#' @export +log_tar_message <- function(stage = c("start", "combining_tests", "all_complete")) { + stage <- match.arg(stage) + + messages <- list( + start = "Starting targets pipeline", + combining_tests = "Combining test results", + all_complete = "All processing complete" + ) + + logger::log_info(messages[[stage]]) +} + +#' Log SLF Intermediate Processing +#' +#' @description Logs messages for read, process, and test stages/scripts +#' +#' @param stage Character: "read", "process", or "test". +#' @param status Character: "start" or "complete". +#' @param type Character: The name of the data (e.g., "Acute", "A&E"). +#' @param year Character: The financial year. +#' @param ... Additional arguments for string interpolation. +#' +#' @export +log_slf_event <- function(stage = c("read", "process", "test"), + status = c("start", "complete"), + type, + year, + ...) { + file_name <- dplyr::case_match( + type, + "acute" ~ "Acute", + "ae" ~ "A&E", + "at" ~ "Alarms Telecare", + "ch" ~ "Care Home", + "client" ~ "Social Care Client", + "cmh" ~ "Community Mental Health", + "dd" ~ "Delayed Discharges", + "deaths" ~ "NRS Deaths", + "dn" ~ "District Nursing", + "gpooh" ~ "GP Out of Hours", + "gp_ooh-c" ~ "GP Out of Hours Consultations", + "gp_ooh-d" ~ "GP Out of Hours Diagnosis", + "gp_ooh-o" ~ "GP Out of Hours Outcomes", + "hc" ~ "Home Care", + "homelessness" ~ "Homelessness", + "it_chi_deaths" ~ "IT_Chi_Deaths", + "ltc" ~ "Long Term Conditions", + "maternity" ~ "Maternity", + "mh" ~ "Mental Health", + "outpatient" ~ "Outpatient", + "pis" ~ "Prescribing", + "sc_demog" ~ "Social Care Demographics", + "sds" ~ "Self Directed Support", + "pis" ~ "Prescribing", + .default = type # use type if file_name not available + ) + + stage <- match.arg(stage) + status <- match.arg(status) + + # List of message templates + messages <- list( + read = list( + start = "Reading {year} {file_name} data from denodo", + complete = "Finished reading and renaming {year} {file_name} data" + ), + process = list( + start = "Processing {year} {file_name} data", + complete = "Finished processing {year} {file_name} data" + ), + test = list( + start = "Processing test on {year} {file_name} data", + complete = "Completed tests on {year} {file_name} data" + ) + ) + + msg <- stringr::str_glue(messages[[stage]][[status]]) + + logger::log_info(msg) +} + +#' Log Create Episode File (create_episode_file.R) Stages +#' +#' @param sub_stage Character: The name of the specific process (e.g., "Join cohort lookups"). +#' @param status Character: "started" or "finished". +#' @param year Character: The financial year. +#' +#' @export +log_ep_substage <- function(sub_stage, status, year) { + msg <- stringr::str_glue("Episode File: {sub_stage} {status} for {year}") + + logger::log_info(msg) +} + +#' Log Create Individual File (create_individual_file.R) Stages +#' +#' @param sub_stage Character: The name of the specific process (e.g., "Add cij columns"). +#' @param status Character: "started" or "finished". +#' @param year Character: The financial year. +#' +#' @export +log_ind_substage <- function(sub_stage, status, year) { + msg <- stringr::str_glue("Individual File: {sub_stage} {status} for {year}") + + logger::log_info(msg) +} diff --git a/R/process_extract_acute.R b/R/process_extract_acute.R index 11e9c4261..6d2005c66 100644 --- a/R/process_extract_acute.R +++ b/R/process_extract_acute.R @@ -17,6 +17,8 @@ process_extract_acute <- function(data, year, acute_cup_path = get_boxi_extract_path(year, "acute_cup"), write_to_disk = TRUE) { + log_slf_event(stage = "process", status = "start", type = "acute", year = year) + # Only run for a single year stopifnot(length(year) == 1L) @@ -168,5 +170,7 @@ process_extract_acute <- function(data, ) } + log_slf_event(stage = "process", status = "complete", type = "acute", year = year) + return(acute_processed) } diff --git a/R/process_extract_ae.R b/R/process_extract_ae.R index cf71578bd..d1bcaec9f 100644 --- a/R/process_extract_ae.R +++ b/R/process_extract_ae.R @@ -13,6 +13,8 @@ #' @export #' @family process extracts process_extract_ae <- function(data, year, write_to_disk = TRUE) { + log_slf_event(stage = "process", status = "start", type = "ae", year = year) + # Only run for a single year stopifnot(length(year) == 1L) @@ -315,5 +317,7 @@ process_extract_ae <- function(data, year, write_to_disk = TRUE) { ) } + log_slf_event(stage = "process", status = "complete", type = "ae", year = year) + return(ae_processed) } diff --git a/R/process_extract_alarms_telecare.R b/R/process_extract_alarms_telecare.R index 6c1b0aa98..20f157f02 100644 --- a/R/process_extract_alarms_telecare.R +++ b/R/process_extract_alarms_telecare.R @@ -14,6 +14,8 @@ process_extract_alarms_telecare <- function( year, write_to_disk = TRUE ) { + log_slf_event(stage = "process", status = "start", type = "at", year = year) + # Only run for a single year stopifnot(length(year) == 1L) @@ -61,5 +63,7 @@ process_extract_alarms_telecare <- function( ) } + log_slf_event(stage = "process", status = "complete", type = "at", year = year) + return(at_data) } diff --git a/R/process_extract_care_home.R b/R/process_extract_care_home.R index feb18be5e..f450da921 100644 --- a/R/process_extract_care_home.R +++ b/R/process_extract_care_home.R @@ -20,6 +20,8 @@ process_extract_care_home <- function( ch_costs, write_to_disk = TRUE ) { + log_slf_event(stage = "process", status = "start", type = "ch", year = year) + # Only run for a single year stopifnot(length(year) == 1L) @@ -142,5 +144,7 @@ process_extract_care_home <- function( ) } + log_slf_event(stage = "process", status = "complete", type = "ch", year = year) + return(ch_processed) } diff --git a/R/process_extract_cmh.R b/R/process_extract_cmh.R index e98b5ab84..0288461aa 100644 --- a/R/process_extract_cmh.R +++ b/R/process_extract_cmh.R @@ -15,6 +15,8 @@ process_extract_cmh <- function(data, year, write_to_disk = TRUE) { + log_slf_event(stage = "process", status = "start", type = "cmh", year = year) + # Only run for a single year stopifnot(length(year) == 1L) @@ -78,5 +80,7 @@ process_extract_cmh <- function(data, ) } + log_slf_event(stage = "process", status = "complete", type = "cmh", year = year) + return(cmh_processed) } diff --git a/R/process_extract_delayed_discharges.R b/R/process_extract_delayed_discharges.R index 81fa48d7e..94c65e717 100644 --- a/R/process_extract_delayed_discharges.R +++ b/R/process_extract_delayed_discharges.R @@ -17,6 +17,8 @@ process_extract_delayed_discharges <- function( year, write_to_disk = TRUE ) { + log_slf_event(stage = "process", status = "start", type = "dd", year = year) + # Only run for a single year stopifnot(length(year) == 1L) @@ -116,5 +118,7 @@ process_extract_delayed_discharges <- function( ) } + log_slf_event(stage = "process", status = "complete", type = "dd", year = year) + return(dd_final) } diff --git a/R/process_extract_district_nursing.R b/R/process_extract_district_nursing.R index a46254b59..8ae10b7c3 100644 --- a/R/process_extract_district_nursing.R +++ b/R/process_extract_district_nursing.R @@ -19,6 +19,8 @@ process_extract_district_nursing <- function( costs = read_file(get_dn_costs_path()), write_to_disk = TRUE ) { + log_slf_event(stage = "process", status = "start", type = "dn", year = year) + # Only run for a single year stopifnot(length(year) == 1L) @@ -144,5 +146,7 @@ process_extract_district_nursing <- function( ) # sourcedev owner } + log_slf_event(stage = "process", status = "complete", type = "dn", year = year) + return(dn_episodes) } diff --git a/R/process_extract_gp_ooh.R b/R/process_extract_gp_ooh.R index b9ca29e43..dbeaca39b 100644 --- a/R/process_extract_gp_ooh.R +++ b/R/process_extract_gp_ooh.R @@ -17,6 +17,8 @@ process_extract_gp_ooh <- function(year, data_list, gp_ooh_cup_path = get_boxi_extract_path(year, "gp_ooh_cup"), write_to_disk = TRUE) { + log_slf_event(stage = "process", status = "start", type = "gpooh", year = year) + diagnosis_extract <- process_extract_ooh_diagnosis(data_list[["diagnosis"]], year) outcomes_extract <- process_extract_ooh_outcomes(data_list[["outcomes"]], year) consultations_extract <- process_extract_ooh_consultations(data_list[["consultations"]], year) @@ -169,5 +171,7 @@ process_extract_gp_ooh <- function(year, ) # sourcedev owner } + log_slf_event(stage = "process", status = "complete", type = "gpooh", year = year) + return(final_data) } diff --git a/R/process_extract_home_care.R b/R/process_extract_home_care.R index 1ae954314..42918a5b0 100644 --- a/R/process_extract_home_care.R +++ b/R/process_extract_home_care.R @@ -14,6 +14,8 @@ process_extract_home_care <- function( year, write_to_disk = TRUE ) { + log_slf_event(stage = "process", status = "start", type = "hc", year = year) + # Only run for a single year stopifnot(length(year) == 1L) @@ -110,5 +112,7 @@ process_extract_home_care <- function( ) } + log_slf_event(stage = "process", status = "complete", type = "hc", year = year) + return(hc_processed) } diff --git a/R/process_extract_homelessness.R b/R/process_extract_homelessness.R index b66f10594..9965c4098 100644 --- a/R/process_extract_homelessness.R +++ b/R/process_extract_homelessness.R @@ -26,6 +26,8 @@ process_extract_homelessness <- function( run_id = NA, run_date_time = NA ) { + log_slf_event(stage = "process", status = "start", type = "homelessness", year = year) + # Only run for a single year stopifnot(length(year) == 1L) @@ -200,5 +202,7 @@ process_extract_homelessness <- function( ) } + log_slf_event(stage = "process", status = "complete", type = "homelessness", year = year) + return(final_data) } diff --git a/R/process_extract_maternity.R b/R/process_extract_maternity.R index 44b3b002e..ff215ba11 100644 --- a/R/process_extract_maternity.R +++ b/R/process_extract_maternity.R @@ -18,6 +18,8 @@ process_extract_maternity <- function(data, BYOC_MODE = FALSE, run_id = NA, run_date_time = NA) { + log_slf_event(stage = "process", status = "start", type = "maternity", year = year) + # Only run for a single year stopifnot(length(year) == 1L) @@ -134,5 +136,7 @@ process_extract_maternity <- function(data, ) } + log_slf_event(stage = "process", status = "complete", type = "maternity", year = year) + return(maternity_processed) } diff --git a/R/process_extract_mental_health.R b/R/process_extract_mental_health.R index ec1b644c5..aca890637 100644 --- a/R/process_extract_mental_health.R +++ b/R/process_extract_mental_health.R @@ -13,6 +13,8 @@ #' @export #' @family process extracts process_extract_mental_health <- function(data, year, write_to_disk = TRUE) { + log_slf_event(stage = "process", status = "start", type = "mh", year = year) + # Only run for a single year stopifnot(length(year) == 1L) @@ -126,5 +128,7 @@ process_extract_mental_health <- function(data, year, write_to_disk = TRUE) { ) } + log_slf_event(stage = "process", status = "complete", type = "mh", year = year) + return(mh_processed) } diff --git a/R/process_extract_nrs_deaths.R b/R/process_extract_nrs_deaths.R index 4e4385bcd..d80d4a750 100644 --- a/R/process_extract_nrs_deaths.R +++ b/R/process_extract_nrs_deaths.R @@ -12,6 +12,8 @@ #' @export #' @family process extracts process_extract_nrs_deaths <- function(data, year, write_to_disk = TRUE) { + log_slf_event(stage = "process", status = "start", type = "deaths", year = year) + stopifnot(length(year) == 1L) year <- check_year_format(year) @@ -32,5 +34,7 @@ process_extract_nrs_deaths <- function(data, year, write_to_disk = TRUE) { ) } + log_slf_event(stage = "process", status = "complete", type = "deaths", year = year) + return(deaths_clean) } diff --git a/R/process_extract_ooh_consultations.R b/R/process_extract_ooh_consultations.R index 076b0b316..d04587e7d 100644 --- a/R/process_extract_ooh_consultations.R +++ b/R/process_extract_ooh_consultations.R @@ -10,6 +10,8 @@ #' @return the final data as a [tibble][tibble::tibble-package]. #' @family process extracts process_extract_ooh_consultations <- function(data, year) { + log_slf_event(stage = "process", status = "start", type = "gp_ooh-c", year = year) + # to skip warning no visible binding for global variable when using data.table distinct_check <- consultation_type <- location <- record_keydate1 <- record_keydate2 <- anon_chi <- @@ -146,5 +148,7 @@ process_extract_ooh_consultations <- function(data, year) { dplyr::as_tibble() # cleaning up overlapping episodes done + log_slf_event(stage = "process", status = "complete", type = "gp_ooh-c", year = year) + return(consultations_clean) } diff --git a/R/process_extract_ooh_diagnosis.R b/R/process_extract_ooh_diagnosis.R index 1b38dd803..306d3d353 100644 --- a/R/process_extract_ooh_diagnosis.R +++ b/R/process_extract_ooh_diagnosis.R @@ -10,6 +10,8 @@ #' @return the final data as a [tibble][tibble::tibble-package]. #' @family process extracts process_extract_ooh_diagnosis <- function(data, year) { + log_slf_event(stage = "process", status = "start", type = "gp_ooh-d", year = year) + # Only run for a single year stopifnot(length(year) == 1L) @@ -157,5 +159,7 @@ process_extract_ooh_diagnosis <- function(data, year) { )) ) + log_slf_event(stage = "process", status = "complete", type = "gp_ooh-d", year = year) + return(diagnosis_clean) } diff --git a/R/process_extract_ooh_outcomes.R b/R/process_extract_ooh_outcomes.R index f188e6de1..77c38db30 100644 --- a/R/process_extract_ooh_outcomes.R +++ b/R/process_extract_ooh_outcomes.R @@ -10,6 +10,8 @@ #' @return the final data as a [tibble][tibble::tibble-package]. #' @family process extracts process_extract_ooh_outcomes <- function(data, year) { + log_slf_event(stage = "process", status = "start", type = "gp_ooh-o", year = year) + # Only run for a single year stopifnot(length(year) == 1L) @@ -61,5 +63,7 @@ process_extract_ooh_outcomes <- function(data, year) { ) %>% dplyr::as_tibble() + log_slf_event(stage = "process", status = "complete", type = "gp_ooh-o", year = year) + return(outcomes_clean) } diff --git a/R/process_extract_outpatients.R b/R/process_extract_outpatients.R index 175c3d9f0..98d5dac74 100644 --- a/R/process_extract_outpatients.R +++ b/R/process_extract_outpatients.R @@ -13,6 +13,8 @@ #' @export #' @family process extracts process_extract_outpatients <- function(data, year, write_to_disk = TRUE) { + log_slf_event(stage = "process", status = "start", type = "outpatients", year = year) + # Only run for a single year stopifnot(length(year) == 1L) @@ -98,5 +100,7 @@ process_extract_outpatients <- function(data, year, write_to_disk = TRUE) { ) } + log_slf_event(stage = "process", status = "complete", type = "outpatients", year = year) + return(outpatients_processed) } diff --git a/R/process_extract_prescribing.R b/R/process_extract_prescribing.R index 364d8c112..b22aa13c2 100644 --- a/R/process_extract_prescribing.R +++ b/R/process_extract_prescribing.R @@ -13,6 +13,8 @@ #' @export #' @family process extracts process_extract_prescribing <- function(data, year, write_to_disk = TRUE) { + log_slf_event(stage = "process", status = "start", type = "pis", year = year) + # Only run for a single year stopifnot(length(year) == 1L) @@ -60,5 +62,7 @@ process_extract_prescribing <- function(data, year, write_to_disk = TRUE) { ) } + log_slf_event(stage = "process", status = "complete", type = "pis", year = year) + return(pis_clean) } diff --git a/R/process_extract_sds.R b/R/process_extract_sds.R index 7b9b3d24c..b096331f1 100644 --- a/R/process_extract_sds.R +++ b/R/process_extract_sds.R @@ -14,6 +14,8 @@ process_extract_sds <- function( year, write_to_disk = TRUE ) { + log_slf_event(stage = "process", status = "start", type = "sds", year = year) + # Only run for a single year stopifnot(length(year) == 1L) @@ -60,5 +62,7 @@ process_extract_sds <- function( ) } + log_slf_event(stage = "process", status = "complete", type = "sds", year = year) + return(outfile) } diff --git a/R/process_it_chi_deaths.R b/R/process_it_chi_deaths.R index a910e9fb4..5dd240ef0 100644 --- a/R/process_it_chi_deaths.R +++ b/R/process_it_chi_deaths.R @@ -11,6 +11,8 @@ #' @export #' @family process extracts process_it_chi_deaths <- function(data, write_to_disk = TRUE) { + log_slf_event(stage = "process", status = "start", type = "it_chi_deaths", year = "all") + it_chi_deaths_clean <- data %>% dplyr::arrange( dplyr::desc(.data$death_date_nrs), @@ -31,5 +33,7 @@ process_it_chi_deaths <- function(data, write_to_disk = TRUE) { ) } + log_slf_event(stage = "process", status = "complete", type = "it_chi_deaths", year = "all") + return(it_chi_deaths_clean) } diff --git a/R/process_lookup_deaths.R b/R/process_lookup_deaths.R index d7423129c..0482a6573 100644 --- a/R/process_lookup_deaths.R +++ b/R/process_lookup_deaths.R @@ -15,6 +15,8 @@ process_slf_deaths_lookup <- function( refined_death = read_file(get_combined_slf_deaths_lookup_path()), write_to_disk = TRUE ) { + log_slf_event(stage = "process", status = "start", type = "slf_deaths_lookup", year = year) + # create slf deaths lookup slf_deaths_lookup <- refined_death %>% # Filter the chi death dates to the FY as the lookup is by FY @@ -32,5 +34,7 @@ process_slf_deaths_lookup <- function( ) } + log_slf_event(stage = "process", status = "complete", type = "slf_deaths_lookup", year = year) + return(slf_deaths_lookup) } diff --git a/R/process_lookup_gpprac.R b/R/process_lookup_gpprac.R index 45de25864..83d4f91c7 100644 --- a/R/process_lookup_gpprac.R +++ b/R/process_lookup_gpprac.R @@ -18,6 +18,8 @@ process_lookup_gpprac <- function( spd_path = get_spd_path(), write_to_disk = TRUE ) { + log_slf_event(stage = "process", status = "start", type = "gpprac_slf_lookup", year = "all") + gpprac_ref_file <- read_file(path = gpprac_ref_path) %>% dplyr::select( "gpprac" = "praccode", @@ -82,5 +84,7 @@ process_lookup_gpprac <- function( ) } + log_slf_event(stage = "process", status = "complete", type = "gpprac_slf_lookup", year = "all") + return(gpprac_slf_lookup) } diff --git a/R/process_lookup_homelessness.R b/R/process_lookup_homelessness.R index 2ebff914f..febb36f86 100644 --- a/R/process_lookup_homelessness.R +++ b/R/process_lookup_homelessness.R @@ -14,6 +14,8 @@ create_homelessness_lookup <- function( year, homelessness_data = read_file(get_source_extract_path(year, "homelessness")) ) { + log_slf_event(stage = "process", status = "start", type = "homelessness_lookup", year = year) + # Specify years available for running if (year <= "1516") { return(NULL) @@ -23,7 +25,7 @@ create_homelessness_lookup <- function( tidyr::drop_na(.data$anon_chi) %>% dplyr::mutate(hl1_in_fy = 1L) - cli::cli_alert_info("Create homelessness lookup function finished at {Sys.time()}") + log_slf_event(stage = "process", status = "complete", type = "homelessness_lookup", year = year) return(homelessness_lookup) } @@ -43,6 +45,8 @@ create_homelessness_lookup <- function( #' @family episode_file add_homelessness_flag <- function(data, year, lookup = create_homelessness_lookup(year)) { + log_slf_event(stage = "process", status = "start", type = "homelessness_flag", year = year) + if (!check_year_valid(year, type = "homelessness")) { data <- data return(data) @@ -57,7 +61,7 @@ add_homelessness_flag <- function(data, year, ) %>% dplyr::mutate(hl1_in_fy = tidyr::replace_na(.data$hl1_in_fy, 0L)) - cli::cli_alert_info("Add homelessness flag function finished at {Sys.time()}") + log_slf_event(stage = "process", status = "complete", type = "homelessness_flag", year = year) return(data) } @@ -74,6 +78,8 @@ add_homelessness_flag <- function(data, year, #' @return the final data as a [tibble][tibble::tibble-package]. #' @export add_homelessness_date_flags <- function(data, year, lookup = create_homelessness_lookup(year)) { + log_slf_event(stage = "process", status = "start", type = "homelessness_date_flags", year = year) + if (!check_year_valid(year, type = "homelessness")) { data <- data return(data) @@ -133,7 +139,7 @@ add_homelessness_date_flags <- function(data, year, lookup = create_homelessness ) ) - cli::cli_alert_info("Add homelessness date flags function finished at {Sys.time()}") + log_slf_event(stage = "process", status = "complete", type = "homelessness_date_flags", year = year) return(data) } diff --git a/R/process_lookup_ltc.R b/R/process_lookup_ltc.R index a68d9a82f..8a22e1451 100644 --- a/R/process_lookup_ltc.R +++ b/R/process_lookup_ltc.R @@ -9,6 +9,8 @@ #' @export #' process_lookup_ltc <- function(data, year, write_to_disk = TRUE) { + log_slf_event(stage = "process", status = "start", type = "ltc", year = year) + # Create LTC flags 1/0------------------------------------ # Set flags to 1 or 0 based on FY @@ -32,5 +34,7 @@ process_lookup_ltc <- function(data, year, write_to_disk = TRUE) { ) } + log_slf_event(stage = "process", status = "complete", type = "ltc", year = year) + return(ltc_flags) } diff --git a/R/process_lookup_postcode.R b/R/process_lookup_postcode.R index 6ae040cc2..0c29566e4 100644 --- a/R/process_lookup_postcode.R +++ b/R/process_lookup_postcode.R @@ -17,6 +17,7 @@ process_lookup_postcode <- function(spd_path = get_spd_path(), locality_path = get_locality_path(), write_to_disk = TRUE) { # Read lookup files ------------------------------------------------------- + log_slf_event(stage = "process", status = "start", type = "slf_pc_lookup", year = "all") # postcode data spd_file <- read_file(spd_path) %>% @@ -102,5 +103,7 @@ process_lookup_postcode <- function(spd_path = get_spd_path(), ) } + log_slf_event(stage = "process", status = "complete", type = "slf_pc_lookup", year = "all") + return(slf_pc_lookup) } diff --git a/R/process_lookup_sc_client.R b/R/process_lookup_sc_client.R index 5065de240..a0c87c9c2 100644 --- a/R/process_lookup_sc_client.R +++ b/R/process_lookup_sc_client.R @@ -19,6 +19,8 @@ process_lookup_sc_client <- sc_demographics = read_file(get_sc_demog_lookup_path()) %>% dplyr::select(c("sending_location", "social_care_id", "anon_chi", "extract_date", "consistent_quality")), write_to_disk = TRUE) { + log_slf_event(stage = "process", status = "start", type = "sc_client", year = year) + # Check if year is valid for sc_client if (!check_year_valid(year, "client")) { return(NULL) @@ -210,5 +212,7 @@ process_lookup_sc_client <- ) } + log_slf_event(stage = "process", status = "complete", type = "sc_client", year = year) + return(sc_client_lookup) } diff --git a/R/process_lookup_sc_demographics.R b/R/process_lookup_sc_demographics.R index b3dc4558e..78430d9fa 100644 --- a/R/process_lookup_sc_demographics.R +++ b/R/process_lookup_sc_demographics.R @@ -21,6 +21,8 @@ process_lookup_sc_demographics <- function( uk_pc_path = get_uk_postcode_path(), write_to_disk = TRUE ) { + log_slf_event(stage = "process", status = "start", type = "sc_demog", year = "all") + data <- data %>% # add per in social_care_id in Renfrewshire fix_scid_renfrewshire() %>% @@ -374,5 +376,7 @@ process_lookup_sc_demographics <- function( ) # hscdiip owner } + log_slf_event(stage = "process", status = "complete", type = "sc_demog", year = "all") + return(sc_demog_lookup) } diff --git a/R/process_refined_death.R b/R/process_refined_death.R index c38c06507..a2c0fccb0 100644 --- a/R/process_refined_death.R +++ b/R/process_refined_death.R @@ -16,6 +16,8 @@ process_refined_death <- function( it_chi_deaths = read_file(get_slf_chi_deaths_path()), write_to_disk = TRUE ) { + log_slf_event(stage = "process", status = "start", type = "refined_death", year = "all") + years_list <- years_to_run() nrs_all_years <- lapply(years_list, (\(year) { @@ -66,5 +68,7 @@ process_refined_death <- function( ) } + log_slf_event(stage = "process", status = "complete", type = "refined_death", year = "all") + return(refined_death) } diff --git a/R/process_sc_all_alarms_telecare.R b/R/process_sc_all_alarms_telecare.R index 295c02496..dd6936840 100644 --- a/R/process_sc_all_alarms_telecare.R +++ b/R/process_sc_all_alarms_telecare.R @@ -16,6 +16,8 @@ process_sc_all_alarms_telecare <- function( sc_demog_lookup = read_file(get_sc_demog_lookup_path()), write_to_disk = TRUE ) { + log_slf_event(stage = "process", status = "start", type = "at", year = "all") + # Data Cleaning----------------------------------------------------- # fix "no visible binding for global variable" @@ -155,5 +157,7 @@ process_sc_all_alarms_telecare <- function( ) } + log_slf_event(stage = "process", status = "complete", type = "at", year = "all") + return(qtr_merge) } diff --git a/R/process_sc_all_care_home.R b/R/process_sc_all_care_home.R index cab867241..9848e5a87 100644 --- a/R/process_sc_all_care_home.R +++ b/R/process_sc_all_care_home.R @@ -27,6 +27,8 @@ process_sc_all_care_home <- function( spd_path = get_spd_path(), write_to_disk = TRUE ) { + log_slf_event(stage = "process", status = "start", type = "ch", year = "all") + sending_location <- social_care_id <- financial_year <- NULL data <- data %>% @@ -494,5 +496,7 @@ process_sc_all_care_home <- function( ) } + log_slf_event(stage = "process", status = "complete", type = "ch", year = "all") + return(ch_data_final) } diff --git a/R/process_sc_all_home_care.R b/R/process_sc_all_home_care.R index 9fc6baab8..781a1be26 100644 --- a/R/process_sc_all_home_care.R +++ b/R/process_sc_all_home_care.R @@ -16,6 +16,8 @@ process_sc_all_home_care <- function( sc_demog_lookup = read_file(get_sc_demog_lookup_path()), write_to_disk = TRUE ) { + log_slf_event(stage = "process", status = "start", type = "hc", year = "all") + sending_location <- social_care_id <- financial_year <- NULL data <- data %>% @@ -214,5 +216,7 @@ process_sc_all_home_care <- function( ) } + log_slf_event(stage = "process", status = "complete", type = "hc", year = "all") + return(all_hc_processed) } diff --git a/R/process_sc_all_sds.R b/R/process_sc_all_sds.R index 72b3bdfca..452ce9f32 100644 --- a/R/process_sc_all_sds.R +++ b/R/process_sc_all_sds.R @@ -15,6 +15,8 @@ process_sc_all_sds <- function( sc_demog_lookup = read_file(get_sc_demog_lookup_path()), write_to_disk = TRUE ) { + log_slf_event(stage = "process", status = "start", type = "sds", year = "all") + # fix "no visible binding for global variable" sds_option_4 <- sds_start_date <- sds_period_start_date <- sds_end_date <- sds_period_end_date <- received <- sds_option <- sending_location <- @@ -200,5 +202,7 @@ process_sc_all_sds <- function( ) } + log_slf_event(stage = "process", status = "complete", type = "sds", year = "all") + return(final_data) } diff --git a/R/process_tests_acute.R b/R/process_tests_acute.R index 9312018f9..b22977b1f 100644 --- a/R/process_tests_acute.R +++ b/R/process_tests_acute.R @@ -10,6 +10,8 @@ #' #' @export process_tests_acute <- function(data, year) { + log_slf_event(stage = "test", status = "start", type = "acute", year = year) + old_data <- get_existing_data_for_tests(data) data <- apply_cost_uplift(data) @@ -22,5 +24,7 @@ process_tests_acute <- function(data, year) { ) %>% write_tests_xlsx(sheet_name = "01b", year, workbook_name = "extract") + log_slf_event(stage = "test", status = "complete", type = "acute", year = year) + return(comparison) } diff --git a/R/process_tests_ae.R b/R/process_tests_ae.R index ecf199cf7..5841d712e 100644 --- a/R/process_tests_ae.R +++ b/R/process_tests_ae.R @@ -7,6 +7,8 @@ #' #' @export process_tests_ae <- function(data, year) { + log_slf_event(stage = "test", status = "start", type = "ae", year = year) + old_data <- get_existing_data_for_tests(data) data <- apply_cost_uplift(data) @@ -25,5 +27,7 @@ process_tests_ae <- function(data, year) { ) %>% write_tests_xlsx(sheet_name = "ae2", year, workbook_name = "extract") + log_slf_event(stage = "test", status = "complete", type = "ae", year = year) + return(comparison) } diff --git a/R/process_tests_alarms_telecare.R b/R/process_tests_alarms_telecare.R index 9303a103f..7c013af6e 100644 --- a/R/process_tests_alarms_telecare.R +++ b/R/process_tests_alarms_telecare.R @@ -8,6 +8,8 @@ #' #' @export process_tests_alarms_telecare <- function(data, year) { + log_slf_event(stage = "test", status = "start", type = "at", year = year) + if (check_year_valid(year, "at")) { old_data <- get_existing_data_for_tests(data) @@ -19,6 +21,8 @@ process_tests_alarms_telecare <- function(data, year) { ) %>% write_tests_xlsx(sheet_name = "at", year, workbook_name = "extract") + log_slf_event(stage = "test", status = "complete", type = "at", year = year) + return(comparison) } else { return(NULL) diff --git a/R/process_tests_care_home.R b/R/process_tests_care_home.R index cc84bdde2..5c1b696cd 100644 --- a/R/process_tests_care_home.R +++ b/R/process_tests_care_home.R @@ -7,6 +7,8 @@ #' #' @export process_tests_care_home <- function(data, year) { + log_slf_event(stage = "test", status = "start", type = "ch", year = year) + if (check_year_valid(year, "ch")) { old_data <- get_existing_data_for_tests(data) @@ -18,6 +20,8 @@ process_tests_care_home <- function(data, year) { ) %>% write_tests_xlsx(sheet_name = "ch", year, workbook_name = "extract") + log_slf_event(stage = "test", status = "complete", type = "ch", year = year) + return(comparison) } else { return(NULL) diff --git a/R/process_tests_cmh.R b/R/process_tests_cmh.R index ea1ae95ed..b684daa64 100644 --- a/R/process_tests_cmh.R +++ b/R/process_tests_cmh.R @@ -7,6 +7,8 @@ #' #' @export process_tests_cmh <- function(data, year) { + log_slf_event(stage = "test", status = "start", type = "cmh", year = year) + if (identical(data, tibble::tibble())) { # Deal with years where we have no data return(data) @@ -22,6 +24,8 @@ process_tests_cmh <- function(data, year) { ) %>% write_tests_xlsx(sheet_name = "cmh", year, workbook_name = "extract") + log_slf_event(stage = "test", status = "complete", type = "cmh", year = year) + return(comparison) } diff --git a/R/process_tests_cross_year.R b/R/process_tests_cross_year.R index 7a726cf26..10abdad17 100644 --- a/R/process_tests_cross_year.R +++ b/R/process_tests_cross_year.R @@ -10,6 +10,8 @@ #' @export #' process_tests_cross_year <- function(year) { + log_slf_event(stage = "test", status = "start", type = "ep_cross_year", year = year) + ep_file <- read_dev_slf_file(year, type = "episode", col_select = c("year", "recid", "anon_chi", "record_keydate1", "record_keydate2") @@ -54,5 +56,7 @@ process_tests_cross_year <- function(year) { dplyr::select(-tidyselect::ends_with("NA")) %>% write_tests_xlsx(sheet_name = "cross_year", workbook_name = "cross_year") + log_slf_event(stage = "test", status = "complete", type = "ep_cross_year", year = year) + return(pivot_tests) } diff --git a/R/process_tests_delayed_discharges.R b/R/process_tests_delayed_discharges.R index 0355ad0e2..5f87a2fc9 100644 --- a/R/process_tests_delayed_discharges.R +++ b/R/process_tests_delayed_discharges.R @@ -10,6 +10,8 @@ #' #' @export process_tests_delayed_discharges <- function(data, year) { + log_slf_event(stage = "test", status = "start", type = "dd", year = year) + old_data <- get_existing_data_for_tests(data) data <- rename_hscp(data) @@ -48,5 +50,7 @@ produce_source_dd_tests <- function(data) { # use function to sum new test flags calculate_measures(measure = "sum") + log_slf_event(stage = "test", status = "complete", type = "dd", year = year) + return(test_flags) } diff --git a/R/process_tests_district_nursing.R b/R/process_tests_district_nursing.R index ff9f50ea4..d11255a15 100644 --- a/R/process_tests_district_nursing.R +++ b/R/process_tests_district_nursing.R @@ -8,6 +8,8 @@ #' #' @export process_tests_district_nursing <- function(data, year) { + log_slf_event(stage = "test", status = "start", type = "dn", year = year) + if (identical(data, tibble::tibble())) { # Deal with years where we have no data return(data) @@ -79,5 +81,7 @@ produce_source_dn_tests <- function(data, ) %>% purrr::reduce(dplyr::full_join, by = c("measure", "value")) + log_slf_event(stage = "test", status = "complete", type = "dn", year = year) + return(join_output) } diff --git a/R/process_tests_episode_file.R b/R/process_tests_episode_file.R index af5befb99..23ab80dd3 100644 --- a/R/process_tests_episode_file.R +++ b/R/process_tests_episode_file.R @@ -7,6 +7,8 @@ #' #' @export process_tests_episode_file <- function(data, year) { + log_slf_event(stage = "test", status = "start", type = "ep", year = year) + data <- data %>% dplyr::select( "year", @@ -44,6 +46,8 @@ process_tests_episode_file <- function(data, year) { workbook_name = "ep_file" ) + log_slf_event(stage = "test", status = "complete", type = "ep", year = year) + return(comparison) } diff --git a/R/process_tests_gp_ooh.R b/R/process_tests_gp_ooh.R index e9778711d..9dacd4b86 100644 --- a/R/process_tests_gp_ooh.R +++ b/R/process_tests_gp_ooh.R @@ -7,6 +7,8 @@ #' #' @export process_tests_gp_ooh <- function(data, year) { + log_slf_event(stage = "test", status = "start", type = "gpooh", year = year) + old_data <- get_existing_data_for_tests(data) data <- rename_hscp(data) @@ -21,5 +23,7 @@ process_tests_gp_ooh <- function(data, year) { ) %>% write_tests_xlsx(sheet_name = "gpooh", year, workbook_name = "extract") + log_slf_event(stage = "test", status = "complete", type = "gpooh", year = year) + return(comparison) } diff --git a/R/process_tests_home_care.R b/R/process_tests_home_care.R index eca647648..6da43d0b9 100644 --- a/R/process_tests_home_care.R +++ b/R/process_tests_home_care.R @@ -7,6 +7,8 @@ #' #' @export process_tests_home_care <- function(data, year) { + log_slf_event(stage = "test", status = "start", type = "hc", year = year) + if (check_year_valid(year, "hc")) { old_data <- get_existing_data_for_tests(data) @@ -18,6 +20,8 @@ process_tests_home_care <- function(data, year) { ) %>% write_tests_xlsx(sheet_name = "hc", year, workbook_name = "extract") + log_slf_event(stage = "test", status = "complete", type = "hc", year = year) + return(comparison) } else { return(NULL) diff --git a/R/process_tests_homelessness.R b/R/process_tests_homelessness.R index c0044ffa2..bdf03533c 100644 --- a/R/process_tests_homelessness.R +++ b/R/process_tests_homelessness.R @@ -8,6 +8,8 @@ #' #' @export process_tests_homelessness <- function(data, year) { + log_slf_event(stage = "test", status = "start", type = "homelessness", year = year) + old_data <- get_existing_data_for_tests(data) data <- rename_hscp(data) @@ -18,6 +20,8 @@ process_tests_homelessness <- function(data, year) { ) %>% write_tests_xlsx(sheet_name = "hl1", year, workbook_name = "extract") + log_slf_event(stage = "test", status = "complete", type = "homelessness", year = year) + return(comparison) } diff --git a/R/process_tests_individual_file.R b/R/process_tests_individual_file.R index 0993e146f..65b99ce69 100644 --- a/R/process_tests_individual_file.R +++ b/R/process_tests_individual_file.R @@ -7,6 +7,8 @@ #' #' @export process_tests_individual_file <- function(data, year) { + log_slf_event(stage = "test", status = "start", type = "ind", year = year) + data <- data %>% dplyr::select( "year", @@ -42,6 +44,8 @@ process_tests_individual_file <- function(data, year) { workbook_name = "indiv_file" ) + log_slf_event(stage = "test", status = "complete", type = "ind", year = year) + return(comparison) } diff --git a/R/process_tests_it_chi_deaths.R b/R/process_tests_it_chi_deaths.R index 597e71fcb..5882d6bcd 100644 --- a/R/process_tests_it_chi_deaths.R +++ b/R/process_tests_it_chi_deaths.R @@ -4,6 +4,8 @@ #' #' @export process_tests_it_chi_deaths <- function(data, update = previous_update()) { + log_slf_event(stage = "test", status = "start", type = "it_chi_deaths", year = "all") + comparison <- produce_test_comparison( old_data = produce_it_chi_deaths_tests( read_file(get_slf_chi_deaths_path(update = update)) @@ -12,6 +14,8 @@ process_tests_it_chi_deaths <- function(data, update = previous_update()) { ) %>% write_tests_xlsx(sheet_name = "it_chi_deaths", workbook_name = "lookup") + log_slf_event(stage = "test", status = "complete", type = "it_chi_deaths", year = "all") + return(comparison) } diff --git a/R/process_tests_lookup_gpprac.R b/R/process_tests_lookup_gpprac.R index 6bfc35356..a19dd500b 100644 --- a/R/process_tests_lookup_gpprac.R +++ b/R/process_tests_lookup_gpprac.R @@ -7,6 +7,8 @@ #' #' @export process_tests_lookup_gpprac <- function(data, update = previous_update()) { + log_slf_event(stage = "test", status = "start", type = "gpprac_lookup", year = "all") + comparison <- produce_test_comparison( old_data = produce_slf_gpprac_tests( read_file(get_slf_gpprac_path(update = update)) @@ -15,6 +17,8 @@ process_tests_lookup_gpprac <- function(data, update = previous_update()) { ) %>% write_tests_xlsx(sheet_name = "source_gpprac_lookup", workbook_name = "lookup") + log_slf_event(stage = "test", status = "complete", type = "gpprac_lookup", year = "all") + return(comparison) } diff --git a/R/process_tests_lookup_pc.R b/R/process_tests_lookup_pc.R index e0d0aeab8..904b4071f 100644 --- a/R/process_tests_lookup_pc.R +++ b/R/process_tests_lookup_pc.R @@ -11,6 +11,8 @@ #' #' @export process_tests_lookup_pc <- function(data, update = previous_update()) { + log_slf_event(stage = "test", status = "start", type = "pc_lookup", year = "all") + comparison <- produce_test_comparison( old_data = produce_slf_postcode_tests( read_file(get_slf_postcode_path(update = update)) @@ -19,6 +21,8 @@ process_tests_lookup_pc <- function(data, update = previous_update()) { ) %>% write_tests_xlsx(sheet_name = "source_pc_lookup", workbook_name = "lookup") + log_slf_event(stage = "test", status = "complete", type = "pc_lookup", year = "all") + return(comparison) } diff --git a/R/process_tests_ltcs.R b/R/process_tests_ltcs.R index d2ba9d6ad..51d03455b 100644 --- a/R/process_tests_ltcs.R +++ b/R/process_tests_ltcs.R @@ -7,6 +7,8 @@ #' #' @export process_tests_ltcs <- function(data, year) { + log_slf_event(stage = "test", status = "start", type = "ltc", year = year) + old_data <- read_file(get_ltcs_path(year, update = previous_update())) comparison <- produce_test_comparison( @@ -16,6 +18,8 @@ process_tests_ltcs <- function(data, year) { dplyr::mutate(recid = "LTCs") %>% write_tests_xlsx(sheet_name = "ltc", year = year, workbook_name = "extract") + log_slf_event(stage = "test", status = "complete", type = "ltc", year = year) + return(comparison) } diff --git a/R/process_tests_maternity.R b/R/process_tests_maternity.R index fa1041750..cc93ebbec 100644 --- a/R/process_tests_maternity.R +++ b/R/process_tests_maternity.R @@ -7,6 +7,8 @@ #' #' @export process_tests_maternity <- function(data, year) { + log_slf_event(stage = "test", status = "start", type = "maternity", year = year) + old_data <- get_existing_data_for_tests(data) data <- apply_cost_uplift(data) @@ -19,5 +21,7 @@ process_tests_maternity <- function(data, year) { ) %>% write_tests_xlsx(sheet_name = "02b", year, workbook_name = "extract") + log_slf_event(stage = "test", status = "complete", type = "maternity", year = year) + return(comparison) } diff --git a/R/process_tests_mental_health.R b/R/process_tests_mental_health.R index e797ede96..92148b5ec 100644 --- a/R/process_tests_mental_health.R +++ b/R/process_tests_mental_health.R @@ -7,6 +7,8 @@ #' #' @export process_tests_mental_health <- function(data, year) { + log_slf_event(stage = "test", status = "start", type = "mh", year = year) + old_data <- get_existing_data_for_tests(data) data <- apply_cost_uplift(data) @@ -19,5 +21,7 @@ process_tests_mental_health <- function(data, year) { ) %>% write_tests_xlsx(sheet_name = "04b", year, workbook_name = "extract") + log_slf_event(stage = "test", status = "complete", type = "mh", year = year) + return(comparison) } diff --git a/R/process_tests_nrs_deaths.R b/R/process_tests_nrs_deaths.R index b79b18c65..c255e2b7b 100644 --- a/R/process_tests_nrs_deaths.R +++ b/R/process_tests_nrs_deaths.R @@ -7,6 +7,8 @@ #' #' @export process_tests_nrs_deaths <- function(data, year) { + log_slf_event(stage = "test", status = "start", type = "deaths", year = year) + old_data <- get_existing_data_for_tests(data) data <- rename_hscp(data) @@ -17,6 +19,8 @@ process_tests_nrs_deaths <- function(data, year) { ) %>% write_tests_xlsx(sheet_name = "nrs", year, workbook_name = "extract") + log_slf_event(stage = "test", status = "complete", type = "deaths", year = year) + return(comparison) } diff --git a/R/process_tests_outpatients.R b/R/process_tests_outpatients.R index 7bad9e517..21f8b6b3d 100644 --- a/R/process_tests_outpatients.R +++ b/R/process_tests_outpatients.R @@ -7,6 +7,8 @@ #' #' @export process_tests_outpatients <- function(data, year) { + log_slf_event(stage = "test", status = "start", type = "outpatients", year = year) + old_data <- get_existing_data_for_tests(data) data <- apply_cost_uplift(data) @@ -29,5 +31,7 @@ process_tests_outpatients <- function(data, year) { ) %>% write_tests_xlsx(sheet_name = "00b", year, workbook_name = "extract") + log_slf_event(stage = "test", status = "complete", type = "outpatients", year = year) + return(comparison) } diff --git a/R/process_tests_prescribing.R b/R/process_tests_prescribing.R index b82682b82..3a6ec2d31 100644 --- a/R/process_tests_prescribing.R +++ b/R/process_tests_prescribing.R @@ -7,6 +7,8 @@ #' #' @export process_tests_prescribing <- function(data, year) { + log_slf_event(stage = "test", status = "start", type = "pis", year = year) + old_data <- get_existing_data_for_tests(data) data <- rename_hscp(data) @@ -17,6 +19,8 @@ process_tests_prescribing <- function(data, year) { ) %>% write_tests_xlsx(sheet_name = "pis", year, workbook_name = "extract") + log_slf_event(stage = "test", status = "complete", type = "pis", year = year) + return(comparison) } diff --git a/R/process_tests_sc_all_at_episodes.R b/R/process_tests_sc_all_at_episodes.R index 8b5580334..be56adfc6 100644 --- a/R/process_tests_sc_all_at_episodes.R +++ b/R/process_tests_sc_all_at_episodes.R @@ -10,6 +10,8 @@ #' #' @export process_tests_sc_all_at_episodes <- function(data) { + log_slf_event(stage = "test", status = "start", type = "sc_at_ep", year = "all") + comparison <- produce_test_comparison( old_data = produce_sc_all_episodes_tests( read_file(get_sc_at_episodes_path(update = previous_update())) @@ -22,5 +24,7 @@ process_tests_sc_all_at_episodes <- function(data) { comparison %>% write_tests_xlsx(sheet_name = "all_at_episodes", workbook_name = "lookup") + log_slf_event(stage = "test", status = "complete", type = "sc_at_ep", year = "all") + return(comparison) } diff --git a/R/process_tests_sc_all_ch_episodes.R b/R/process_tests_sc_all_ch_episodes.R index 7e9655c06..090286e5a 100644 --- a/R/process_tests_sc_all_ch_episodes.R +++ b/R/process_tests_sc_all_ch_episodes.R @@ -10,6 +10,8 @@ #' #' @export process_tests_sc_all_ch_episodes <- function(data) { + log_slf_event(stage = "test", status = "start", type = "sc_ch_ep", year = "all") + comparison <- produce_test_comparison( old_data = produce_sc_all_episodes_tests( read_file(get_sc_ch_episodes_path(update = previous_update())) @@ -22,5 +24,7 @@ process_tests_sc_all_ch_episodes <- function(data) { comparison %>% write_tests_xlsx(sheet_name = "all_ch_episodes", workbook_name = "lookup") + log_slf_event(stage = "test", status = "complete", type = "sc_ch_ep", year = "all") + return(comparison) } diff --git a/R/process_tests_sc_all_hc_episodes.R b/R/process_tests_sc_all_hc_episodes.R index 7194790c0..d9ec96529 100644 --- a/R/process_tests_sc_all_hc_episodes.R +++ b/R/process_tests_sc_all_hc_episodes.R @@ -10,6 +10,8 @@ #' #' @export process_tests_sc_all_hc_episodes <- function(data) { + log_slf_event(stage = "test", status = "start", type = "sc_hc_ep", year = "all") + comparison <- produce_test_comparison( old_data = produce_sc_all_episodes_tests( read_file(get_sc_hc_episodes_path(update = previous_update())) @@ -22,5 +24,7 @@ process_tests_sc_all_hc_episodes <- function(data) { comparison %>% write_tests_xlsx(sheet_name = "all_hc_episodes", workbook_name = "lookup") + log_slf_event(stage = "test", status = "complete", type = "sc_hc_ep", year = "all") + return(comparison) } diff --git a/R/process_tests_sc_all_sds_episodes.R b/R/process_tests_sc_all_sds_episodes.R index cf87a671c..6bd7e7190 100644 --- a/R/process_tests_sc_all_sds_episodes.R +++ b/R/process_tests_sc_all_sds_episodes.R @@ -10,6 +10,8 @@ #' #' @export process_tests_sc_all_sds_episodes <- function(data) { + log_slf_event(stage = "test", status = "start", type = "sc_sds_ep", year = "all") + comparison <- produce_test_comparison( old_data = produce_sc_all_episodes_tests( read_file(get_sc_sds_episodes_path(update = previous_update())) @@ -22,5 +24,7 @@ process_tests_sc_all_sds_episodes <- function(data) { comparison %>% write_tests_xlsx(sheet_name = "all_sds_episodes", workbook_name = "lookup") + log_slf_event(stage = "test", status = "complete", type = "sc_sds_ep", year = "all") + return(comparison) } diff --git a/R/process_tests_sc_client_lookup.R b/R/process_tests_sc_client_lookup.R index 475c4416e..661149ab4 100644 --- a/R/process_tests_sc_client_lookup.R +++ b/R/process_tests_sc_client_lookup.R @@ -8,6 +8,8 @@ #' #' @export process_tests_sc_client_lookup <- function(data, year) { + log_slf_event(stage = "test", status = "start", type = "client", year = year) + if (check_year_valid(year, "client")) { comparison <- produce_test_comparison( old_data = produce_tests_sc_client_lookup( @@ -19,6 +21,8 @@ process_tests_sc_client_lookup <- function(data, year) { comparison %>% write_tests_xlsx(sheet_name = "sc_client", year, workbook_name = "extract") + log_slf_event(stage = "test", status = "complete", type = "client", year = year) + return(comparison) } else { return(NULL) diff --git a/R/process_tests_sc_demographics.R b/R/process_tests_sc_demographics.R index c9c13d7ab..671de1243 100644 --- a/R/process_tests_sc_demographics.R +++ b/R/process_tests_sc_demographics.R @@ -10,6 +10,8 @@ #' #' @export process_tests_sc_demographics <- function(data) { + log_slf_event(stage = "test", status = "start", type = "sc_demog", year = "all") + comparison <- produce_test_comparison( old_data = produce_sc_demog_lookup_tests( read_file(get_sc_demog_lookup_path(update = previous_update())) @@ -22,6 +24,8 @@ process_tests_sc_demographics <- function(data) { comparison %>% write_tests_xlsx(sheet_name = "sc_demographics", workbook_name = "lookup") + log_slf_event(stage = "test", status = "complete", type = "sc_demog", year = "all") + return(comparison) } diff --git a/R/process_tests_sc_sandpit.R b/R/process_tests_sc_sandpit.R index 9d1b35f8a..7fd3d6467 100644 --- a/R/process_tests_sc_sandpit.R +++ b/R/process_tests_sc_sandpit.R @@ -7,6 +7,8 @@ #' @export #' process_tests_sc_sandpit <- function(type = c("at", "hc", "ch", "sds", "demographics", "client"), year = NULL) { + log_slf_event(stage = "test", status = "start", type = "sc_sandpit", year = "all") + data <- data %>% slfhelper::get_chi() @@ -24,6 +26,8 @@ process_tests_sc_sandpit <- function(type = c("at", "hc", "ch", "sds", "demograp comparison %>% write_tests_xlsx(sheet_name = {{ type }}, year = year, workbook_name = "sandpit") + log_slf_event(stage = "test", status = "complete", type = "sc_sandpit", year = "all") + return(comparison) } diff --git a/R/process_tests_sds.R b/R/process_tests_sds.R index 6f4db9e9e..f51d7a9b9 100644 --- a/R/process_tests_sds.R +++ b/R/process_tests_sds.R @@ -7,6 +7,8 @@ #' #' @export process_tests_sds <- function(data, year) { + log_slf_event(stage = "test", status = "start", type = "sds", year = year) + if (check_year_valid(year, "sds")) { old_data <- get_existing_data_for_tests(data) @@ -18,6 +20,8 @@ process_tests_sds <- function(data, year) { ) %>% write_tests_xlsx(sheet_name = "sds", year, workbook_name = "extract") + log_slf_event(stage = "test", status = "complete", type = "sds", year = year) + return(comparison) } else { return(NULL) diff --git a/R/read_extract_acute.R b/R/read_extract_acute.R index 063f625a4..6987bb4a8 100644 --- a/R/read_extract_acute.R +++ b/R/read_extract_acute.R @@ -8,6 +8,8 @@ #' @export read_extract_acute <- function(year, file_path = get_boxi_extract_path(year = year, type = "acute")) { # Read BOXI extract + log_slf_event(stage = "read", status = "start", type = "acute", year = year) + extract_acute <- read_file(file_path, col_type = readr::cols( "Costs Financial Year (01)" = readr::col_integer(), @@ -155,5 +157,7 @@ read_extract_acute <- function(year, file_path = get_boxi_extract_path(year = ye cost_total_net = tidyr::replace_na(.data[["cost_total_net"]], 0.0) ) + log_slf_event(stage = "read", status = "complete", type = "acute", year = year) + return(extract_acute) } diff --git a/R/read_extract_ae.R b/R/read_extract_ae.R index c0fd9ed6e..708d743ab 100644 --- a/R/read_extract_ae.R +++ b/R/read_extract_ae.R @@ -8,6 +8,8 @@ read_extract_ae <- function( year, file_path = get_boxi_extract_path(year = year, type = "ae") ) { + log_slf_event(stage = "read", status = "start", type = "ae", year = year) + extract_ae <- read_file(file_path, col_type = readr::cols( "Arrival Date" = readr::col_date(format = "%Y/%m/%d %T"), @@ -88,5 +90,7 @@ read_extract_ae <- function( commhosp = "Community Hospital Flag" ) + log_slf_event(stage = "read", status = "complete", type = "ae", year = year) + return(extract_ae) } diff --git a/R/read_extract_cmh.R b/R/read_extract_cmh.R index 08e23d554..faebd181f 100644 --- a/R/read_extract_cmh.R +++ b/R/read_extract_cmh.R @@ -7,6 +7,8 @@ read_extract_cmh <- function( year, file_path = get_boxi_extract_path(year = year, type = "cmh") ) { + log_slf_event(stage = "read", status = "start", type = "cmh", year = year) + # Specify years available for running if (file_path == get_dummy_boxi_extract_path()) { return(tibble::tibble()) @@ -55,5 +57,7 @@ read_extract_cmh <- function( diag5 = "Other Aim of Contact (4)" ) + log_slf_event(stage = "read", status = "complete", type = "cmh", year = year) + return(extract_cmh) } diff --git a/R/read_extract_delayed_discharges.R b/R/read_extract_delayed_discharges.R index d1b9c62a1..dee2a6353 100644 --- a/R/read_extract_delayed_discharges.R +++ b/R/read_extract_delayed_discharges.R @@ -6,6 +6,8 @@ #' @export #' read_extract_delayed_discharges <- function(file_path = get_dd_path()) { + log_slf_event(stage = "read", status = "start", type = "dd", year = year) + extract_delayed_discharges <- read_file(file_path) %>% janitor::clean_names() %>% dplyr::mutate( @@ -14,5 +16,7 @@ read_extract_delayed_discharges <- function(file_path = get_dd_path()) { ) %>% dplyr::select(-.data[["cennum"]]) + log_slf_event(stage = "read", status = "complete", type = "dd", year = year) + return(extract_delayed_discharges) } diff --git a/R/read_extract_district_nursing.R b/R/read_extract_district_nursing.R index d6d94876d..7524aa101 100644 --- a/R/read_extract_district_nursing.R +++ b/R/read_extract_district_nursing.R @@ -7,6 +7,8 @@ read_extract_district_nursing <- function( year, file_path = get_boxi_extract_path(year = year, type = "dn") ) { + log_slf_event(stage = "read", status = "start", type = "dn", year = year) + if (file_path == get_dummy_boxi_extract_path()) { return(tibble::tibble()) } @@ -55,5 +57,7 @@ read_extract_district_nursing <- function( location_contact = "Location of Contact" ) + log_slf_event(stage = "read", status = "complete", type = "dn", year = year) + return(extract_district_nursing) } diff --git a/R/read_extract_gp_ooh.R b/R/read_extract_gp_ooh.R index ca7d32b51..11a95d37b 100644 --- a/R/read_extract_gp_ooh.R +++ b/R/read_extract_gp_ooh.R @@ -16,11 +16,15 @@ read_extract_gp_ooh <- function(year, diagnosis_path = get_boxi_extract_path(year = year, type = "gp_ooh-d"), outcomes_path = get_boxi_extract_path(year = year, type = "gp_ooh-o"), consultations_path = get_boxi_extract_path(year = year, type = "gp_ooh-c")) { + log_slf_event(stage = "read", status = "start", type = "gpooh", year = year) + ooh_extracts <- list( "diagnosis" = read_extract_ooh_diagnosis(year, diagnosis_path), "outcomes" = read_extract_ooh_outcomes(year, outcomes_path), "consultations" = read_extract_ooh_consultations(year, consultations_path) ) + log_slf_event(stage = "read", status = "complete", type = "gpooh", year = year) + return(ooh_extracts) } diff --git a/R/read_extract_homelessness.R b/R/read_extract_homelessness.R index aa03c7db7..2aaf2825d 100644 --- a/R/read_extract_homelessness.R +++ b/R/read_extract_homelessness.R @@ -9,6 +9,8 @@ read_extract_homelessness <- function( file_path = get_boxi_extract_path(year, type = "homelessness", BYOC_MODE), BYOC_MODE ) { + log_slf_event(stage = "read", status = "start", type = "homelessness", year = year) + year <- check_year_format(year, format = "fyyear") c_year <- convert_fyyear_to_year(year) @@ -59,5 +61,7 @@ read_extract_homelessness <- function( dplyr::collect() %>% slfhelper::get_anon_chi("chi") + log_slf_event(stage = "read", status = "complete", type = "homelessness", year = year) + return(extract_homelessness) } diff --git a/R/read_extract_maternity.R b/R/read_extract_maternity.R index 8a905ed9a..51e1228da 100644 --- a/R/read_extract_maternity.R +++ b/R/read_extract_maternity.R @@ -9,6 +9,8 @@ read_extract_maternity <- function( file_path = get_boxi_extract_path(year, type = "maternity", BYOC_MODE = BYOC_MODE), BYOC_MODE ) { + log_slf_event(stage = "read", status = "start", type = "maternity", year = year) + year <- check_year_format(year, format = "fyyear") c_year <- convert_fyyear_to_year(year) @@ -19,7 +21,7 @@ read_extract_maternity <- function( on.exit(try(DBI::dbDisconnect(denodo_connect), silent = TRUE), add = TRUE) - # Read BOXI extract + logger::log_info("Read maternity data from Denodo") extract_maternity <- dplyr::tbl( denodo_connect, dbplyr::in_schema("sdl", "sdl_maternity_episode_source") @@ -83,5 +85,7 @@ read_extract_maternity <- function( dplyr::collect() %>% slfhelper::get_anon_chi("chi") + log_slf_event(stage = "read", status = "complete", type = "maternity", year = year) + return(extract_maternity) } diff --git a/R/read_extract_mental_health.R b/R/read_extract_mental_health.R index 8556a50f0..1aa0ca5dd 100644 --- a/R/read_extract_mental_health.R +++ b/R/read_extract_mental_health.R @@ -7,6 +7,8 @@ read_extract_mental_health <- function( year, file_path = get_boxi_extract_path(year = year, type = "mh") ) { + log_slf_event(stage = "read", status = "start", type = "mh", year = year) + # Read BOXI extract extract_mental_health <- read_file(file_path, col_types = readr::cols_only( @@ -132,5 +134,7 @@ read_extract_mental_health <- function( cost_total_net = tidyr::replace_na(.data[["cost_total_net"]], 0.0) ) + log_slf_event(stage = "read", status = "complete", type = "mh", year = year) + return(extract_mental_health) } diff --git a/R/read_extract_nrs_deaths.R b/R/read_extract_nrs_deaths.R index 1524411a9..8eaf4f264 100644 --- a/R/read_extract_nrs_deaths.R +++ b/R/read_extract_nrs_deaths.R @@ -7,6 +7,8 @@ read_extract_nrs_deaths <- function( year, file_path = get_boxi_extract_path(year = year, type = "deaths") ) { + log_slf_event(stage = "read", status = "start", type = "deaths", year = year) + extract_nrs_deaths <- read_file(file_path, col_types = readr::cols_only( "Death Location Code" = readr::col_character(), @@ -64,5 +66,7 @@ read_extract_nrs_deaths <- function( gpprac = "GP practice code(99)" ) + log_slf_event(stage = "read", status = "complete", type = "deaths", year = year) + return(extract_nrs_deaths) } diff --git a/R/read_extract_ooh_consultations.R b/R/read_extract_ooh_consultations.R index 2b507d410..7b3b929bf 100644 --- a/R/read_extract_ooh_consultations.R +++ b/R/read_extract_ooh_consultations.R @@ -7,6 +7,8 @@ read_extract_ooh_consultations <- function( year, file_path = get_boxi_extract_path(year = year, type = "gp_ooh-c") ) { + log_slf_event(stage = "read", status = "start", type = "gp_ooh-c", year = year) + # Read consultations data consultations_extract <- read_file(file_path, col_types = readr::cols( @@ -56,5 +58,7 @@ read_extract_ooh_consultations <- function( # change back to anon_chi slfhelper::get_anon_chi() + log_slf_event(stage = "read", status = "complete", type = "gp_ooh-c", year = year) + return(consultations_extract) } diff --git a/R/read_extract_ooh_diagnosis.R b/R/read_extract_ooh_diagnosis.R index 9843fab97..4556546e1 100644 --- a/R/read_extract_ooh_diagnosis.R +++ b/R/read_extract_ooh_diagnosis.R @@ -8,6 +8,8 @@ read_extract_ooh_diagnosis <- function( year, file_path = get_boxi_extract_path(year = year, type = "gp_ooh-d") ) { + log_slf_event(stage = "read", status = "start", type = "gp_ooh-d", year = year) + # Load extract file diagnosis_extract <- read_file(file_path, # All columns are character type @@ -22,5 +24,7 @@ read_extract_ooh_diagnosis <- function( tidyr::drop_na(.data$readcode) %>% dplyr::distinct() + log_slf_event(stage = "read", status = "complete", type = "gp_ooh-d", year = year) + return(diagnosis_extract) } diff --git a/R/read_extract_ooh_outcomes.R b/R/read_extract_ooh_outcomes.R index d24a41eb8..33fa15077 100644 --- a/R/read_extract_ooh_outcomes.R +++ b/R/read_extract_ooh_outcomes.R @@ -7,6 +7,8 @@ read_extract_ooh_outcomes <- function( year, file_path = get_boxi_extract_path(year = year, type = "gp_ooh-o") ) { + log_slf_event(stage = "read", status = "start", type = "gp_ooh-o", year = year) + ## Load extract file outcomes_extract <- read_file(file_path, # All columns are character type @@ -21,5 +23,7 @@ read_extract_ooh_outcomes <- function( dplyr::filter(.data$outcome != "") %>% dplyr::distinct() + log_slf_event(stage = "read", status = "complete", type = "gp_ooh-o", year = year) + return(outcomes_extract) } diff --git a/R/read_extract_outpatients.R b/R/read_extract_outpatients.R index ae6c1713d..4a22df647 100644 --- a/R/read_extract_outpatients.R +++ b/R/read_extract_outpatients.R @@ -7,6 +7,8 @@ read_extract_outpatients <- function( year, file_path = get_boxi_extract_path(year = year, type = "outpatient") ) { + log_slf_event(stage = "read", status = "start", type = "outpatient", year = year) + # Read BOXI extract extract_outpatients <- read_file(file_path, col_type = readr::cols( @@ -86,5 +88,7 @@ read_extract_outpatients <- function( hbtreatcode = "Treatment NHS Board Code - current" ) + log_slf_event(stage = "read", status = "complete", type = "outpatient", year = year) + return(extract_outpatients) } diff --git a/R/read_extract_prescribing.R b/R/read_extract_prescribing.R index c99fde654..acb39bb4b 100644 --- a/R/read_extract_prescribing.R +++ b/R/read_extract_prescribing.R @@ -4,6 +4,8 @@ #' #' @export read_extract_prescribing <- function(year, file_path = get_it_prescribing_path(year)) { + log_slf_event(stage = "read", status = "start", type = "pis", year = year) + pis_file <- read_file(file_path) %>% # Rename variables dplyr::select( @@ -20,5 +22,7 @@ read_extract_prescribing <- function(year, file_path = get_it_prescribing_path(y dob = lubridate::dmy(.data$dob) ) + log_slf_event(stage = "read", status = "complete", type = "pis", year = year) + return(pis_file) } diff --git a/R/read_it_chi_deaths.R b/R/read_it_chi_deaths.R index c9b3601eb..6d96ead28 100644 --- a/R/read_it_chi_deaths.R +++ b/R/read_it_chi_deaths.R @@ -7,6 +7,8 @@ #' @export #' @family process extracts read_it_chi_deaths <- function(file_path = get_it_deaths_path()) { + log_slf_event(stage = "read", status = "start", type = "it_chi_deaths", year = "all") + it_chi_deaths <- read_file(file_path) %>% dplyr::select( anon_chi = "anon_chi", @@ -18,5 +20,7 @@ read_it_chi_deaths <- function(file_path = get_it_deaths_path()) { death_date_chi = lubridate::dmy(.data$death_date_chi) ) + log_slf_event(stage = "read", status = "complete", type = "it_chi_deaths", year = "all") + return(it_chi_deaths) } diff --git a/R/read_lookup_ltc.R b/R/read_lookup_ltc.R index 36629ba40..f0396e8c8 100644 --- a/R/read_lookup_ltc.R +++ b/R/read_lookup_ltc.R @@ -6,6 +6,8 @@ #' @export #' read_lookup_ltc <- function(file_path = get_it_ltc_path()) { + log_slf_event(stage = "read", status = "start", type = "ltc", year = "all") + # Read data------------------------------------------------ ltc_file <- read_file(file_path) %>% # Rename variables @@ -47,6 +49,7 @@ read_lookup_ltc <- function(file_path = get_it_ltc_path()) { stop("There were bad CHI numbers in the LTC file") } + log_slf_event(stage = "read", status = "complete", type = "ltc", year = "all") return(ltc_file) } diff --git a/R/read_lookup_sc_client.R b/R/read_lookup_sc_client.R index 8a973b07e..a381c01a8 100644 --- a/R/read_lookup_sc_client.R +++ b/R/read_lookup_sc_client.R @@ -11,6 +11,8 @@ #' @family process extracts read_lookup_sc_client <- function(fyyear, sc_dvprod_connection = phs_db_connection(dsn = "DVPROD")) { + log_slf_event(stage = "read", status = "start", type = "client", year = fyyear) + check_year_format(fyyear) year <- convert_fyyear_to_year(fyyear) @@ -111,5 +113,7 @@ read_lookup_sc_client <- function(fyyear, client_data <- client_data } + log_slf_event(stage = "read", status = "complete", type = "client", year = fyyear) + return(client_data) } diff --git a/R/read_lookup_sc_demographics.R b/R/read_lookup_sc_demographics.R index 5b4e20876..987361b2b 100644 --- a/R/read_lookup_sc_demographics.R +++ b/R/read_lookup_sc_demographics.R @@ -6,6 +6,8 @@ #' @export #' read_lookup_sc_demographics <- function(sc_dvprod_connection = phs_db_connection(dsn = "DVPROD")) { + log_slf_event(stage = "read", status = "start", type = "sc_demog", year = "all") + sc_demog <- dplyr::tbl( sc_dvprod_connection, dbplyr::in_schema("social_care_2", "demographic_snapshot") @@ -55,5 +57,7 @@ read_lookup_sc_demographics <- function(sc_dvprod_connection = phs_db_connection ) %>% dplyr::distinct() + log_slf_event(stage = "read", status = "complete", type = "sc_demog", year = "all") + return(sc_demog) } diff --git a/R/read_sc_all_alarms_telecare.R b/R/read_sc_all_alarms_telecare.R index 48ca93a48..08b1156f0 100644 --- a/R/read_sc_all_alarms_telecare.R +++ b/R/read_sc_all_alarms_telecare.R @@ -8,6 +8,7 @@ #' read_sc_all_alarms_telecare <- function(sc_dvprod_connection = phs_db_connection(dsn = "DVPROD")) { # Read in data--------------------------------------- + log_slf_event(stage = "read", status = "start", type = "at", year = "all") ## read in data - social care 2 demographic at_full_data <- dplyr::tbl( @@ -68,5 +69,7 @@ read_sc_all_alarms_telecare <- function(sc_dvprod_connection = phs_db_connection dplyr::arrange(.data$sending_location, .data$social_care_id) %>% dplyr::filter(.data$service_start_date_after_period_end_date != 1) + log_slf_event(stage = "read", status = "complete", type = "at", year = "all") + return(at_full_data) } diff --git a/R/read_sc_all_care_home.R b/R/read_sc_all_care_home.R index 55600b4f8..b3a2cba92 100644 --- a/R/read_sc_all_care_home.R +++ b/R/read_sc_all_care_home.R @@ -7,6 +7,8 @@ #' @export #' read_sc_all_care_home <- function(sc_dvprod_connection = phs_db_connection(dsn = "DVPROD")) { + log_slf_event(stage = "read", status = "start", type = "ch", year = "all") + ch_data <- dplyr::tbl( sc_dvprod_connection, dbplyr::in_schema("social_care_2", "carehome_snapshot") @@ -73,5 +75,7 @@ read_sc_all_care_home <- function(sc_dvprod_connection = phs_db_connection(dsn = ), as.integer) ) + log_slf_event(stage = "read", status = "complete", type = "ch", year = "all") + return(ch_data) } diff --git a/R/read_sc_all_home_care.R b/R/read_sc_all_home_care.R index fdaa5c915..e38019afd 100644 --- a/R/read_sc_all_home_care.R +++ b/R/read_sc_all_home_care.R @@ -7,6 +7,8 @@ #' @export #' read_sc_all_home_care <- function(sc_dvprod_connection = phs_db_connection(dsn = "DVPROD")) { + log_slf_event(stage = "read", status = "start", type = "hc", year = "all") + home_care_data <- dplyr::tbl( sc_dvprod_connection, dbplyr::in_schema("social_care_2", "homecare_snapshot") @@ -74,5 +76,7 @@ read_sc_all_home_care <- function(sc_dvprod_connection = phs_db_connection(dsn = "reablement" ), as.integer)) + log_slf_event(stage = "read", status = "complete", type = "hc", year = "all") + return(home_care_data) } diff --git a/R/read_sc_all_sds.R b/R/read_sc_all_sds.R index eac8d24e3..b9deff063 100644 --- a/R/read_sc_all_sds.R +++ b/R/read_sc_all_sds.R @@ -7,6 +7,8 @@ #' @export #' read_sc_all_sds <- function(sc_dvprod_connection = phs_db_connection(dsn = "DVPROD")) { + log_slf_event(stage = "read", status = "start", type = "sds", year = "all") + sds_full_data <- dplyr::tbl( sc_dvprod_connection, dbplyr::in_schema("social_care_2", "sds_snapshot") @@ -54,5 +56,7 @@ read_sc_all_sds <- function(sc_dvprod_connection = phs_db_connection(dsn = "DVPR "sds_option_3" ), as.integer)) + log_slf_event(stage = "read", status = "complete", type = "sds", year = "all") + return(sds_full_data) } diff --git a/R/write_console_output.R b/R/write_console_output.R deleted file mode 100644 index 346b9a3ad..000000000 --- a/R/write_console_output.R +++ /dev/null @@ -1,68 +0,0 @@ -#' Write Console Output to File -#' -#' @description Sets up sink to capture console output and messages to .txt file. -#' -#' @param console_outputs If TRUE, capture console output and messages to .txt file. -#' @param file_type Type of file being processed: "episode", "individual", or "targets". -#' @param year Financial year. -write_console_output <- function(console_outputs = TRUE, - file_type = c("episode", "individual", "targets"), - year = NULL) { - if (!console_outputs) { - return(invisible(NULL)) - } - - file_type <- match.arg(file_type) - - # update information - update <- latest_update() - update_year <- as.integer(substr(phsmethods::extract_fin_year(end_date()), 1, 4)) - update_quarter <- qtr() - - # output directory path - con_output_dir <- file.path( - "/conf/sourcedev/Source_Linkage_File_Updates/_console_output", - update_year, - update_quarter - ) - if (!dir.exists(con_output_dir)) { - dir.create(con_output_dir, recursive = TRUE) - } - - base_filename <- switch(file_type, - "episode" = stringr::str_glue("ep_{year}_{update}_update"), - "individual" = stringr::str_glue("ind_{year}_{update}_update"), - "targets" = stringr::str_glue("targets_console_{update}_update") - ) - existing_files <- list.files( - path = con_output_dir, - pattern = stringr::str_glue("^{base_filename}_[0-9]+\\.txt$"), - full.names = FALSE - ) - if (length(existing_files) == 0) { - increment <- 1 - } else { - numbers <- stringr::str_extract(existing_files, "[0-9]+(?=\\.txt$)") %>% - as.integer() - increment <- max(numbers, na.rm = TRUE) + 1 - } - - file_name <- stringr::str_glue("{base_filename}_{increment}.txt") - file_path <- file.path(con_output_dir, file_name) - - # sink connection - con <- file(file_path, open = "wt") - - sink(con, type = "output", split = TRUE) - sink(con, type = "message", append = TRUE) - - on.exit( - { - sink(type = "message") - sink(type = "output") - close(con) - cat("\n Console output saved to:", file_path, "\n") - }, - add = TRUE - ) -} diff --git a/Run_SLF_Files_manually/run_episode_file_1415.R b/Run_SLF_Files_manually/run_episode_file_1415.R index ab9bb0c69..28f064852 100644 --- a/Run_SLF_Files_manually/run_episode_file_1415.R +++ b/Run_SLF_Files_manually/run_episode_file_1415.R @@ -43,7 +43,11 @@ write_console_output( ) #------------------------------------------------------------------------------- -## Read processed data from sourcedev +log_ep_message("start", year) + +## Read processed data and create episode file +log_ep_message("read_data", year) + processed_data_list <- list( "ae" = read_file(get_source_extract_path(year, "ae")), "acute" = read_file(get_source_extract_path(year, "acute")), @@ -62,13 +66,15 @@ processed_data_list <- list( "sds" = read_file(get_source_extract_path(year, "sds")) ) -# Run the episode file and tests +log_ep_message("creating", year) + create_episode_file(processed_data_list, year = year, write_temp_to_disk = write_temp_to_disk ) %>% process_tests_episode_file(year = year) -#------------------------------------------------------------------------------- +log_ep_message("complete", year) +#------------------------------------------------------------------------------- ## End of Script ## diff --git a/Run_SLF_Files_manually/run_episode_file_1516.R b/Run_SLF_Files_manually/run_episode_file_1516.R index 25c7b645a..c97698e63 100644 --- a/Run_SLF_Files_manually/run_episode_file_1516.R +++ b/Run_SLF_Files_manually/run_episode_file_1516.R @@ -43,7 +43,11 @@ write_console_output( ) #------------------------------------------------------------------------------- -## Read processed data from sourcedev +log_ep_message("start", year) + +## Read processed data and create episode file +log_ep_message("read_data", year) + processed_data_list <- list( "ae" = read_file(get_source_extract_path(year, "ae")), "acute" = read_file(get_source_extract_path(year, "acute")), @@ -62,13 +66,15 @@ processed_data_list <- list( "sds" = read_file(get_source_extract_path(year, "sds")) ) -# Run the episode file and tests +log_ep_message("creating", year) + create_episode_file(processed_data_list, year = year, write_temp_to_disk = write_temp_to_disk ) %>% process_tests_episode_file(year = year) -#------------------------------------------------------------------------------- +log_ep_message("complete", year) +#------------------------------------------------------------------------------- ## End of Script ## diff --git a/Run_SLF_Files_manually/run_episode_file_1617.R b/Run_SLF_Files_manually/run_episode_file_1617.R index fad21bd97..ea991cbd9 100644 --- a/Run_SLF_Files_manually/run_episode_file_1617.R +++ b/Run_SLF_Files_manually/run_episode_file_1617.R @@ -43,7 +43,11 @@ write_console_output( ) #------------------------------------------------------------------------------- -## Read processed data from sourcedev +log_ep_message("start", year) + +## Read processed data and create episode file +log_ep_message("read_data", year) + processed_data_list <- list( "ae" = read_file(get_source_extract_path(year, "ae")), "acute" = read_file(get_source_extract_path(year, "acute")), @@ -62,13 +66,15 @@ processed_data_list <- list( "sds" = read_file(get_source_extract_path(year, "sds")) ) -# Run the episode file and tests +log_ep_message("creating", year) + create_episode_file(processed_data_list, year = year, write_temp_to_disk = write_temp_to_disk ) %>% process_tests_episode_file(year = year) -#------------------------------------------------------------------------------- +log_ep_message("complete", year) +#------------------------------------------------------------------------------- ## End of Script ## diff --git a/Run_SLF_Files_manually/run_episode_file_1718.R b/Run_SLF_Files_manually/run_episode_file_1718.R index 448c3e43e..6d4c08865 100644 --- a/Run_SLF_Files_manually/run_episode_file_1718.R +++ b/Run_SLF_Files_manually/run_episode_file_1718.R @@ -43,7 +43,11 @@ write_console_output( ) #------------------------------------------------------------------------------- -## Read processed data from sourcedev +log_ep_message("start", year) + +## Read processed data and create episode file +log_ep_message("read_data", year) + processed_data_list <- list( "ae" = read_file(get_source_extract_path(year, "ae")), "acute" = read_file(get_source_extract_path(year, "acute")), @@ -62,13 +66,15 @@ processed_data_list <- list( "sds" = read_file(get_source_extract_path(year, "sds")) ) -# Run the episode file and tests +log_ep_message("creating", year) + create_episode_file(processed_data_list, year = year, write_temp_to_disk = write_temp_to_disk ) %>% process_tests_episode_file(year = year) -#------------------------------------------------------------------------------- +log_ep_message("complete", year) +#------------------------------------------------------------------------------- ## End of Script ## diff --git a/Run_SLF_Files_manually/run_episode_file_1819.R b/Run_SLF_Files_manually/run_episode_file_1819.R index fe6489401..a057b6603 100644 --- a/Run_SLF_Files_manually/run_episode_file_1819.R +++ b/Run_SLF_Files_manually/run_episode_file_1819.R @@ -43,7 +43,11 @@ write_console_output( ) #------------------------------------------------------------------------------- -## Read processed data from sourcedev +log_ep_message("start", year) + +## Read processed data and create episode file +log_ep_message("read_data", year) + processed_data_list <- list( "ae" = read_file(get_source_extract_path(year, "ae")), "acute" = read_file(get_source_extract_path(year, "acute")), @@ -62,13 +66,15 @@ processed_data_list <- list( "sds" = read_file(get_source_extract_path(year, "sds")) ) -# Run the episode file and tests +log_ep_message("creating", year) + create_episode_file(processed_data_list, year = year, write_temp_to_disk = write_temp_to_disk ) %>% process_tests_episode_file(year = year) -#------------------------------------------------------------------------------- +log_ep_message("complete", year) +#------------------------------------------------------------------------------- ## End of Script ## diff --git a/Run_SLF_Files_manually/run_episode_file_1920.R b/Run_SLF_Files_manually/run_episode_file_1920.R index 5b56912ef..8458a52a6 100644 --- a/Run_SLF_Files_manually/run_episode_file_1920.R +++ b/Run_SLF_Files_manually/run_episode_file_1920.R @@ -43,7 +43,11 @@ write_console_output( ) #------------------------------------------------------------------------------- -## Read processed data from sourcedev +log_ep_message("start", year) + +## Read processed data and create episode file +log_ep_message("read_data", year) + processed_data_list <- list( "ae" = read_file(get_source_extract_path(year, "ae")), "acute" = read_file(get_source_extract_path(year, "acute")), @@ -62,13 +66,15 @@ processed_data_list <- list( "sds" = read_file(get_source_extract_path(year, "sds")) ) -# Run the episode file and tests +log_ep_message("creating", year) + create_episode_file(processed_data_list, year = year, write_temp_to_disk = write_temp_to_disk ) %>% process_tests_episode_file(year = year) -#------------------------------------------------------------------------------- +log_ep_message("complete", year) +#------------------------------------------------------------------------------- ## End of Script ## diff --git a/Run_SLF_Files_manually/run_episode_file_2021.R b/Run_SLF_Files_manually/run_episode_file_2021.R index f1bf54a02..43834fc97 100644 --- a/Run_SLF_Files_manually/run_episode_file_2021.R +++ b/Run_SLF_Files_manually/run_episode_file_2021.R @@ -43,7 +43,11 @@ write_console_output( ) #------------------------------------------------------------------------------- -## Read processed data from sourcedev +log_ep_message("start", year) + +## Read processed data and create episode file +log_ep_message("read_data", year) + processed_data_list <- list( "ae" = read_file(get_source_extract_path(year, "ae")), "acute" = read_file(get_source_extract_path(year, "acute")), @@ -62,13 +66,15 @@ processed_data_list <- list( "sds" = read_file(get_source_extract_path(year, "sds")) ) -# Run the episode file and tests +log_ep_message("creating", year) + create_episode_file(processed_data_list, year = year, write_temp_to_disk = write_temp_to_disk ) %>% process_tests_episode_file(year = year) -#------------------------------------------------------------------------------- +log_ep_message("complete", year) +#------------------------------------------------------------------------------- ## End of Script ## diff --git a/Run_SLF_Files_manually/run_episode_file_2122.R b/Run_SLF_Files_manually/run_episode_file_2122.R index a7cd6609c..5384b339c 100644 --- a/Run_SLF_Files_manually/run_episode_file_2122.R +++ b/Run_SLF_Files_manually/run_episode_file_2122.R @@ -43,7 +43,11 @@ write_console_output( ) #------------------------------------------------------------------------------- -## Read processed data from sourcedev +log_ep_message("start", year) + +## Read processed data and create episode file +log_ep_message("read_data", year) + processed_data_list <- list( "ae" = read_file(get_source_extract_path(year, "ae")), "acute" = read_file(get_source_extract_path(year, "acute")), @@ -62,13 +66,15 @@ processed_data_list <- list( "sds" = read_file(get_source_extract_path(year, "sds")) ) -# Run the episode file and tests +log_ep_message("creating", year) + create_episode_file(processed_data_list, year = year, write_temp_to_disk = write_temp_to_disk ) %>% process_tests_episode_file(year = year) -#------------------------------------------------------------------------------- +log_ep_message("complete", year) +#------------------------------------------------------------------------------- ## End of Script ## diff --git a/Run_SLF_Files_manually/run_episode_file_2223.R b/Run_SLF_Files_manually/run_episode_file_2223.R index 5aaff0c28..ed740ebd6 100644 --- a/Run_SLF_Files_manually/run_episode_file_2223.R +++ b/Run_SLF_Files_manually/run_episode_file_2223.R @@ -43,7 +43,11 @@ write_console_output( ) #------------------------------------------------------------------------------- -## Read processed data from sourcedev +log_ep_message("start", year) + +## Read processed data and create episode file +log_ep_message("read_data", year) + processed_data_list <- list( "ae" = read_file(get_source_extract_path(year, "ae")), "acute" = read_file(get_source_extract_path(year, "acute")), @@ -62,13 +66,15 @@ processed_data_list <- list( "sds" = read_file(get_source_extract_path(year, "sds")) ) -# Run the episode file and tests +log_ep_message("creating", year) + create_episode_file(processed_data_list, year = year, write_temp_to_disk = write_temp_to_disk ) %>% process_tests_episode_file(year = year) -#------------------------------------------------------------------------------- +log_ep_message("complete", year) +#------------------------------------------------------------------------------- ## End of Script ## diff --git a/Run_SLF_Files_manually/run_episode_file_2324.R b/Run_SLF_Files_manually/run_episode_file_2324.R index 633ce9b22..9bc942c27 100644 --- a/Run_SLF_Files_manually/run_episode_file_2324.R +++ b/Run_SLF_Files_manually/run_episode_file_2324.R @@ -43,7 +43,11 @@ write_console_output( ) #------------------------------------------------------------------------------- -## Read processed data from sourcedev +log_ep_message("start", year) + +## Read processed data and create episode file +log_ep_message("read_data", year) + processed_data_list <- list( "ae" = read_file(get_source_extract_path(year, "ae")), "acute" = read_file(get_source_extract_path(year, "acute")), @@ -62,13 +66,15 @@ processed_data_list <- list( "sds" = read_file(get_source_extract_path(year, "sds")) ) -# Run the episode file and tests +log_ep_message("creating", year) + create_episode_file(processed_data_list, year = year, write_temp_to_disk = write_temp_to_disk ) %>% process_tests_episode_file(year = year) -#------------------------------------------------------------------------------- +log_ep_message("complete", year) +#------------------------------------------------------------------------------- ## End of Script ## diff --git a/Run_SLF_Files_manually/run_episode_file_2425.R b/Run_SLF_Files_manually/run_episode_file_2425.R index febd6a1f6..688e2ba5b 100644 --- a/Run_SLF_Files_manually/run_episode_file_2425.R +++ b/Run_SLF_Files_manually/run_episode_file_2425.R @@ -43,7 +43,11 @@ write_console_output( ) #------------------------------------------------------------------------------- -## Read processed data from sourcedev +log_ep_message("start", year) + +## Read processed data and create episode file +log_ep_message("read_data", year) + processed_data_list <- list( "ae" = read_file(get_source_extract_path(year, "ae")), "acute" = read_file(get_source_extract_path(year, "acute")), @@ -62,13 +66,15 @@ processed_data_list <- list( "sds" = read_file(get_source_extract_path(year, "sds")) ) -# Run the episode file and tests +log_ep_message("creating", year) + create_episode_file(processed_data_list, year = year, write_temp_to_disk = write_temp_to_disk ) %>% process_tests_episode_file(year = year) -#------------------------------------------------------------------------------- +log_ep_message("complete", year) +#------------------------------------------------------------------------------- ## End of Script ## diff --git a/Run_SLF_Files_manually/run_episode_file_2526.R b/Run_SLF_Files_manually/run_episode_file_2526.R index 9ac9b40d7..8ad6120c8 100644 --- a/Run_SLF_Files_manually/run_episode_file_2526.R +++ b/Run_SLF_Files_manually/run_episode_file_2526.R @@ -43,7 +43,11 @@ write_console_output( ) #------------------------------------------------------------------------------- -## Read processed data from sourcedev +log_ep_message("start", year) + +## Read processed data and create episode file +log_ep_message("read_data", year) + processed_data_list <- list( "ae" = read_file(get_source_extract_path(year, "ae")), "acute" = read_file(get_source_extract_path(year, "acute")), @@ -62,13 +66,15 @@ processed_data_list <- list( "sds" = read_file(get_source_extract_path(year, "sds")) ) -# Run the episode file and tests +log_ep_message("creating", year) + create_episode_file(processed_data_list, year = year, write_temp_to_disk = write_temp_to_disk ) %>% process_tests_episode_file(year = year) -#------------------------------------------------------------------------------- +log_ep_message("complete", year) +#------------------------------------------------------------------------------- ## End of Script ## diff --git a/Run_SLF_Files_manually/run_individual_file_1415.R b/Run_SLF_Files_manually/run_individual_file_1415.R index 980985ede..869f66770 100644 --- a/Run_SLF_Files_manually/run_individual_file_1415.R +++ b/Run_SLF_Files_manually/run_individual_file_1415.R @@ -43,16 +43,20 @@ write_console_output( ) #------------------------------------------------------------------------------- -# Clean temporary files -# clean_temp_data(year, "ep") +log_ind_message("start", year) + +## Read the episode file and create the individual file +log_ind_message("read_data", year) -# Read the episode file episode_file <- arrow::read_parquet(get_slf_episode_path(year)) # Run the individual file and tests +log_ind_message("creating", year) + create_individual_file(episode_file, year = year, write_temp_to_disk = write_temp_to_disk) %>% process_tests_individual_file(year = year) -#------------------------------------------------------------------------------- +log_ind_message("complete", year) +#------------------------------------------------------------------------------- ## End of Script ## diff --git a/Run_SLF_Files_manually/run_individual_file_1516.R b/Run_SLF_Files_manually/run_individual_file_1516.R index d4b855028..9cb77b603 100644 --- a/Run_SLF_Files_manually/run_individual_file_1516.R +++ b/Run_SLF_Files_manually/run_individual_file_1516.R @@ -43,16 +43,20 @@ write_console_output( ) #------------------------------------------------------------------------------- -# Clean temporary files -# clean_temp_data(year, "ep") +log_ind_message("start", year) + +## Read the episode file and create the individual file +log_ind_message("read_data", year) -# Read the episode file episode_file <- arrow::read_parquet(get_slf_episode_path(year)) # Run the individual file and tests +log_ind_message("creating", year) + create_individual_file(episode_file, year = year, write_temp_to_disk = write_temp_to_disk) %>% process_tests_individual_file(year = year) -#------------------------------------------------------------------------------- +log_ind_message("complete", year) +#------------------------------------------------------------------------------- ## End of Script ## diff --git a/Run_SLF_Files_manually/run_individual_file_1617.R b/Run_SLF_Files_manually/run_individual_file_1617.R index 21f2da77a..6a9d85f50 100644 --- a/Run_SLF_Files_manually/run_individual_file_1617.R +++ b/Run_SLF_Files_manually/run_individual_file_1617.R @@ -43,16 +43,20 @@ write_console_output( ) #------------------------------------------------------------------------------- -# Clean temporary files -# clean_temp_data(year, "ep") +log_ind_message("start", year) + +## Read the episode file and create the individual file +log_ind_message("read_data", year) -# Read the episode file episode_file <- arrow::read_parquet(get_slf_episode_path(year)) # Run the individual file and tests +log_ind_message("creating", year) + create_individual_file(episode_file, year = year, write_temp_to_disk = write_temp_to_disk) %>% process_tests_individual_file(year = year) -#------------------------------------------------------------------------------- +log_ind_message("complete", year) +#------------------------------------------------------------------------------- ## End of Script ## diff --git a/Run_SLF_Files_manually/run_individual_file_1718.R b/Run_SLF_Files_manually/run_individual_file_1718.R index 11c74ae58..088349ffa 100644 --- a/Run_SLF_Files_manually/run_individual_file_1718.R +++ b/Run_SLF_Files_manually/run_individual_file_1718.R @@ -43,16 +43,20 @@ write_console_output( ) #------------------------------------------------------------------------------- -# Clean temporary files -# clean_temp_data(year, "ep") +log_ind_message("start", year) + +## Read the episode file and create the individual file +log_ind_message("read_data", year) -# Read the episode file episode_file <- arrow::read_parquet(get_slf_episode_path(year)) # Run the individual file and tests +log_ind_message("creating", year) + create_individual_file(episode_file, year = year, write_temp_to_disk = write_temp_to_disk) %>% process_tests_individual_file(year = year) -#------------------------------------------------------------------------------- +log_ind_message("complete", year) +#------------------------------------------------------------------------------- ## End of Script ## diff --git a/Run_SLF_Files_manually/run_individual_file_1819.R b/Run_SLF_Files_manually/run_individual_file_1819.R index ef38b7cb5..88f905509 100644 --- a/Run_SLF_Files_manually/run_individual_file_1819.R +++ b/Run_SLF_Files_manually/run_individual_file_1819.R @@ -43,16 +43,20 @@ write_console_output( ) #------------------------------------------------------------------------------- -# Clean temporary files -# clean_temp_data(year, "ep") +log_ind_message("start", year) + +## Read the episode file and create the individual file +log_ind_message("read_data", year) -# Read the episode file episode_file <- arrow::read_parquet(get_slf_episode_path(year)) # Run the individual file and tests +log_ind_message("creating", year) + create_individual_file(episode_file, year = year, write_temp_to_disk = write_temp_to_disk) %>% process_tests_individual_file(year = year) -#------------------------------------------------------------------------------- +log_ind_message("complete", year) +#------------------------------------------------------------------------------- ## End of Script ## diff --git a/Run_SLF_Files_manually/run_individual_file_1920.R b/Run_SLF_Files_manually/run_individual_file_1920.R index f93832f62..447ba1f75 100644 --- a/Run_SLF_Files_manually/run_individual_file_1920.R +++ b/Run_SLF_Files_manually/run_individual_file_1920.R @@ -43,16 +43,20 @@ write_console_output( ) #------------------------------------------------------------------------------- -# Clean temporary files -# clean_temp_data(year, "ep") +log_ind_message("start", year) + +## Read the episode file and create the individual file +log_ind_message("read_data", year) -# Read the episode file episode_file <- arrow::read_parquet(get_slf_episode_path(year)) # Run the individual file and tests +log_ind_message("creating", year) + create_individual_file(episode_file, year = year, write_temp_to_disk = write_temp_to_disk) %>% process_tests_individual_file(year = year) -#------------------------------------------------------------------------------- +log_ind_message("complete", year) +#------------------------------------------------------------------------------- ## End of Script ## diff --git a/Run_SLF_Files_manually/run_individual_file_2021.R b/Run_SLF_Files_manually/run_individual_file_2021.R index a408d0cdd..25678bf17 100644 --- a/Run_SLF_Files_manually/run_individual_file_2021.R +++ b/Run_SLF_Files_manually/run_individual_file_2021.R @@ -43,16 +43,20 @@ write_console_output( ) #------------------------------------------------------------------------------- -# Clean temporary files -# clean_temp_data(year, "ep") +log_ind_message("start", year) + +## Read the episode file and create the individual file +log_ind_message("read_data", year) -# Read the episode file episode_file <- arrow::read_parquet(get_slf_episode_path(year)) # Run the individual file and tests +log_ind_message("creating", year) + create_individual_file(episode_file, year = year, write_temp_to_disk = write_temp_to_disk) %>% process_tests_individual_file(year = year) -#------------------------------------------------------------------------------- +log_ind_message("complete", year) +#------------------------------------------------------------------------------- ## End of Script ## diff --git a/Run_SLF_Files_manually/run_individual_file_2122.R b/Run_SLF_Files_manually/run_individual_file_2122.R index 6f2ba447d..883ab5a6e 100644 --- a/Run_SLF_Files_manually/run_individual_file_2122.R +++ b/Run_SLF_Files_manually/run_individual_file_2122.R @@ -43,16 +43,20 @@ write_console_output( ) #------------------------------------------------------------------------------- -# Clean temporary files -# clean_temp_data(year, "ep") +log_ind_message("start", year) + +## Read the episode file and create the individual file +log_ind_message("read_data", year) -# Read the episode file episode_file <- arrow::read_parquet(get_slf_episode_path(year)) # Run the individual file and tests +log_ind_message("creating", year) + create_individual_file(episode_file, year = year, write_temp_to_disk = write_temp_to_disk) %>% process_tests_individual_file(year = year) -#------------------------------------------------------------------------------- +log_ind_message("complete", year) +#------------------------------------------------------------------------------- ## End of Script ## diff --git a/Run_SLF_Files_manually/run_individual_file_2223.R b/Run_SLF_Files_manually/run_individual_file_2223.R index 78a455f33..323bec908 100644 --- a/Run_SLF_Files_manually/run_individual_file_2223.R +++ b/Run_SLF_Files_manually/run_individual_file_2223.R @@ -43,16 +43,20 @@ write_console_output( ) #------------------------------------------------------------------------------- -# Clean temporary files -# clean_temp_data(year, "ep") +log_ind_message("start", year) + +## Read the episode file and create the individual file +log_ind_message("read_data", year) -# Read the episode file episode_file <- arrow::read_parquet(get_slf_episode_path(year)) # Run the individual file and tests +log_ind_message("creating", year) + create_individual_file(episode_file, year = year, write_temp_to_disk = write_temp_to_disk) %>% process_tests_individual_file(year = year) -#------------------------------------------------------------------------------- +log_ind_message("complete", year) +#------------------------------------------------------------------------------- ## End of Script ## diff --git a/Run_SLF_Files_manually/run_individual_file_2324.R b/Run_SLF_Files_manually/run_individual_file_2324.R index ce18d979b..a971d38c6 100644 --- a/Run_SLF_Files_manually/run_individual_file_2324.R +++ b/Run_SLF_Files_manually/run_individual_file_2324.R @@ -43,16 +43,20 @@ write_console_output( ) #------------------------------------------------------------------------------- -# Clean temporary files -# clean_temp_data(year, "ep") +log_ind_message("start", year) + +## Read the episode file and create the individual file +log_ind_message("read_data", year) -# Read the episode file episode_file <- arrow::read_parquet(get_slf_episode_path(year)) # Run the individual file and tests +log_ind_message("creating", year) + create_individual_file(episode_file, year = year, write_temp_to_disk = write_temp_to_disk) %>% process_tests_individual_file(year = year) -#------------------------------------------------------------------------------- +log_ind_message("complete", year) +#------------------------------------------------------------------------------- ## End of Script ## diff --git a/Run_SLF_Files_manually/run_individual_file_2425.R b/Run_SLF_Files_manually/run_individual_file_2425.R index 3a281afc2..47dbe565a 100644 --- a/Run_SLF_Files_manually/run_individual_file_2425.R +++ b/Run_SLF_Files_manually/run_individual_file_2425.R @@ -43,16 +43,20 @@ write_console_output( ) #------------------------------------------------------------------------------- -# Clean temporary files -# clean_temp_data(year, "ep") +log_ind_message("start", year) + +## Read the episode file and create the individual file +log_ind_message("read_data", year) -# Read the episode file episode_file <- arrow::read_parquet(get_slf_episode_path(year)) # Run the individual file and tests +log_ind_message("creating", year) + create_individual_file(episode_file, year = year, write_temp_to_disk = write_temp_to_disk) %>% process_tests_individual_file(year = year) -#------------------------------------------------------------------------------- +log_ind_message("complete", year) +#------------------------------------------------------------------------------- ## End of Script ## diff --git a/Run_SLF_Files_manually/run_individual_file_2526.R b/Run_SLF_Files_manually/run_individual_file_2526.R index bf00e307a..39059ed04 100644 --- a/Run_SLF_Files_manually/run_individual_file_2526.R +++ b/Run_SLF_Files_manually/run_individual_file_2526.R @@ -43,16 +43,20 @@ write_console_output( ) #------------------------------------------------------------------------------- -# Clean temporary files -# clean_temp_data(year, "ep") +log_ind_message("start", year) + +## Read the episode file and create the individual file +log_ind_message("read_data", year) -# Read the episode file episode_file <- arrow::read_parquet(get_slf_episode_path(year)) # Run the individual file and tests +log_ind_message("creating", year) + create_individual_file(episode_file, year = year, write_temp_to_disk = write_temp_to_disk) %>% process_tests_individual_file(year = year) -#------------------------------------------------------------------------------- +log_ind_message("complete", year) +#------------------------------------------------------------------------------- ## End of Script ## diff --git a/Run_SLF_Files_targets/run_all_targets.R b/Run_SLF_Files_targets/run_all_targets.R index 4bdaadd76..841748f81 100644 --- a/Run_SLF_Files_targets/run_all_targets.R +++ b/Run_SLF_Files_targets/run_all_targets.R @@ -46,15 +46,17 @@ write_console_output( # Run the targets pipeline "_Targets.R" #------------------------------------------------------------------------------- +log_tar_message("start") # tar_make() will run the pipeline and use crew() for parallel processing. tar_make() # Run combine_tests() for outputting the test workbooks. +log_tar_message("combining_tests") + createslf::combine_tests() -#------------------------------------------------------------------------------- - -# END OF SCRIPT # +log_tar_message("all_complete") #------------------------------------------------------------------------------- +# END OF SCRIPT # diff --git a/_pkgdown.yml b/_pkgdown.yml index 66b9b9b25..9c0e38a12 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -105,7 +105,6 @@ reference: - store_ep_file_vars - load_ep_file_vars - write_temp_data - - write_console_output - title: Initialisation @@ -237,3 +236,14 @@ reference: - link_ch_with_adms - update_uk_postcode_directory + + - title: Logger + desc: Functions that initialise logger for logging messages during SLF processing. + - contents: + - write_console_output + - log_ep_message + - log_ind_message + - log_tar_message + - log_slf_event + - log_ep_substage + - log_ind_substage diff --git a/man/log_ep_message.Rd b/man/log_ep_message.Rd new file mode 100644 index 000000000..184378290 --- /dev/null +++ b/man/log_ep_message.Rd @@ -0,0 +1,16 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/logger_utils.R +\name{log_ep_message} +\alias{log_ep_message} +\title{Log Episode File Messages} +\usage{ +log_ep_message(stage = c("start", "read_data", "creating", "complete"), year) +} +\arguments{ +\item{stage}{Stage of episode file (run_episode_file_fy.R) creation.} + +\item{year}{Financial year.} +} +\description{ +Log Episode File Messages +} diff --git a/man/log_ep_substage.Rd b/man/log_ep_substage.Rd new file mode 100644 index 000000000..7ec3b0d56 --- /dev/null +++ b/man/log_ep_substage.Rd @@ -0,0 +1,18 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/logger_utils.R +\name{log_ep_substage} +\alias{log_ep_substage} +\title{Log Create Episode File (create_episode_file.R) Stages} +\usage{ +log_ep_substage(sub_stage, status, year) +} +\arguments{ +\item{sub_stage}{Character: The name of the specific process (e.g., "Join cohort lookups").} + +\item{status}{Character: "started" or "finished".} + +\item{year}{Character: The financial year.} +} +\description{ +Log Create Episode File (create_episode_file.R) Stages +} diff --git a/man/log_ind_message.Rd b/man/log_ind_message.Rd new file mode 100644 index 000000000..0b3c8e706 --- /dev/null +++ b/man/log_ind_message.Rd @@ -0,0 +1,16 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/logger_utils.R +\name{log_ind_message} +\alias{log_ind_message} +\title{Log Individual File Messages} +\usage{ +log_ind_message(stage = c("start", "read_data", "creating", "complete"), year) +} +\arguments{ +\item{stage}{Stage of individual file (run_individual_file_fy.R) creation.} + +\item{year}{Financial year.} +} +\description{ +Log Individual File Messages +} diff --git a/man/log_ind_substage.Rd b/man/log_ind_substage.Rd new file mode 100644 index 000000000..42aff2035 --- /dev/null +++ b/man/log_ind_substage.Rd @@ -0,0 +1,18 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/logger_utils.R +\name{log_ind_substage} +\alias{log_ind_substage} +\title{Log Create Individual File (create_individual_file.R) Stages} +\usage{ +log_ind_substage(sub_stage, status, year) +} +\arguments{ +\item{sub_stage}{Character: The name of the specific process (e.g., "Add cij columns").} + +\item{status}{Character: "started" or "finished".} + +\item{year}{Character: The financial year.} +} +\description{ +Log Create Individual File (create_individual_file.R) Stages +} diff --git a/man/log_slf_event.Rd b/man/log_slf_event.Rd new file mode 100644 index 000000000..414336bf3 --- /dev/null +++ b/man/log_slf_event.Rd @@ -0,0 +1,28 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/logger_utils.R +\name{log_slf_event} +\alias{log_slf_event} +\title{Log SLF Intermediate Processing} +\usage{ +log_slf_event( + stage = c("read", "process", "test"), + status = c("start", "complete"), + type, + year, + ... +) +} +\arguments{ +\item{stage}{Character: "read", "process", or "test".} + +\item{status}{Character: "start" or "complete".} + +\item{type}{Character: The name of the data (e.g., "Acute", "A&E").} + +\item{year}{Character: The financial year.} + +\item{...}{Additional arguments for string interpolation.} +} +\description{ +Logs messages for read, process, and test stages/scripts +} diff --git a/man/log_tar_message.Rd b/man/log_tar_message.Rd new file mode 100644 index 000000000..bc7b7cdfd --- /dev/null +++ b/man/log_tar_message.Rd @@ -0,0 +1,14 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/logger_utils.R +\name{log_tar_message} +\alias{log_tar_message} +\title{Log Targets Messages} +\usage{ +log_tar_message(stage = c("start", "combining_tests", "all_complete")) +} +\arguments{ +\item{stage}{Stage of targets (run_targets_fy.R) creation.} +} +\description{ +Log Targets Messages +} diff --git a/man/write_console_output.Rd b/man/write_console_output.Rd index aa2a066e1..95a118d70 100644 --- a/man/write_console_output.Rd +++ b/man/write_console_output.Rd @@ -1,5 +1,5 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/write_console_output.R +% Please edit documentation in R/logger_utils.R \name{write_console_output} \alias{write_console_output} \title{Write Console Output to File} @@ -11,12 +11,12 @@ write_console_output( ) } \arguments{ -\item{console_outputs}{If TRUE, capture console output and messages to .txt file.} +\item{console_outputs}{If TRUE, capture console output and messages to file.} \item{file_type}{Type of file being processed: "episode", "individual", or "targets".} \item{year}{Financial year.} } \description{ -Sets up sink to capture console output and messages to .txt file. +Initialises logger for run_episode_file_fy.R and run_individual_file_fy.R. }