Merge pull request #86 from PopHIVE/feat/add-ilinet-respiratory-bundle

Add ILINet (FluView) data to respiratory bundle

Author: DanWeinberger

data/bundle_respiratory/build.R

@@ -31,7 +31,8 @@ bundle_files  <- list( '../epic/standard/weekly.csv.gz',
                        '../wastewater/standard/data.csv.gz',
                        '../delphi_doctors_claims/standard/data.csv.gz',
                        '../delphi_hospital_claims/standard/data.csv.gz',
-                       '../delphi_nhsn/standard/data.csv.gz'
+                       '../delphi_nhsn/standard/data.csv.gz',
+                       '../delphi_ili_fluview/standard/data.csv.gz'
 )
 
 start_time <- "2020"
@@ -60,43 +61,6 @@ combined <- Reduce(
 #colnames(combined) <- sub("n_", "epic_", colnames(combined), fixed = TRUE)
 
 
-
-############################
-############################
-#Experimental: try to just create one big output table
-# output_table <- combined %>%
-#   pivot_longer(
-#     cols = where(is.numeric),
-#     names_to = "metric",
-#     values_to = "value"
-#   ) %>%
-#   arrange(geography, metric, time) %>%
-#   group_by(geography, metric) %>%
-#   mutate(
-#     value_smooth = zoo::rollapplyr(value, 3, mean, partial = TRUE, na.rm = TRUE),
-#     value_smooth_scale = value_smooth / max(value_smooth, na.rm = TRUE) * 100
-#   ) %>%
-#   ungroup() %>%
-#   pivot_wider(
-#     names_from = metric,
-#     values_from = c(value, value_smooth, value_smooth_scale),
-#     names_sep = "_"
-#   )
-# vroom::vroom_write(
-#   output_table,
-#   "dist/TEST_mega.csv.gz",
-#   ","
-# )
-# arrow::write_parquet(output_table,
-#                      "dist/TEST_mega.parquet")
-# 
-# jsonlite::write_json(output_table, gzfile("dist/TEST_mega.json.gz"), dataframe = "columns")  #way too big
-
-####################################
-####################################
-####################################
-
-
 overall_trends <-   combined %>%
   filter( (time >= max(time) - 365*2) & geography %in% state_fips) %>%
   rename(fips= geography) %>%
@@ -175,7 +139,7 @@ overall_trends %>%
 
 overall_trends %>% 
   filter(grepl('flu',variable) & !is.na(value)) %>%
-  filter(variable %in% c('epic_pct_flu', 'percent_visits_flu', 'rate_flu','wastewater_flua','delphi_nhsn_flu' ,'delphi_hospital_flu_smooth')) %>%
+  filter(variable %in% c('epic_pct_flu', 'percent_visits_flu', 'rate_flu','wastewater_flua','delphi_nhsn_flu' ,'delphi_hospital_flu_smooth','delphi_fluview_wili')) %>%
   mutate( source = if_else(variable=='epic_pct_flu', 'Epic Cosmos, ED',
                                    if_else(variable=='percent_visits_flu', 'CDC NSSP',
                                            if_else(variable=='rate_flu', 'CDC RespNET',

data/bundle_respiratory/dist/covid_overall_trends.parquet

@@ -63,19 +63,8 @@ if (!identical(process$raw_state, raw_state)) {
         !is.na(geography) ~ geography,  # Use merged FIPS code
         TRUE ~ NA_character_  # Should not happen if lookup is complete
       ),
-      # Convert epiweek to date (Saturday at end of week)
-      # Epiweek format: YYYYWW (e.g., 202301 = 2023 week 1)
-      epiweek_year = as.integer(substr(as.character(epiweek), 1, 4)),
-      epiweek_week = as.integer(substr(as.character(epiweek), 5, 6)),
-      # Calculate Saturday ending the epiweek
-      # MMWR week 1 always contains Jan 4, and weeks run Sunday-Saturday
-      # Find Jan 4 of the year, get its MMWR week start (Sunday), then add weeks
-      jan4 = as.Date(paste0(epiweek_year, "-01-04")),
-      # Find the Sunday of the week containing Jan 4
-      jan4_wday = lubridate::wday(jan4, week_start = 7),  # Sunday = 1
-      week1_sunday = jan4 - (jan4_wday - 1),
-      # Calculate the Saturday of the requested epiweek
-      time = week1_sunday + (epiweek_week - 1) * 7 + 6  # +6 to get Saturday
+      # Convert epiweek (YYYY-mm-dd Sunday) to Saturday end-of-week
+      time = as.Date(epiweek) + 6
     ) %>%
     # Select and rename columns to standard format
     # wili = weighted ILI percentage, ili = unweighted ILI percentage