Update age data

Author: HughParsonage

.Rbuildignore

@@ -12,6 +12,7 @@
 ^timings$
 ^docs$
 ^vignettes/lito-analysis-budget2018\\.R$
+^vignettes/long-vignettes$
 
 ^vignettes/avg-impact-vs-fy-ending-vs-Quintile\.pptx$
 ^vignettes/avg_tax_rates-by-facet-percentile\.csv$

R/aus_pop_qtr.R

@@ -14,9 +14,9 @@ aus_pop_qtr <- function(date_quarter,
     switch(min(length(bad_qtrs), 3),
            stop("Entry ", bad_qtrs, " was not in the correct form."),
            stop("Entries ", bad_qtrs[1], " and ", bad_qtrs[2], " were not in the correct form."),
-           stop("Entry ", bad_qtrs[1], " were not in the right form. ", 
+           stop("Entry ", bad_qtrs[1], " was not in the right form. ", 
                 "There were ", length(bad_qtrs) - 1,
-                "other bad entries."))
+                " other bad entries."))
   }
 
   pop_data <- aus_pop_by_yearqtr

R/sysdata.rda

@@ -690,15 +690,54 @@ Age_pension_deeming_rates_by_Date <-
   .[, type := gsub("_", " ", gsub("threshold_", "", type, fixed = TRUE))] %>%
   .[, .(Date, type, threshold, deeming_rate_below, deeming_rate_above)]
 
+.aus_pop_by_age_yearqtr <-
+  rsdmx::readSDMX(paste0("http://stat.data.abs.gov.au/restsdmx/sdmx.ashx/GetData/ERP_QUARTERLY/1.0.3.0",
+                         paste0(c("", 1:100), collapse = "+"),
+                         ".Q/all?startTime=1981-Q3")) %>%
+  as.data.frame %>%
+  as.data.table 
+
+if ("AGE" %in% names(.aus_pop_by_age_yearqtr)) {
+  setnames(.aus_pop_by_age_yearqtr, "AGE", "Age")
+}
+
+if (nrow(.aus_pop_by_age_yearqtr) > 14e3L) {
+  fwrite(drop_constant_cols(.aus_pop_by_age_yearqtr), 
+         file = "./data-raw/Estim-Resi-Pop-by-age-1981-present.csv")
+}
+
+.aus_pop_by_yearqtr <-
+  rsdmx::readSDMX("http://stat.data.abs.gov.au/restsdmx/sdmx.ashx/GetData/ERP_QUARTERLY/1.0.3.TT.Q/all?startTime=1981-Q3&endTime=2017-Q3") %>%
+  as.data.frame %>%
+  as.data.table
+
+
+
+if (nrow(.aus_pop_by_yearqtr) > 145) {
+  fwrite(drop_constant_cols(.aus_pop_by_yearqtr), 
+         file = "./data-raw/Estim-Res-Pop-1981-present.tsv",
+         sep = "\t")
+}
+
+austres_reshaped <- 
+  data.table(obsTime = paste0(floor(1971 + seq_along(austres) / 4),
+                              "-Q", 1 + seq_along(austres) %% 4),
+             obsValue = as.integer(1000 * as.numeric(austres))) %>%
+  .[obsTime < "1981-Q3"]
+
 aus_pop_by_yearqtr <- 
-  fread("./data-raw/Estim-Res-Pop-1981-2017.tsv")
+  rbind(austres_reshaped, 
+        fread("./data-raw/Estim-Res-Pop-1981-present.tsv")) %>%
+  setkey(obsTime)
 
 aust_pop_by_age_yearqtr <- 
-  fread("./data-raw/Estim-Resi-Pop-by-age-1981-2016.csv", 
-        select = c("Age", "Time", "Value")) %>%
+  fread("./data-raw/Estim-Resi-Pop-by-age-1981-present.csv") %>%
+  .[, Q := as.integer(substr(obsTime, 7, 7))] %>%
+  .[, Y := substr(obsTime, 0, 4)] %>%
+  .[, date_char := paste0("01-", month.name[(Q)*3], "-", Y)] %>%
   .[, .(Age, 
-        Date = as.Date(paste0("01-", Time), format = "%d-%b-%y"),
-        Value)] %>%
+        Date = as.Date(date_char, format = "%d-%B-%Y"),
+        Value = obsValue)] %>%
   setkey(Age, Date)
 
 download.file("http://www.ausstats.abs.gov.au/ausstats/meisubs.nsf/LatestTimeSeries/5206001_key_aggregates/$FILE/5206001_key_aggregates.xls",

data-raw/5206001_key_aggregates-latest-release.xls

@@ -1,4 +1,45 @@
 obsTime	obsValue
+1971-Q2	13067300
+1971-Q3	13130500
+1971-Q4	13198400
+1972-Q1	13254200
+1972-Q2	13303700
+1972-Q3	13353900
+1972-Q4	13409300
+1973-Q1	13459200
+1973-Q2	13504500
+1973-Q3	13552600
+1973-Q4	13614300
+1974-Q1	13669500
+1974-Q2	13722600
+1974-Q3	13772100
+1974-Q4	13832000
+1975-Q1	13862600
+1975-Q2	13893000
+1975-Q3	13926800
+1975-Q4	13968900
+1976-Q1	14004700
+1976-Q2	14033100
+1976-Q3	14066000
+1976-Q4	14110100
+1977-Q1	14155600
+1977-Q2	14192200
+1977-Q3	14231700
+1977-Q4	14281500
+1978-Q1	14330300
+1978-Q2	14359300
+1978-Q3	14396600
+1978-Q4	14430800
+1979-Q1	14478400
+1979-Q2	14515700
+1979-Q3	14554900
+1979-Q4	14602500
+1980-Q1	14646400
+1980-Q2	14695400
+1980-Q3	14746600
+1980-Q4	14807400
+1981-Q1	14874400
+1981-Q2	14923300
 1981-Q3	14988677
 1981-Q4	15054117
 1982-Q1	15121698