Continuous

DESCRIPTION

@@ -25,7 +25,6 @@ Authors@R: c(
             email = "[email protected]",
             comment=c(ORCID = "0000-0003-2521-5043")),
     person("Lee", "Stanish", role="ctb",
-            email = "[email protected]",
             comment=c(ORCID = "0000-0002-9775-6861")),
     person("Joeseph", "Zemmels", role="ctb",
             email = "[email protected]",

NAMESPACE

@@ -46,6 +46,7 @@ export(readWQPqw)
 export(readWQPsummary)
 export(read_USGS_samples)
 export(read_waterdata)
+export(read_waterdata_continuous)
 export(read_waterdata_daily)
 export(read_waterdata_field_measurements)
 export(read_waterdata_latest_continuous)

NEWS

@@ -1,7 +1,10 @@
 dataRetrieval 2.7.22
 ===================
 * Added read_waterdata_latest_daily to access latest daily USGS water data.
+* Added read_waterdata_continuous to access continuous USGS water data.
 * Added state_name and hydrologic_unit_code to read_waterdata_ts_meta
+* Removed daily_id from read_waterdata_daily output. Currently it
+is not stable over time. 
 
 dataRetrieval 2.7.21
 ===================

R/AAA.R

@@ -8,7 +8,8 @@ pkg.env <- new.env()
 
   services <- c("server", "daily", "time-series-metadata",
                "monitoring-locations", "latest-continuous",
-               "field-measurements", "latest-daily")
+               "field-measurements", "latest-daily",
+               "continuous")
   collections <- c("parameter-codes", "agency-codes", "altitude-datums", "aquifer-codes",
                    "aquifer-types", "coordinate-accuracy-codes", "coordinate-datum-codes",
                    "coordinate-method-codes", "medium-codes", 

R/construct_api_requests.R

@@ -68,7 +68,7 @@ construct_api_requests <- function(service,
     if(!is.na(max_results)){
       get_list[["limit"]] <- max_results
     } else {
-      get_list[["limit"]] <- 10000
+      get_list[["limit"]] <- 50000
     }
   } else {
     if(!is.na(max_results)){

R/readNWISunit.R

@@ -91,6 +91,11 @@ readNWISuv <- function(siteNumbers, parameterCd, startDate = "", endDate = "", t
     service <- "iv_recent"
   }
 
+  .Deprecated(new = "read_waterdata_continuous",
+              package = "dataRetrieval", 
+              msg = "NWIS servers are slated for decommission. Please begin to migrate to read_waterdata_continuous.")
+
+
   url <- constructNWISURL(siteNumbers,
     parameterCd,
     startDate,
@@ -360,7 +365,10 @@ readNWISmeas <- function(siteNumbers,
                          expanded = FALSE,
                          convertType = TRUE) {
 
-  message(new_nwis_message())
+  .Deprecated(new = "read_waterdata_field_measurements",
+              package = "dataRetrieval", 
+              msg = "NWIS servers are slated for decommission. Please begin to migrate to read_waterdata_field_measurements.")
+
   # Doesn't seem to be a WaterML1 format option
   url <- constructNWISURL(
     siteNumbers = siteNumbers,

R/read_waterdata_continuous.R

@@ -0,0 +1,113 @@
+#' Get Continuous USGS Water Data
+#' 
+#' @description `r get_description("continuous")`
+#' 
+#' Currently, the services only allow up to 3 years of data to be requested with
+#' a single request. If no "time" is specified, the service will return the 
+#' last single year of data. If this is a bottleneck, please check back 
+#' for new direct download functions that are expected to be available sometime
+#' in 2026.
+#' 
+#' @export
+#' @param monitoring_location_id `r get_params("continuous")$monitoring_location_id`
+#' @param parameter_code `r get_params("continuous")$parameter_code`
+#' @param time `r get_params("continuous")$time`
+#' @param value `r get_params("continuous")$value`
+#' @param unit_of_measure `r get_params("continuous")$unit_of_measure`
+#' @param approval_status `r get_params("continuous")$approval_status`
+#' @param last_modified `r get_params("continuous")$last_modified`
+#' @param time_series_id `r get_params("continuous")$time_series_id`
+#' @param qualifier `r get_params("continuous")$qualifier`
+#' @param statistic_id `r get_params("continuous")$statistic_id`. Note that 
+#' for continuous data, the statistic_id is almost universally 00011. 
+#' Requesting anything else will most-likely cause a timeout. 
+#' @param properties A vector of requested columns to be returned from the query.
+#' Available options are: 
+#' `r schema <- check_OGC_requests(endpoint = "continuous", type = "schema"); paste(names(schema$properties)[!names(schema$properties) %in% c("id", "internal_id")], collapse = ", ")`
+#' @param limit The optional limit parameter is used to control the subset of the 
+#' selected features that should be returned in each page. The maximum allowable
+#' limit is 50000. It may be beneficial to set this number lower if your internet
+#' connection is spotty. The default (`NA`) will set the limit to the maximum
+#' allowable limit for the service.
+#' @param max_results The optional maximum number of rows to return. This value
+#' must be less than the requested limit. 
+#' @param convertType logical, defaults to `TRUE`. If `TRUE`, the function
+#' will convert the data to dates and qualifier to string vector, and sepcifically
+#' order the returning data frame by time and monitoring_location_id.
+#' @examplesIf is_dataRetrieval_user()
+#' 
+#' \donttest{
+#' site <- "USGS-451605097071701"
+#' pcode <- "72019"
+#'
+#' uv_data_trim <- read_waterdata_continuous(monitoring_location_id = site,
+#'                           parameter_code = pcode, 
+#'                           properties = c("value",
+#'                                          "time"))
+#'
+#' uv_data <- read_waterdata_continuous(monitoring_location_id = site,
+#'                            parameter_code = pcode,
+#'                            time = "P2D")
+#' 
+#' 
+#' # Only return data that has been modified in last 7 days         
+#' multi_site2 <- read_waterdata_continuous(monitoring_location_id =  c("USGS-451605097071701",
+#'                                                                      "USGS-14181500"),
+#'                                                parameter_code = c("00060", "72019"),
+#'                                                last_modified = "P7D")
+#' 
+#' }
+read_waterdata_continuous <- function(monitoring_location_id = NA_character_,
+                                      parameter_code = NA_character_,
+                                      properties = NA_character_,
+                                      time_series_id = NA_character_,
+                                      approval_status = NA_character_,
+                                      unit_of_measure = NA_character_,
+                                      qualifier = NA_character_,
+                                      statistic_id = NA_character_,
+                                      value = NA,
+                                      last_modified = NA_character_,
+                                      time = NA_character_,
+                                      limit = NA,
+                                      max_results = NA,
+                                      convertType = TRUE){
+
+  service <- "continuous"
+  output_id <- "continuous_id"
+
+  args <- mget(names(formals()))
+  args[["skipGeometry"]] <- TRUE
+
+  if(!is.na(statistic_id) & !all(statistic_id == "00011")){
+    warning("With few if any exceptions, statistic_id is always 00011 for continuous data, and requesting other statistic ids will likely return no data.")
+  }
+
+  return_list <- get_ogc_data(args,
+                              output_id, 
+                              service)
+
+  if(convertType){
+    return_list <- order_results(return_list, properties)
+    return_list <- return_list[, names(return_list)[names(return_list)!= output_id]]
+    if("time_series_id" %in% names(return_list)){
+      return_list <- return_list[, c( names(return_list)[names(return_list)!= "time_series_id"],
+                                      "time_series_id")]
+    }
+  }
+
+  return(return_list)
+}
+
+order_results <- function(return_list, properties){
+
+  if(all(is.na(properties)) | 
+                    all(c("time", "monitoring_location_id") %in% properties)){
+    return_list <- return_list[order(return_list$time, 
+                                     return_list$monitoring_location_id), ]
+  } else if ("time" %in% properties) {
+    return_list <- return_list[order(return_list$time), ]    
+  }
+
+  return(return_list)
+}
+

R/read_waterdata_daily.R

@@ -13,10 +13,9 @@
 #' @param last_modified `r get_params("daily")$last_modified`
 #' @param time_series_id `r get_params("daily")$time_series_id`
 #' @param qualifier `r get_params("daily")$qualifier`
-#' @param daily_id `r get_params("daily")$id`
 #' @param properties A vector of requested columns to be returned from the query.
 #' Available options are: 
-#' `r schema <- check_OGC_requests(endpoint = "daily", type = "schema"); paste(names(schema$properties), collapse = ", ")`
+#' `r schema <- check_OGC_requests(endpoint = "daily", type = "schema"); paste(names(schema$properties)[!names(schema$properties) %in% c("id")], collapse = ", ")`
 #' @param bbox Only features that have a geometry that intersects the bounding
 #' box are selected.The bounding box is provided as four or six numbers, depending
 #' on whether the coordinate reference system includes a vertical axis (height or
@@ -25,7 +24,7 @@
 #' Southern-most latitude, Eastern-most longitude, Northern-most longitude).
 #' @param limit The optional limit parameter is used to control the subset of the 
 #' selected features that should be returned in each page. The maximum allowable
-#' limit is 10000. It may be beneficial to set this number lower if your internet
+#' limit is 50000. It may be beneficial to set this number lower if your internet
 #' connection is spotty. The default (`NA`) will set the limit to the maximum
 #' allowable limit for the service.
 #' @param max_results The optional maximum number of rows to return. This value
@@ -39,7 +38,6 @@
 #' 
 #' \donttest{
 #' site <- "USGS-02238500"
-#' pcode <- "00060"
 #' dv_data_sf <- read_waterdata_daily(monitoring_location_id = site,
 #'                               parameter_code = "00060", 
 #'                               time = c("2021-01-01", "2022-01-01"))
@@ -50,8 +48,7 @@
 #'
 #' dv_data_trim <- read_waterdata_daily(monitoring_location_id = site,
 #'                           parameter_code = "00060", 
-#'                           properties = c("monitoring_location_id",
-#'                                          "value",
+#'                           properties = c("value",
 #'                                          "time"),
 #'                           time = c("2021-01-01", "2022-01-01"))
 #'
@@ -71,22 +68,21 @@
 #' 
 #' }
 read_waterdata_daily <- function(monitoring_location_id = NA_character_,
-                            parameter_code = NA_character_,
-                            statistic_id = NA_character_,
-                            properties = NA_character_,
-                            time_series_id = NA_character_,
-                            daily_id = NA_character_,
-                            approval_status = NA_character_,
-                            unit_of_measure = NA_character_,
-                            qualifier = NA_character_,
-                            value = NA,
-                            last_modified = NA_character_,
-                            skipGeometry = NA,
-                            time = NA_character_,
-                            bbox = NA,
-                            limit = NA,
-                            max_results = NA,
-                            convertType = TRUE){
+                                 parameter_code = NA_character_,
+                                 statistic_id = NA_character_,
+                                 properties = NA_character_,
+                                 time_series_id = NA_character_,
+                                 approval_status = NA_character_,
+                                 unit_of_measure = NA_character_,
+                                 qualifier = NA_character_,
+                                 value = NA,
+                                 last_modified = NA_character_,
+                                 skipGeometry = NA,
+                                 time = NA_character_,
+                                 bbox = NA,
+                                 limit = NA,
+                                 max_results = NA,
+                                 convertType = TRUE){
 
   service <- "daily"
   output_id <- "daily_id"
@@ -96,7 +92,14 @@ read_waterdata_daily <- function(monitoring_location_id = NA_character_,
                               output_id, 
                               service)
 
-  return_list <- return_list[order(return_list$time, return_list$monitoring_location_id), ]
+  if(convertType){
+    return_list <- order_results(return_list, properties)
+    return_list <- return_list[,names(return_list)[names(return_list)!= output_id]]
+    if("time_series_id" %in% names(return_list)){
+      return_list <- return_list[, c( names(return_list)[names(return_list)!= "time_series_id"],
+                                      "time_series_id")]
+    }
+  }
 
   return(return_list)
 }

R/read_waterdata_field_measurements.R

@@ -18,7 +18,7 @@
 #' @param measuring_agency `r get_params("field-measurements")$measuring_agency`
 #' @param properties A vector of requested columns to be returned from the query.
 #' Available options are: 
-#' `r schema <- check_OGC_requests(endpoint = "field-measurements", type = "schema"); paste(names(schema$properties), collapse = ", ")`
+#' `r schema <- check_OGC_requests(endpoint = "field-measurements", type = "schema"); paste(names(schema$properties)[!names(schema$properties) %in% c("id")], collapse = ", ")`
 #' @param bbox Only features that have a geometry that intersects the bounding
 #' box are selected.The bounding box is provided as four or six numbers, depending
 #' on whether the coordinate reference system includes a vertical axis (height or
@@ -27,7 +27,7 @@
 #' Southern-most latitude, Eastern-most longitude, Northern-most longitude).
 #' @param limit The optional limit parameter is used to control the subset of the 
 #' selected features that should be returned in each page. The maximum allowable
-#' limit is 10000. It may be beneficial to set this number lower if your internet
+#' limit is 50000. It may be beneficial to set this number lower if your internet
 #' connection is spotty. The default (`NA`) will set the limit to the maximum
 #' allowable limit for the service.
 #' @param max_results The optional maximum number of rows to return. This value
@@ -99,7 +99,14 @@ read_waterdata_field_measurements <- function(monitoring_location_id = NA_charac
                               output_id, 
                               service)
 
-  return_list <- return_list[order(return_list$time, return_list$monitoring_location_id), ]
+  if(convertType){
+    return_list <- order_results(return_list, properties)
+    return_list <- return_list[,names(return_list)[names(return_list)!= output_id]]
+    if("field_visit_id" %in% names(return_list)){
+      return_list <- return_list[, c( names(return_list)[names(return_list)!= "field_visit_id"],
+                                      "field_visit_id")]
+    }
+  }
 
   return(return_list)
 }

R/read_waterdata_latest_continuous.R

@@ -5,18 +5,19 @@
 #' @export
 #' @param monitoring_location_id `r get_params("latest-continuous")$monitoring_location_id`
 #' @param parameter_code `r get_params("latest-continuous")$parameter_code`
-#' @param statistic_id `r get_params("latest-continuous")$statistic_id`
 #' @param time `r get_params("latest-continuous")$time`
 #' @param value `r get_params("latest-continuous")$value`
 #' @param unit_of_measure `r get_params("latest-continuous")$unit_of_measure`
 #' @param approval_status `r get_params("latest-continuous")$approval_status`
 #' @param last_modified `r get_params("latest-continuous")$last_modified`
 #' @param time_series_id `r get_params("latest-continuous")$time_series_id`
 #' @param qualifier `r get_params("latest-continuous")$qualifier`
-#' @param latest_continuous_id `r get_params("latest-continuous")$id`
+#' @param statistic_id `r get_params("latest-continuous")$statistic_id`. Note that 
+#' for continuous data, the statistic_id is almost universally 00011. 
+#' Requesting anything else will most-likely cause a timeout. 
 #' @param properties A vector of requested columns to be returned from the query.
 #' Available options are: 
-#' `r schema <- check_OGC_requests(endpoint = "latest-continuous", type = "schema"); paste(names(schema$properties), collapse = ", ")`
+#' `r schema <- check_OGC_requests(endpoint = "latest-continuous", type = "schema"); paste(names(schema$properties)[!names(schema$properties) %in% c("id")], collapse = ", ")`
 #' @param bbox Only features that have a geometry that intersects the bounding
 #' box are selected.The bounding box is provided as four or six numbers, depending
 #' on whether the coordinate reference system includes a vertical axis (height or
@@ -25,7 +26,7 @@
 #' Southern-most latitude, Eastern-most longitude, Northern-most longitude).
 #' @param limit The optional limit parameter is used to control the subset of the 
 #' selected features that should be returned in each page. The maximum allowable
-#' limit is 10000. It may be beneficial to set this number lower if your internet
+#' limit is 50000. It may be beneficial to set this number lower if your internet
 #' connection is spotty. The default (`NA`) will set the limit to the maximum
 #' allowable limit for the service.
 #' @param max_results The optional maximum number of rows to return. This value
@@ -74,7 +75,6 @@ read_waterdata_latest_continuous <- function(monitoring_location_id = NA_charact
                             statistic_id = NA_character_,
                             properties = NA_character_,
                             time_series_id = NA_character_,
-                            latest_continuous_id = NA_character_,
                             approval_status = NA_character_,
                             unit_of_measure = NA_character_,
                             qualifier = NA_character_,
@@ -95,7 +95,14 @@ read_waterdata_latest_continuous <- function(monitoring_location_id = NA_charact
                               output_id, 
                               service)
 
-  return_list <- return_list[order(return_list$time, return_list$monitoring_location_id), ]
+  if(convertType){
+    return_list <- order_results(return_list, properties)
+    return_list <- return_list[, names(return_list)[names(return_list)!= output_id]]
+    if("time_series_id" %in% names(return_list)){
+      return_list <- return_list[, c( names(return_list)[names(return_list)!= "time_series_id"],
+                                      "time_series_id")]
+    }
+  }
 
   return(return_list)
 }