Skip to content

Commit 872d4fe

Browse files
ldecicco-USGSldecicco-USGS
authored
Merge pull request #835 from ldecicco-USGS/max_results
Remove max_results
2 parents e397bbc + e7cd2dc commit 872d4fe

29 files changed

+165
-307
lines changed

‎NEWS‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ is not stable over time. Moved other "id" columns to end of returned data frames
99
encouraged to migrate to the "read_waterdata_metadata" functions.
1010
* Added no_paging argument. This will make the request more efficient, but is not
1111
recommended because it will silently cut off data after 50,000 rows.
12+
* Removed max_results argument. Was confusing and redundant with the combination
13+
of no_paging and limit.
1214

1315
dataRetrieval 2.7.21
1416
===================

‎R/construct_api_requests.R‎

Lines changed: 25 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -136,28 +136,9 @@ check_limits <- function(args){
136136
current_api_limit <- 50000
137137

138138
if(is.na(args[["limit"]])){
139-
if(!is.na(args[["max_results"]])){
140-
# we can leave limit empty unless we're doing no paging and the max is > limit
141-
if(args[["max_results"]] > current_api_limit){
142-
args[["limit"]] <- current_api_limit
143-
if(args[["no_paging"]]){
144-
warning("no_paging option is capped at ", current_api_limit, " max_results")
145-
args[["max_results"]] <- current_api_limit
146-
}
147-
} else {
148-
args[["limit"]] <- args[["max_results"]]
149-
}
150-
151-
} else {
152-
args[["limit"]] <- current_api_limit
153-
}
154-
} else {
155-
if(!is.na(args[["max_results"]])){
156-
if(args[["limit"]] > args[["max_results"]]) stop("limit cannot be greater than max_result")
157-
} else if (args[["limit"]] > current_api_limit){
158-
args[["limit"]] <- current_api_limit
159-
}
160-
}
139+
args[["limit"]] <- current_api_limit
140+
}
141+
161142
return(args)
162143
}
163144

@@ -283,6 +264,11 @@ switch_arg_id <- function(ls, id_name, service){
283264
#' start_end2 <- c("2021-01-01T12:15:00-0500", "")
284265
#' dataRetrieval:::format_api_dates(start_end2)
285266
#'
267+
#' time = c("2014-05-01T00:00:00Z", "2014-05-01T12:00:00Z")
268+
#' dataRetrieval:::format_api_dates(time)
269+
#'
270+
#' time = c("2014-05-01T00:00Z", "2014-05-01T12:00Z")
271+
#' dataRetrieval:::format_api_dates(time)
286272
format_api_dates <- function(datetime, date = FALSE){
287273

288274
if(is.character(datetime)){
@@ -296,19 +282,31 @@ format_api_dates <- function(datetime, date = FALSE){
296282
grepl("/", datetime)){
297283
return(datetime)
298284
} else {
285+
datetime1 <- tryCatch({
286+
lubridate::as_datetime(datetime)
287+
},
288+
warning = function(w) {
289+
strptime(datetime, format = "%Y-%m-%dT%H:%MZ", tz = "UTC")
290+
})
299291
if(date){
300-
datetime <- format(lubridate::as_datetime(datetime), "%Y-%m-%d")
292+
datetime <- format(datetime1, "%Y-%m-%d")
301293
} else {
302-
datetime <- lubridate::format_ISO8601(lubridate::as_datetime(datetime), usetz = "Z")
294+
datetime <- lubridate::format_ISO8601(datetime1, usetz = "Z")
303295
}
304296
}
305297
} else if (length(datetime) == 2) {
306298

299+
datetime1 <- tryCatch({
300+
lubridate::as_datetime(datetime)
301+
},
302+
warning = function(w) {
303+
strptime(datetime, format = "%Y-%m-%dT%H:%MZ", tz = "UTC")
304+
})
305+
307306
if(date){
308-
datetime <- paste0(format(lubridate::as_datetime(datetime), "%Y-%m-%d"), collapse = "/")
307+
datetime <- paste0(format(datetime1, "%Y-%m-%d"), collapse = "/")
309308
} else {
310-
datetime <- paste0(lubridate::format_ISO8601(lubridate::as_datetime(datetime),
311-
usetz = "Z"),
309+
datetime <- paste0(lubridate::format_ISO8601(datetime1, usetz = "Z"),
312310
collapse = "/")
313311
}
314312

‎R/get_ogc_data.R‎

Lines changed: 21 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,7 @@
99
get_ogc_data <- function(args,
1010
output_id,
1111
service){
12-
13-
args[["service"]] <- service
14-
12+
1513
args <- switch_arg_id(args,
1614
id_name = output_id,
1715
service = service)
@@ -23,9 +21,7 @@ get_ogc_data <- function(args,
2321
id = output_id)
2422
convertType <- args[["convertType"]]
2523
args[["convertType"]] <- NULL
26-
27-
max_results <- args[["max_results"]]
28-
args[["max_results"]] <- NULL
24+
args[["service"]] <- service
2925

3026
req <- do.call(construct_api_requests, args)
3127

@@ -37,9 +33,9 @@ get_ogc_data <- function(args,
3733
}
3834

3935
if(no_paging){
40-
return_list <- get_csv(req, max_results)
36+
return_list <- get_csv(req, limit = args[["limit"]])
4137
} else {
42-
return_list <- walk_pages(req, max_results)
38+
return_list <- walk_pages(req)
4339
}
4440

4541
if(is.na(args[["skipGeometry"]])){
@@ -51,10 +47,23 @@ get_ogc_data <- function(args,
5147
return_list <- deal_with_empty(return_list, properties, service,
5248
skipGeometry, convertType, no_paging)
5349

54-
if(convertType) return_list <- cleanup_cols(return_list, service = service)
55-
5650
return_list <- rejigger_cols(return_list, properties, output_id)
5751

52+
if(convertType){
53+
return_list <- cleanup_cols(return_list, service)
54+
return_list <- order_results(return_list)
55+
56+
# Mostly drop the id column except ts-meta, monitoring location:
57+
if(!service %in% c("monitoring-locations",
58+
"time-series-metadata",
59+
"parameter-codes")){
60+
return_list <- return_list[, names(return_list)[names(return_list)!= output_id]]
61+
}
62+
# Move other id columns:
63+
return_list <- move_id_col(return_list,
64+
output_id)
65+
}
66+
5867
attr(return_list, "request") <- req
5968
attr(return_list, "queryTime") <- Sys.time()
6069
return_list
@@ -73,11 +82,7 @@ order_results <- function(df){
7382
}
7483

7584
move_id_col <- function(df, output_id){
76-
# attributes get dropped
77-
req <- attr(df, "request")
78-
queryTime <- attr(df, "queryTime")
79-
80-
df <- df[, names(df)[names(df)!= output_id]]
85+
8186
if("time_series_id" %in% names(df)){
8287
df <- df[, c(names(df)[names(df)!= "time_series_id"],
8388
"time_series_id")]
@@ -87,10 +92,7 @@ move_id_col <- function(df, output_id){
8792
df <- df[, c(names(df)[names(df)!= "field_visit_id"],
8893
"field_visit_id")]
8994
}
90-
91-
attr(df, "request") <- req
92-
attr(df, "queryTime") <- queryTime
93-
95+
9496
return(df)
9597
}
9698

‎R/read_waterdata.R‎

Lines changed: 14 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,6 @@ read_waterdata <- function(service,
5959
match.arg(service, pkg.env$api_endpoints)
6060

6161
args <- list(...)
62-
args[["service"]] <- service
6362

6463
output_id <- switch(service,
6564
"daily" = "daily_id",
@@ -75,19 +74,20 @@ read_waterdata <- function(service,
7574
args[["properties"]] <- NA_character_
7675
}
7776

77+
if(!"limit" %in% names(args)){
78+
args[["limit"]] <- NA_character_
79+
}
80+
81+
args[["service"]] <- service
82+
args <- check_limits(args)
83+
7884
data_req <- suppressWarnings(do.call(construct_api_requests, args))
7985

8086
data_req <- data_req |>
8187
httr2::req_headers(`Content-Type` = "application/query-cql-json") |>
8288
httr2::req_body_raw(CQL)
8389

84-
if("max_results" %in% names(args)){
85-
max_results <- args[["max_results"]]
86-
} else {
87-
max_results <- NA
88-
}
89-
90-
return_list <- walk_pages(data_req, max_results)
90+
return_list <- walk_pages(data_req)
9191

9292
if(is.null(args[["skipGeometry"]])){
9393
skipGeometry <- FALSE
@@ -101,16 +101,15 @@ read_waterdata <- function(service,
101101
service,
102102
skipGeometry,
103103
convertType)
104+
105+
return_list <- rejigger_cols(return_list, args[["properties"]], output_id)
104106

105-
if(convertType) return_list <- cleanup_cols(return_list)
106-
107-
# Add other time series services when they come online
108-
if(service %in% c("daily")){
109-
return_list <- return_list[order(return_list$time, return_list$monitoring_location_id), ]
107+
if(convertType){
108+
return_list <- cleanup_cols(return_list, service)
109+
return_list <- order_results(return_list)
110+
return_list <- move_id_col(return_list, output_id)
110111
}
111112

112-
return_list <- rejigger_cols(return_list, args[["properties"]], output_id)
113-
114113
return(return_list)
115114
}
116115

‎R/read_waterdata_continuous.R‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,6 @@
3636
#' limit is 50000. It may be beneficial to set this number lower if your internet
3737
#' connection is spotty. The default (`NA`) will set the limit to the maximum
3838
#' allowable limit for the service.
39-
#' @param max_results The optional maximum number of rows to return. This value
40-
#' must be less than the requested limit.
4139
#' @param convertType logical, defaults to `TRUE`. If `TRUE`, the function
4240
#' will convert the data to dates and qualifier to string vector, and sepcifically
4341
#' order the returning data frame by time and monitoring_location_id.
@@ -80,7 +78,6 @@ read_waterdata_continuous <- function(monitoring_location_id = NA_character_,
8078
last_modified = NA_character_,
8179
time = NA_character_,
8280
limit = NA,
83-
max_results = NA,
8481
convertType = TRUE,
8582
no_paging = FALSE){
8683

@@ -97,12 +94,7 @@ read_waterdata_continuous <- function(monitoring_location_id = NA_character_,
9794
return_list <- get_ogc_data(args,
9895
output_id,
9996
service)
100-
101-
if(convertType){
102-
return_list <- order_results(return_list)
103-
return_list <- move_id_col(return_list, output_id)
104-
}
105-
97+
10698
return(return_list)
10799
}
108100

‎R/read_waterdata_daily.R‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,6 @@
3232
#' limit is 50000. It may be beneficial to set this number lower if your internet
3333
#' connection is spotty. The default (`NA`) will set the limit to the maximum
3434
#' allowable limit for the service.
35-
#' @param max_results The optional maximum number of rows to return. This value
36-
#' must be less than the requested limit.
3735
#' @param skipGeometry This option can be used to skip response geometries for
3836
#' each feature. The returning object will be a data frame with no spatial
3937
#' information.
@@ -72,7 +70,6 @@
7270
#' multi_site <- read_waterdata_daily(monitoring_location_id = c("USGS-01491000",
7371
#' "USGS-01645000"),
7472
#' parameter_code = c("00060", "00010"),
75-
#' limit = 500,
7673
#' time = c("2023-01-01", "2024-01-01"))
7774
#'
7875
#' dv_data_quick <- read_waterdata_daily(monitoring_location_id = site,
@@ -94,7 +91,6 @@ read_waterdata_daily <- function(monitoring_location_id = NA_character_,
9491
time = NA_character_,
9592
bbox = NA,
9693
limit = NA,
97-
max_results = NA,
9894
convertType = TRUE,
9995
no_paging = FALSE){
10096

@@ -106,11 +102,6 @@ read_waterdata_daily <- function(monitoring_location_id = NA_character_,
106102
output_id,
107103
service)
108104

109-
if(convertType){
110-
return_list <- order_results(return_list)
111-
return_list <- move_id_col(return_list, output_id)
112-
}
113-
114105
return(return_list)
115106
}
116107

‎R/read_waterdata_field_measurements.R‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,6 @@
3535
#' limit is 50000. It may be beneficial to set this number lower if your internet
3636
#' connection is spotty. The default (`NA`) will set the limit to the maximum
3737
#' allowable limit for the service.
38-
#' @param max_results The optional maximum number of rows to return. This value
39-
#' must be less than the requested limit.
4038
#' @param skipGeometry This option can be used to skip response geometries for
4139
#' each feature. The returning object will be a data frame with no spatial
4240
#' information.
@@ -97,7 +95,6 @@ read_waterdata_field_measurements <- function(monitoring_location_id = NA_charac
9795
time = NA_character_,
9896
bbox = NA,
9997
limit = NA,
100-
max_results = NA,
10198
convertType = TRUE,
10299
no_paging = FALSE){
103100

@@ -109,11 +106,6 @@ read_waterdata_field_measurements <- function(monitoring_location_id = NA_charac
109106
output_id,
110107
service)
111108

112-
if(convertType){
113-
return_list <- order_results(return_list)
114-
return_list <- move_id_col(return_list, output_id)
115-
}
116-
117109
return(return_list)
118110
}
119111

‎R/read_waterdata_latest_continuous.R‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,6 @@
3434
#' limit is 50000. It may be beneficial to set this number lower if your internet
3535
#' connection is spotty. The default (`NA`) will set the limit to the maximum
3636
#' allowable limit for the service.
37-
#' @param max_results The optional maximum number of rows to return. This value
38-
#' must be less than the requested limit.
3937
#' @param skipGeometry This option can be used to skip response geometries for
4038
#' each feature. The returning object will be a data frame with no spatial
4139
#' information.
@@ -93,7 +91,6 @@ read_waterdata_latest_continuous <- function(monitoring_location_id = NA_charact
9391
time = NA_character_,
9492
bbox = NA,
9593
limit = NA,
96-
max_results = NA,
9794
convertType = TRUE,
9895
no_paging = FALSE){
9996

@@ -104,11 +101,6 @@ read_waterdata_latest_continuous <- function(monitoring_location_id = NA_charact
104101
return_list <- get_ogc_data(args,
105102
output_id,
106103
service)
107-
108-
if(convertType){
109-
return_list <- order_results(return_list)
110-
return_list <- move_id_col(return_list, output_id)
111-
}
112104

113105
return(return_list)
114106
}

‎R/read_waterdata_latest_daily.R‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,6 @@
3232
#' limit is 50000. It may be beneficial to set this number lower if your internet
3333
#' connection is spotty. The default (`NA`) will set the limit to the maximum
3434
#' allowable limit for the service.
35-
#' @param max_results The optional maximum number of rows to return. This value
36-
#' must be less than the requested limit.
3735
#' @param skipGeometry This option can be used to skip response geometries for
3836
#' each feature. The returning object will be a data frame with no spatial
3937
#' information.
@@ -84,7 +82,6 @@ read_waterdata_latest_daily <- function(monitoring_location_id = NA_character_,
8482
time = NA_character_,
8583
bbox = NA,
8684
limit = NA,
87-
max_results = NA,
8885
convertType = TRUE,
8986
no_paging = FALSE){
9087

@@ -96,10 +93,6 @@ read_waterdata_latest_daily <- function(monitoring_location_id = NA_character_,
9693
output_id,
9794
service)
9895

99-
if(convertType){
100-
return_list <- order_results(return_list)
101-
return_list <- move_id_col(return_list, output_id)
102-
}
10396
return(return_list)
10497
}
10598

0 commit comments

Comments
 (0)