Skip to content

Commit 38b7317

Browse files
authored
Better handling of compressed bodies (#664)
* In `req_dry_run()`, print headers/body from response * In `req_perform()`, print body after response is received. Fixes #91. Fixes #656. Closes #427.
1 parent b337cfb commit 38b7317

17 files changed

+397
-240
lines changed

NEWS.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# httr2 (development version)
22

3+
* `req_dry_run()` and `req_verbose()` now do a better job of displaying compressed bodies (#91, #656).
34
* `resp_link_url()` once again ignores the case of headers (@DavidRLovell, #655)
45
* `oauth_client()` and `oauth_token()` gain refreshed print methods that use bulleted lists, like other httr2 objects. Additionally, print a `oauth_client()` with a custom `auth` function no longer errors (#648).
56
* `req_headers()` always redacts `Authorization` (#649).

R/content-type.R

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,3 +115,29 @@ check_content_type <- function(content_type,
115115
call = call
116116
)
117117
}
118+
119+
120+
is_text_type <- function(content_type) {
121+
if (is.null(content_type)) {
122+
return(FALSE)
123+
}
124+
125+
parsed <- parse_content_type(content_type)
126+
if (parsed$type == "text") {
127+
return(TRUE)
128+
}
129+
130+
special_cases <- c(
131+
"application/xml",
132+
"application/x-www-form-urlencoded",
133+
"application/json",
134+
"application/ld+json",
135+
"multipart/form-data"
136+
)
137+
base_type <- paste0(parsed$type, "/", parsed$subtype)
138+
if (base_type %in% special_cases) {
139+
return(TRUE)
140+
}
141+
142+
FALSE
143+
}

R/req-dry-run.R

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
#' Perform a dry run
2+
#'
3+
#' This shows you exactly what httr2 will send to the server, without
4+
#' actually sending anything. It requires the httpuv package because it
5+
#' works by sending the real HTTP request to a local webserver, thanks to
6+
#' the magic of [curl::curl_echo()].
7+
#'
8+
#' ## Limitations
9+
#'
10+
#' * The `Host` header is not respected.
11+
#'
12+
#' @inheritParams req_verbose
13+
#' @param quiet If `TRUE` doesn't print anything.
14+
#' @returns Invisibly, a list containing information about the request,
15+
#' including `method`, `path`, and `headers`.
16+
#' @export
17+
#' @examples
18+
#' # httr2 adds default User-Agent, Accept, and Accept-Encoding headers
19+
#' request("http://example.com") |> req_dry_run()
20+
#'
21+
#' # the Authorization header is automatically redacted to avoid leaking
22+
#' # credentials on the console
23+
#' req <- request("http://example.com") |> req_auth_basic("user", "password")
24+
#' req |> req_dry_run()
25+
#'
26+
#' # if you need to see it, use redact_headers = FALSE
27+
#' req |> req_dry_run(redact_headers = FALSE)
28+
req_dry_run <- function(req, quiet = FALSE, redact_headers = TRUE) {
29+
check_request(req)
30+
check_installed("httpuv")
31+
32+
req <- req_prepare(req)
33+
handle <- req_handle(req)
34+
curl::handle_setopt(handle, url = req$url)
35+
resp <- curl::curl_echo(handle, progress = FALSE)
36+
37+
if (!quiet) {
38+
cli::cat_line(resp$method, " ", resp$path, " HTTP/1.1")
39+
40+
headers <- headers_redact(
41+
as_headers(as.list(resp$headers)),
42+
redact = redact_headers,
43+
to_redact = attr(req$headers, "redact")
44+
)
45+
cli::cat_line(cli::style_bold(names(headers)), ": ", headers)
46+
cli::cat_line()
47+
show_body(resp$body, headers$`content-type`)
48+
}
49+
50+
invisible(list(
51+
method = resp$method,
52+
path = resp$path,
53+
headers = as.list(resp$headers)
54+
))
55+
}
56+
57+
show_body <- function(body, content_type, prefix = "") {
58+
if (is.null(body)) {
59+
return(invisible())
60+
}
61+
62+
if (is_text_type(content_type)) {
63+
body <- rawToChar(body)
64+
body <- gsub("\n", paste0("\n", prefix), body)
65+
Encoding(body) <- "UTF-8"
66+
cli::cat_line(prefix, body)
67+
} else {
68+
cli::cat_line(prefix, "<", length(body), " bytes>")
69+
}
70+
71+
invisible()
72+
}

R/req-options.R

Lines changed: 0 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -127,95 +127,6 @@ req_proxy <- function(req, url, port = NULL, username = NULL, password = NULL, a
127127
}
128128

129129

130-
#' Show extra output when request is performed
131-
#'
132-
#' @description
133-
#' `req_verbose()` uses the following prefixes to distinguish between
134-
#' different components of the HTTP requests and responses:
135-
#'
136-
#' * `* ` informative curl messages
137-
#' * `->` request headers
138-
#' * `>>` request body
139-
#' * `<-` response headers
140-
#' * `<<` response body
141-
#'
142-
#' @inheritParams req_perform
143-
#' @param header_req,header_resp Show request/response headers?
144-
#' @param body_req,body_resp Should request/response bodies? When the response
145-
#' body is compressed, this will show the number of bytes received in
146-
#' each "chunk".
147-
#' @param info Show informational text from curl? This is mainly useful
148-
#' for debugging https and auth problems, so is disabled by default.
149-
#' @param redact_headers Redact confidential data in the headers? Currently
150-
#' redacts the contents of the Authorization header to prevent you from
151-
#' accidentally leaking credentials when debugging/reprexing.
152-
#' @seealso [req_perform()] which exposes a limited subset of these options
153-
#' through the `verbosity` argument and [with_verbosity()] which allows you
154-
#' to control the verbosity of requests deeper within the call stack.
155-
#' @returns A modified HTTP [request].
156-
#' @export
157-
#' @examples
158-
#' # Use `req_verbose()` to see the headers that are sent back and forth when
159-
#' # making a request
160-
#' resp <- request("https://httr2.r-lib.org") |>
161-
#' req_verbose() |>
162-
#' req_perform()
163-
#'
164-
#' # Or use one of the convenient shortcuts:
165-
#' resp <- request("https://httr2.r-lib.org") |>
166-
#' req_perform(verbosity = 1)
167-
req_verbose <- function(req,
168-
header_req = TRUE,
169-
header_resp = TRUE,
170-
body_req = FALSE,
171-
body_resp = FALSE,
172-
info = FALSE,
173-
redact_headers = TRUE) {
174-
check_request(req)
175-
176-
to_redact <- attr(req$headers, "redact")
177-
debug <- function(type, msg) {
178-
switch(type + 1,
179-
text = if (info) verbose_message("* ", msg),
180-
headerOut = if (header_resp) verbose_header("<- ", msg),
181-
headerIn = if (header_req) verbose_header("-> ", msg, redact_headers, to_redact = to_redact),
182-
dataOut = if (body_resp) verbose_message("<< ", msg),
183-
dataIn = if (body_req) verbose_message(">> ", msg)
184-
)
185-
}
186-
req_options(req, debugfunction = debug, verbose = TRUE)
187-
}
188-
189-
190-
# helpers -----------------------------------------------------------------
191-
192-
verbose_message <- function(prefix, x) {
193-
if (any(x > 128)) {
194-
# This doesn't handle unicode, but it seems like most output
195-
# will be compressed in some way, so displaying bodies is unlikely
196-
# to be useful anyway.
197-
lines <- paste0(length(x), " bytes of binary data")
198-
} else {
199-
x <- readBin(x, character())
200-
lines <- unlist(strsplit(x, "\r?\n", useBytes = TRUE))
201-
}
202-
cli::cat_line(prefix, lines)
203-
}
204-
205-
verbose_header <- function(prefix, x, redact = TRUE, to_redact = NULL) {
206-
x <- readBin(x, character())
207-
lines <- unlist(strsplit(x, "\r?\n", useBytes = TRUE))
208-
209-
for (line in lines) {
210-
if (grepl("^[-a-zA-z0-9]+:", line)) {
211-
header <- headers_redact(as_headers(line), redact, to_redact = to_redact)
212-
cli::cat_line(prefix, cli::style_bold(names(header)), ": ", header)
213-
} else {
214-
cli::cat_line(prefix, line)
215-
}
216-
}
217-
}
218-
219130
auth_flags <- function(x = "basic") {
220131
constants <- c(
221132
basic = 1,

R/req-perform.R

Lines changed: 8 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,10 @@ req_perform <- function(
149149
}
150150

151151
handle_resp <- function(req, resp, error_call = caller_env()) {
152+
if (resp_show_body(resp)) {
153+
show_body(resp$body, resp$headers$`content-type`, prefix = "<< ")
154+
}
155+
152156
if (is_error(resp)) {
153157
cnd_signal(resp)
154158
} else if (error_is_error(req, resp)) {
@@ -224,58 +228,6 @@ last_request <- function() {
224228
the$last_request
225229
}
226230

227-
#' Perform a dry run
228-
#'
229-
#' This shows you exactly what httr2 will send to the server, without
230-
#' actually sending anything. It requires the httpuv package because it
231-
#' works by sending the real HTTP request to a local webserver, thanks to
232-
#' the magic of [curl::curl_echo()].
233-
#'
234-
#' ## Limitations
235-
#'
236-
#' * The `Host` header is not respected.
237-
#'
238-
#' @inheritParams req_verbose
239-
#' @param quiet If `TRUE` doesn't print anything.
240-
#' @returns Invisibly, a list containing information about the request,
241-
#' including `method`, `path`, and `headers`.
242-
#' @export
243-
#' @examples
244-
#' # httr2 adds default User-Agent, Accept, and Accept-Encoding headers
245-
#' request("http://example.com") |> req_dry_run()
246-
#'
247-
#' # the Authorization header is automatically redacted to avoid leaking
248-
#' # credentials on the console
249-
#' req <- request("http://example.com") |> req_auth_basic("user", "password")
250-
#' req |> req_dry_run()
251-
#'
252-
#' # if you need to see it, use redact_headers = FALSE
253-
#' req |> req_dry_run(redact_headers = FALSE)
254-
req_dry_run <- function(req, quiet = FALSE, redact_headers = TRUE) {
255-
check_request(req)
256-
check_installed("httpuv")
257-
258-
if (!quiet) {
259-
to_redact <- attr(req$headers, "redact")
260-
debug <- function(type, msg) {
261-
if (type == 2L) verbose_header("", msg, redact = redact_headers, to_redact = to_redact)
262-
if (type == 4L) verbose_message("", msg)
263-
}
264-
req <- req_options(req, debugfunction = debug, verbose = TRUE)
265-
}
266-
267-
req <- req_prepare(req)
268-
handle <- req_handle(req)
269-
curl::handle_setopt(handle, url = req$url)
270-
resp <- curl::curl_echo(handle, progress = FALSE)
271-
272-
invisible(list(
273-
method = resp$method,
274-
path = resp$path,
275-
headers = as.list(resp$headers)
276-
))
277-
}
278-
279231
# Must call req_prepare(), then req_handle(), then after the request has been
280232
# performed, req_completed()
281233
req_prepare <- function(req) {
@@ -304,3 +256,7 @@ req_completed <- function(req) {
304256

305257
new_path <- function(x) structure(x, class = "httr2_path")
306258
is_path <- function(x) inherits(x, "httr2_path")
259+
260+
resp_show_body <- function(resp) {
261+
resp$request$policies$show_body %||% FALSE
262+
}

R/req-verbose.R

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
2+
#' Show extra output when request is performed
3+
#'
4+
#' @description
5+
#' `req_verbose()` uses the following prefixes to distinguish between
6+
#' different components of the HTTP requests and responses:
7+
#'
8+
#' * `* ` informative curl messages
9+
#' * `->` request headers
10+
#' * `>>` request body
11+
#' * `<-` response headers
12+
#' * `<<` response body
13+
#'
14+
#' @inheritParams req_perform
15+
#' @param header_req,header_resp Show request/response headers?
16+
#' @param body_req,body_resp Should request/response bodies? When the response
17+
#' body is compressed, this will show the number of bytes received in
18+
#' each "chunk".
19+
#' @param info Show informational text from curl? This is mainly useful
20+
#' for debugging https and auth problems, so is disabled by default.
21+
#' @param redact_headers Redact confidential data in the headers? Currently
22+
#' redacts the contents of the Authorization header to prevent you from
23+
#' accidentally leaking credentials when debugging/reprexing.
24+
#' @seealso [req_perform()] which exposes a limited subset of these options
25+
#' through the `verbosity` argument and [with_verbosity()] which allows you
26+
#' to control the verbosity of requests deeper within the call stack.
27+
#' @returns A modified HTTP [request].
28+
#' @export
29+
#' @examples
30+
#' # Use `req_verbose()` to see the headers that are sent back and forth when
31+
#' # making a request
32+
#' resp <- request("https://httr2.r-lib.org") |>
33+
#' req_verbose() |>
34+
#' req_perform()
35+
#'
36+
#' # Or use one of the convenient shortcuts:
37+
#' resp <- request("https://httr2.r-lib.org") |>
38+
#' req_perform(verbosity = 1)
39+
req_verbose <- function(req,
40+
header_req = TRUE,
41+
header_resp = TRUE,
42+
body_req = FALSE,
43+
body_resp = FALSE,
44+
info = FALSE,
45+
redact_headers = TRUE) {
46+
check_request(req)
47+
48+
to_redact <- attr(req$headers, "redact")
49+
debug <- function(type, msg) {
50+
switch(type + 1,
51+
text = if (info) verbose_message("* ", msg),
52+
headerOut = if (header_resp) verbose_header("<- ", msg),
53+
headerIn = if (header_req) verbose_header("-> ", msg, redact_headers, to_redact = to_redact),
54+
dataOut = NULL, # displayed in handle_resp()
55+
dataIn = if (body_req) verbose_message(">> ", msg)
56+
)
57+
}
58+
req <- req_policies(req, show_body = body_resp)
59+
req <- req_options(req, debugfunction = debug, verbose = TRUE)
60+
req
61+
}
62+
63+
# helpers -----------------------------------------------------------------
64+
65+
verbose_message <- function(prefix, x) {
66+
if (any(x > 128)) {
67+
# This doesn't handle unicode, but it seems like most output
68+
# will be compressed in some way, so displaying bodies is unlikely
69+
# to be useful anyway.
70+
lines <- paste0(length(x), " bytes of binary data")
71+
} else {
72+
x <- readBin(x, character())
73+
lines <- unlist(strsplit(x, "\r?\n", useBytes = TRUE))
74+
}
75+
cli::cat_line(prefix, lines)
76+
}
77+
78+
verbose_header <- function(prefix, x, redact = TRUE, to_redact = NULL) {
79+
x <- readBin(x, character())
80+
lines <- unlist(strsplit(x, "\r?\n", useBytes = TRUE))
81+
82+
for (line in lines) {
83+
if (grepl("^[-a-zA-z0-9]+:", line)) {
84+
header <- headers_redact(as_headers(line), redact, to_redact = to_redact)
85+
cli::cat_line(prefix, cli::style_bold(names(header)), ": ", header)
86+
} else {
87+
cli::cat_line(prefix, line)
88+
}
89+
}
90+
}
91+
92+
# Testing helpers -------------------------------------------------------------
93+
94+
# Reset all headers that otherwise might vary
95+
req_headers_reset <- function(req) {
96+
req_headers(req, `Accept-Encoding` = "", Host = "http://example.com", `User-Agent` = "")
97+
}
98+
99+
transform_resp_headers <- function(lines) {
100+
lines <- gsub(example_url(), "<webfakes>", lines, fixed = TRUE)
101+
lines <- lines[!grepl("^<- (Date|ETag|Content-Length):", lines)]
102+
lines
103+
}

man/req_dry_run.Rd

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)