Allow recording the raw HTTP/1 headers sent and received

This allows feeding them into a WARC file
(https://en.wikipedia.org/wiki/WARC_(file_format)). The rest of the
request and response is already available as it's either the explicitly
set body or the received response body.

This includes the final `\r\n\r\n` between the headers and the response
(so that it can be distinguished from bare `\n\n`).

Needed for a similar request in reqwest:
seanmonstar/reqwest#1229.

Author: dis-archive

src/client/conn/http1.rs

@@ -112,6 +112,8 @@ pub struct Builder {
     h1_parser_config: ParserConfig,
     h1_writev: Option<bool>,
     h1_title_case_headers: bool,
+    h1_record_raw_request_headers: bool,
+    h1_record_raw_response_headers: bool,
     h1_preserve_header_case: bool,
     h1_max_headers: Option<usize>,
     #[cfg(feature = "ffi")]
@@ -312,6 +314,8 @@ impl Builder {
             h1_read_buf_exact_size: None,
             h1_parser_config: Default::default(),
             h1_title_case_headers: false,
+            h1_record_raw_request_headers: false,
+            h1_record_raw_response_headers: false,
             h1_preserve_header_case: false,
             h1_max_headers: None,
             #[cfg(feature = "ffi")]
@@ -428,6 +432,22 @@ impl Builder {
         self
     }
 
+    /// Set whether to record the raw headers sent.
+    ///
+    /// Default is false.
+    pub fn record_raw_request_headers(&mut self, enabled: bool) -> &mut Builder {
+        self.h1_record_raw_request_headers = enabled;
+        self
+    }
+
+    /// Set whether to record the raw headers received.
+    ///
+    /// Default is false.
+    pub fn record_raw_response_headers(&mut self, enabled: bool) -> &mut Builder {
+        self.h1_record_raw_response_headers = enabled;
+        self
+    }
+
     /// Set whether to support preserving original header cases.
     ///
     /// Currently, this will record the original cases received, and store them
@@ -539,6 +559,12 @@ impl Builder {
             if opts.h1_title_case_headers {
                 conn.set_title_case_headers();
             }
+            if opts.h1_record_raw_request_headers {
+                conn.set_record_raw_request_headers();
+            }
+            if opts.h1_record_raw_response_headers {
+                conn.set_record_raw_response_headers();
+            }
             if opts.h1_preserve_header_case {
                 conn.set_preserve_header_case();
             }

src/ext/mod.rs

@@ -86,6 +86,39 @@ impl fmt::Debug for Protocol {
     }
 }
 
+/// Raw request headers as sent over the TLS or TCP connection.
+#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
+#[derive(Clone, Debug)]
+pub struct RawRequestHeaders(Bytes);
+
+#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
+impl RawRequestHeaders {
+    /// Returns the raw bytes sent for the header of the request.
+    pub fn as_bytes(&self) -> &[u8] {
+        &self.0
+    }
+    pub(crate) fn from(bytes: Bytes) -> Self {
+        Self(bytes)
+    }
+}
+
+/// Raw response headers as sent over the TLS or TCP connection.
+#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
+#[derive(Clone, Debug)]
+pub struct RawResponseHeaders(Bytes);
+
+#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
+impl RawResponseHeaders {
+    /// Returns the raw bytes received for the header of the response.
+    pub fn as_bytes(&self) -> &[u8] {
+        &self.0
+    }
+    #[cfg(feature = "client")]
+    pub(crate) fn from(bytes: Bytes) -> Self {
+        Self(bytes)
+    }
+}
+
 /// A map from header names to their original casing as received in an HTTP message.
 ///
 /// If an HTTP/1 response `res` is parsed on a connection whose option

src/proto/h1/conn.rs

@@ -68,6 +68,9 @@ where
                 date_header: true,
                 #[cfg(feature = "server")]
                 timer: Time::Empty,
+                raw_request_headers: None,
+                record_raw_request_headers: false,
+                record_raw_response_headers: false,
                 preserve_header_case: false,
                 #[cfg(feature = "ffi")]
                 preserve_header_order: false,
@@ -123,6 +126,16 @@ where
         self.state.title_case_headers = true;
     }
 
+    #[cfg(feature = "client")]
+    pub(crate) fn set_record_raw_request_headers(&mut self) {
+        self.state.record_raw_request_headers = true;
+    }
+
+    #[cfg(feature = "client")]
+    pub(crate) fn set_record_raw_response_headers(&mut self) {
+        self.state.record_raw_response_headers = true;
+    }
+
     pub(crate) fn set_preserve_header_case(&mut self) {
         self.state.preserve_header_case = true;
     }
@@ -241,6 +254,8 @@ where
                 req_method: &mut self.state.method,
                 h1_parser_config: self.state.h1_parser_config.clone(),
                 h1_max_headers: self.state.h1_max_headers,
+                raw_request_headers: self.state.raw_request_headers.as_ref(),
+                record_raw_headers: self.state.record_raw_response_headers,
                 preserve_header_case: self.state.preserve_header_case,
                 #[cfg(feature = "ffi")]
                 preserve_header_order: self.state.preserve_header_order,
@@ -617,6 +632,7 @@ where
         self.enforce_version(&mut head);
 
         let buf = self.io.headers_buf();
+        let headers_start = buf.len();
         match super::role::encode_headers::<T>(
             Encode {
                 head: &mut head,
@@ -633,6 +649,13 @@ where
             Ok(encoder) => {
                 debug_assert!(self.state.cached_headers.is_none());
                 debug_assert!(head.headers.is_empty());
+                if self.state.record_raw_request_headers {
+                    self.state.raw_request_headers = Some(crate::ext::RawRequestHeaders::from(
+                        Bytes::copy_from_slice(&buf[headers_start..]),
+                    ));
+                } else {
+                    self.state.raw_request_headers = None;
+                }
                 self.state.cached_headers = Some(head.headers);
 
                 #[cfg(feature = "client")]
@@ -934,6 +957,9 @@ struct State {
     date_header: bool,
     #[cfg(feature = "server")]
     timer: Time,
+    raw_request_headers: Option<crate::ext::RawRequestHeaders>,
+    record_raw_request_headers: bool,
+    record_raw_response_headers: bool,
     preserve_header_case: bool,
     #[cfg(feature = "ffi")]
     preserve_header_order: bool,

src/proto/h1/io.rs

@@ -184,6 +184,8 @@ where
                     req_method: parse_ctx.req_method,
                     h1_parser_config: parse_ctx.h1_parser_config.clone(),
                     h1_max_headers: parse_ctx.h1_max_headers,
+                    raw_request_headers: parse_ctx.raw_request_headers,
+                    record_raw_headers: parse_ctx.record_raw_headers,
                     preserve_header_case: parse_ctx.preserve_header_case,
                     #[cfg(feature = "ffi")]
                     preserve_header_order: parse_ctx.preserve_header_order,
@@ -706,6 +708,8 @@ mod tests {
                 req_method: &mut None,
                 h1_parser_config: Default::default(),
                 h1_max_headers: None,
+                raw_request_headers: None,
+                record_raw_headers: false,
                 preserve_header_case: false,
                 #[cfg(feature = "ffi")]
                 preserve_header_order: false,

src/proto/h1/mod.rs

@@ -73,6 +73,8 @@ pub(crate) struct ParseContext<'a> {
     req_method: &'a mut Option<Method>,
     h1_parser_config: ParserConfig,
     h1_max_headers: Option<usize>,
+    record_raw_headers: bool,
+    raw_request_headers: Option<&'a crate::ext::RawRequestHeaders>,
     preserve_header_case: bool,
     #[cfg(feature = "ffi")]
     preserve_header_order: bool,