Fixes #127: Added ability to record http headers (#300)

* Fixes #127: Added ability to record http headers

* Normalized request info collectors

* Added TODO for getting web api post data

Author: niemyjski

src/Exceptionless/Configuration/ExceptionlessConfiguration.cs

@@ -228,6 +228,7 @@ public bool IncludePrivateInformation {
                 IncludeUserName = value;
                 IncludeMachineName = value;
                 IncludeIpAddress = value;
+                IncludeHeaders = value;
                 IncludeCookies = value;
                 IncludePostData = value;
                 IncludeQueryString = value;
@@ -247,6 +248,11 @@ public bool IncludePrivateInformation {
         /// </summary>
         public bool IncludeIpAddress { get; set; }
         /// <summary>
+        /// Gets or sets a value indicating whether to include Headers.
+        /// NOTE: DataExclusions are applied to all Headers keys when enabled.
+        /// </summary>
+        public bool IncludeHeaders { get; set; }
+        /// <summary>
         /// Gets or sets a value indicating whether to include Cookies.
         /// NOTE: DataExclusions are applied to all Cookie keys when enabled.
         /// </summary>

src/Exceptionless/Models/Client/Data/RequestInfo.cs

@@ -5,6 +5,7 @@ namespace Exceptionless.Models.Data {
     public class RequestInfo : IData {
         public RequestInfo() {
             Data = new DataDictionary();
+            Headers = new Dictionary<string, string[]>();
             Cookies = new Dictionary<string, string>();
             QueryString = new Dictionary<string, string>();
         }
@@ -49,6 +50,11 @@ public RequestInfo() {
         /// </summary>
         public string ClientIpAddress { get; set; }
 
+        /// <summary>
+        /// The header values from the request.
+        /// </summary>
+        public Dictionary<string, string[]> Headers { get; set; }
+
         /// <summary>
         /// The request cookies.
         /// </summary>
@@ -70,7 +76,7 @@ public RequestInfo() {
         public DataDictionary Data { get; set; }
 
         protected bool Equals(RequestInfo other) {
-            return string.Equals(UserAgent, other.UserAgent) && string.Equals(HttpMethod, other.HttpMethod) && IsSecure == other.IsSecure && string.Equals(Host, other.Host) && Port == other.Port && string.Equals(Path, other.Path) && string.Equals(Referrer, other.Referrer) && string.Equals(ClientIpAddress, other.ClientIpAddress) && Cookies.CollectionEquals(other.Cookies) && QueryString.CollectionEquals(other.QueryString) && Equals(Data, other.Data);
+            return string.Equals(UserAgent, other.UserAgent) && string.Equals(HttpMethod, other.HttpMethod) && IsSecure == other.IsSecure && string.Equals(Host, other.Host) && Port == other.Port && string.Equals(Path, other.Path) && string.Equals(Referrer, other.Referrer) && string.Equals(ClientIpAddress, other.ClientIpAddress) && Headers.CollectionEquals(other.Headers) && Cookies.CollectionEquals(other.Cookies) && QueryString.CollectionEquals(other.QueryString) && Equals(Data, other.Data);
         }
 
         public override bool Equals(object obj) {
@@ -95,6 +101,7 @@ public override int GetHashCode() {
                 hashCode = (hashCode * 397) ^ (Path == null ? 0 : Path.GetHashCode());
                 hashCode = (hashCode * 397) ^ (Referrer == null ? 0 : Referrer.GetHashCode());
                 hashCode = (hashCode * 397) ^ (ClientIpAddress == null ? 0 : ClientIpAddress.GetHashCode());
+                hashCode = (hashCode * 397) ^ (Headers == null ? 0 : Headers.GetCollectionHashCode());
                 hashCode = (hashCode * 397) ^ (Cookies == null ? 0 : Cookies.GetCollectionHashCode(_cookieHashCodeExclusions));
                 hashCode = (hashCode * 397) ^ (QueryString == null ? 0 : QueryString.GetCollectionHashCode());
                 hashCode = (hashCode * 397) ^ (Data == null ? 0 : Data.GetCollectionHashCode());

src/Platforms/Exceptionless.AspNetCore/RequestInfoCollector.cs

@@ -14,14 +14,16 @@
 namespace Exceptionless.AspNetCore {
     public static class RequestInfoCollector {
         private const int MAX_BODY_SIZE = 50 * 1024;
+        private const int MAX_DATA_ITEM_LENGTH = 1000;
+
         public static RequestInfo Collect(HttpContext context, ExceptionlessConfiguration config) {
             if (context == null)
                 return null;
 
             var info = new RequestInfo {
                 HttpMethod = context.Request.Method,
                 IsSecure = context.Request.IsHttps,
-                Path = context.Request.Path.HasValue ? context.Request.Path.Value : "/",
+                Path = context.Request.Path.HasValue ? context.Request.Path.Value : "/"
             };
 
             if (config.IncludeIpAddress)
@@ -32,13 +34,16 @@ public static RequestInfo Collect(HttpContext context, ExceptionlessConfiguratio
 
             info.Port = context.Request.Host.Port.GetValueOrDefault(info.IsSecure ? 443 : 80);
 
-            if (context.Request.Headers.ContainsKey(HeaderNames.UserAgent))
-                info.UserAgent = context.Request.Headers[HeaderNames.UserAgent].ToString();
+            if (context.Request.Headers.TryGetValue(HeaderNames.UserAgent, out var userAgentHeader))
+                info.UserAgent = userAgentHeader.ToString();
 
-            if (context.Request.Headers.ContainsKey(HeaderNames.Referer))
-                info.Referrer = context.Request.Headers[HeaderNames.Referer].ToString();
+            if (context.Request.Headers.TryGetValue(HeaderNames.Referer, out var refererHeader))
+                info.Referrer = refererHeader.ToString();
 
             var exclusionList = config.DataExclusions as string[] ?? config.DataExclusions.ToArray();
+            if (config.IncludeHeaders)
+                info.Headers = context.Request.Headers.ToHeaderDictionary(exclusionList);
+
             if (config.IncludeCookies)
                 info.Cookies = context.Request.Cookies.ToDictionary(exclusionList);
 
@@ -93,17 +98,15 @@ private static object GetPostData(HttpContext context, ExceptionlessConfiguratio
                     return message;
                 }
 
-                var maxDataToRead = contentLength == 0 ? MAX_BODY_SIZE : contentLength;
-
                 // pass default values, except for leaveOpen: true. This prevents us from disposing the underlying stream
                 using (var inputStream = new StreamReader(context.Request.Body, Encoding.UTF8, true, 1024, true)) {
                     var sb = new StringBuilder();
                     int numRead;
 
-                    int bufferSize = (int)Math.Min(1024, maxDataToRead);
+                    int bufferSize = (int)Math.Min(1024, contentLength);
 
                     char[] buffer = new char[bufferSize];
-                    while ((numRead = inputStream.ReadBlock(buffer, 0, bufferSize)) > 0 && (sb.Length + numRead) <= maxDataToRead) {
+                    while ((numRead = inputStream.ReadBlock(buffer, 0, bufferSize)) > 0 && (sb.Length + numRead) <= contentLength) {
                         sb.Append(buffer, 0, numRead);
                     }
                     string postData = sb.ToString();
@@ -121,8 +124,15 @@ private static object GetPostData(HttpContext context, ExceptionlessConfiguratio
             }
         }
 
-        private static readonly List<string> _ignoredFormFields = new List<string> {
-            "__*"
+        private static readonly List<string> _ignoredHeaders = new List<string> {
+            HeaderNames.Authorization,
+            HeaderNames.Cookie,
+            HeaderNames.Host,
+            HeaderNames.Method,
+            HeaderNames.Path,
+            HeaderNames.ProxyAuthorization,
+            HeaderNames.Referer,
+            HeaderNames.UserAgent
         };
 
         private static readonly List<string> _ignoredCookies = new List<string> {
@@ -131,20 +141,44 @@ private static object GetPostData(HttpContext context, ExceptionlessConfiguratio
             "*SessionId*"
         };
 
-        private static Dictionary<string, string> ToDictionary(this IRequestCookieCollection cookies, IList<string> exclusions) {
+        private static readonly List<string> _ignoredFormFields = new List<string> {
+            "__*"
+        };
+
+        private static Dictionary<string, string[]> ToHeaderDictionary(this IEnumerable<KeyValuePair<string, StringValues>> headers, string[] exclusions) {
+            var d = new Dictionary<string, string[]>();
+
+            foreach (var header in headers) {
+                if (String.IsNullOrEmpty(header.Key) || _ignoredHeaders.Contains(header.Key) || header.Key.AnyWildcardMatches(exclusions))
+                    continue;
+
+                string[] values = header.Value.Where(hv => hv != null && hv.Length < MAX_DATA_ITEM_LENGTH).ToArray();
+                if (values.Length == 0)
+                    continue;
+
+                d[header.Key] = values;
+            }
+
+            return d;
+        }
+
+        private static Dictionary<string, string> ToDictionary(this IRequestCookieCollection cookies, string[] exclusions) {
             var d = new Dictionary<string, string>();
 
             foreach (var kvp in cookies) {
                 if (String.IsNullOrEmpty(kvp.Key) || kvp.Key.AnyWildcardMatches(_ignoredCookies) || kvp.Key.AnyWildcardMatches(exclusions))
                     continue;
 
-                d.Add(kvp.Key, kvp.Value);
+                if (kvp.Value == null || kvp.Value.Length >= MAX_DATA_ITEM_LENGTH)
+                    continue;
+                
+                d[kvp.Key] = kvp.Value;
             }
 
             return d;
         }
 
-        private static Dictionary<string, string> ToDictionary(this IEnumerable<KeyValuePair<string, StringValues>> values, IEnumerable<string> exclusions) {
+        private static Dictionary<string, string> ToDictionary(this IEnumerable<KeyValuePair<string, StringValues>> values, string[] exclusions) {
             var d = new Dictionary<string, string>();
 
             foreach (var kvp in values) {
@@ -153,10 +187,12 @@ private static Dictionary<string, string> ToDictionary(this IEnumerable<KeyValue
 
                 try {
                     string value = kvp.Value.ToString();
-                    d.Add(kvp.Key, value);
+                    if (value.Length >= MAX_DATA_ITEM_LENGTH)
+                        continue;
+                    
+                    d[kvp.Key] = value;
                 } catch (Exception ex) {
-                    if (!d.ContainsKey(kvp.Key))
-                        d.Add(kvp.Key, ex.Message);
+                    d[kvp.Key] = $"EXCEPTION: {ex.Message}";
                 }
             }
 

src/Platforms/Exceptionless.Web/RequestInfoCollector.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.Specialized;
-using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Text;
@@ -13,8 +12,8 @@
 
 namespace Exceptionless.ExtendedData {
     internal static class RequestInfoCollector {
+        private const int MAX_BODY_SIZE = 50 * 1024;
         private const int MAX_DATA_ITEM_LENGTH = 1000;
-        private const int MAX_BODY_SIZE = 50*1024;
 
         public static RequestInfo Collect(HttpContextBase context, ExceptionlessConfiguration config) {
             if (context == null)
@@ -50,13 +49,12 @@ public static RequestInfo Collect(HttpContextBase context, ExceptionlessConfigur
                 info.Port = context.Request.Url.Port;
 
             var exclusionList = config.DataExclusions as string[] ?? config.DataExclusions.ToArray();
+            if (config.IncludeHeaders)
+                info.Headers = context.Request.Headers.ToHeaderDictionary(exclusionList);
 
             if (config.IncludeCookies)
                 info.Cookies = context.Request.Cookies.ToDictionary(exclusionList);
-
-            if (config.IncludePostData && !String.Equals(context.Request.HttpMethod, "GET", StringComparison.OrdinalIgnoreCase))
-                info.PostData = GetPostData(context, config, exclusionList);
-
+            
             if (config.IncludeQueryString) {
                 try {
                     info.QueryString = context.Request.QueryString.ToDictionary(exclusionList);
@@ -65,6 +63,9 @@ public static RequestInfo Collect(HttpContextBase context, ExceptionlessConfigur
                 }
             }
 
+            if (config.IncludePostData && !String.Equals(context.Request.HttpMethod, "GET", StringComparison.OrdinalIgnoreCase))
+                info.PostData = GetPostData(context, config, exclusionList);
+
             return info;
         }
 
@@ -139,8 +140,15 @@ private static object GetPostData(HttpContextBase context, ExceptionlessConfigur
             }
         }
 
-        private static readonly List<string> _ignoredFormFields = new List<string> {
-            "__*"
+        private static readonly List<string> _ignoredHeaders = new List<string> {
+            "Authorization",
+            "Cookie",
+            "Host",
+            "Method",
+            "Path",
+            "Proxy-Authorization",
+            "Referer",
+            "User-Agent"
         };
 
         private static readonly List<string> _ignoredCookies = new List<string> {
@@ -149,38 +157,68 @@ private static object GetPostData(HttpContextBase context, ExceptionlessConfigur
             "*SessionId*"
         };
 
-        private static Dictionary<string, string> ToDictionary(this HttpCookieCollection cookies, IEnumerable<string> exclusions) {
+        private static readonly List<string> _ignoredFormFields = new List<string> {
+            "__*"
+        };
+
+        private static Dictionary<string, string[]> ToHeaderDictionary(this NameValueCollection headers, string[] exclusions) {
+            var result = new Dictionary<string, string[]>();
+
+            foreach (string key in headers.AllKeys) {
+                if (String.IsNullOrEmpty(key) || _ignoredHeaders.Contains(key) || key.AnyWildcardMatches(exclusions))
+                    continue;
+
+                try {
+                    string value = headers.Get(key);
+                    if (value == null || value.Length >= MAX_DATA_ITEM_LENGTH)
+                        continue;
+
+                    result[key] = value.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
+                }
+                catch (Exception ex) {
+                    result[key] = new[] { $"EXCEPTION: {ex.Message}" };
+                }
+            }
+
+            return result;
+        }
+
+        private static Dictionary<string, string> ToDictionary(this HttpCookieCollection cookies, string[] exclusions) {
             var d = new Dictionary<string, string>();
 
-            foreach (string key in cookies.AllKeys.Distinct().Where(k => !String.IsNullOrEmpty(k) && !k.AnyWildcardMatches(_ignoredCookies) && !k.AnyWildcardMatches(exclusions))) {
+            foreach (string key in cookies.AllKeys.Distinct()) {
+                if (String.IsNullOrEmpty(key) || key.AnyWildcardMatches(_ignoredCookies) || key.AnyWildcardMatches(exclusions))
+                    continue;
+
                 try {
-                    HttpCookie cookie = cookies.Get(key);
-                    if (cookie != null && cookie.Value != null && cookie.Value.Length < MAX_DATA_ITEM_LENGTH && !d.ContainsKey(key))
-                        d.Add(key, cookie.Value);
+                    var cookie = cookies.Get(key);
+                    if (cookie == null || cookie.Value == null || cookie.Value.Length >= MAX_DATA_ITEM_LENGTH)
+                        continue;
+
+                    d[key] = cookie.Value;
                 } catch (Exception ex) {
-                    if (!d.ContainsKey(key))
-                        d.Add(key, ex.Message);
+                    d[key] = $"EXCEPTION: {ex.Message}";
                 }
             }
 
             return d;
         }
 
-        private static Dictionary<string, string> ToDictionary(this NameValueCollection values, IEnumerable<string> exclusions) {
+        private static Dictionary<string, string> ToDictionary(this NameValueCollection values, string[] exclusions) {
             var d = new Dictionary<string, string>();
-
-            var exclusionsArray = exclusions as string[] ?? exclusions.ToArray();
+            
             foreach (string key in values.AllKeys) {
-                if (String.IsNullOrEmpty(key) || key.AnyWildcardMatches(_ignoredFormFields) || key.AnyWildcardMatches(exclusionsArray))
+                if (String.IsNullOrEmpty(key) || key.AnyWildcardMatches(_ignoredFormFields) || key.AnyWildcardMatches(exclusions))
                     continue;
 
                 try {
                     string value = values.Get(key);
-                    if (value != null && !d.ContainsKey(key) && value.Length < MAX_DATA_ITEM_LENGTH)
-                        d.Add(key, value);
+                    if (value == null || d.ContainsKey(key) || value.Length >= MAX_DATA_ITEM_LENGTH)
+                        continue;
+
+                    d[key] = value;
                 } catch (Exception ex) {
-                    if (!d.ContainsKey(key))
-                        d.Add(key, "EXCEPTION: " + ex.Message);
+                    d[key] = $"EXCEPTION: {ex.Message}";
                 }
             }