Fix request bodies, improve censoring (#3)

- Fix HTTP accidentally using HTTPS under the hood
- Delay setting request method to the last second to avoid conflict with setting request body
- Fix parsing body/error from HTTP response
- Fix body not included on requests
- Enhance censoring to handle nested data
- Throw exception when trying to apply censors to non-JSON data

Author: nwithan8

README.md

@@ -90,6 +90,8 @@ Now when tests are run, no real HTTP calls will be made. Instead, the HTTP respo
 
 Censor sensitive data in the request and response bodies and headers, such as API keys and auth tokens.
 
+NOTE: Censors can only be applied to JSON request and response bodies. Attempting to apply censors to non-JSON data will throw an exception.
+
 **Default**: *Disabled*
 
 ```java
@@ -106,7 +108,9 @@ public class Example {
         Cassette cassette = new Cassette("path/to/cassettes", "my_cassette");
 
         AdvancedSettings advancedSettings = new AdvancedSettings();
-        advancedSettings.censors = new Censors().hideHeader("Authorization"); // Hide the Authorization header
+        List<String> headersToCensor = new ArrayList<>();
+        headersToCensor.add("Authorization"); // Hide the Authorization header
+        advancedSettings.censors = new Censors().hideHeader(headersToCensor);
         // or
         advancedSettings.censors = Censors.strict(); // use the built-in strict censoring mode (hides common sensitive data)
 
@@ -196,7 +200,9 @@ import com.easypost.easyvcr.clients.httpurlconnection.RecordableURL;
 public class Example {
     public static void main(String[] args) {
         AdvancedSettings advancedSettings = new AdvancedSettings();
-        advancedSettings.censors = new Censors().hideQueryParameter("api_key"); // hide the api_key query parameter
+        List<String> censoredQueryParams = new ArrayList<String>();
+        censoredQueryParams.add("api_key"); // hide the api_key query parameter
+        advancedSettings.censors = new Censors().hideQueryParameter(censoredQueryParams);
 
         // Create a VCR with the advanced settings applied
         VCR vcr = new VCR(advancedSettings);

src/main/java/com/easypost/easyvcr/Censors.java

@@ -2,7 +2,7 @@
 
 import com.easypost.easyvcr.internalutilities.Tools;
 import com.easypost.easyvcr.internalutilities.json.Serialization;
-import com.google.gson.JsonSyntaxException;
+import com.google.gson.JsonParseException;
 import org.apache.http.NameValuePair;
 import org.apache.http.client.utils.URLEncodedUtils;
 
@@ -22,6 +22,10 @@ public final class Censors {
      * The body parameters to censor.
      */
     private final List<String> bodyParamsToCensor;
+    /**
+     * Whether censor keys are case sensitive.
+     */
+    private final boolean caseSensitive;
     /**
      * The string to replace censored data with.
      */
@@ -48,10 +52,21 @@ public Censors() {
      * @param censorString The string to use to censor sensitive information.
      */
     public Censors(String censorString) {
-        queryParamsToCensor = new ArrayList<>();
-        bodyParamsToCensor = new ArrayList<>();
-        headersToCensor = new ArrayList<>();
-        censorText = censorString;
+        this(censorString, false);
+    }
+
+    /**
+     * Initialize a new instance of the Censors factory.
+     *
+     * @param censorString  The string to use to censor sensitive information.
+     * @param caseSensitive Whether to use case sensitive censoring.
+     */
+    public Censors(String censorString, boolean caseSensitive) {
+        this.queryParamsToCensor = new ArrayList<>();
+        this.bodyParamsToCensor = new ArrayList<>();
+        this.headersToCensor = new ArrayList<>();
+        this.censorText = censorString;
+        this.caseSensitive = caseSensitive;
     }
 
     /**
@@ -70,48 +85,44 @@ public static Censors regular() {
      */
     public static Censors strict() {
         Censors censors = new Censors();
-        for (String key : Statics.DEFAULT_CREDENTIAL_HEADERS_TO_HIDE) {
-            censors.hideHeader(key);
-        }
-        for (String key : Statics.DEFAULT_CREDENTIAL_PARAMETERS_TO_HIDE) {
-            censors.hideQueryParameter(key);
-            censors.hideBodyParameter(key);
-        }
+        censors.hideHeaders(Statics.DEFAULT_CREDENTIAL_HEADERS_TO_HIDE);
+        censors.hideBodyParameters(Statics.DEFAULT_CREDENTIAL_PARAMETERS_TO_HIDE);
+        censors.hideQueryParameters(Statics.DEFAULT_CREDENTIAL_PARAMETERS_TO_HIDE);
         return censors;
     }
 
     /**
-     * Add a rule to censor a specified body parameter.
+     * Add a rule to censor specified body parameters.
      * Note: Only top-level pairs can be censored.
      *
-     * @param parameterKey Key of body parameter to censor.
+     * @param parameterKeys Keys of body parameters to censor.
      * @return This Censors factory.
      */
-    public Censors hideBodyParameter(String parameterKey) {
-        bodyParamsToCensor.add(parameterKey);
+    public Censors hideBodyParameters(List<String> parameterKeys) {
+        bodyParamsToCensor.addAll(parameterKeys);
         return this;
     }
 
     /**
-     * Add a rule to censor a specified header key.
-     * Note: This will censor the header key in both the request and response.
+     * Add a rule to censor specified header keys.
+     * Note: This will censor the header keys in both the request and response.
      *
-     * @param headerKey Key of header to censor.
+     * @param headerKeys Keys of headers to censor.
      * @return This Censors factory.
      */
-    public Censors hideHeader(String headerKey) {
-        headersToCensor.add(headerKey);
+    public Censors hideHeaders(List<String> headerKeys) {
+        headersToCensor.addAll(headerKeys);
         return this;
     }
 
     /**
-     * Add a rule to censor a specified query parameter.
+     * Add a rule to censor specified query parameters.
      *
-     * @param parameterKey Key of query parameter to censor.
+     * @param parameterKeys Keys of query parameters to censor.
      * @return This Censors factory.
      */
-    public Censors hideQueryParameter(String parameterKey) {
-        queryParamsToCensor.add(parameterKey);
+    public Censors hideQueryParameters(List<String> parameterKeys) {
+        queryParamsToCensor.addAll(parameterKeys);
         return this;
     }
 
@@ -121,32 +132,19 @@ public Censors hideQueryParameter(String parameterKey) {
      * @param body String representation of request body to apply censors to.
      * @return Censored string representation of request body.
      */
-    public String applyBodyParametersCensors(String body) {
+    public String censorBodyParameters(String body) {
         if (body == null || body.length() == 0) {
             // short circuit if body is null or empty
             return body;
         }
 
-        Map<String, Object> bodyParameters;
-        try {
-            bodyParameters = Serialization.convertJsonToObject(body, Map.class);
-        } catch (JsonSyntaxException ignored) {
-            // short circuit if body is not a JSON dictionary
-            return body;
-        }
-
-        if (bodyParameters == null || bodyParameters.size() == 0) {
-            // short circuit if there are no body parameters
+        if (bodyParamsToCensor.size() == 0) {
+            // short circuit if there are no censors to apply
             return body;
         }
 
-        for (String parameterKey : bodyParamsToCensor) {
-            if (bodyParameters.containsKey(parameterKey)) {
-                bodyParameters.put(parameterKey, censorText);
-            }
-        }
-
-        return Serialization.convertObjectToJson(bodyParameters);
+        // TODO: Future different content type support here, only JSON is supported currently
+        return censorJsonBodyParameters(body);
     }
 
     /**
@@ -155,12 +153,17 @@ public String applyBodyParametersCensors(String body) {
      * @param headers Map of headers to apply censors to.
      * @return Censored map of headers.
      */
-    public Map<String, List<String>> applyHeadersCensors(Map<String, List<String>> headers) {
+    public Map<String, List<String>> censorHeaders(Map<String, List<String>> headers) {
         if (headers == null || headers.size() == 0) {
             // short circuit if there are no headers to censor
             return headers;
         }
 
+        if (headersToCensor.size() == 0) {
+            // short circuit if there are no censors to apply
+            return headers;
+        }
+
         final Map<String, List<String>> headersCopy = new HashMap<>(headers);
 
         for (String headerKey : headersToCensor) {
@@ -177,11 +180,17 @@ public Map<String, List<String>> applyHeadersCensors(Map<String, List<String>> h
      * @param url Full URL string to apply censors to.
      * @return Censored URL string.
      */
-    public String applyQueryParametersCensors(String url) {
+    public String censorQueryParameters(String url) {
         if (url == null || url.length() == 0) {
             // short circuit if url is null
             return url;
         }
+
+        if (queryParamsToCensor.size() == 0) {
+            // short circuit if there are no censors to apply
+            return url;
+        }
+
         URI uri = URI.create(url);
         Map<String, String> queryParameters = Tools.queryParametersToMap(uri);
         if (queryParameters.size() == 0) {
@@ -204,4 +213,118 @@ public String applyQueryParametersCensors(String url) {
 
         return uri.getScheme() + "://" + uri.getHost() + uri.getPath() + "?" + formattedQueryParameters;
     }
+
+    private List<Object> applyBodyCensors(List<Object> list) {
+        if (list == null || list.size() == 0) {
+            // short circuit if list is null or empty
+            return list;
+        }
+
+        List<Object> censoredList = new ArrayList<>();
+
+        for (Object object : list) {
+            Object value = object;
+            if (Utilities.isDictionary(value)) {
+                // recursively censor inner dictionaries
+                try {
+                    // change the value if can be parsed as a dictionary
+                    value = applyBodyCensors((Map<String, Object>) value);
+                } catch (ClassCastException e) {
+                    // otherwise, skip censoring
+                }
+            } else if (Utilities.isList(value)) {
+                // recursively censor list elements
+                try {
+                    // change the value if can be parsed as a list
+                    value = applyBodyCensors((List<Object>) value);
+                } catch (ClassCastException e) {
+                    // otherwise, skip censoring
+                }
+            }  // either a primitive or null, no censoring needed
+
+            censoredList.add(value);
+        }
+
+        return censoredList;
+
+    }
+
+    private Map<String, Object> applyBodyCensors(Map<String, Object> dictionary) {
+        if (dictionary == null || dictionary.size() == 0) {
+            // short circuit if dictionary is null or empty
+            return dictionary;
+        }
+
+        Map<String, Object> censoredBodyDictionary = new HashMap<>();
+
+        for (Map.Entry<String, Object> entry : dictionary.entrySet()) {
+            String key = entry.getKey();
+            Object value = entry.getValue();
+            if (keyShouldBeCensored(key, this.bodyParamsToCensor)) {
+                if (value == null) {
+                    // don't need to worry about censoring something that's null
+                    // (don't replace null with the censor string)
+                    continue;
+                } else if (Utilities.isDictionary(value)) {
+                    // replace with empty dictionary
+                    censoredBodyDictionary.put(key, new HashMap<>());
+                } else if (Utilities.isList(value)) {
+                    // replace with empty array
+                    censoredBodyDictionary.put(key, new ArrayList<>());
+                } else {
+                    // replace with censor text
+                    censoredBodyDictionary.put(key, this.censorText);
+                }
+            } else {
+                if (Utilities.isDictionary(value)) {
+                    // recursively censor inner dictionaries
+                    try {
+                        // change the value if can be parsed as a dictionary
+                        value = applyBodyCensors((Map<String, Object>) value);
+                    } catch (ClassCastException e) {
+                        // otherwise, skip censoring
+                    }
+                } else if (Utilities.isList(value)) {
+                    // recursively censor list elements
+                    try {
+                        // change the value if can be parsed as a list
+                        value = applyBodyCensors((List<Object>) value);
+                    } catch (ClassCastException e) {
+                        // otherwise, skip censoring
+                    }
+                }
+
+                censoredBodyDictionary.put(key, value);
+            }
+        }
+
+        return censoredBodyDictionary;
+    }
+
+    private String censorJsonBodyParameters(String body) {
+        Map<String, Object> bodyDictionary;
+        try {
+            bodyDictionary = Serialization.convertJsonToObject(body, Map.class);
+            Map<String, Object> censoredBodyDictionary = applyBodyCensors(bodyDictionary);
+            return censoredBodyDictionary == null ? body : Serialization.convertObjectToJson(censoredBodyDictionary);
+        } catch (Exception ignored) {
+            // body is not a JSON dictionary
+            try {
+                List<Object> bodyList = Serialization.convertJsonToObject(body, List.class);
+                List<Object> censoredBodyList = applyBodyCensors(bodyList);
+                return censoredBodyList == null ? body : Serialization.convertObjectToJson(censoredBodyList);
+            } catch (Exception notJsonData) {
+                throw new JsonParseException("Body is not a JSON dictionary or list");
+            }
+        }
+    }
+
+    private boolean keyShouldBeCensored(String foundKey, List<String> keysToCensor) {
+        // keysToCensor are already cased as needed
+        if (!this.caseSensitive) {
+            foundKey = foundKey.toLowerCase();
+        }
+
+        return keysToCensor.contains(foundKey);
+    }
 }

src/main/java/com/easypost/easyvcr/Utilities.java

@@ -30,4 +30,22 @@ public static boolean responseCameFromRecording(HttpURLConnection connection) {
         }
         return false;
     }
+
+    /**
+     * Check if the object is a dictionary.
+     * @param obj The object to check.
+     * @return True if the object is a dictionary.
+     */
+    public static boolean isDictionary(Object obj) {
+        return obj instanceof Map;
+    }
+
+    /**
+     * Check if the object is a list.
+     * @param obj The object to check.
+     * @return True if the object is a list.
+     */
+    public static boolean isList(Object obj) {
+        return obj instanceof List;
+    }
 }