Add ability to censor elements in the URL path and host (#21)

nwithan8 · web-flow · commit 1f7c8b2dfa51 · 2022-12-19T11:14:32.000-07:00
- Add ability to censor elements in the URL path and host
- Add unit test to confirm feature working
- Add CHANGELOG.md and README.md details for path censoring
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # CHANGELOG
 
+## Next Release
+
+- [ADDED] Ability to censor parts of a URL path using regex patterns.
+
 ## v0.4.2 (2022-10-20)
 
 - Fix a bug where the error data of a bad HTTP request (4xx or 5xx) was not stored as expected in cassettes, causing
diff --git a/README.md b/README.md
@@ -91,9 +91,13 @@ Now when tests are run, no real HTTP calls will be made. Instead, the HTTP respo
 
 ### Censoring
 
-Censor sensitive data in the request and response bodies and headers, such as API keys and auth tokens.
+Censor sensitive data in the request and response, such as API keys and auth tokens.
 
-NOTE: Censors can only be applied to JSON request and response bodies. Attempting to apply censors to non-JSON data will throw an exception.
+Can censor:
+- Request and response headers (via key name)
+- Request and response bodies (via key name) (JSON only)
+- Request query parameters (via key name)
+- Request URL path elements (via regex pattern matching)
 
 **Default**: *Disabled*
 
@@ -114,12 +118,17 @@ public class Example {
         Cassette cassette = new Cassette("path/to/cassettes", "my_cassette");
 
         AdvancedSettings advancedSettings = new AdvancedSettings();
+        
         List<String> headersToCensor = new ArrayList<>();
         headersToCensor.add("Authorization"); // Hide the Authorization header
         advancedSettings.censors = new Censors().censorHeadersByKeys(headersToCensor);
         advancedSettings.censors.censorBodyElements(new ArrayList<>() {{
             add(new CensorElement("table", true)); // Hide the table element (case-sensitive) in the request and response body
         }});
+        advancedSettings.censors.censorPathElementsByPattern(new ArrayList<>() {{
+            add(".*\\d{4}.*"); // Hide any path element that contains 4 digits
+        }});
+        
         // or
         advancedSettings.censors =
                 Censors.strict(); // use the built-in strict censoring mode (hides common sensitive data)
diff --git a/src/main/java/com/easypost/easyvcr/CensorElement.java b/src/main/java/com/easypost/easyvcr/CensorElement.java
@@ -2,21 +2,21 @@
 
 public class CensorElement {
     /**
-     * The name of the element to censor.
+     * The value of the element to censor.
      */
-    private final String name;
+    protected final String value;
     /**
-     * Whether the name must match exactly to trigger a censor.
+     * Whether the value must match exactly to trigger a censor.
      */
-    private final boolean caseSensitive;
+    protected final boolean caseSensitive;
 
     /**
      * Constructor.
      * @param name The name of the element to censor.
      * @param caseSensitive Whether the name must match exactly to trigger a censor.
      */
     public CensorElement(String name, boolean caseSensitive) {
-        this.name = name;
+        this.value = name;
         this.caseSensitive = caseSensitive;
     }
 
@@ -27,9 +27,9 @@ public CensorElement(String name, boolean caseSensitive) {
      */
     public boolean matches(String key) {
         if (caseSensitive) {
-            return key.equals(name);
+            return key.equals(value);
         } else {
-            return key.equalsIgnoreCase(name);
+            return key.equalsIgnoreCase(value);
         }
     }
 }
diff --git a/src/main/java/com/easypost/easyvcr/Censors.java b/src/main/java/com/easypost/easyvcr/Censors.java
@@ -36,6 +36,11 @@ public final class Censors {
      */
     private final List<CensorElement> queryParamsToCensor;
 
+    /**
+     * The URL path elements to censor.
+     */
+    private final List<RegexCensorElement> pathElementsToCensor;
+
     /**
      * Initialize a new instance of the Censors factory, using default censor string.
      */
@@ -52,6 +57,7 @@ public Censors(String censorString) {
         this.queryParamsToCensor = new ArrayList<>();
         this.bodyElementsToCensor = new ArrayList<>();
         this.headersToCensor = new ArrayList<>();
+        this.pathElementsToCensor = new ArrayList<>();
         this.censorText = censorString;
     }
 
@@ -126,7 +132,7 @@ private static List<Object> applyJsonCensors(List<Object> list, String censorTex
      * @param dictionary       JSON dictionary to process.
      * @param censorText       Text to use when censoring an element.
      * @param elementsToCensor List of elements to find and censor.
-     * @return Censored JSON dicstionary.
+     * @return Censored JSON dictionary.
      */
     private static Map<String, Object> applyJsonCensors(Map<String, Object> dictionary, String censorText,
                                                         List<CensorElement> elementsToCensor) {
@@ -280,45 +286,73 @@ public static Map<String, List<String>> applyHeaderCensors(Map<String, List<Stri
     /**
      * Censor the appropriate query parameters.
      *
-     * @param url                 Full URL string to apply censors to.
-     * @param censorText          The string to use to censor sensitive information.
-     * @param queryParamsToCensor The query parameters to censor.
+     * @param url                  Full URL string to apply censors to.
+     * @param censorText           The string to use to censor sensitive information.
+     * @param queryParamsToCensor  The query parameters to censor.
+     * @param pathElementsToCensor The path elements to censor.
      * @return Censored URL string.
      */
-    public static String applyQueryParameterCensors(String url, String censorText,
-                                                    List<CensorElement> queryParamsToCensor) {
+    public static String applyUrlCensors(String url, String censorText,
+                                         List<CensorElement> queryParamsToCensor,
+                                         List<RegexCensorElement> pathElementsToCensor) {
         if (url == null || url.length() == 0) {
             // short circuit if url is null
             return url;
         }
 
-        if (queryParamsToCensor.size() == 0) {
+        if (queryParamsToCensor.size() == 0 && pathElementsToCensor.size() == 0) {
             // short circuit if there are no censors to apply
             return url;
         }
 
         URI uri = URI.create(url);
-        Map<String, String> queryParameters = Tools.queryParametersToMap(uri);
+
+        String path = Utilities.extractPathFromUri(uri);
+        Map<String, String> queryParameters = Utilities.queryParametersToMap(uri);
+
+        String censoredPath;
+        String censoredQueryString;
+
+        if (pathElementsToCensor.size() == 0) {
+            // don't need to censor path elements
+            censoredPath = path;
+        } else {
+            // censor path elements
+            String tempPath = path;
+            for (RegexCensorElement regexCensorElement : pathElementsToCensor) {
+                tempPath = regexCensorElement.matchAndReplaceAsNeeded(tempPath, censorText);
+            }
+
+            censoredPath = tempPath;
+        }
+
         if (queryParameters.size() == 0) {
             // short circuit if there are no query parameters to censor
-            return url;
-        }
+            censoredQueryString = null;
+        } else {
+            if (queryParamsToCensor.size() == 0) {
+                // don't need to censor query parameters
+                censoredQueryString = uri.getQuery();
+            } else {
+                // censor query parameters
+                List<String> queryKeys = new ArrayList<>(queryParameters.keySet());
+                for (String queryKey : queryKeys) {
+                    if (elementShouldBeCensored(queryKey, queryParamsToCensor)) {
+                        queryParameters.put(queryKey, censorText);
+                    }
+                }
 
-        List<String> queryKeys = new ArrayList<>(queryParameters.keySet());
-        for (String queryKey : queryKeys) {
-            if (elementShouldBeCensored(queryKey, queryParamsToCensor)) {
-                queryParameters.put(queryKey, censorText);
+                List<NameValuePair> censoredQueryParametersList = Tools.mapToQueryParameters(queryParameters);
+                censoredQueryString = URLEncodedUtils.format(censoredQueryParametersList, StandardCharsets.UTF_8);
             }
         }
 
-        List<NameValuePair> censoredQueryParametersList = Tools.mapToQueryParameters(queryParameters);
-        String formattedQueryParameters = URLEncodedUtils.format(censoredQueryParametersList, StandardCharsets.UTF_8);
-        if (formattedQueryParameters.length() == 0) {
-            // short circuit if there are no query parameters to censor
-            return url;
+        String censoredUrl = censoredPath;
+        if (censoredQueryString != null) {
+            censoredUrl += "?" + censoredQueryString;
         }
 
-        return uri.getScheme() + "://" + uri.getHost() + uri.getPath() + "?" + formattedQueryParameters;
+        return uri.getScheme() + "://" + censoredUrl;
     }
 
     /**
@@ -360,7 +394,7 @@ public Censors censorBodyElementsByKeys(List<String> elementKeys) {
      * Add a rule to censor specified headers.
      * Note: This will censor the header keys in both the request and response.
      *
-     * @param headers List of Headers to censor.
+     * @param headers List of headers to censor.
      * @return This Censors factory.
      */
     public Censors censorHeaders(List<CensorElement> headers) {
@@ -397,7 +431,7 @@ public Censors censorHeadersByKeys(List<String> headerKeys) {
     /**
      * Add a rule to censor specified query parameters.
      *
-     * @param elements List of QueryElements to censor.
+     * @param elements List of query parameters to censor.
      * @return This Censors factory.
      */
     public Censors censorQueryParameters(List<CensorElement> elements) {
@@ -429,6 +463,41 @@ public Censors censorQueryParametersByKeys(List<String> parameterKeys) {
         return censorQueryParametersByKeys(parameterKeys, false);
     }
 
+    /**
+     * Add a rule to censor specified path elements.
+     *
+     * @param elements List of path elements to censor.
+     * @return This Censors factory.
+     */
+    public Censors censorPathElements(List<RegexCensorElement> elements) {
+        pathElementsToCensor.addAll(elements);
+        return this;
+    }
+
+    /**
+     * Add a rule to censor specified path elements by regular expression patterns.
+     *
+     * @param patterns      Patterns of path elements to censor.
+     * @param caseSensitive Whether to use case-sensitive pattern matching.
+     * @return This Censors factory.
+     */
+    public Censors censorPathElementsByPattern(List<String> patterns, boolean caseSensitive) {
+        for (String pattern : patterns) {
+            pathElementsToCensor.add(new RegexCensorElement(pattern, caseSensitive));
+        }
+        return this;
+    }
+
+    /**
+     * Add a rule to censor specified path elements by regular expression patterns.
+     *
+     * @param patterns Patterns of path elements to censor.
+     * @return This Censors factory.
+     */
+    public Censors censorPathElementsByPattern(List<String> patterns) {
+        return censorPathElementsByPattern(patterns, false);
+    }
+
     /**
      * Censor the appropriate body elements.
      *
@@ -455,7 +524,7 @@ public Map<String, List<String>> applyHeaderCensors(Map<String, List<String>> he
      * @param url Full URL string to apply censors to.
      * @return Censored URL string.
      */
-    public String applyQueryParameterCensors(String url) {
-        return applyQueryParameterCensors(url, this.censorText, this.queryParamsToCensor);
+    public String applyUrlCensors(String url) {
+        return applyUrlCensors(url, this.censorText, this.queryParamsToCensor, this.pathElementsToCensor);
     }
 }
diff --git a/src/main/java/com/easypost/easyvcr/MatchRules.java b/src/main/java/com/easypost/easyvcr/MatchRules.java
@@ -142,8 +142,8 @@ public MatchRules byFullUrl(boolean exact) {
         } else {
             byBaseUrl();
             by((received, recorded) -> {
-                Map<String, String> receivedQuery = Tools.queryParametersToMap(received.getUri());
-                Map<String, String> recordedQuery = Tools.queryParametersToMap(recorded.getUri());
+                Map<String, String> receivedQuery = Utilities.queryParametersToMap(received.getUri());
+                Map<String, String> recordedQuery = Utilities.queryParametersToMap(recorded.getUri());
                 if (receivedQuery.size() != recordedQuery.size()) {
                     return false;
                 }
diff --git a/src/main/java/com/easypost/easyvcr/RegexCensorElement.java b/src/main/java/com/easypost/easyvcr/RegexCensorElement.java
@@ -0,0 +1,44 @@
+package com.easypost.easyvcr;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class RegexCensorElement extends CensorElement {
+    /**
+     * Constructor.
+     *
+     * @param pattern       The regular expression pattern of the element to censor.
+     * @param caseSensitive Whether the pattern is case sensitive.
+     */
+    public RegexCensorElement(String pattern, boolean caseSensitive) {
+        super(pattern, caseSensitive);
+    }
+
+    /**
+     * Replace the provided value with the provided replacement if the value matches the pattern.
+     *
+     * @param value       Value to apply the replacement to.
+     * @param replacement Replacement for a detected matching section.
+     * @return The replacement if the value matches the pattern, otherwise the original value.
+     */
+    public String matchAndReplaceAsNeeded(String value, String replacement) {
+        Pattern pattern = Pattern.compile(this.value, caseSensitive ? 0 : Pattern.CASE_INSENSITIVE);
+        Matcher matcher = pattern.matcher(value);
+        return matcher.replaceAll(replacement);
+    }
+
+    /**
+     * Return whether the provided element matches the name, accounting for case sensitivity.
+     *
+     * @param key The value to check.
+     * @return True if the element matches the pattern.
+     */
+    @Override
+    public boolean matches(String key) {
+        Pattern pattern = Pattern.compile(this.value, caseSensitive ? 0 : Pattern.CASE_INSENSITIVE);
+        Matcher matcher = pattern.matcher(key);
+
+        // a portion of the key matches the pattern, find() == true, matches() == false (whole key must match)
+        return matcher.find();
+    }
+}
diff --git a/src/main/java/com/easypost/easyvcr/Utilities.java b/src/main/java/com/easypost/easyvcr/Utilities.java
@@ -1,7 +1,13 @@
 package com.easypost.easyvcr;
 
+import org.apache.http.NameValuePair;
+import org.apache.http.client.utils.URLEncodedUtils;
+
 import javax.net.ssl.HttpsURLConnection;
 import java.net.HttpURLConnection;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -62,4 +68,44 @@ public static String removeJsonElements(String json, List<CensorElement> element
 
         return Censors.censorJsonData(json, "FILTERED", elements);
     }
+
+    /**
+     * Convert a URI's query parameters to a Map.
+     *
+     * @param uri The URI.
+     * @return The Map of query parameters.
+     */
+    public static Map<String, String> queryParametersToMap(URI uri) {
+        List<NameValuePair> receivedQueryDict = URLEncodedUtils.parse(uri, StandardCharsets.UTF_8);
+        if (receivedQueryDict == null || receivedQueryDict.size() == 0) {
+            return Collections.emptyMap();
+        }
+        Map<String, String> queryDict = new java.util.Hashtable<>();
+        for (NameValuePair pair : receivedQueryDict) {
+            queryDict.put(pair.getName(), pair.getValue());
+        }
+        return queryDict;
+    }
+
+    /**
+     * Extract the path from a URI.
+     *
+     * @param uri The URI to extract the path from.
+     * @return The path.
+     */
+    public static String extractPathFromUri(URI uri) {
+        String uriString = uri.toString();
+
+        // strip the query parameters
+        uriString = uriString.replace(uri.getQuery(), "");
+
+        if (uriString.endsWith("?")) {
+            uriString = uriString.substring(0, uriString.length() - 1);
+        }
+
+        // strip the scheme
+        uriString = uriString.replace(uri.getScheme() + "://", "");
+
+        return uriString;
+    }
 }
diff --git a/src/main/java/com/easypost/easyvcr/interactionconverters/HttpUrlConnectionInteractionConverter.java b/src/main/java/com/easypost/easyvcr/interactionconverters/HttpUrlConnectionInteractionConverter.java
diff --git a/src/main/java/com/easypost/easyvcr/internalutilities/Tools.java b/src/main/java/com/easypost/easyvcr/internalutilities/Tools.java
diff --git a/src/test/java/HttpUrlConnectionTest.java b/src/test/java/HttpUrlConnectionTest.java
