diff --git a/.gitignore b/.gitignore index 74cbf5e..f04f266 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ /.project /pom.xml.bak /.settings/ +.idea # source https://github.com/github/gitignore/blob/master/Global/JetBrains.gitignore # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider diff --git a/.idea/compiler.xml b/.idea/compiler.xml deleted file mode 100644 index 35e8594..0000000 --- a/.idea/compiler.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/encodings.xml b/.idea/encodings.xml deleted file mode 100644 index b26911b..0000000 --- a/.idea/encodings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 0ae54f4..0000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index da579d0..0000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/log_file_filter__hpi_run_.xml b/.idea/runConfigurations/log_file_filter__hpi_run_.xml deleted file mode 100644 index d04b477..0000000 --- a/.idea/runConfigurations/log_file_filter__hpi_run_.xml +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 35eb1dd..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/log-file-filter.iml b/log-file-filter.iml deleted file mode 100644 index d6843d0..0000000 --- a/log-file-filter.iml +++ /dev/null @@ -1,177 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/src/main/java/com/tsystems/sbs/DefaultRegexpPairs.java b/src/main/java/com/tsystems/sbs/DefaultRegexpPairs.java index 9c9c816..45a3486 100644 --- a/src/main/java/com/tsystems/sbs/DefaultRegexpPairs.java +++ b/src/main/java/com/tsystems/sbs/DefaultRegexpPairs.java @@ -13,22 +13,28 @@ public final class DefaultRegexpPairs { private final static List DEFAULT_REGEXES - = Collections.unmodifiableList( - Arrays.asList( + = Collections.unmodifiableList( + Arrays.asList( new RegexpPair("(https?+://[^:\\s]++):[^@\\s]++@", "$1:********@"),//Passwd URL MASKING new RegexpPair("password=\\S*", "password=********") //PASSWORD MASKING )); private final static List DEFAULT_REGEXES_AWS - = Collections.unmodifiableList( - Arrays.asList( + = Collections.unmodifiableList( + Arrays.asList( new RegexpPair("(AWS_[a-zA-Z_]+=)(\\S+)", "$1********"), // AWS RegExp MASKING - new RegexpPair("(aws_[a-zA-Z_]+=)(\\S+)", "$1********") + new RegexpPair("(aws_[a-zA-Z_]+=)(\\S+)", "$1********"), + regexpMaskForJsonKeyEndingWith("accesskeyid"), + regexpMaskForJsonKeyEndingWith("secretaccesskey"), + regexpMaskForJsonKeyEndingWith("token"), + regexpMaskForEscapedJsonKeyWithSuffix("AccessKeyId"), + regexpMaskForEscapedJsonKeyWithSuffix("SecretAccessKey"), + regexpMaskForEscapedJsonKeyWithSuffix("Token") )); private final static List DEFAULT_REGEXES_DD - = Collections.unmodifiableList( - Arrays.asList( + = Collections.unmodifiableList( + Arrays.asList( new RegexpPair("((?i)(\\bdatadog|dd|dogapi\\b).*)(\\b([a-zA-Z-0-9]{32})\\b)", "$1********"), // Datadog RegExp MASKING API KEY new RegexpPair("((?i)(\\bdatadog|dd|dogapi\\b).*)(\\b([a-zA-Z-0-9]{40})\\b)", "$1********") // Datadog RegExp MASKING APP KEY )); @@ -44,4 +50,23 @@ public static List getDefaultRegexesAWS() { public static List getDefaultRegexesDD() { return DEFAULT_REGEXES_DD; } + + /** + * Case-insensitive regexp that matches output key ending with specified word, value of which should be masked + * Sample input: + * "ansible_ec2_iam_security_credentials_instance_profiles_accesskeyid": "ASIAU5QBETVVXEXAMPLE" + */ + private static RegexpPair regexpMaskForJsonKeyEndingWith(String keySuffix) { + return new RegexpPair("(?i)\"\\w*" + keySuffix + "\\\":\\s*\\\"[\\w\\d/+=*]+\\\"", "\"" + keySuffix + "\": \"********\""); + } + + /** + * Case-sensitive regexp that matches escaped output key, value of which should be masked + * Typically it is masked output of escaped json + * Sample input: + * \"AccessKeyId\" : \"ASIAU5QBETVVXEXAMPLE\" + */ + private static RegexpPair regexpMaskForEscapedJsonKeyWithSuffix(String key) { + return new RegexpPair("\\\\\\\"" + key + "\\\\\\\"\\s*:\\s*\\\\\\\"[\\w\\d/+=*]+\\\\\\\"", "\\\"" + key + "\\\": \\\"********\\\""); + } } diff --git a/src/test/java/com/tsystems/sbs/DefaultRegexpPairsAWSTest.java b/src/test/java/com/tsystems/sbs/DefaultRegexpPairsAWSTest.java index 7fd51d1..bd399a9 100644 --- a/src/test/java/com/tsystems/sbs/DefaultRegexpPairsAWSTest.java +++ b/src/test/java/com/tsystems/sbs/DefaultRegexpPairsAWSTest.java @@ -1,16 +1,13 @@ package com.tsystems.sbs; -import hudson.console.LineTransformationOutputStream; import org.junit.Test; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static org.codehaus.groovy.runtime.ResourceGroovyMethods.filterLine; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; @@ -34,6 +31,57 @@ public void testDefaultPairs() { String expected = "AWS_ACCESS_KEY_ID=******** AWS_SECRET_ACCESS_KEY=******** AWS_SESSION_TOKEN=********"; + String replacedInputString = maskIntput(input, defaultRegexpPairs); + System.out.println("Replaced input result: " + replacedInputString); + + // Test the behavior + assertEquals(expected, replacedInputString); + } + + @Test + public void testDefaultJsonKeyValuePairsMasking() { + List defaultRegexpPairs = getDefaultRegexpPairs(); + + String input = "log message with json {" + + "\"ansible_ec2_iam_security_credentials_instance_profiles_accesskeyid\": \"ASIAU5QBETVVXEXAMPLE\"," + + "\"ansible_ec2_iam_security_credentials_instance_profiles_secretaccesskey\": \"/lD8T9bXuZUW/F/8MutOB1vDXK2uG/gNHUe/d8bG\"," + + "\"ansible_ec2_iam_security_credentials_instance_profiles_token\": \"Z1XKqTnKIHd7eLJhBZb9QWVcG0Rj3f8z1uYgO4Xm6vNiD5F7cM9pA\"" + + "}"; + String expected = "log message with json {" + + "\"accesskeyid\": \"********\"," + + "\"secretaccesskey\": \"********\"," + + "\"token\": \"********\"" + + "}"; + + String replacedInputString = maskIntput(input, defaultRegexpPairs); + + // Test the behavior + assertEquals(expected, replacedInputString); + } + + + @Test + public void testDefaultEscapedJsonKeyValuePairsMasking() { + List defaultRegexpPairs = getDefaultRegexpPairs(); + + String input = "log message with escaped json {" + + "\\\"AccessKeyId\\\" : \\\"ASIAU5QBETVVXEXAMPLE\\\", " + + "\\\"SecretAccessKey\\\" : \\\"/lD8T9bXuZUW/F/8MutOB1vDXK2uG/gNHUe/d8bG\\\"," + + "\\\"Token\\\" : \\\"Z1XKqTnKIHd7eLJhBZb9QWVcG0Rj3f8z1uYgO4Xm6vNiD5F7cM9pA\\\"" + + "}"; + String expected = "log message with escaped json {" + + "\\\"AccessKeyId\\\": \\\"********\\\", " + + "\\\"SecretAccessKey\\\": \\\"********\\\"," + + "\\\"Token\\\": \\\"********\\\"" + + "}"; + + String replacedInputString = maskIntput(input, defaultRegexpPairs); + + // Test the behavior + assertEquals(expected, replacedInputString); + } + + private static String maskIntput(String input, List defaultRegexpPairs) { StringBuilder replacedInput = new StringBuilder(input); for (RegexpPair pair : defaultRegexpPairs) { @@ -44,7 +92,6 @@ public void testDefaultPairs() { Matcher matcher = regexPattern.matcher(replacedInput); while (matcher.find()) { - String matchedPattern = matcher.group(); String replacedString = replacement; // Replace all occurrences of $n with the matched groups @@ -58,11 +105,7 @@ public void testDefaultPairs() { } } - String replacedInputString = replacedInput.toString(); - System.out.println("Replaced input result: " + replacedInputString); - - // Test the behavior - assertEquals(expected, replacedInputString); + return replacedInput.toString(); } }