Merge pull request #33 from paypal/issue-32-sanitized-string-charset-java8

fix wrong charset of sanitized strings in Java8 heap dumps

Author: maxxedev

src/main/java/com/paypal/heapdumptool/sanitizer/HeapDumpSanitizer.java

@@ -9,6 +9,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.io.UnsupportedEncodingException;
 import java.nio.charset.StandardCharsets;
 import java.util.Collection;
 import java.util.HashMap;
@@ -68,6 +69,7 @@ public class HeapDumpSanitizer {
     private final Map<String, ClassObject> classNameToClassObjectsMap = new HashMap<>();
     private final Set<Long> excludeStringObjectIds = new HashSet<>();
     private final Set<Long> excludeStringValueArrayObjectIds = new HashSet<>();
+    private boolean isLikelyJdk9Plus;
 
     public void setInputStream(final InputStream inputStream) {
         this.inputStream = inputStream;
@@ -260,6 +262,10 @@ private void copyHeapDumpClassDump(final Pipe pipe, final long classObjectId) th
             final String fieldName = stringIdToStringMap.getOrDefault(fieldNameStringId, "");
             final BasicType basicType = BasicType.findByU1Code(fieldType).orElseThrow(IllegalStateException::new);
             classObject.fields.add(new Field(fieldName, basicType));
+
+            if (isStringClass(classObjectId) && STRING_CODER_FIELD.equals(fieldName)) {
+                isLikelyJdk9Plus = true;
+            }
         }
     }
 
@@ -457,13 +463,25 @@ private boolean shouldApplyArraySanitization(final long objectId, final int elem
 
     private void applySanitization(final Pipe pipe, final long numBytes) throws IOException {
         pipe.skipInput(numBytes);
+        final byte[] replacementData = getSanitizationTextBytes();
 
-        final byte[] replacementData = sanitizeCommand.getSanitizationText().getBytes(StandardCharsets.UTF_8);
         try (final InputStream replacementDataStream = new InfiniteCircularInputStream(replacementData)) {
             pipe.copyFrom(replacementDataStream, numBytes);
         }
     }
 
+    private byte[] getSanitizationTextBytes() throws UnsupportedEncodingException {
+        if (!sanitizeCommand.isSanitizationTextCharsetAutoDetect()) {
+            final String sanitizationTextCharset = sanitizeCommand.getSanitizationTextCharset();
+            return sanitizeCommand.getSanitizationText().getBytes(sanitizationTextCharset);
+        }
+
+        if (isLikelyJdk9Plus) {
+            return sanitizeCommand.getSanitizationText().getBytes(StandardCharsets.UTF_8);
+        }
+        return sanitizeCommand.getSanitizationText().getBytes(StandardCharsets.UTF_16BE);
+    }
+
     private static boolean isLatin1(final String input) {
         for (final char c : input.toCharArray()) {
             if (c > 0xFF) {

src/main/java/com/paypal/heapdumptool/sanitizer/SanitizeOrCaptureCommandBase.java

@@ -28,7 +28,7 @@ public abstract class SanitizeOrCaptureCommandBase implements CliCommand {
     @Option(names = {"-d", DOCKER_REGISTRY_OPTION}, description = "docker registry hostname for bootstrapping heap-dump-tool docker image")
     private String dockerRegistry;
 
-    @Option(names = {"-a", "--tar-input"}, description = "Treat input as tar archive")
+    @Option(names = {"-a", "--tar-input"}, description = "Treat input as tar archive", arity = "1")
     private boolean tarInput;
 
     @Option(names = {"-e", "--exclude-string-fields"},
@@ -47,15 +47,29 @@ public abstract class SanitizeOrCaptureCommandBase implements CliCommand {
     // match, all strings would be displayed as "*"
     private boolean forceMatchStringCoder;
 
-    @Option(names = {"-s", "--sanitize-byte-char-arrays-only"}, description = "Sanitize byte/char arrays only", defaultValue = "true", showDefaultValue = ALWAYS)
+    @Option(names = {"-s", "--sanitize-byte-char-arrays-only"},
+            description = "Sanitize byte/char arrays only",
+            defaultValue = "true",
+            arity = "1",
+            showDefaultValue = ALWAYS)
     private boolean sanitizeByteCharArraysOnly = true;
 
-    @Option(names = {"-S", "--sanitize-arrays-only"}, description = "Sanitize arrays only", defaultValue = "false", showDefaultValue = ALWAYS)
+    @Option(names = {"-S", "--sanitize-arrays-only"},
+            description = "Sanitize arrays only",
+            arity = "1",
+            defaultValue = "false",
+            showDefaultValue = ALWAYS)
     private boolean sanitizeArraysOnly;
 
     @Option(names = {"-t", "--text"}, description = "Sanitization text to replace with", defaultValue = "\\0", showDefaultValue = ALWAYS)
     private String sanitizationText = "\\0";
 
+    @Option(names = {"-T", "--text-charset"},
+            description = "Sanitization text charset",
+            defaultValue = "<auto-detect>",
+            showDefaultValue = ALWAYS)
+    private String sanitizationTextCharset = "<auto-detect>";
+
     private StringFieldMap excludeStringFieldMap;
 
     @Option(names = {"-b", "--buffer-size"}, description = "Buffer size for reading and writing", defaultValue = "100MB", showDefaultValue = ALWAYS)
@@ -113,6 +127,18 @@ public void setSanitizationText(final String sanitizationText) {
         this.sanitizationText = StringEscapeUtils.unescapeJava(sanitizationText);
     }
 
+    public boolean isSanitizationTextCharsetAutoDetect() {
+        return new SanitizeCommand().getSanitizationTextCharset().equals(getSanitizationTextCharset());
+    }
+
+    public String getSanitizationTextCharset() {
+        return sanitizationTextCharset;
+    }
+
+    public void setSanitizationTextCharset(final String sanitizationTextCharset) {
+        this.sanitizationTextCharset = sanitizationTextCharset;
+    }
+
     public boolean isForceMatchStringCoder() {
         return forceMatchStringCoder;
     }

src/test/java/com/paypal/heapdumptool/sanitizer/HeapDumpSanitizerTest.java

@@ -20,9 +20,9 @@
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.ThreadLocalRandom;
@@ -33,17 +33,16 @@
 import static com.paypal.heapdumptool.fixture.ByteArrayTool.lengthen;
 import static com.paypal.heapdumptool.fixture.ByteArrayTool.nCopiesLongToBytes;
 import static java.nio.ByteOrder.BIG_ENDIAN;
+import static java.nio.charset.StandardCharsets.UTF_16BE;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.util.Arrays.asList;
 import static org.apache.commons.io.FileUtils.byteCountToDisplaySize;
 import static org.apache.commons.lang3.ArrayUtils.EMPTY_BYTE_ARRAY;
 import static org.apache.commons.lang3.ArrayUtils.EMPTY_STRING_ARRAY;
-import static org.apache.commons.lang3.JavaVersion.JAVA_9;
-import static org.apache.commons.lang3.JavaVersion.JAVA_RECENT;
-import static org.apache.commons.lang3.SystemUtils.isJavaVersionAtLeast;
+import static org.apache.commons.lang3.JavaVersion.JAVA_1_8;
+import static org.apache.commons.lang3.SystemUtils.isJavaVersionAtMost;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatCode;
-import static org.assertj.core.api.Assumptions.assumeThat;
 
 @TestMethodOrder(Random.class)
 class HeapDumpSanitizerTest {
@@ -215,11 +214,6 @@ void testSanitizeArraysOnly() throws Exception {
 
     @Test
     void testThreadNameExcludedFromSanitization() throws Exception {
-        // verify java 8 manually ...
-        assumeThat(isJavaVersionAtLeast(JAVA_9))
-                .withFailMessage(JAVA_RECENT + "")
-                .isTrue();
-
         // "xN-classified-value" with each letter incremented by 1
         final String x2ClassifiedValue = "y3.dmbttjgjfe.wbmvf";
         final String x5ClassifiedValue = "y6.dmbttjgjfe.wbmvf";
@@ -228,12 +222,13 @@ void testThreadNameExcludedFromSanitization() throws Exception {
         thread.setDaemon(true);
         thread.setName(adjustLetters(x2ClassifiedValue));
 
+        final Charset charset = isJavaVersionAtMost(JAVA_1_8) ? UTF_16BE : UTF_8;
         final byte[] sanitizedHeapDump = loadSanitizedHeapDump();
         assertThat(sanitizedHeapDump)
                 .withFailMessage("threadGroupName " + threadGroup.getName())
-                .containsSequence(butLast(threadGroup.getName()).getBytes(UTF_8))
+                .containsSequence(butLast(threadGroup.getName()).getBytes(charset))
                 .withFailMessage("threadName " + thread.getName())
-                .containsSequence(butLast(thread.getName()).getBytes(UTF_8));
+                .containsSequence(butLast(thread.getName()).getBytes(charset));
     }
 
     private String butLast(final String input) {
@@ -257,12 +252,12 @@ void testThreadNameIncludedInSanitization() throws Exception {
 
     // 0xDEADBEEF
     private long deadcow() {
-        return 0xDEADBEEE + Long.parseLong("1");
+        return 0xDEADBEEE + Integer.parseInt("1");
     }
 
     // 0xCAFEBABE
     private long cafegirl() {
-        return 0XCAFEBABD + Long.parseLong("1");
+        return 0xCAFEBABD + Integer.parseInt("1");
     }
 
     private void verifyDoesNotContainsSequence(final byte[] big, final byte[] small) {