Merge branch 'master' of https://github.com/FasterXML/jackson-dataformats-text

Author: cowtowncoder

README.md

@@ -23,7 +23,7 @@ Currently included backends are:
 Standard supported formats that are not yet included here (but are likely added
 in future)  are:
 
-* [XML](../../jackson-dataformt-xml)
+* [XML](../../../jackson-dataformat-xml)
 
 ## License
 

csv/README.md

@@ -10,7 +10,7 @@ Since version 2.3 this module is considered complete and production ready.
 All Jackson layers (streaming, databind, tree model) are supported.
 
 [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.fasterxml.jackson.dataformat/jackson-dataformat-csv/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.fasterxml.jackson.dataformat/jackson-dataformat-csv/)
-[![Javadoc](https://javadoc-emblem.rhcloud.com/doc/com.fasterxml.jackson.dataformat/jackson-dataformat-csv/badge.svg)](http://www.javadoc.io/doc/com.fasterxml.jackson.dataformat/jackson-dataformat-csv)
+[![Javadoc](https://javadoc.io/badge/com.fasterxml.jackson.dataformat/jackson-dataformat-csv.svg)](http://www.javadoc.io/doc/com.fasterxml.jackson.dataformat/jackson-dataformat-csv)
 
 ## Maven dependency
 

release-notes/VERSION

@@ -8,12 +8,16 @@ Modules:
 === Releases ===
 ------------------------------------------------------------------------
 
-2.9.3 (not yet released)
+2.9.3 (09-Dec-2017)
 
+#39 (yamk): Binary data not recognized by YAML parser
+ (repoted by tmoschou@github)
 #42 (csv): Add support for escaping double quotes with the configured escape character
  (contributed by frankgrimes97@github)
 #51 (csv): Set of custom objects with `IGNORE_UNKNOWN` brokes silently csv
  (reported by Simone L)
+#53: (yaml) Binary values written without type tag
+ (reported by arulrajnet@github)
 
 2.9.2 (14-Oct-2017)
 

yaml/README.md

@@ -10,7 +10,7 @@ Project is licensed under [Apache License 2.0](http://www.apache.org/licenses/LI
 
 [![Build Status](https://travis-ci.org/FasterXML/jackson-dataformat-yaml.svg?branch=master)](https://travis-ci.org/FasterXML/jackson-dataformat-yaml)
 [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml/)
-[![Javadoc](https://javadoc-emblem.rhcloud.com/doc/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml/badge.svg)](http://www.javadoc.io/doc/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml)
+[![Javadoc](https://javadoc.io/badge/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml.svg)](http://www.javadoc.io/doc/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml)
 
 Module has been production ready since version 2.5.
 

yaml/pom.xml

@@ -30,7 +30,7 @@
     <dependency>
       <groupId>org.yaml</groupId>
       <artifactId>snakeyaml</artifactId>
-      <version>1.18</version>
+      <version>1.19</version>
     </dependency>
 
      <!-- and for testing need annotations -->

yaml/src/main/java/com/fasterxml/jackson/dataformat/yaml/YAMLGenerator.java

@@ -12,6 +12,8 @@
 import org.yaml.snakeyaml.DumperOptions.FlowStyle;
 import org.yaml.snakeyaml.emitter.Emitter;
 import org.yaml.snakeyaml.events.*;
+import org.yaml.snakeyaml.external.biz.base64Coder.Base64Coder;
+import org.yaml.snakeyaml.nodes.Tag;
 
 import com.fasterxml.jackson.core.*;
 import com.fasterxml.jackson.core.base.GeneratorBase;
@@ -139,6 +141,7 @@ private Feature(boolean defaultState) {
     protected final static long MIN_INT_AS_LONG = (long) Integer.MIN_VALUE;
     protected final static long MAX_INT_AS_LONG = (long) Integer.MAX_VALUE;
     protected final static Pattern PLAIN_NUMBER_P = Pattern.compile("[0-9]*(\\.[0-9]*)?");
+    protected final static String TAG_BINARY = Tag.BINARY.toString();
 
     /*
     /**********************************************************
@@ -170,7 +173,8 @@ private Feature(boolean defaultState) {
     private final static Character STYLE_LITERAL = Character.valueOf('|');
 
     // Which flow style to use for Base64? Maybe basic quoted?
-    private final static Character STYLE_BASE64 = Character.valueOf('"');
+    // 29-Nov-2017, tatu: Actually SnakeYAML uses block style so:
+    private final static Character STYLE_BASE64 = STYLE_LITERAL;
 
     private final static Character STYLE_PLAIN = null;
 
@@ -593,12 +597,10 @@ public void writeBinary(Base64Variant b64variant, byte[] data, int offset, int l
             return;
         }
         _verifyValueWrite("write Binary value");
-        // ok, better just Base64 encode as a String...
         if (offset > 0 || (offset+len) != data.length) {
             data = Arrays.copyOfRange(data, offset, offset+len);
         }
-        String encoded = b64variant.encode(data);
-        _writeScalar(encoded, "byte[]", STYLE_BASE64);
+        _writeScalarBinary(data);
     }
 
     /*
@@ -762,13 +764,24 @@ protected void _releaseBuffers() {
      */
 
     // Implicit means that (type) tags won't be shown, right?
-    private final static ImplicitTuple DEFAULT_IMPLICIT = new ImplicitTuple(true, true);
+    private final static ImplicitTuple NO_TAGS = new ImplicitTuple(true, true);
+
+    // ... and sometimes we specifically DO want explicit tag:
+    private final static ImplicitTuple EXPLICIT_TAGS = new ImplicitTuple(false, false);
 
     protected void _writeScalar(String value, String type, Character style) throws IOException
     {
         _emitter.emit(_scalarEvent(value, style));
     }
 
+    private void _writeScalarBinary(byte[] data) throws IOException
+    {
+        // 29-Nov-2017, tatu: Use SnakeYAML encoder instead of Jackson's
+        String encoded = Base64Coder.encodeLines(data);
+        _emitter.emit(new ScalarEvent(null, TAG_BINARY, EXPLICIT_TAGS, encoded,
+                null, null, STYLE_BASE64));
+    }
+
     protected ScalarEvent _scalarEvent(String value, Character style)
     {
         String yamlTag = _typeId;
@@ -779,7 +792,9 @@ protected ScalarEvent _scalarEvent(String value, Character style)
         if (anchor != null) {
             _objectId = null;
         }
-        return new ScalarEvent(anchor, yamlTag, DEFAULT_IMPLICIT, value,
+        // 29-Nov-2017, tatu: Not 100% sure why we don't force explicit tags for
+        //    type id, but trying to do so seems to double up tag output...
+        return new ScalarEvent(anchor, yamlTag, NO_TAGS, value,
                 null, null, style);
     }
 }

yaml/src/main/java/com/fasterxml/jackson/dataformat/yaml/YAMLParser.java

@@ -7,18 +7,17 @@
 
 import org.yaml.snakeyaml.error.Mark;
 import org.yaml.snakeyaml.events.*;
+import org.yaml.snakeyaml.external.biz.base64Coder.Base64Coder;
 import org.yaml.snakeyaml.nodes.NodeId;
 import org.yaml.snakeyaml.nodes.Tag;
 import org.yaml.snakeyaml.parser.ParserImpl;
 import org.yaml.snakeyaml.reader.StreamReader;
+import org.yaml.snakeyaml.resolver.Resolver;
 
 import com.fasterxml.jackson.core.*;
 import com.fasterxml.jackson.core.base.ParserBase;
 import com.fasterxml.jackson.core.io.IOContext;
 import com.fasterxml.jackson.core.util.BufferRecycler;
-import com.fasterxml.jackson.core.util.ByteArrayBuilder;
-
-import org.yaml.snakeyaml.resolver.Resolver;
 
 /**
  * {@link JsonParser} implementation used to expose YAML documents
@@ -419,7 +418,7 @@ public JsonToken nextToken() throws IOException
         }
     }
 
-    protected JsonToken _decodeScalar(ScalarEvent scalar)
+    protected JsonToken _decodeScalar(ScalarEvent scalar) throws IOException
     {
         String value = scalar.getValue();
         _textValue = value;
@@ -429,7 +428,6 @@ protected JsonToken _decodeScalar(ScalarEvent scalar)
 
         if (typeTag == null || typeTag.equals("!")) { // no, implicit
             Tag nodeTag = _yamlResolver.resolve(NodeId.scalar, value, scalar.getImplicit().canOmitTagInPlainScalar());
-
             if (nodeTag == Tag.STR) {
                 return JsonToken.VALUE_STRING;
             }
@@ -458,6 +456,14 @@ protected JsonToken _decodeScalar(ScalarEvent scalar)
                     typeTag = typeTag.split(",")[0];
                 }
             }
+            // [dataformats-text#39]: support binary type
+            if ("binary".equals(typeTag)) {
+                // 29-Nov-2017, tatu: two choices for base64 codecs (or 3 with Java 8):
+                //   Jackson's own or SnakeYAML. Former is faster, but maybe makes sense
+                //   to use latter for sake of consistency.
+                _binaryValue = Base64Coder.decodeLines(value);
+                return JsonToken.VALUE_EMBEDDED_OBJECT;
+            }
             // canonical values by YAML are actually 'y' and 'n'; but plenty more unofficial:
             if ("bool".equals(typeTag)) { // must be "true" or "false"
                 Boolean B = _matchYAMLBoolean(value, len);
@@ -623,23 +629,21 @@ public int getText(Writer writer) throws IOException
 
     @Override
     public Object getEmbeddedObject() throws IOException {
+        if (_currToken == JsonToken.VALUE_EMBEDDED_OBJECT ) {
+            return _binaryValue;
+        }
         return null;
     }
 
-    // TODO: can remove from 2.9 or so (base impl added in 2.8)
-    @SuppressWarnings("resource")
+    // Base impl from `ParserBase` works fine here:
+//    public byte[] getBinaryValue(Base64Variant variant) throws IOException
+
     @Override
-    public byte[] getBinaryValue(Base64Variant variant) throws IOException
+    public int readBinaryValue(Base64Variant b64variant, OutputStream out) throws IOException
     {
-        if (_binaryValue == null) {
-            if (_currToken != JsonToken.VALUE_STRING) {
-                _reportError("Current token ("+_currToken+") not VALUE_STRING, can not access as binary");
-            }
-            ByteArrayBuilder builder = _getByteArrayBuilder();
-            _decodeBase64(getText(), builder, variant);
-            _binaryValue = builder.toByteArray();
-        }
-        return _binaryValue;
+        byte[] b = getBinaryValue(b64variant);
+        out.write(b);
+        return b.length;
     }
 
     /*

yaml/src/test/java/com/fasterxml/jackson/dataformat/yaml/EventsTest.java

@@ -10,7 +10,7 @@ public class ExceptionConversionTest extends ModuleTestBase
 {
     public void testSimpleParsingLeakage() throws Exception
     {
-        YAMLMapper mapper = mapperForYAML();
+        YAMLMapper mapper = newObjectMapper();
         try {
              mapper.readTree("foo:\nbar: true\n  baz: false");
              fail("Should not pass with invalid YAML");

yaml/src/test/java/com/fasterxml/jackson/dataformat/yaml/ExceptionConversionTest.java

@@ -67,8 +67,8 @@ protected ModuleTestBase() { }
     /* Helper methods, setup
     /**********************************************************************
      */
-    
-    protected YAMLMapper mapperForYAML()
+
+    protected YAMLMapper newObjectMapper()
     {
         return new YAMLMapper();
     }