Fixed base64/xml decode an empty string #1209

Author: mikefarah

pkg/yqlib/decoder_base64.go

@@ -9,9 +9,10 @@ import (
 )
 
 type base64Decoder struct {
-	reader   io.Reader
-	finished bool
-	encoding base64.Encoding
+	reader       io.Reader
+	finished     bool
+	readAnything bool
+	encoding     base64.Encoding
 }
 
 func NewBase64Decoder() Decoder {
@@ -20,6 +21,7 @@ func NewBase64Decoder() Decoder {
 
 func (dec *base64Decoder) Init(reader io.Reader) {
 	dec.reader = reader
+	dec.readAnything = false
 	dec.finished = false
 }
 
@@ -35,7 +37,15 @@ func (dec *base64Decoder) Decode(rootYamlNode *yaml.Node) error {
 	}
 	if buf.Len() == 0 {
 		dec.finished = true
+
+		// if we've read _only_ an empty string, lets return that
+		// otherwise if we've already read some bytes, and now we get
+		// an empty string, then we are done.
+		if dec.readAnything {
+			return io.EOF
+		}
 	}
+	dec.readAnything = true
 	rootYamlNode.Kind = yaml.ScalarNode
 	rootYamlNode.Tag = "!!str"
 	rootYamlNode.Value = buf.String()

pkg/yqlib/decoder_xml.go

@@ -13,6 +13,7 @@ import (
 
 type xmlDecoder struct {
 	reader          io.Reader
+	readAnything    bool
 	attributePrefix string
 	contentName     string
 	strictMode      bool
@@ -28,6 +29,7 @@ func NewXMLDecoder(attributePrefix string, contentName string, strictMode bool)
 
 func (dec *xmlDecoder) Init(reader io.Reader) {
 	dec.reader = reader
+	dec.readAnything = false
 	dec.finished = false
 }
 
@@ -137,8 +139,11 @@ func (dec *xmlDecoder) Decode(rootYamlNode *yaml.Node) error {
 		return err
 	} else if firstNode.Tag == "!!null" {
 		dec.finished = true
-		return io.EOF
+		if dec.readAnything {
+			return io.EOF
+		}
 	}
+	dec.readAnything = true
 	rootYamlNode.Kind = yaml.DocumentNode
 	rootYamlNode.Content = []*yaml.Node{firstNode}
 	dec.finished = true

pkg/yqlib/expression_tokeniser.go

@@ -403,12 +403,18 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`fromyaml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat}))
 	lexer.Add([]byte(`fromjson`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat}))
 	lexer.Add([]byte(`fromxml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: XMLInputFormat}))
+	lexer.Add([]byte(`fromprops`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: PropertiesInputFormat}))
 
 	lexer.Add([]byte(`from_yaml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat}))
 	lexer.Add([]byte(`from_json`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat}))
 	lexer.Add([]byte(`from_xml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: XMLInputFormat}))
 	lexer.Add([]byte(`from_props`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: PropertiesInputFormat}))
 
+	lexer.Add([]byte(`@yamld`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat}))
+	lexer.Add([]byte(`@jsond`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat}))
+	lexer.Add([]byte(`@xmld`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: XMLInputFormat}))
+	lexer.Add([]byte(`@propsd`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: PropertiesInputFormat}))
+
 	lexer.Add([]byte(`sortKeys`), opToken(sortKeysOpType))
 	lexer.Add([]byte(`sort_keys`), opToken(sortKeysOpType))
 

pkg/yqlib/operator_encoder_decoder_test.go

@@ -251,6 +251,14 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!str)::\n",
 		},
 	},
+	{
+		description: "empty xml decode",
+		skipDoc:     true,
+		expression:  `"" | @xmld`,
+		expected: []string{
+			"D0, P[], (!!null)::\n",
+		},
+	},
 }
 
 func TestEncoderDecoderOperatorScenarios(t *testing.T) {

pkg/yqlib/xml_test.go

@@ -216,7 +216,7 @@ var xmlScenarios = []formatScenario{
 		description:  "Empty doc",
 		skipDoc:      true,
 		input:        "",
-		expected:     "",
+		expected:     "\n",
 		scenarioType: "decode",
 	},
 	{