Fixed json decode to maintain key order

Author: mikefarah

pkg/yqlib/decoder_json.go

@@ -22,13 +22,13 @@ func (dec *jsonDecoder) Init(reader io.Reader) {
 
 func (dec *jsonDecoder) Decode(rootYamlNode *yaml.Node) error {
 
-	var dataBucket interface{}
+	var dataBucket orderedMap
 	log.Debug("going to decode")
 	err := dec.decoder.Decode(&dataBucket)
 	if err != nil {
 		return err
 	}
-	node, err := dec.convertToYamlNode(dataBucket)
+	node, err := dec.convertToYamlNode(&dataBucket)
 
 	if err != nil {
 		return err
@@ -38,39 +38,36 @@ func (dec *jsonDecoder) Decode(rootYamlNode *yaml.Node) error {
 	return nil
 }
 
-func (dec *jsonDecoder) convertToYamlNode(data interface{}) (*yaml.Node, error) {
-	switch data := data.(type) {
-	case nil:
-		return createScalarNode(nil, "null"), nil
-	case float64, float32:
-		// json decoder returns ints as float.
-		return parseSnippet(fmt.Sprintf("%v", data))
-	case int, int64, int32, string, bool:
-		return createScalarNode(data, fmt.Sprintf("%v", data)), nil
-	case map[string]interface{}:
-		return dec.parseMap(data)
-	case []interface{}:
-		return dec.parseArray(data)
-	default:
-		return nil, fmt.Errorf("unrecognised type :(")
+func (dec *jsonDecoder) convertToYamlNode(data *orderedMap) (*yaml.Node, error) {
+	if data.kv == nil {
+		switch rawData := data.altVal.(type) {
+		case nil:
+			return createScalarNode(nil, "null"), nil
+		case float64, float32:
+			// json decoder returns ints as float.
+			return parseSnippet(fmt.Sprintf("%v", rawData))
+		case int, int64, int32, string, bool:
+			return createScalarNode(rawData, fmt.Sprintf("%v", rawData)), nil
+		case []*orderedMap:
+			return dec.parseArray(rawData)
+		default:
+			return nil, fmt.Errorf("unrecognised type :( %v", rawData)
+		}
 	}
-}
-
-func (dec *jsonDecoder) parseMap(dataMap map[string]interface{}) (*yaml.Node, error) {
 
 	var yamlMap = &yaml.Node{Kind: yaml.MappingNode}
-
-	for key, value := range dataMap {
-		yamlValue, err := dec.convertToYamlNode(value)
+	for _, keyValuePair := range data.kv {
+		yamlValue, err := dec.convertToYamlNode(&keyValuePair.V)
 		if err != nil {
 			return nil, err
 		}
-		yamlMap.Content = append(yamlMap.Content, createScalarNode(key, fmt.Sprintf("%v", key)), yamlValue)
+		yamlMap.Content = append(yamlMap.Content, createScalarNode(keyValuePair.K, keyValuePair.K), yamlValue)
 	}
 	return yamlMap, nil
+
 }
 
-func (dec *jsonDecoder) parseArray(dataArray []interface{}) (*yaml.Node, error) {
+func (dec *jsonDecoder) parseArray(dataArray []*orderedMap) (*yaml.Node, error) {
 
 	var yamlMap = &yaml.Node{Kind: yaml.SequenceNode}
 

pkg/yqlib/encoder.go

@@ -66,8 +66,13 @@ func (o *orderedMap) UnmarshalJSON(data []byte) error {
 		}
 		return nil
 	case '[':
-		var arr []orderedMap
-		return json.Unmarshal(data, &arr)
+		var res []*orderedMap
+		if err := json.Unmarshal(data, &res); err != nil {
+			return err
+		}
+		o.altVal = res
+		o.kv = nil
+		return nil
 	}
 
 	return json.Unmarshal(data, &o.altVal)

pkg/yqlib/json_test.go

@@ -22,6 +22,13 @@ const sampleNdJson = `{"this": "is a multidoc json file"}
 {"a number": 4}
 `
 
+const sampleNdJsonKey = `{"a": "first", "b": "next", "ab": "last"}`
+
+const expectedJsonKeysInOrder = `a: first
+b: next
+ab: last
+`
+
 const expectedNdJsonYaml = `this: is a multidoc json file
 ---
 each:
@@ -157,13 +164,34 @@ var jsonScenarios = []formatScenario{
 		expected:     expectedNdJsonYaml,
 		scenarioType: "decode-ndjson",
 	},
+	{
+		description:  "Decode NDJSON, maintain key order",
+		skipDoc:      true,
+		input:        sampleNdJsonKey,
+		expected:     expectedJsonKeysInOrder,
+		scenarioType: "decode-ndjson",
+	},
 	{
 		description:  "numbers",
 		skipDoc:      true,
 		input:        "[3, 3.0, 3.1, -1]",
 		expected:     "- 3\n- 3\n- 3.1\n- -1\n",
 		scenarioType: "decode-ndjson",
 	},
+	{
+		description:  "number single",
+		skipDoc:      true,
+		input:        "3",
+		expected:     "3\n",
+		scenarioType: "decode-ndjson",
+	},
+	{
+		description:  "empty string",
+		skipDoc:      true,
+		input:        `""`,
+		expected:     "\"\"\n",
+		scenarioType: "decode-ndjson",
+	},
 	{
 		description:  "strings",
 		skipDoc:      true,