Fixed loading yaml with header issue #1445

Author: mikefarah

examples/small.yaml

@@ -0,0 +1,4 @@
+---
+# comment
+# about things
+a: cat

pkg/yqlib/decoder_yaml.go

@@ -2,6 +2,7 @@ package yqlib
 
 import (
 	"bufio"
+	"bytes"
 	"errors"
 	"io"
 	"regexp"
@@ -12,11 +13,15 @@ import (
 
 type yamlDecoder struct {
 	decoder yaml.Decoder
+
+	prefs YamlPreferences
+
 	// work around of various parsing issues by yaml.v3 with document headers
-	prefs          YamlPreferences
 	leadingContent string
-	readAnything   bool
-	firstFile      bool
+	bufferRead     bytes.Buffer
+
+	readAnything bool
+	firstFile    bool
 }
 
 func NewYamlDecoder(prefs YamlPreferences) Decoder {
@@ -59,6 +64,7 @@ func (dec *yamlDecoder) processReadStream(reader *bufio.Reader) (io.Reader, stri
 func (dec *yamlDecoder) Init(reader io.Reader) error {
 	readerToUse := reader
 	leadingContent := ""
+	dec.bufferRead = bytes.Buffer{}
 	var err error
 	// if we 'evaluating together' - we only process the leading content
 	// of the first file - this ensures comments from subsequent files are
@@ -68,6 +74,12 @@ func (dec *yamlDecoder) Init(reader io.Reader) error {
 		if err != nil {
 			return err
 		}
+	} else if !dec.prefs.LeadingContentPreProcessing {
+		// if we're not process the leading content
+		// keep a copy of what we've read. This is incase its a
+		// doc with only comments - the decoder will return nothing
+		// then we can read the comments from bufferRead
+		readerToUse = io.TeeReader(reader, &dec.bufferRead)
 	}
 	dec.leadingContent = leadingContent
 	dec.readAnything = false
@@ -78,13 +90,20 @@ func (dec *yamlDecoder) Init(reader io.Reader) error {
 
 func (dec *yamlDecoder) Decode() (*CandidateNode, error) {
 	var dataBucket yaml.Node
-
 	err := dec.decoder.Decode(&dataBucket)
 	if errors.Is(err, io.EOF) && dec.leadingContent != "" && !dec.readAnything {
 		// force returning an empty node with a comment.
 		dec.readAnything = true
 		return dec.blankNodeWithComment(), nil
-
+	} else if errors.Is(err, io.EOF) && !dec.prefs.LeadingContentPreProcessing && !dec.readAnything {
+		// didn't find any yaml,
+		// check the tee buffer, maybe there were comments
+		dec.readAnything = true
+		dec.leadingContent = dec.bufferRead.String()
+		if dec.leadingContent != "" {
+			return dec.blankNodeWithComment(), nil
+		}
+		return nil, err
 	} else if err != nil {
 		return nil, err
 	}
@@ -97,6 +116,7 @@ func (dec *yamlDecoder) Decode() (*CandidateNode, error) {
 		candidateNode.LeadingContent = dec.leadingContent
 		dec.leadingContent = ""
 	}
+	dec.readAnything = true
 	// move document comments into candidate node
 	// otherwise unwrap drops them.
 	candidateNode.TrailingContent = dataBucket.FootComment

pkg/yqlib/lexer_participle.go

@@ -86,7 +86,7 @@ var participleYqRules = []*participleYqRule{
 
 	{"LoadString", `load_?str|str_?load`, loadOp(nil, true), 0},
 
-	{"LoadYaml", `load`, loadOp(NewYamlDecoder(ConfiguredYamlPreferences), false), 0},
+	{"LoadYaml", `load`, loadOp(NewYamlDecoder(LoadYamlPreferences), false), 0},
 
 	{"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0},
 

pkg/yqlib/operator_load.go

@@ -9,6 +9,13 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
+var LoadYamlPreferences = YamlPreferences{
+	LeadingContentPreProcessing: false,
+	PrintDocSeparators:          true,
+	UnwrapScalar:                true,
+	EvaluateTogether:            false,
+}
+
 type loadPrefs struct {
 	loadAsString bool
 	decoder      Decoder
@@ -43,7 +50,10 @@ func loadYaml(filename string, decoder Decoder) (*CandidateNode, error) {
 		// return null candidate
 		return &CandidateNode{Node: &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!null"}}, nil
 	} else if documents.Len() == 1 {
-		return documents.Front().Value.(*CandidateNode), nil
+		candidate := documents.Front().Value.(*CandidateNode)
+		log.Debug("first comment:", candidate.LeadingContent)
+		// candidate.Node.Content[0].Content[0].HeadComment = candidate.LeadingContent
+		return candidate, nil
 
 	} else {
 		sequenceNode := &CandidateNode{Node: &yaml.Node{Kind: yaml.SequenceNode}}

pkg/yqlib/operator_load_test.go

@@ -13,6 +13,15 @@ var loadScenarios = []expressionScenario{
 			"D0, P[], (doc)::# comment\n\n",
 		},
 	},
+	{
+		skipDoc:     true,
+		description: "Load file with a header comment into an array",
+		document:    `- "../../examples/small.yaml"`,
+		expression:  `.[] |= load(.)`,
+		expected: []string{
+			"D0, P[], (doc)::- # comment\n  # about things\n  a: cat\n",
+		},
+	},
 	{
 		skipDoc:     true,
 		description: "Load empty file with no comment",

pkg/yqlib/operators_test.go

@@ -31,7 +31,7 @@ type expressionScenario struct {
 }
 
 func TestMain(m *testing.M) {
-	logging.SetLevel(logging.ERROR, "")
+	logging.SetLevel(logging.DEBUG, "")
 	Now = func() time.Time {
 		return time.Date(2021, time.May, 19, 1, 2, 3, 4, time.UTC)
 	}