Merge branch 'master' into add-comment-style

mikefarah · web-flow · commit 1a4b2485ff27 · 2024-11-16T15:16:12.000+11:00
diff --git a/.golangci.yml b/.golangci.yml
@@ -10,7 +10,9 @@ linters:
     - gofmt
     - goimports
     - gosec
-    - megacheck
+    - gosimple
+    - staticcheck
+    - unused
     - misspell
     - nakedret
     - nolintlint
@@ -31,13 +33,6 @@ linters-settings:
             desc: "replaced by io and os packages since Go 1.16: https://tip.golang.org/doc/go1.16#ioutil"
 issues:
   exclude-rules:
-  - linters:
-    - gosec
-    text:  "Implicit memory aliasing in for loop."
-    path: _test\.go
-  - linters:
-    - revive
-    text: "unexported-return"
   - linters:
     - revive
     text: "var-naming"
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.23.2 as builder
+FROM golang:1.23.3 as builder
 
 WORKDIR /go/src/mikefarah/yq
 
diff --git a/Dockerfile.dev b/Dockerfile.dev
@@ -1,4 +1,4 @@
-FROM golang:1.23.2
+FROM golang:1.23.3
 
 RUN apt-get update && \
     apt-get install -y npm && \
diff --git a/cmd/constant.go b/cmd/constant.go
@@ -12,9 +12,6 @@ var outputFormat = ""
 var inputFormat = ""
 
 var exitStatus = false
-var forceColor = false
-var forceNoColor = false
-var colorsEnabled = false
 var indent = 2
 var noDocSeparators = false
 var nullInput = false
@@ -23,6 +20,10 @@ var verbose = false
 var version = false
 var prettyPrint = false
 
+var forceColor = false
+var forceNoColor = false
+var colorsEnabled = false
+
 // can be either "" (off), "extract" or "process"
 var frontMatter = ""
 
diff --git a/cmd/evaluate_all_command.go b/cmd/evaluate_all_command.go
@@ -12,7 +12,7 @@ func createEvaluateAllCommand() *cobra.Command {
 		Use:     "eval-all [expression] [yaml_file1]...",
 		Aliases: []string{"ea"},
 		Short:   "Loads _all_ yaml documents of _all_ yaml files and runs expression once",
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+		ValidArgsFunction: func(_ *cobra.Command, args []string, _ string) ([]string, cobra.ShellCompDirective) {
 			if len(args) == 0 {
 				return nil, cobra.ShellCompDirectiveNoFileComp
 			}
diff --git a/cmd/evaluate_sequence_command.go b/cmd/evaluate_sequence_command.go
@@ -13,7 +13,7 @@ func createEvaluateSequenceCommand() *cobra.Command {
 		Use:     "eval [expression] [yaml_file1]...",
 		Aliases: []string{"e"},
 		Short:   "(default) Apply the expression to each document in each yaml file in sequence",
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+		ValidArgsFunction: func(_ *cobra.Command, args []string, _ string) ([]string, cobra.ShellCompDirective) {
 			if len(args) == 0 {
 				return nil, cobra.ShellCompDirectiveNoFileComp
 			}
diff --git a/cmd/root.go b/cmd/root.go
@@ -66,7 +66,7 @@ yq -P -oy sample.json
 			return evaluateSequence(cmd, args)
 
 		},
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+		PersistentPreRunE: func(cmd *cobra.Command, _ []string) error {
 			cmd.SetOut(cmd.OutOrStdout())
 			level := logging.WARNING
 			stringFormat := `[%{level}] %{color}%{time:15:04:05}%{color:reset} %{message}`
@@ -90,6 +90,10 @@ yq -P -oy sample.json
 			logging.SetBackend(backend)
 			yqlib.InitExpressionParser()
 
+			// when NO_COLOR environment variable presents and not an empty string the coloured output should be disabled;
+			// refer to no-color.org
+			forceNoColor = os.Getenv("NO_COLOR") != ""
+
 			return nil
 		},
 	}
@@ -182,7 +186,7 @@ yq -P -oy sample.json
 	rootCmd.PersistentFlags().BoolVarP(&exitStatus, "exit-status", "e", false, "set exit status if there are no matches or null or false is returned")
 
 	rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors")
-	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
+	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", forceNoColor, "force print with no colors")
 	rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
 	if err = rootCmd.RegisterFlagCompletionFunc("front-matter", cobra.FixedCompletions([]string{"extract", "process"}, cobra.ShellCompDirectiveNoFileComp)); err != nil {
 		panic(err)
diff --git a/go.mod b/go.mod
@@ -16,8 +16,8 @@ require (
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
 	github.com/yuin/gopher-lua v1.1.1
-	golang.org/x/net v0.30.0
-	golang.org/x/text v0.19.0
+	golang.org/x/net v0.31.0
+	golang.org/x/text v0.20.0
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -26,7 +26,8 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/sys v0.27.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 )
 
 go 1.21.0
diff --git a/go.sum b/go.sum
@@ -62,16 +62,18 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
 github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
-golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
-golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
-golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
-golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
+golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ=
+golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg=
+golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo=
+golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
-golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
-golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
+golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
+golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
+golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
+golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW8s2qTSe3wGBtvo0MbVQG/c5k8RE=
diff --git a/pkg/yqlib/decoder_toml.go b/pkg/yqlib/decoder_toml.go
@@ -286,7 +286,7 @@ func (dec *tomlDecoder) processTable(currentNode *toml.Node) (bool, error) {
 		}
 
 		runAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue)
-		if err != nil && !errors.Is(io.EOF, err) {
+		if err != nil && !errors.Is(err, io.EOF) {
 			return false, err
 		}
 	}
@@ -343,7 +343,7 @@ func (dec *tomlDecoder) processArrayTable(currentNode *toml.Node) (bool, error)
 	tableValue := dec.parser.Expression()
 	runAgainstCurrentExp, err := dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue)
 	log.Debugf("table node err: %w", err)
-	if err != nil && !errors.Is(io.EOF, err) {
+	if err != nil && !errors.Is(err, io.EOF) {
 		return false, err
 	}
 	c := Context{}
diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go
@@ -311,7 +311,7 @@ func opTokenWithPrefs(opType *operationType, assignOpType *operationType, prefer
 }
 
 func expressionOpToken(expression string) yqAction {
-	return func(rawToken lexer.Token) (*token, error) {
+	return func(_ lexer.Token) (*token, error) {
 		prefs := expressionOpPreferences{expression: expression}
 		expressionOp := &Operation{OperationType: expressionOpType, Preferences: prefs}
 		return &token{TokenType: operationToken, Operation: expressionOp}, nil
@@ -522,7 +522,7 @@ func parentWithLevel() yqAction {
 }
 
 func parentWithDefaultLevel() yqAction {
-	return func(rawToken lexer.Token) (*token, error) {
+	return func(_ lexer.Token) (*token, error) {
 		prefs := parentOpPreferences{Level: 1}
 		op := &Operation{OperationType: getParentOpType, Value: getParentOpType.Type, StringValue: getParentOpType.Type, Preferences: prefs}
 		return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil
diff --git a/pkg/yqlib/operator_assign.go b/pkg/yqlib/operator_assign.go
@@ -7,7 +7,7 @@ type assignPreferences struct {
 }
 
 func assignUpdateFunc(prefs assignPreferences) crossFunctionCalculation {
-	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	return func(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 		if !prefs.OnlyWriteNull || lhs.Tag == "!!null" {
 			lhs.UpdateFrom(rhs, prefs)
 		}
diff --git a/pkg/yqlib/operator_compare.go b/pkg/yqlib/operator_compare.go
@@ -18,7 +18,7 @@ func compareOperator(d *dataTreeNavigator, context Context, expressionNode *Expr
 }
 
 func compare(prefs compareTypePref) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
-	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	return func(_ *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 		log.Debugf("compare cross function")
 		if lhs == nil && rhs == nil {
 			owner := &CandidateNode{}
diff --git a/pkg/yqlib/operator_create_map.go b/pkg/yqlib/operator_create_map.go
@@ -53,7 +53,7 @@ func sequenceFor(d *dataTreeNavigator, context Context, matchingNode *CandidateN
 	log.Debugf("**********sequenceFor %v", NodeToString(matchingNode))
 
 	mapPairs, err := crossFunction(d, context.ChildContext(matches), expressionNode,
-		func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+		func(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 			node := &CandidateNode{Kind: MappingNode, Tag: "!!map"}
 
 			log.Debugf("**********adding key %v and value %v", NodeToString(lhs), NodeToString(rhs))
diff --git a/pkg/yqlib/operator_equals.go b/pkg/yqlib/operator_equals.go
@@ -6,7 +6,7 @@ func equalsOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 }
 
 func isEquals(flip bool) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
-	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	return func(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 		value := false
 		log.Debugf("isEquals cross function")
 		if lhs == nil && rhs == nil {
diff --git a/pkg/yqlib/operator_error.go b/pkg/yqlib/operator_error.go
@@ -1,7 +1,7 @@
 package yqlib
 
 import (
-	"fmt"
+	"errors"
 )
 
 func errorOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
@@ -16,5 +16,5 @@ func errorOperator(d *dataTreeNavigator, context Context, expressionNode *Expres
 	if rhs.MatchingNodes.Len() > 0 {
 		errorMessage = rhs.MatchingNodes.Front().Value.(*CandidateNode).Value
 	}
-	return Context{}, fmt.Errorf(errorMessage)
+	return Context{}, errors.New(errorMessage)
 }
diff --git a/pkg/yqlib/operator_sort.go b/pkg/yqlib/operator_sort.go
@@ -69,6 +69,7 @@ func (a sortableNodeArray) Less(i, j int) bool {
 	rhsContext := a[j].CompareContext
 
 	rhsEl := rhsContext.MatchingNodes.Front()
+
 	for lhsEl := lhsContext.MatchingNodes.Front(); lhsEl != nil && rhsEl != nil; lhsEl = lhsEl.Next() {
 		lhs := lhsEl.Value.(*CandidateNode)
 		rhs := rhsEl.Value.(*CandidateNode)
@@ -83,7 +84,7 @@ func (a sortableNodeArray) Less(i, j int) bool {
 
 		rhsEl = rhsEl.Next()
 	}
-	return false
+	return lhsContext.MatchingNodes.Len() < rhsContext.MatchingNodes.Len()
 }
 
 func (a sortableNodeArray) compare(lhs *CandidateNode, rhs *CandidateNode, dateTimeLayout string) int {
diff --git a/pkg/yqlib/operator_sort_test.go b/pkg/yqlib/operator_sort_test.go
@@ -21,6 +21,17 @@ var sortByOperatorScenarios = []expressionScenario{
 			"D0, P[0], (!!map)::{a: banana}\n",
 		},
 	},
+	{
+		description: "Sort by with null",
+		skipDoc:     true,
+		document:    "[{a: banana},null,{a: apple}]",
+		expression:  `sort_by(.a)[]`,
+		expected: []string{
+			"D0, P[1], (!!null)::null\n",
+			"D0, P[2], (!!map)::{a: apple}\n",
+			"D0, P[0], (!!map)::{a: banana}\n",
+		},
+	},
 	{
 		description: "Sort by multiple fields",
 		document:    "[{a: dog},{a: cat, b: banana},{a: cat, b: apple}]",
diff --git a/scripts/devtools.sh b/scripts/devtools.sh
@@ -1,5 +1,5 @@
 #!/bin/sh
 set -ex
 go mod download golang.org/x/tools@latest
-curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.55.2
+curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.62.0
 wget -O- -nv https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-FROM golang:1.23.2 as builder`
	`1`	`+FROM golang:1.23.3 as builder`
`2`	`2`
`3`	`3`	`WORKDIR /go/src/mikefarah/yq`
`4`	`4`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-FROM golang:1.23.2`
	`1`	`+FROM golang:1.23.3`
`2`	`2`
`3`	`3`	`RUN apt-get update && \`
`4`	`4`	`apt-get install -y npm && \`
Original file line number	Diff line number	Diff line change
`@@ -12,7 +12,7 @@ func createEvaluateAllCommand() *cobra.Command {`
`12`	`12`	`Use: "eval-all [expression] [yaml_file1]...",`
`13`	`13`	`Aliases: []string{"ea"},`
`14`	`14`	`Short: "Loads _all_ yaml documents of _all_ yaml files and runs expression once",`
`15`		`- ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {`
	`15`	`+ ValidArgsFunction: func(_ *cobra.Command, args []string, _ string) ([]string, cobra.ShellCompDirective) {`
`16`	`16`	`if len(args) == 0 {`
`17`	`17`	`return nil, cobra.ShellCompDirectiveNoFileComp`
`18`	`18`	`}`
Original file line number	Diff line number	Diff line change
`@@ -13,7 +13,7 @@ func createEvaluateSequenceCommand() *cobra.Command {`
`13`	`13`	`Use: "eval [expression] [yaml_file1]...",`
`14`	`14`	`Aliases: []string{"e"},`
`15`	`15`	`Short: "(default) Apply the expression to each document in each yaml file in sequence",`
`16`		`- ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {`
	`16`	`+ ValidArgsFunction: func(_ *cobra.Command, args []string, _ string) ([]string, cobra.ShellCompDirective) {`
`17`	`17`	`if len(args) == 0 {`
`18`	`18`	`return nil, cobra.ShellCompDirectiveNoFileComp`
`19`	`19`	`}`
Original file line number	Diff line number	Diff line change
`@@ -286,7 +286,7 @@ func (dec tomlDecoder) processTable(currentNode toml.Node) (bool, error) {`
`286`	`286`	`}`
`287`	`287`
`288`	`288`	`runAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue)`
`289`		`- if err != nil && !errors.Is(io.EOF, err) {`
	`289`	`+ if err != nil && !errors.Is(err, io.EOF) {`
`290`	`290`	`return false, err`
`291`	`291`	`}`
`292`	`292`	`}`
`@@ -343,7 +343,7 @@ func (dec tomlDecoder) processArrayTable(currentNode toml.Node) (bool, error)`
`343`	`343`	`tableValue := dec.parser.Expression()`
`344`	`344`	`runAgainstCurrentExp, err := dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue)`
`345`	`345`	`log.Debugf("table node err: %w", err)`
`346`		`- if err != nil && !errors.Is(io.EOF, err) {`
	`346`	`+ if err != nil && !errors.Is(err, io.EOF) {`
`347`	`347`	`return false, err`
`348`	`348`	`}`
`349`	`349`	`c := Context{}`