Skip to content

Commit 7103b78

Browse files
mikefarahmikefarah
authored
Draft: Toml (#1439)
* toml wip * wip * Fixed auto parsing toml * Added build flag not to include toml * Parse toml docs and tests * minor updates
1 parent 47f4ddc commit 7103b78

File tree

16 files changed

+828
-23
lines changed

16 files changed

+828
-23
lines changed

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ yq*.snap
4141

4242
test.yml
4343
test*.yml
44+
test*.xml
45+
test*.toml
4446
test*.yaml
4547
0.yml
4648
1.yml

‎acceptance_tests/inputs-format-auto.sh

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
setUp() {
44
rm test*.yml 2>/dev/null || true
5+
rm test*.toml 2>/dev/null || true
56
rm test*.tfstate 2>/dev/null || true
67
rm test*.json 2>/dev/null || true
78
rm test*.properties 2>/dev/null || true
@@ -30,6 +31,26 @@ EOM
3031
assertEquals "$expected" "$X"
3132
}
3233

34+
testInputToml() {
35+
cat >test.toml <<EOL
36+
[owner]
37+
name = "Tom Preston-Werner"
38+
dob = 1979-05-27T07:32:00-08:00
39+
EOL
40+
41+
read -r -d '' expected << EOM
42+
owner:
43+
name: Tom Preston-Werner
44+
dob: 1979-05-27T07:32:00-08:00
45+
EOM
46+
47+
X=$(./yq -oy test.toml)
48+
assertEquals "$expected" "$X"
49+
50+
X=$(./yq ea -oy test.toml)
51+
assertEquals "$expected" "$X"
52+
}
53+
3354
testInputTfstate() {
3455
cat >test.tfstate <<EOL
3556
{ "mike" : { "things": "cool" } }

‎cmd/root.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,8 @@ yq -P sample.json
6565
panic(err)
6666
}
6767

68-
rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "auto", "[auto|a|yaml|y|json|j|props|p|xml|x] output format type.")
69-
rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "auto", "[auto|a|yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.")
68+
rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "auto", "[auto|a|yaml|y|json|j|props|p|xml|x|tsv|t|csv|c] output format type.")
69+
rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "auto", "[auto|a|yaml|y|props|p|xml|x|tsv|t|csv|c|toml] parse format for input. Note that json is a subset of yaml.")
7070

7171
rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", yqlib.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes")
7272
rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", yqlib.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).")

‎cmd/utils.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
7575
}
7676
} else if isAutomaticOutputFormat() {
7777
// automatic input worked, we can do it for output too unless specified
78+
if inputFormat == "toml" {
79+
return "", nil, fmt.Errorf("toml is not yet supported as an output format. Please specify another output format using the [--output-format/-o] flag")
80+
}
7881
outputFormat = inputFormat
7982
}
8083
} else if isAutomaticOutputFormat() {
@@ -137,6 +140,8 @@ func createDecoder(format yqlib.InputFormat, evaluateTogether bool) (yqlib.Decod
137140
return yqlib.NewCSVObjectDecoder(','), nil
138141
case yqlib.TSVObjectInputFormat:
139142
return yqlib.NewCSVObjectDecoder('\t'), nil
143+
case yqlib.TomlInputFormat:
144+
return yqlib.NewTomlDecoder(), nil
140145
case yqlib.YamlInputFormat:
141146
prefs := yqlib.ConfiguredYamlPreferences
142147
prefs.EvaluateTogether = evaluateTogether

‎examples/sample.toml

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
2+
3+
# This is a TOML document
4+
5+
title = "TOML Example"
6+
7+
[owner]
8+
name = "Tom Preston-Werner"
9+
dob = 1979-05-27T07:32:00-08:00
10+
11+
[database]
12+
enabled = true
13+
ports = [ 8000, 8001, 8002 ]
14+
data = [ ["delta", "phi"], [3.14] ]
15+
temp_targets = { cpu = 79.5, case = 72.0 }
16+
17+
[servers]
18+
19+
[servers.alpha]
20+
ip = "10.0.0.1"
21+
role = "frontend"
22+
23+
[servers.beta]
24+
ip = "10.0.0.2"
25+
role = "backend"
26+

‎go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ require (
1111
github.com/goccy/go-yaml v1.10.0
1212
github.com/jinzhu/copier v0.3.5
1313
github.com/magiconair/properties v1.8.7
14+
github.com/pelletier/go-toml/v2 v2.0.6
1415
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
1516
github.com/spf13/cobra v1.6.1
1617
github.com/spf13/pflag v1.0.5

‎go.sum

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,9 @@ github.com/alecthomas/participle/v2 v2.0.0-beta.5/go.mod h1:RC764t6n4L8D8ITAJv0q
66
github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk=
77
github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
88
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
9-
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
109
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
10+
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
11+
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
1112
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
1213
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
1314
github.com/elliotchance/orderedmap v1.5.0 h1:1IsExUsjv5XNBD3ZdC7jkAAqLWOOKdbPTmkHx63OsBg=
@@ -36,6 +37,8 @@ github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope
3637
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
3738
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
3839
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
40+
github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU=
41+
github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek=
3942
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
4043
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
4144
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@@ -49,9 +52,14 @@ github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUq
4952
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
5053
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
5154
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
55+
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
56+
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
5257
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
53-
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
5458
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
59+
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
60+
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
61+
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
62+
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
5563
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
5664
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
5765
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=

‎pkg/yqlib/data_tree_navigator.go

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,16 @@ import (
44
"fmt"
55

66
logging "gopkg.in/op/go-logging.v1"
7+
yaml "gopkg.in/yaml.v3"
78
)
89

910
type DataTreeNavigator interface {
1011
// given the context and a expressionNode,
1112
// this will process the against the given expressionNode and return
1213
// a new context of matching candidates
1314
GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error)
15+
16+
DeeplyAssign(context Context, path []interface{}, rhsNode *yaml.Node) error
1417
}
1518

1619
type dataTreeNavigator struct {
@@ -20,6 +23,27 @@ func NewDataTreeNavigator() DataTreeNavigator {
2023
return &dataTreeNavigator{}
2124
}
2225

26+
func (d *dataTreeNavigator) DeeplyAssign(context Context, path []interface{}, rhsNode *yaml.Node) error {
27+
28+
rhsCandidateNode := &CandidateNode{
29+
Path: path,
30+
Node: rhsNode,
31+
}
32+
33+
assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
34+
35+
rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}
36+
37+
assignmentOpNode := &ExpressionNode{
38+
Operation: assignmentOp,
39+
LHS: createTraversalTree(path, traversePreferences{}, false),
40+
RHS: &ExpressionNode{Operation: rhsOp},
41+
}
42+
43+
_, err := d.GetMatchingNodes(context, assignmentOpNode)
44+
return err
45+
}
46+
2347
func (d *dataTreeNavigator) GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error) {
2448
if expressionNode == nil {
2549
log.Debugf("getMatchingNodes - nothing to do")

‎pkg/yqlib/decoder.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ const (
1616
JsonInputFormat
1717
CSVObjectInputFormat
1818
TSVObjectInputFormat
19+
TomlInputFormat
1920
UriInputFormat
2021
)
2122

@@ -38,8 +39,10 @@ func InputFormatFromString(format string) (InputFormat, error) {
3839
return CSVObjectInputFormat, nil
3940
case "tsv", "t":
4041
return TSVObjectInputFormat, nil
42+
case "toml":
43+
return TomlInputFormat, nil
4144
default:
42-
return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml]", format)
45+
return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml|toml]", format)
4346
}
4447
}
4548

‎pkg/yqlib/decoder_properties.go

Lines changed: 1 addition & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -95,23 +95,7 @@ func (dec *propertiesDecoder) applyProperty(context Context, properties *propert
9595

9696
rhsNode.Tag = guessTagFromCustomType(rhsNode)
9797

98-
rhsCandidateNode := &CandidateNode{
99-
Path: path,
100-
Node: rhsNode,
101-
}
102-
103-
assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
104-
105-
rhsOp := &Operation{OperationType: referenceOpType, CandidateNode: rhsCandidateNode}
106-
107-
assignmentOpNode := &ExpressionNode{
108-
Operation: assignmentOp,
109-
LHS: createTraversalTree(path, traversePreferences{}, false),
110-
RHS: &ExpressionNode{Operation: rhsOp},
111-
}
112-
113-
_, err := dec.d.GetMatchingNodes(context, assignmentOpNode)
114-
return err
98+
return dec.d.DeeplyAssign(context, path, rhsNode)
11599
}
116100

117101
func (dec *propertiesDecoder) Decode() (*CandidateNode, error) {

0 commit comments

Comments
 (0)