Skip to content

Commit eb286a1

Browse files
skmcgrailskmcgrail
authored
Support Op and Sep Tokens in Expression Values (#1324)
* Support Op and Sep Tokens in Expression Values * Additional Test Cases
1 parent 431a408 commit eb286a1

File tree

11 files changed

+177
-33
lines changed

11 files changed

+177
-33
lines changed

‎.changelog/5082ddd069fe4504932a1a45b3eff82a.json‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"id": "5082ddd0-69fe-4504-932a-1a45b3eff82a",
3+
"type": "feature",
4+
"description": "Support for `:`, `=`, `[`, `]` being present in expression values.",
5+
"modules": [
6+
"internal/ini"
7+
]
8+
}

‎internal/ini/doc.go‎

Lines changed: 23 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,17 +13,30 @@
1313
// }
1414
//
1515
// Below is the BNF that describes this parser
16-
// Grammar:
17-
// stmt -> value stmt'
18-
// stmt' -> epsilon | op stmt
19-
// value -> number | string | boolean | quoted_string
16+
// Grammar:
17+
// stmt -> section | stmt'
18+
// stmt' -> epsilon | expr
19+
// expr -> value (stmt)* | equal_expr (stmt)*
20+
// equal_expr -> value ( ':' | '=' ) equal_expr'
21+
// equal_expr' -> number | string | quoted_string
22+
// quoted_string -> " quoted_string'
23+
// quoted_string' -> string quoted_string_end
24+
// quoted_string_end -> "
2025
//
21-
// section -> [ section'
22-
// section' -> value section_close
23-
// section_close -> ]
26+
// section -> [ section'
27+
// section' -> section_value section_close
28+
// section_value -> number | string_subset | boolean | quoted_string_subset
29+
// quoted_string_subset -> " quoted_string_subset'
30+
// quoted_string_subset' -> string_subset quoted_string_end
31+
// quoted_string_subset -> "
32+
// section_close -> ]
2433
//
25-
// SkipState will skip (NL WS)+
34+
// value -> number | string_subset | boolean
35+
// string -> ? UTF-8 Code-Points except '\n' (U+000A) and '\r\n' (U+000D U+000A) ?
36+
// string_subset -> ? Code-points excepted by <string> grammar except ':' (U+003A), '=' (U+003D), '[' (U+005B), and ']' (U+005D) ?
2637
//
27-
// comment -> # comment' | ; comment'
28-
// comment' -> epsilon | value
38+
// SkipState will skip (NL WS)+
39+
//
40+
// comment -> # comment' | ; comment'
41+
// comment' -> epsilon | value
2942
package ini

‎internal/ini/ini_parser.go‎

Lines changed: 9 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,12 @@ import (
55
"io"
66
)
77

8+
// ParseState represents the current state of the parser.
9+
type ParseState uint
10+
811
// State enums for the parse table
912
const (
10-
InvalidState = iota
13+
InvalidState ParseState = iota
1114
// stmt -> value stmt'
1215
StatementState
1316
// stmt' -> MarkComplete | op stmt
@@ -36,7 +39,7 @@ const (
3639
)
3740

3841
// parseTable is a state machine to dictate the grammar above.
39-
var parseTable = map[ASTKind]map[TokenType]int{
42+
var parseTable = map[ASTKind]map[TokenType]ParseState{
4043
ASTKindStart: {
4144
TokenLit: StatementState,
4245
TokenSep: OpenScopeState,
@@ -64,6 +67,8 @@ var parseTable = map[ASTKind]map[TokenType]int{
6467
},
6568
ASTKindEqualExpr: {
6669
TokenLit: ValueState,
70+
TokenSep: ValueState,
71+
TokenOp: ValueState,
6772
TokenWS: SkipTokenState,
6873
TokenNL: SkipState,
6974
},
@@ -77,7 +82,7 @@ var parseTable = map[ASTKind]map[TokenType]int{
7782
},
7883
ASTKindExprStatement: {
7984
TokenLit: ValueState,
80-
TokenSep: OpenScopeState,
85+
TokenSep: ValueState,
8186
TokenOp: ValueState,
8287
TokenWS: ValueState,
8388
TokenNL: MarkCompleteState,
@@ -204,18 +209,6 @@ loop:
204209
case ValueState:
205210
// ValueState requires the previous state to either be an equal expression
206211
// or an expression statement.
207-
//
208-
// This grammar occurs when the RHS is a number, word, or quoted string.
209-
// equal_expr -> lit op equal_expr'
210-
// equal_expr' -> number | string | quoted_string
211-
// quoted_string -> " quoted_string'
212-
// quoted_string' -> string quoted_string_end
213-
// quoted_string_end -> "
214-
//
215-
// otherwise
216-
// expr_stmt -> equal_expr (expr_stmt')*
217-
// expr_stmt' -> ws S | op S | MarkComplete
218-
// S -> equal_expr' expr_stmt'
219212
switch k.Kind {
220213
case ASTKindEqualExpr:
221214
// assigning a value to some key
@@ -242,7 +235,7 @@ loop:
242235
}
243236

244237
children[len(children)-1] = rhs
245-
k.SetChildren(children)
238+
root.SetChildren(children)
246239

247240
stack.Push(k)
248241
}

‎internal/ini/ini_parser_test.go‎

Lines changed: 55 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,9 @@ func TestParser(t *testing.T) {
2323
outputID, _, _ := newLitToken([]rune("output"))
2424
outputLit, _, _ := newLitToken([]rune("json"))
2525

26+
sepInValueID, _, _ := newLitToken([]rune("sepInValue"))
27+
sepInValueLit := newToken(TokenOp, []rune("=:[foo]]bar["), StringType)
28+
2629
equalOp, _, _ := newOpToken([]rune("= 1234"))
2730
equalColonOp, _, _ := newOpToken([]rune(": 1234"))
2831
numLit, _, _ := newLitToken([]rune("1234"))
@@ -51,6 +54,9 @@ func TestParser(t *testing.T) {
5154
outputEQExpr := newEqualExpr(newExpression(outputID), equalOp)
5255
outputEQExpr.AppendChild(newExpression(outputLit))
5356

57+
sepInValueExpr := newEqualExpr(newExpression(sepInValueID), equalOp)
58+
sepInValueExpr.AppendChild(newExpression(sepInValueLit))
59+
5460
cases := []struct {
5561
name string
5662
r io.Reader
@@ -65,24 +71,48 @@ func TestParser(t *testing.T) {
6571
},
6672
},
6773
{
68-
name: "0==0",
69-
r: bytes.NewBuffer([]byte(`0==0`)),
70-
expectedError: true,
74+
name: "0==0",
75+
r: bytes.NewBuffer([]byte(`0==0`)),
76+
expectedStack: []AST{
77+
func() AST {
78+
equalExpr := newEqualExpr(newExpression(newToken(TokenLit, []rune("0"), StringType)), equalOp)
79+
equalExpr.AppendChild(newExpression(newToken(TokenOp, []rune("=0"), StringType)))
80+
return newExprStatement(equalExpr)
81+
}(),
82+
},
7183
},
7284
{
7385
name: "0=:0",
7486
r: bytes.NewBuffer([]byte(`0=:0`)),
75-
expectedError: true,
87+
expectedStack: []AST{
88+
func() AST {
89+
equalExpr := newEqualExpr(newExpression(newToken(TokenLit, []rune("0"), StringType)), equalOp)
90+
equalExpr.AppendChild(newExpression(newToken(TokenOp, []rune(":0"), StringType)))
91+
return newExprStatement(equalExpr)
92+
}(),
93+
},
7694
},
7795
{
7896
name: "0:=0",
7997
r: bytes.NewBuffer([]byte(`0:=0`)),
80-
expectedError: true,
98+
expectedStack: []AST{
99+
func() AST {
100+
equalExpr := newEqualExpr(newExpression(newToken(TokenLit, []rune("0"), StringType)), equalColonOp)
101+
equalExpr.AppendChild(newExpression(newToken(TokenOp, []rune("=0"), StringType)))
102+
return newExprStatement(equalExpr)
103+
}(),
104+
},
81105
},
82106
{
83107
name: "0::0",
84108
r: bytes.NewBuffer([]byte(`0::0`)),
85-
expectedError: true,
109+
expectedStack: []AST{
110+
func() AST {
111+
equalExpr := newEqualExpr(newExpression(newToken(TokenLit, []rune("0"), StringType)), equalColonOp)
112+
equalExpr.AppendChild(newExpression(newToken(TokenOp, []rune(":0"), StringType)))
113+
return newExprStatement(equalExpr)
114+
}(),
115+
},
86116
},
87117
{
88118
name: "section with variable",
@@ -287,6 +317,25 @@ output = json
287317
newExprStatement(outputEQExpr),
288318
},
289319
},
320+
{
321+
name: "token seperators [ and ] in values",
322+
r: bytes.NewBuffer([]byte(
323+
`[default]
324+
sepInValue = =:[foo]]bar[
325+
output = json
326+
[assumerole]
327+
sepInValue==:[foo]]bar[
328+
output = json
329+
`)),
330+
expectedStack: []AST{
331+
newCompletedSectionStatement(defaultProfileStmt),
332+
newExprStatement(sepInValueExpr),
333+
newExprStatement(outputEQExpr),
334+
newCompletedSectionStatement(assumeProfileStmt),
335+
newExprStatement(sepInValueExpr),
336+
newExprStatement(outputEQExpr),
337+
},
338+
},
290339
}
291340

292341
for i, c := range cases {

‎internal/ini/testdata/invalid/bad_section_name‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[ :=foo ]

‎internal/ini/testdata/invalid/bad_syntax_2‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[ foo ]]

‎internal/ini/testdata/invalid/invalid_keys‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[assumerole]
2+
key[id] = value

‎internal/ini/testdata/valid/op_sep_in_values‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
[case1]
2+
sepInValue = =:[foo]]bar[
3+
key:= value1
4+
5+
[case2]
6+
sepInValue==:[foo]]bar[
7+
key = value2
8+
9+
[case3]
10+
sepInValue = []
11+
key== value3
12+
13+
[case4]
14+
sepInValue = [value] x=a
15+
key:=value4
16+
17+
[case5]
18+
key : value5
19+
20+
[case6]
21+
s3 =
22+
[nested6]
23+
key = valuen6
24+
key :=value6
25+
26+
[case7]
27+
s3 =
28+
key :value7
29+
[sub7]
30+
key ==values7

‎internal/ini/testdata/valid/op_sep_in_values_expected‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
{
2+
"case1": {
3+
"sepinvalue": "=:[foo]]bar[",
4+
"key": "= value1"
5+
},
6+
"case2": {
7+
"sepinvalue": "=:[foo]]bar[",
8+
"key": "value2"
9+
},
10+
"case3": {
11+
"sepinvalue": "[]",
12+
"key": "= value3"
13+
},
14+
"case4": {
15+
"sepinvalue": "[value] x=a",
16+
"key": "=value4"
17+
},
18+
"case5": {
19+
"key": "value5"
20+
},
21+
"case6": {
22+
"s3": "",
23+
"key": "=value6"
24+
},
25+
"case7": {
26+
"s3": "",
27+
"key": "value7"
28+
},
29+
"sub7": {
30+
"key": "=values7"
31+
}
32+
}

‎internal/ini/visitor.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,10 @@ func (v *DefaultVisitor) VisitExpr(expr AST) error {
6363

6464
rhs := children[1]
6565

66-
if rhs.Root.Type() != TokenLit {
66+
// The right-hand value side the equality expression is allowed to contain '[', ']', ':', '=' in the values.
67+
// If the token is not either a literal or one of the token types that identifies those four additional
68+
// tokens then error.
69+
if !(rhs.Root.Type() == TokenLit || rhs.Root.Type() == TokenOp || rhs.Root.Type() == TokenSep) {
6770
return NewParseError("unexpected token type")
6871
}
6972

0 commit comments

Comments
 (0)