Skip to content

Commit c8d95b9

Browse files
authored
Gracefully handle encountered regular expression when running jsonnetfmt (#724)
* Gracefully handle encountered regular expression when running jsonnetfmt, adding tests. * Do not validate verbatim strings. * Also do not validate string blocks. * Change golden prefix for formatter tests to fmt.golden to be consistant with cpp version.
1 parent 6838b0a commit c8d95b9

File tree

4 files changed

+160
-8
lines changed

4 files changed

+160
-8
lines changed

formatter/formatter_test.go

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
package formatter
2+
3+
import (
4+
"flag"
5+
"fmt"
6+
"github.com/google/go-jsonnet/internal/testutils"
7+
"io"
8+
"io/ioutil"
9+
"path/filepath"
10+
"regexp"
11+
"strings"
12+
"testing"
13+
)
14+
15+
var update = flag.Bool("update", false, "update .golden files")
16+
17+
// ErrorWriter encapsulates a writer and an error state indicating when at least
18+
// one error has been written to the writer.
19+
type ErrorWriter struct {
20+
ErrorsFound bool
21+
Writer io.Writer
22+
}
23+
24+
type formatterTest struct {
25+
name string
26+
input string
27+
output string
28+
}
29+
30+
type ChangedGoldensList struct {
31+
changedGoldens []string
32+
}
33+
34+
func runTest(t *testing.T, test *formatterTest, changedGoldensList *ChangedGoldensList) {
35+
read := func(file string) []byte {
36+
bytz, err := ioutil.ReadFile(file)
37+
if err != nil {
38+
t.Fatalf("reading file: %s: %v", file, err)
39+
}
40+
return bytz
41+
}
42+
43+
input := read(test.input)
44+
var outBuilder strings.Builder
45+
output, err := Format(test.name, string(input), Options{})
46+
if err != nil {
47+
errWriter := ErrorWriter{
48+
Writer: &outBuilder,
49+
ErrorsFound: false,
50+
}
51+
52+
_, writeErr := errWriter.Writer.Write([]byte(err.Error()))
53+
if writeErr != nil {
54+
panic(writeErr)
55+
}
56+
} else {
57+
outBuilder.Write([]byte(output))
58+
}
59+
60+
outData := outBuilder.String()
61+
62+
if *update {
63+
changed, err := testutils.UpdateGoldenFile(test.output, []byte(outData), 0666)
64+
if err != nil {
65+
t.Error(err)
66+
}
67+
if changed {
68+
changedGoldensList.changedGoldens = append(changedGoldensList.changedGoldens, test.output)
69+
}
70+
} else {
71+
golden, err := ioutil.ReadFile(test.output)
72+
if err != nil {
73+
t.Error(err)
74+
return
75+
}
76+
if diff, hasDiff := testutils.CompareWithGolden(outData, golden); hasDiff {
77+
t.Error(fmt.Errorf("golden file %v has diff:\n%v", test.input, diff))
78+
}
79+
}
80+
}
81+
82+
func TestFormatter(t *testing.T) {
83+
flag.Parse()
84+
85+
var tests []*formatterTest
86+
87+
match, err := filepath.Glob("testdata/*.jsonnet")
88+
if err != nil {
89+
t.Fatal(err)
90+
}
91+
92+
jsonnetExtRE := regexp.MustCompile(`\.jsonnet$`)
93+
94+
for _, input := range match {
95+
// Skip escaped filenames.
96+
if strings.ContainsRune(input, '%') {
97+
continue
98+
}
99+
name := jsonnetExtRE.ReplaceAllString(input, "")
100+
golden := jsonnetExtRE.ReplaceAllString(input, ".fmt.golden")
101+
tests = append(tests, &formatterTest{
102+
name: name,
103+
input: input,
104+
output: golden,
105+
})
106+
}
107+
108+
changedGoldensList := ChangedGoldensList{}
109+
for _, test := range tests {
110+
t.Run(test.name, func(t *testing.T) {
111+
runTest(t, test, &changedGoldensList)
112+
})
113+
}
114+
115+
if *update {
116+
// Little hack: a failed test which prints update stats.
117+
t.Run("Goldens Updated", func(t *testing.T) {
118+
t.Logf("Expected failure, for printing update stats. Does not appear without `-update`.")
119+
t.Logf("%d formatter goldens updated:\n", len(changedGoldensList.changedGoldens))
120+
for _, golden := range changedGoldensList.changedGoldens {
121+
t.Log(golden)
122+
}
123+
t.Fail()
124+
})
125+
}
126+
}
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
testdata/regular_expression:3:11-29 testdata/regular_expression:3:11-29 Unknown escape sequence in string literal: \d
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
x: {
3+
data: '([^:]+)(?::\d+)?',
4+
},
5+
}

internal/parser/parser.go

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -420,6 +420,8 @@ func (p *parser) parseObjectRemainderField(literalFields *LiteralFieldSet, tok *
420420
var expr1 ast.Node
421421
var id *ast.Identifier
422422
var fodder2 ast.Fodder
423+
var err errors.StaticError
424+
423425
switch next.kind {
424426
case tokenIdentifier:
425427
kind = ast.ObjectFieldID
@@ -428,7 +430,10 @@ func (p *parser) parseObjectRemainderField(literalFields *LiteralFieldSet, tok *
428430
case tokenStringDouble, tokenStringSingle,
429431
tokenStringBlock, tokenVerbatimStringDouble, tokenVerbatimStringSingle:
430432
kind = ast.ObjectFieldStr
431-
expr1 = tokenStringToAst(next)
433+
expr1, err = tokenStringToAst(next)
434+
if err != nil {
435+
return nil, err
436+
}
432437
default:
433438
fodder1 = next.fodder
434439
kind = ast.ObjectFieldExpr
@@ -827,43 +832,58 @@ func (p *parser) parseArray(tok *token) (ast.Node, errors.StaticError) {
827832
}, nil
828833
}
829834

830-
func tokenStringToAst(tok *token) *ast.LiteralString {
835+
func tokenStringToAst(tok *token) (*ast.LiteralString, errors.StaticError) {
836+
var node *ast.LiteralString
837+
var validate bool = true
838+
831839
switch tok.kind {
832840
case tokenStringSingle:
833-
return &ast.LiteralString{
841+
node = &ast.LiteralString{
834842
NodeBase: ast.NewNodeBaseLoc(tok.loc, tok.fodder),
835843
Value: tok.data,
836844
Kind: ast.StringSingle,
837845
}
838846
case tokenStringDouble:
839-
return &ast.LiteralString{
847+
node = &ast.LiteralString{
840848
NodeBase: ast.NewNodeBaseLoc(tok.loc, tok.fodder),
841849
Value: tok.data,
842850
Kind: ast.StringDouble,
843851
}
844852
case tokenStringBlock:
845-
return &ast.LiteralString{
853+
node = &ast.LiteralString{
846854
NodeBase: ast.NewNodeBaseLoc(tok.loc, tok.fodder),
847855
Value: tok.data,
848856
Kind: ast.StringBlock,
849857
BlockIndent: tok.stringBlockIndent,
850858
BlockTermIndent: tok.stringBlockTermIndent,
851859
}
860+
validate = false
852861
case tokenVerbatimStringDouble:
853-
return &ast.LiteralString{
862+
node = &ast.LiteralString{
854863
NodeBase: ast.NewNodeBaseLoc(tok.loc, tok.fodder),
855864
Value: tok.data,
856865
Kind: ast.VerbatimStringDouble,
857866
}
867+
validate = false
858868
case tokenVerbatimStringSingle:
859-
return &ast.LiteralString{
869+
node = &ast.LiteralString{
860870
NodeBase: ast.NewNodeBaseLoc(tok.loc, tok.fodder),
861871
Value: tok.data,
862872
Kind: ast.VerbatimStringSingle,
863873
}
874+
validate = false
864875
default:
865876
panic(fmt.Sprintf("Not a string token %#+v", tok))
866877
}
878+
879+
if validate {
880+
_, err := StringUnescape((*node).Loc(), (*node).Value)
881+
if err != nil {
882+
return node, errors.MakeStaticError(err.Error(), tok.loc)
883+
}
884+
}
885+
886+
return node, nil
867887
}
868888

869889
func (p *parser) parseTerminal() (ast.Node, errors.StaticError) {
@@ -907,7 +927,7 @@ func (p *parser) parseTerminal() (ast.Node, errors.StaticError) {
907927
}, nil
908928
case tokenStringDouble, tokenStringSingle,
909929
tokenStringBlock, tokenVerbatimStringDouble, tokenVerbatimStringSingle:
910-
return tokenStringToAst(tok), nil
930+
return tokenStringToAst(tok)
911931
case tokenFalse:
912932
return &ast.LiteralBoolean{
913933
NodeBase: ast.NewNodeBaseLoc(tok.loc, tok.fodder),

0 commit comments

Comments
 (0)