Skip to content

Commit ae3fe91

Browse files
kyleconroykyleconroy
authored
Add the experiment parser in tests (#500)
It's gated behind a environment variable that has no effect in production builds.
1 parent 6826ce0 commit ae3fe91

File tree

6 files changed

+178
-41
lines changed

6 files changed

+178
-41
lines changed

‎Makefile

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@ build:
44
test:
55
go test --tags=exp ./...
66

7+
exp:
8+
SQLC_EXPERIMENTAL_PARSER=on go test --tags=exp ./...
9+
710
sqlc-dev:
811
go build -o ~/bin/sqlc-dev --tags=exp ./cmd/sqlc/
912

‎internal/cmd/cmd.go

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,16 @@ var initCmd = &cobra.Command{
6868
},
6969
}
7070

71+
type Env struct {
72+
ExperimentalParser bool
73+
}
74+
75+
func ParseEnv() Env {
76+
return Env{
77+
ExperimentalParser: os.Getenv("SQLC_EXPERIMENTAL_PARSER") == "on",
78+
}
79+
}
80+
7181
var genCmd = &cobra.Command{
7282
Use: "generate",
7383
Short: "Generate Go code from SQL",
@@ -79,7 +89,7 @@ var genCmd = &cobra.Command{
7989
os.Exit(1)
8090
}
8191

82-
output, err := Generate(dir, stderr)
92+
output, err := Generate(Env{}, dir, stderr)
8393
if err != nil {
8494
os.Exit(1)
8595
}
@@ -104,7 +114,7 @@ var checkCmd = &cobra.Command{
104114
fmt.Fprintln(stderr, "error parsing sqlc.json: file does not exist")
105115
os.Exit(1)
106116
}
107-
if _, err := Generate(dir, stderr); err != nil {
117+
if _, err := Generate(Env{}, dir, stderr); err != nil {
108118
os.Exit(1)
109119
}
110120
return nil

‎internal/cmd/generate.go

Lines changed: 46 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
"github.com/kyleconroy/sqlc/internal/dinosql/kotlin"
1717
"github.com/kyleconroy/sqlc/internal/multierr"
1818
"github.com/kyleconroy/sqlc/internal/mysql"
19+
"github.com/kyleconroy/sqlc/internal/pg"
1920
)
2021

2122
const errMessageNoVersion = `The configuration file must have a version number.
@@ -43,7 +44,7 @@ type outPair struct {
4344
config.SQL
4445
}
4546

46-
func Generate(dir string, stderr io.Writer) (map[string]string, error) {
47+
func Generate(e Env, dir string, stderr io.Writer) (map[string]string, error) {
4748
var yamlMissing, jsonMissing bool
4849
yamlPath := filepath.Join(dir, "sqlc.yaml")
4950
jsonPath := filepath.Join(dir, "sqlc.json")
@@ -135,7 +136,7 @@ func Generate(dir string, stderr io.Writer) (map[string]string, error) {
135136
name = combo.Kotlin.Package
136137
}
137138

138-
result, errored = parse(name, dir, sql.SQL, combo, parseOpts, stderr)
139+
result, errored = parse(e, name, dir, sql.SQL, combo, parseOpts, stderr)
139140
if errored {
140141
break
141142
}
@@ -173,7 +174,40 @@ func Generate(dir string, stderr io.Writer) (map[string]string, error) {
173174
return output, nil
174175
}
175176

176-
func parse(name, dir string, sql config.SQL, combo config.CombinedSettings, parserOpts dinosql.ParserOpts, stderr io.Writer) (dinosql.Generateable, bool) {
177+
type postgreEngine interface {
178+
ParseCatalog([]string) error
179+
ParseQueries([]string, dinosql.ParserOpts) error
180+
Result() dinosql.Generateable
181+
}
182+
183+
type dinosqlEngine struct {
184+
catalog pg.Catalog
185+
result *dinosql.Result
186+
}
187+
188+
func (d *dinosqlEngine) ParseCatalog(schema []string) error {
189+
c, err := dinosql.ParseCatalog(schema)
190+
if err != nil {
191+
return err
192+
}
193+
d.catalog = c
194+
return nil
195+
}
196+
197+
func (d *dinosqlEngine) ParseQueries(queries []string, opts dinosql.ParserOpts) error {
198+
q, err := dinosql.ParseQueries(d.catalog, queries, opts)
199+
if err != nil {
200+
return err
201+
}
202+
d.result = q
203+
return nil
204+
}
205+
206+
func (d *dinosqlEngine) Result() dinosql.Generateable {
207+
return &kotlin.Result{Result: d.result}
208+
}
209+
210+
func parse(e Env, name, dir string, sql config.SQL, combo config.CombinedSettings, parserOpts dinosql.ParserOpts, stderr io.Writer) (dinosql.Generateable, bool) {
177211
switch sql.Engine {
178212
case config.EngineMySQL:
179213
// Experimental MySQL support
@@ -192,8 +226,13 @@ func parse(name, dir string, sql config.SQL, combo config.CombinedSettings, pars
192226
return q, false
193227

194228
case config.EnginePostgreSQL:
195-
c, err := dinosql.ParseCatalog(sql.Schema)
196-
if err != nil {
229+
var eng postgreEngine
230+
if e.ExperimentalParser {
231+
eng = compiler.NewEngine(sql, combo)
232+
} else {
233+
eng = &dinosqlEngine{}
234+
}
235+
if err := eng.ParseCatalog(sql.Schema); err != nil {
197236
fmt.Fprintf(stderr, "# package %s\n", name)
198237
if parserErr, ok := err.(*multierr.Error); ok {
199238
for _, fileErr := range parserErr.Errs() {
@@ -204,9 +243,7 @@ func parse(name, dir string, sql config.SQL, combo config.CombinedSettings, pars
204243
}
205244
return nil, true
206245
}
207-
208-
q, err := dinosql.ParseQueries(c, sql.Queries, parserOpts)
209-
if err != nil {
246+
if err := eng.ParseQueries(sql.Queries, parserOpts); err != nil {
210247
fmt.Fprintf(stderr, "# package %s\n", name)
211248
if parserErr, ok := err.(*multierr.Error); ok {
212249
for _, fileErr := range parserErr.Errs() {
@@ -217,7 +254,7 @@ func parse(name, dir string, sql config.SQL, combo config.CombinedSettings, pars
217254
}
218255
return nil, true
219256
}
220-
return &kotlin.Result{Result: q}, false
257+
return eng.Result(), false
221258

222259
case config.EngineXLemon, config.EngineXDolphin, config.EngineXElephant:
223260
r, err := compiler.Run(sql, combo)

‎internal/compiler/compile.go

Lines changed: 42 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"github.com/kyleconroy/sqlc/internal/dinosql"
1515
"github.com/kyleconroy/sqlc/internal/dolphin"
1616
"github.com/kyleconroy/sqlc/internal/migrations"
17+
"github.com/kyleconroy/sqlc/internal/multierr"
1718
"github.com/kyleconroy/sqlc/internal/pg"
1819
"github.com/kyleconroy/sqlc/internal/postgresql"
1920
"github.com/kyleconroy/sqlc/internal/sql/ast"
@@ -54,50 +55,38 @@ func enumValueName(value string) string {
5455
}
5556

5657
// end copypasta
57-
58-
func Run(conf config.SQL, combo config.CombinedSettings) (*Result, error) {
59-
var c *catalog.Catalog
60-
var p Parser
61-
62-
switch conf.Engine {
63-
case config.EngineXLemon:
64-
p = sqlite.NewParser()
65-
c = catalog.New("main")
66-
case config.EngineXDolphin:
67-
p = dolphin.NewParser()
68-
c = catalog.New("public") // TODO: What is the default database for MySQL?
69-
case config.EngineXElephant:
70-
p = postgresql.NewParser()
71-
c = postgresql.NewCatalog()
72-
default:
73-
return nil, fmt.Errorf("unknown engine: %s", conf.Engine)
74-
}
75-
76-
files, err := sqlpath.Glob(conf.Schema)
58+
func parseCatalog(p Parser, c *catalog.Catalog, schema []string) error {
59+
files, err := sqlpath.Glob(schema)
7760
if err != nil {
78-
return nil, err
61+
return err
7962
}
80-
63+
merr := multierr.New()
8164
for _, filename := range files {
8265
blob, err := ioutil.ReadFile(filename)
8366
if err != nil {
84-
// merr.Add(filename, "", 0, err)
85-
return nil, err
67+
merr.Add(filename, "", 0, err)
68+
continue
8669
}
8770
contents := migrations.RemoveRollbackStatements(string(blob))
8871
stmts, err := p.Parse(strings.NewReader(contents))
8972
if err != nil {
90-
// merr.Add(filename, , 0, err)
91-
return nil, err
73+
merr.Add(filename, contents, 0, err)
74+
continue
9275
}
9376
for i := range stmts {
9477
if err := c.Update(stmts[i]); err != nil {
95-
// merr.Add(filename, contents, location(stmt), err)
96-
return nil, err
78+
merr.Add(filename, contents, stmts[i].Pos(), err)
79+
continue
9780
}
9881
}
9982
}
83+
if len(merr.Errs()) > 0 {
84+
return merr
85+
}
86+
return nil
87+
}
10088

89+
func buildResult(c *catalog.Catalog) (*Result, error) {
10190
var structs []dinosql.GoStruct
10291
var enums []dinosql.GoEnum
10392
for _, schema := range c.Schemas {
@@ -150,3 +139,28 @@ func Run(conf config.SQL, combo config.CombinedSettings) (*Result, error) {
150139
}
151140
return &Result{structs: structs, enums: enums}, nil
152141
}
142+
143+
func Run(conf config.SQL, combo config.CombinedSettings) (*Result, error) {
144+
var c *catalog.Catalog
145+
var p Parser
146+
147+
switch conf.Engine {
148+
case config.EngineXLemon:
149+
p = sqlite.NewParser()
150+
c = catalog.New("main")
151+
case config.EngineXDolphin:
152+
p = dolphin.NewParser()
153+
c = catalog.New("public") // TODO: What is the default database for MySQL?
154+
case config.EngineXElephant:
155+
p = postgresql.NewParser()
156+
c = postgresql.NewCatalog()
157+
default:
158+
return nil, fmt.Errorf("unknown engine: %s", conf.Engine)
159+
}
160+
161+
if err := parseCatalog(p, c, conf.Schema); err != nil {
162+
return nil, err
163+
}
164+
165+
return buildResult(c)
166+
}

‎internal/compiler/engine.go

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
package compiler
2+
3+
import (
4+
"fmt"
5+
6+
"github.com/kyleconroy/sqlc/internal/config"
7+
"github.com/kyleconroy/sqlc/internal/dinosql"
8+
"github.com/kyleconroy/sqlc/internal/dolphin"
9+
"github.com/kyleconroy/sqlc/internal/postgresql"
10+
"github.com/kyleconroy/sqlc/internal/sql/catalog"
11+
"github.com/kyleconroy/sqlc/internal/sqlite"
12+
)
13+
14+
// The Engine type only exists as a compatibility shim between the old dinosql
15+
// package and the new compiler package.
16+
type Engine struct {
17+
conf config.SQL
18+
combo config.CombinedSettings
19+
catalog *catalog.Catalog
20+
parser Parser
21+
result *Result
22+
}
23+
24+
func NewEngine(conf config.SQL, combo config.CombinedSettings) *Engine {
25+
e := &Engine{conf: conf, combo: combo}
26+
switch conf.Engine {
27+
case config.EngineXLemon:
28+
e.parser = sqlite.NewParser()
29+
e.catalog = catalog.New("main")
30+
case config.EngineXDolphin:
31+
e.parser = dolphin.NewParser()
32+
e.catalog = catalog.New("public") // TODO: What is the default database for MySQL?
33+
case config.EngineXElephant, config.EnginePostgreSQL:
34+
e.parser = postgresql.NewParser()
35+
e.catalog = postgresql.NewCatalog()
36+
default:
37+
panic(fmt.Sprintf("unknown engine: %s", conf.Engine))
38+
}
39+
return e
40+
}
41+
42+
func (e *Engine) ParseCatalog(schema []string) error {
43+
return parseCatalog(e.parser, e.catalog, schema)
44+
}
45+
46+
func (e *Engine) ParseQueries(queries []string, opts dinosql.ParserOpts) error {
47+
r, err := buildResult(e.catalog)
48+
if err != nil {
49+
return err
50+
}
51+
e.result = r
52+
return nil
53+
}
54+
55+
func (e *Engine) Result() dinosql.Generateable {
56+
return e.result
57+
}

‎internal/endtoend/endtoend_test.go

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111

1212
"github.com/google/go-cmp/cmp"
1313
"github.com/google/go-cmp/cmp/cmpopts"
14+
1415
"github.com/kyleconroy/sqlc/internal/cmd"
1516
)
1617

@@ -35,7 +36,7 @@ func TestExamples(t *testing.T) {
3536
t.Parallel()
3637
path := filepath.Join(examples, tc)
3738
var stderr bytes.Buffer
38-
output, err := cmd.Generate(path, &stderr)
39+
output, err := cmd.Generate(cmd.Env{}, path, &stderr)
3940
if err != nil {
4041
t.Fatalf("sqlc generate failed: %s", stderr.String())
4142
}
@@ -47,6 +48,11 @@ func TestExamples(t *testing.T) {
4748
func TestReplay(t *testing.T) {
4849
t.Parallel()
4950

51+
experimental := map[string]bool{
52+
"ondeck": true,
53+
}
54+
env := cmd.ParseEnv()
55+
5056
files, err := ioutil.ReadDir("testdata")
5157
if err != nil {
5258
t.Fatal(err)
@@ -62,14 +68,24 @@ func TestReplay(t *testing.T) {
6268
path, _ := filepath.Abs(filepath.Join("testdata", tc))
6369
var stderr bytes.Buffer
6470
expected := expectedStderr(t, path)
65-
output, err := cmd.Generate(path, &stderr)
71+
output, err := cmd.Generate(cmd.Env{}, path, &stderr)
6672
if len(expected) == 0 && err != nil {
6773
t.Fatalf("sqlc generate failed: %s", stderr.String())
6874
}
6975
cmpDirectory(t, path, output)
7076
if diff := cmp.Diff(expected, stderr.String()); diff != "" {
7177
t.Errorf("stderr differed (-want +got):\n%s", diff)
7278
}
79+
if env.ExperimentalParser && experimental[tc] {
80+
output, err := cmd.Generate(env, path, &stderr)
81+
if len(expected) == 0 && err != nil {
82+
t.Fatalf("EXPERIMENTAL sqlc generate failed: %s", stderr.String())
83+
}
84+
cmpDirectory(t, path, output)
85+
if diff := cmp.Diff(expected, stderr.String()); diff != "" {
86+
t.Errorf("EXPERIMENTAL stderr differed (-want +got):\n%s", diff)
87+
}
88+
}
7389
})
7490
}
7591
}

0 commit comments

Comments
 (0)