Merge pull request #431 from kuba--/update-regression

parse and analyze regression queries

Author: ajnavarro

_testdata/regression.yml

@@ -29,11 +29,13 @@
   Name: 'Create pilosa index on language UDF'
   Statements:
     - CREATE INDEX language_idx ON files USING pilosa (language(file_path, blob_content)) WITH (async = false)
+    - DROP INDEX language_idx ON files
 -
   ID: 'query6'
   Name: 'Create pilosalib index on language UDF'
   Statements:
     - CREATE INDEX language_idx ON files USING pilosalib (language(file_path, blob_content)) WITH (async = false)
+    - DROP INDEX language_idx ON files
 -
   ID: 'query7'
   Name: 'Query by language using the pilosa index'
@@ -52,19 +54,21 @@
   ID: 'query9'
   Name: 'Query all files from HEAD'
   Statements:
-    - SELECT cf.file_path, f.blob_content FROM ref_commits r NATURAL JOIN commit_files cf NATURAL JOIN files f WHERE r.ref_name = 'HEAD' AND r.index = 0
+    - SELECT cf.file_path, f.blob_content FROM ref_commits r NATURAL JOIN commit_files cf NATURAL JOIN files f WHERE r.ref_name = 'HEAD' AND r.history_index = 0
 -
   ID: 'query10'
   Name: 'Get all LICENSE blobs using pilosa index'
   Statements:
     - CREATE INDEX file_path_idx ON files USING pilosa (file_path) WITH (async = false)
     - SELECT blob_content FROM files WHERE file_path = 'LICENSE'
+    - DROP INDEX file_path_idx ON files
 -
   ID: 'query11'
   Name: 'Get all LICENSE blobs using pilosalib index'
   Statements:
     - CREATE INDEX file_path_idx ON files USING pilosalib (file_path) WITH (async = false)
     - SELECT blob_content FROM files WHERE file_path = 'LICENSE'
+    - DROP INDEX file_path_idx ON files
 -
   ID: 'query12'
   Name: '10 top repos by file count in HEAD'

cmd/gitbase/command/server.go

@@ -66,6 +66,20 @@ func (l *jaegerLogrus) Error(s string) {
 	l.Entry.Error(s)
 }
 
+func NewDatabaseEngine(readonly bool, version string) *sqle.Engine {
+	catalog := sql.NewCatalog()
+	ab := analyzer.NewBuilder(catalog)
+	if readonly {
+		ab = ab.ReadOnly()
+	}
+	a := ab.Build()
+	engine := sqle.New(catalog, a, &sqle.Config{
+		VersionPostfix: version,
+	})
+
+	return engine
+}
+
 // Execute starts a new gitbase server based on provided configuration, it
 // honors the go-flags.Commander interface.
 func (c *Server) Execute(args []string) error {
@@ -74,7 +88,7 @@ func (c *Server) Execute(args []string) error {
 	}
 
 	if err := c.buildDatabase(); err != nil {
-		logrus.WithField("error", err).Fatal("unable to start database server")
+		logrus.WithField("error", err).Fatal("unable to initialize database engine")
 		return err
 	}
 
@@ -136,15 +150,7 @@ func (c *Server) Execute(args []string) error {
 
 func (c *Server) buildDatabase() error {
 	if c.engine == nil {
-		catalog := sql.NewCatalog()
-		ab := analyzer.NewBuilder(catalog)
-		if c.ReadOnly {
-			ab = ab.ReadOnly()
-		}
-		a := ab.Build()
-		c.engine = sqle.New(catalog, a, &sqle.Config{
-			VersionPostfix: c.Version,
-		})
+		c.engine = NewDatabaseEngine(c.ReadOnly, c.Version)
 	}
 
 	c.pool = gitbase.NewRepositoryPool()

integration_test.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/src-d/gitbase/cmd/gitbase/command"
 	"github.com/src-d/gitbase/internal/rule"
 
 	"github.com/src-d/gitbase"
@@ -793,7 +794,6 @@ func deleteIndex(
 
 func setup(t testing.TB) (*sqle.Engine, *gitbase.RepositoryPool, func()) {
 	t.Helper()
-	engine := newBaseEngine()
 	require.NoError(t, fixtures.Init())
 	cleanup := func() {
 		require.NoError(t, fixtures.Clean())
@@ -804,24 +804,26 @@ func setup(t testing.TB) (*sqle.Engine, *gitbase.RepositoryPool, func()) {
 		pool.AddGitWithID("worktree", f.Worktree().Root())
 	}
 
-	return engine, pool, cleanup
+	return newBaseEngine(), pool, cleanup
 }
 
 func newSquashEngine() *sqle.Engine {
-	catalog := sql.NewCatalog()
-	analyzer := analyzer.NewBuilder(catalog).
+	engine := newBaseEngine()
+	dbname := engine.Analyzer.CurrentDatabase
+
+	engine.Catalog.RegisterFunctions(sqlfunction.Defaults)
+	engine.Analyzer = analyzer.NewBuilder(engine.Catalog).
 		AddPostAnalyzeRule(rule.SquashJoinsRule, rule.SquashJoins).
 		Build()
-	e := sqle.New(catalog, analyzer, &sqle.Config{VersionPostfix: "test"})
-	e.AddDatabase(gitbase.NewDatabase("foo"))
-	e.Catalog.RegisterFunctions(sqlfunction.Defaults)
-	e.Catalog.RegisterFunctions(function.Functions)
-	return e
+	engine.Analyzer.CurrentDatabase = dbname
+	return engine
 }
 
 func newBaseEngine() *sqle.Engine {
-	engine := sqle.NewDefault()
-	engine.AddDatabase(gitbase.NewDatabase("foo"))
+	foo := gitbase.NewDatabase("foo")
+	engine := command.NewDatabaseEngine(false, "test")
+
+	engine.AddDatabase(foo)
 	engine.Catalog.RegisterFunctions(function.Functions)
 	return engine
 }

regression_test.go

@@ -3,12 +3,14 @@ package gitbase_test
 import (
 	"context"
 	"io/ioutil"
+	"os"
 	"testing"
 
 	"github.com/src-d/gitbase"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/src-d/go-mysql-server.v0/sql"
-	"gopkg.in/src-d/go-mysql-server.v0/sql/parse"
+	"gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosa"
+	"gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosalib"
 	yaml "gopkg.in/yaml.v2"
 )
 
@@ -18,20 +20,35 @@ type Query struct {
 	Statements []string `yaml:"Statements"`
 }
 
-func TestParseRegressionQueries(t *testing.T) {
+func TestRegressionQueries(t *testing.T) {
 	require := require.New(t)
 
-	queries, err := loadQueriesYaml("./_testdata/regression.yml")
+	engine, pool, cleanup := setup(t)
+	defer cleanup()
+
+	tmpDir, err := ioutil.TempDir(os.TempDir(), "pilosa-idx-gitbase")
 	require.NoError(err)
+	defer os.RemoveAll(tmpDir)
+	engine.Catalog.RegisterIndexDriver(pilosa.NewIndexDriver(tmpDir))
+	engine.Catalog.RegisterIndexDriver(pilosalib.NewDriver(tmpDir))
 
 	ctx := sql.NewContext(
 		context.TODO(),
-		sql.WithSession(gitbase.NewSession(gitbase.NewRepositoryPool())),
+		sql.WithSession(gitbase.NewSession(pool)),
 	)
 
+	queries, err := loadQueriesYaml("./_testdata/regression.yml")
+	require.NoError(err)
+
 	for _, q := range queries {
 		for _, stmt := range q.Statements {
-			if _, err := parse.Parse(ctx, stmt); err != nil {
+			_, iter, err := engine.Query(ctx, stmt)
+			if err != nil {
+				require.Failf(err.Error(), "ID: %s, Name: %s, Statement: %s", q.ID, q.Name, stmt)
+			}
+
+			_, err = sql.RowIterToRows(iter)
+			if err != nil {
 				require.Failf(err.Error(), "ID: %s, Name: %s, Statement: %s", q.ID, q.Name, stmt)
 			}
 		}