*: implement rule to squash chainable tables into a single one

Fixes #197

This commit introduces the new rule to squash inner joins between tables that are chainable. This rule is yet experimental and thus can be used by using the `UNSTABLE_SQUASH_ENABLE` environment variable until it is proven to be stable and the results match the ones retrieved without this optimisation.

What is a join between chainable tables? That's an inner join between two tables that can be joined using some logic in which one table records are retrieved from some data in the other table rows because there is some sort of relationship between them. To assert such logic can be applied, the join needs to have certain filters in its condition.

- `repositories` is chainable with `remotes` iff `repositories.id = remotes.repository_id`
- `repositories` is chainable with `refs` iff `repositories.id = refs.repository_id`
- `remotes` is chainable with `refs` iff `refs.repository_id = remotes.repository_id`
- `refs` is chainable with `commits` iff `refs.hash = commits.hash`, `history_idx(refs.hash, commits.hash) >= 0` or `0 <= history_idx(refs.hash, commits.hash)`
- `refs` is chainable with `tree_entries` iff `commit_has_tree(refs.hash, tree_entries.tree_hash)`
- `refs` is chainable with `blobs` iff `commit_has_blob(refs.hash, blob.hash)`
- `commits` is chainable with `tree_entries` iff `commit_has_tree(commits.hash, tree_entries.tree_hash)` or `commits.tree_hash = tree_entries.tree_hash`
- `commits` is chainable with `blobs` iff `commit_has_blob(commits.hash, blob.hash)`
- `tree_entries` is chainable with `blobs` iff `tree_entries.entry_hash = blobs.hash`

Since chains are built hierarchically using the table hierarchy (repositories -> remotes -> refs -> commits -> tree_entries -> blobs), any chain can be joined with any table iff the last part of the chain and that table match the conditions stated before.

To allow this, there are several iterators, some for the same table, depending on the position in the chain that table is, that are composed in the analyzer rule.

Even if the chain is built hierarchically and thus it has a different schema ordering than the original schema, a mapping between the chain schema and the original one is computed so that the result of the chain is exactly the same as the one without the optimisation.

Notes about the experimental status of this feature:

- Lacks some integration tests that will be done on subsequent PRs.
- Tree entries table is currently under examination and depending on what we decide, the results it returns right now might need to change.
- Probably better to test it thoroughly and do some performance measurements before making this the default behaviour.

Signed-off-by: Miguel Molina <[email protected]>

Author: erizocosmico

blobs.go

@@ -27,10 +27,11 @@ var (
 
 type blobsTable struct{}
 
-var blobsSchema = sql.Schema{
-	{Name: "hash", Type: sql.Text, Nullable: false, Source: blobsTableName},
-	{Name: "size", Type: sql.Int64, Nullable: false, Source: blobsTableName},
-	{Name: "content", Type: sql.Blob, Nullable: false, Source: blobsTableName},
+// BlobsSchema is the schema for the blobs table.
+var BlobsSchema = sql.Schema{
+	{Name: "hash", Type: sql.Text, Nullable: false, Source: BlobsTableName},
+	{Name: "size", Type: sql.Int64, Nullable: false, Source: BlobsTableName},
+	{Name: "content", Type: sql.Blob, Nullable: false, Source: BlobsTableName},
 }
 
 var _ sql.PushdownProjectionAndFiltersTable = (*blobsTable)(nil)
@@ -39,20 +40,24 @@ func newBlobsTable() sql.Table {
 	return new(blobsTable)
 }
 
+var _ Table = (*blobsTable)(nil)
+
+func (blobsTable) isGitbaseTable() {}
+
 func (blobsTable) String() string {
-	return printTable(blobsTableName, blobsSchema)
+	return printTable(BlobsTableName, BlobsSchema)
 }
 
 func (blobsTable) Resolved() bool {
 	return true
 }
 
 func (blobsTable) Name() string {
-	return blobsTableName
+	return BlobsTableName
 }
 
 func (blobsTable) Schema() sql.Schema {
-	return blobsSchema
+	return BlobsSchema
 }
 
 func (r *blobsTable) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) {
@@ -79,15 +84,15 @@ func (blobsTable) Children() []sql.Node {
 }
 
 func (blobsTable) HandledFilters(filters []sql.Expression) []sql.Expression {
-	return handledFilters(blobsTableName, blobsSchema, filters)
+	return handledFilters(BlobsTableName, BlobsSchema, filters)
 }
 
 func (r *blobsTable) WithProjectAndFilters(
 	ctx *sql.Context,
 	_, filters []sql.Expression,
 ) (sql.RowIter, error) {
 	return rowIterWithSelectors(
-		ctx, blobsSchema, blobsTableName, filters,
+		ctx, BlobsSchema, BlobsTableName, filters,
 		[]string{"hash"},
 		func(selectors selectors) (RowRepoIter, error) {
 			if len(selectors["hash"]) == 0 {

blobs_test.go

@@ -11,19 +11,19 @@ import (
 func TestBlobsTable_Name(t *testing.T) {
 	require := require.New(t)
 
-	table := getTable(require, blobsTableName)
-	require.Equal(blobsTableName, table.Name())
+	table := getTable(require, BlobsTableName)
+	require.Equal(BlobsTableName, table.Name())
 
 	// Check that each column source is the same as table name
 	for _, c := range table.Schema() {
-		require.Equal(blobsTableName, c.Source)
+		require.Equal(BlobsTableName, c.Source)
 	}
 }
 
 func TestBlobsTable_Children(t *testing.T) {
 	require := require.New(t)
 
-	table := getTable(require, blobsTableName)
+	table := getTable(require, BlobsTableName)
 	require.Equal(0, len(table.Children()))
 }
 
@@ -32,7 +32,7 @@ func TestBlobsTable_RowIter(t *testing.T) {
 	ctx, _, cleanup := setup(t)
 	defer cleanup()
 
-	table := getTable(require, blobsTableName)
+	table := getTable(require, BlobsTableName)
 
 	rows, err := sql.NodeToRows(ctx, table)
 	require.NoError(err)
@@ -105,7 +105,7 @@ func TestBlobsPushdown(t *testing.T) {
 
 	iter, err = table.WithProjectAndFilters(session, nil, []sql.Expression{
 		expression.NewEquals(
-			expression.NewGetFieldWithTable(0, sql.Text, blobsTableName, "hash", false),
+			expression.NewGetFieldWithTable(0, sql.Text, BlobsTableName, "hash", false),
 			expression.NewLiteral("32858aad3c383ed1ff0a0f9bdf231d54a00c9e88", sql.Text),
 		),
 	})
@@ -117,15 +117,15 @@ func TestBlobsPushdown(t *testing.T) {
 
 	iter, err = table.WithProjectAndFilters(session, nil, []sql.Expression{
 		expression.NewLessThan(
-			expression.NewGetFieldWithTable(1, sql.Int64, blobsTableName, "size", false),
+			expression.NewGetFieldWithTable(1, sql.Int64, BlobsTableName, "size", false),
 			expression.NewLiteral(int64(10), sql.Int64),
 		),
 	})
 	require.NoError(err)
 
 	iter, err = table.WithProjectAndFilters(session, nil, []sql.Expression{
 		expression.NewEquals(
-			expression.NewGetFieldWithTable(0, sql.Text, blobsTableName, "hash", false),
+			expression.NewGetFieldWithTable(0, sql.Text, BlobsTableName, "hash", false),
 			expression.NewLiteral("not exists", sql.Text),
 		),
 	})

cmd/gitquery/server.go

@@ -2,15 +2,23 @@ package main
 
 import (
 	"net"
+	"os"
 	"strconv"
 
 	"github.com/src-d/gitbase"
 	"github.com/src-d/gitbase/internal/function"
+	"github.com/src-d/gitbase/internal/rule"
+
 	sqle "gopkg.in/src-d/go-mysql-server.v0"
 	"gopkg.in/src-d/go-mysql-server.v0/server"
 	"gopkg.in/src-d/go-vitess.v0/mysql"
 )
 
+// Squashing tables and pushing down join conditions is still a work in
+// progress and unstable. To enable it, the UNSTABLE_SQUASH_ENABLE must
+// not be empty.
+var enableUnstableSquash = os.Getenv("UNSTABLE_SQUASH_ENABLE") != ""
+
 // CmdServer defines server command
 type CmdServer struct {
 	cmd
@@ -44,6 +52,11 @@ func (c *CmdServer) buildDatabase() error {
 
 	c.engine.AddDatabase(gitbase.NewDatabase(c.name))
 	c.engine.Catalog.RegisterFunctions(function.Functions)
+
+	if enableUnstableSquash {
+		c.engine.Analyzer.AddRule(rule.SquashJoinsRule, rule.SquashJoins)
+	}
+
 	return nil
 }
 

commits.go

@@ -12,16 +12,17 @@ import (
 type commitsTable struct {
 }
 
-var commitsSchema = sql.Schema{
-	{Name: "hash", Type: sql.Text, Nullable: false, Source: commitsTableName},
-	{Name: "author_name", Type: sql.Text, Nullable: false, Source: commitsTableName},
-	{Name: "author_email", Type: sql.Text, Nullable: false, Source: commitsTableName},
-	{Name: "author_when", Type: sql.Timestamp, Nullable: false, Source: commitsTableName},
-	{Name: "committer_name", Type: sql.Text, Nullable: false, Source: commitsTableName},
-	{Name: "committer_email", Type: sql.Text, Nullable: false, Source: commitsTableName},
-	{Name: "committer_when", Type: sql.Timestamp, Nullable: false, Source: commitsTableName},
-	{Name: "message", Type: sql.Text, Nullable: false, Source: commitsTableName},
-	{Name: "tree_hash", Type: sql.Text, Nullable: false, Source: commitsTableName},
+// CommitsSchema is the schema for the commits table.
+var CommitsSchema = sql.Schema{
+	{Name: "hash", Type: sql.Text, Nullable: false, Source: CommitsTableName},
+	{Name: "author_name", Type: sql.Text, Nullable: false, Source: CommitsTableName},
+	{Name: "author_email", Type: sql.Text, Nullable: false, Source: CommitsTableName},
+	{Name: "author_when", Type: sql.Timestamp, Nullable: false, Source: CommitsTableName},
+	{Name: "committer_name", Type: sql.Text, Nullable: false, Source: CommitsTableName},
+	{Name: "committer_email", Type: sql.Text, Nullable: false, Source: CommitsTableName},
+	{Name: "committer_when", Type: sql.Timestamp, Nullable: false, Source: CommitsTableName},
+	{Name: "message", Type: sql.Text, Nullable: false, Source: CommitsTableName},
+	{Name: "tree_hash", Type: sql.Text, Nullable: false, Source: CommitsTableName},
 }
 
 var _ sql.PushdownProjectionAndFiltersTable = (*commitsTable)(nil)
@@ -30,20 +31,24 @@ func newCommitsTable() sql.Table {
 	return new(commitsTable)
 }
 
+var _ Table = (*commitsTable)(nil)
+
+func (commitsTable) isGitbaseTable() {}
+
 func (commitsTable) String() string {
-	return printTable(commitsTableName, commitsSchema)
+	return printTable(CommitsTableName, CommitsSchema)
 }
 
 func (commitsTable) Resolved() bool {
 	return true
 }
 
 func (commitsTable) Name() string {
-	return commitsTableName
+	return CommitsTableName
 }
 
 func (commitsTable) Schema() sql.Schema {
-	return commitsSchema
+	return CommitsSchema
 }
 
 func (r *commitsTable) TransformUp(f sql.TransformNodeFunc) (sql.Node, error) {
@@ -70,15 +75,15 @@ func (commitsTable) Children() []sql.Node {
 }
 
 func (commitsTable) HandledFilters(filters []sql.Expression) []sql.Expression {
-	return handledFilters(commitsTableName, commitsSchema, filters)
+	return handledFilters(CommitsTableName, CommitsSchema, filters)
 }
 
 func (r *commitsTable) WithProjectAndFilters(
 	ctx *sql.Context,
 	_, filters []sql.Expression,
 ) (sql.RowIter, error) {
 	return rowIterWithSelectors(
-		ctx, commitsSchema, commitsTableName, filters,
+		ctx, CommitsSchema, CommitsTableName, filters,
 		[]string{"hash"},
 		func(selectors selectors) (RowRepoIter, error) {
 			if len(selectors["hash"]) == 0 {

commits_test.go

@@ -11,19 +11,19 @@ import (
 func TestCommitsTable_Name(t *testing.T) {
 	require := require.New(t)
 
-	table := getTable(require, commitsTableName)
-	require.Equal(commitsTableName, table.Name())
+	table := getTable(require, CommitsTableName)
+	require.Equal(CommitsTableName, table.Name())
 
 	// Check that each column source is the same as table name
 	for _, c := range table.Schema() {
-		require.Equal(commitsTableName, c.Source)
+		require.Equal(CommitsTableName, c.Source)
 	}
 }
 
 func TestCommitsTable_Children(t *testing.T) {
 	require := require.New(t)
 
-	table := getTable(require, commitsTableName)
+	table := getTable(require, CommitsTableName)
 	require.Equal(0, len(table.Children()))
 }
 
@@ -32,7 +32,7 @@ func TestCommitsTable_RowIter(t *testing.T) {
 	session, _, cleanup := setup(t)
 	defer cleanup()
 
-	table := getTable(require, commitsTableName)
+	table := getTable(require, CommitsTableName)
 
 	rows, err := sql.NodeToRows(session, table)
 	require.Nil(err)
@@ -61,7 +61,7 @@ func TestCommitsPushdown(t *testing.T) {
 
 	iter, err = table.WithProjectAndFilters(session, nil, []sql.Expression{
 		expression.NewEquals(
-			expression.NewGetFieldWithTable(0, sql.Text, commitsTableName, "hash", false),
+			expression.NewGetFieldWithTable(0, sql.Text, CommitsTableName, "hash", false),
 			expression.NewLiteral("918c48b83bd081e863dbe1b80f8998f058cd8294", sql.Text),
 		),
 	})
@@ -73,7 +73,7 @@ func TestCommitsPushdown(t *testing.T) {
 
 	iter, err = table.WithProjectAndFilters(session, nil, []sql.Expression{
 		expression.NewEquals(
-			expression.NewGetFieldWithTable(0, sql.Text, commitsTableName, "hash", false),
+			expression.NewGetFieldWithTable(0, sql.Text, CommitsTableName, "hash", false),
 			expression.NewLiteral("not exists", sql.Text),
 		),
 	})
@@ -85,7 +85,7 @@ func TestCommitsPushdown(t *testing.T) {
 
 	iter, err = table.WithProjectAndFilters(session, nil, []sql.Expression{
 		expression.NewEquals(
-			expression.NewGetFieldWithTable(2, sql.Text, commitsTableName, "author_email", false),
+			expression.NewGetFieldWithTable(2, sql.Text, CommitsTableName, "author_email", false),
 			expression.NewLiteral("[email protected]", sql.Text),
 		),
 	})
@@ -97,11 +97,11 @@ func TestCommitsPushdown(t *testing.T) {
 
 	iter, err = table.WithProjectAndFilters(session, nil, []sql.Expression{
 		expression.NewEquals(
-			expression.NewGetFieldWithTable(2, sql.Text, commitsTableName, "author_email", false),
+			expression.NewGetFieldWithTable(2, sql.Text, CommitsTableName, "author_email", false),
 			expression.NewLiteral("[email protected]", sql.Text),
 		),
 		expression.NewEquals(
-			expression.NewGetFieldWithTable(7, sql.Text, commitsTableName, "message", false),
+			expression.NewGetFieldWithTable(7, sql.Text, CommitsTableName, "message", false),
 			expression.NewLiteral("vendor stuff\n", sql.Text),
 		),
 	})

database.go

@@ -5,13 +5,18 @@ import (
 )
 
 const (
-	// TODO 'references' is a reserved keyword into the parser
-	referencesTableName   = "refs"
-	commitsTableName      = "commits"
-	blobsTableName        = "blobs"
-	treeEntriesTableName  = "tree_entries"
-	repositoriesTableName = "repositories"
-	remotesTableName      = "remotes"
+	// ReferencesTableName is the name of the refs table.
+	ReferencesTableName = "refs"
+	// CommitsTableName is the name of the commits table.
+	CommitsTableName = "commits"
+	// BlobsTableName is the name of the blobs table.
+	BlobsTableName = "blobs"
+	// TreeEntriesTableName is the name of the tree entries table.
+	TreeEntriesTableName = "tree_entries"
+	// RepositoriesTableName is the name of the repositories table.
+	RepositoriesTableName = "repositories"
+	// RemotesTableName is the name of the remotes table.
+	RemotesTableName = "remotes"
 )
 
 // Database holds all git repository tables
@@ -47,11 +52,11 @@ func (d *Database) Name() string {
 // Tables returns a map with all initialized tables
 func (d *Database) Tables() map[string]sql.Table {
 	return map[string]sql.Table{
-		commitsTableName:      d.commits,
-		referencesTableName:   d.references,
-		blobsTableName:        d.blobs,
-		treeEntriesTableName:  d.treeEntries,
-		repositoriesTableName: d.repositories,
-		remotesTableName:      d.remotes,
+		CommitsTableName:      d.commits,
+		ReferencesTableName:   d.references,
+		BlobsTableName:        d.blobs,
+		TreeEntriesTableName:  d.treeEntries,
+		RepositoriesTableName: d.repositories,
+		RemotesTableName:      d.remotes,
 	}
 }

database_test.go

@@ -31,12 +31,12 @@ func TestDatabase_Tables(t *testing.T) {
 
 	sort.Strings(tableNames)
 	expected := []string{
-		commitsTableName,
-		referencesTableName,
-		treeEntriesTableName,
-		blobsTableName,
-		repositoriesTableName,
-		remotesTableName,
+		CommitsTableName,
+		ReferencesTableName,
+		TreeEntriesTableName,
+		BlobsTableName,
+		RepositoriesTableName,
+		RemotesTableName,
 	}
 	sort.Strings(expected)