rule: make commit_files squashable with blobs (#873)

rule: make commit_files squashable with blobs

Author: ajnavarro

CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added `commit_file_stats` function.
 - Added documentation about `commit_stats`.
 - Add metrics (engine, analyzer, regex, pilosa) based on go-kit interface. ([#744](https://github.com/src-d/go-mysql-server/pull/744))
+- `commit_files` is now squashable with `blobs`.
 
 ### Changed
 

docs/using-gitbase/optimize-queries.md

@@ -234,6 +234,7 @@ You can find the full list of conditions that need to be met for the squash to b
 ```
 repositories -> refs -> ref_commits -> commits -> commit_trees -> tree_entries -> blobs
 repositories -> refs -> ref_commits -> commits -> commit_blobs -> blobs
+repositories -> refs -> ref_commits -> commits -> commit_files -> blobs
 repositories -> refs -> ref_commits -> commits -> commit_files -> files
 repositories -> remotes -> refs -> (any of the other hierarchies)
 ```
@@ -324,6 +325,10 @@ This will pretty-print the analyzed tree of your query. If you see a node named
 
 - `tree_entries.blob_hash = blobs.blob_hash`
 
+### `commit_files` with `blobs`
+
+- `commit_files.blob_hash = blobs.blob_hash`
+
 ### `commit_files` with `files`
 
 - `commit_files.file_path = files.file_path`

internal/rule/squashjoins.go

@@ -5,11 +5,11 @@ import (
 	"reflect"
 
 	"github.com/src-d/gitbase"
-	errors "gopkg.in/src-d/go-errors.v1"
 	"github.com/src-d/go-mysql-server/sql"
 	"github.com/src-d/go-mysql-server/sql/analyzer"
 	"github.com/src-d/go-mysql-server/sql/expression"
 	"github.com/src-d/go-mysql-server/sql/plan"
+	errors "gopkg.in/src-d/go-errors.v1"
 )
 
 // SquashJoinsRule name.
@@ -764,6 +764,19 @@ func buildSquashedTable(
 				}
 
 				iter = gitbase.NewTreeEntryBlobsIter(it, f, readContent)
+			case gitbase.FilesIter:
+				var f sql.Expression
+				f, filters, err = filtersForJoin(
+					gitbase.CommitFilesTableName,
+					gitbase.BlobsTableName,
+					filters,
+					append(it.Schema(), gitbase.BlobsSchema...),
+				)
+				if err != nil {
+					return nil, err
+				}
+
+				iter = gitbase.NewCommitFileBlobsIter(it, f, readContent)
 			default:
 				addUnsquashable(gitbase.BlobsTableName)
 				continue
@@ -985,8 +998,8 @@ var tableHierarchy = []string{
 	gitbase.CommitTreesTableName,
 	gitbase.TreeEntriesTableName,
 	gitbase.CommitBlobsTableName,
-	gitbase.BlobsTableName,
 	gitbase.CommitFilesTableName,
+	gitbase.BlobsTableName,
 	gitbase.FilesTableName,
 }
 
@@ -1540,6 +1553,11 @@ func isRedundantFilter(f sql.Expression, t1, t2 string) bool {
 			isCol(gitbase.ReferencesTableName, "commit_hash"),
 			isCol(gitbase.CommitFilesTableName, "commit_hash"),
 		)(f)
+	case t1 == gitbase.CommitFilesTableName && t2 == gitbase.BlobsTableName:
+		return isEq(
+			isCol(gitbase.CommitFilesTableName, "blob_hash"),
+			isCol(gitbase.BlobsTableName, "blob_hash"),
+		)(f)
 	}
 	return false
 }

internal/rule/squashjoins_test.go

@@ -5,13 +5,13 @@ import (
 	"testing"
 
 	"github.com/src-d/gitbase"
-	"github.com/stretchr/testify/require"
-	errors "gopkg.in/src-d/go-errors.v1"
 	"github.com/src-d/go-mysql-server/sql"
 	"github.com/src-d/go-mysql-server/sql/analyzer"
 	"github.com/src-d/go-mysql-server/sql/expression"
 	"github.com/src-d/go-mysql-server/sql/parse"
 	"github.com/src-d/go-mysql-server/sql/plan"
+	"github.com/stretchr/testify/require"
+	errors "gopkg.in/src-d/go-errors.v1"
 )
 
 func TestAnalyzeSquashJoinsExchange(t *testing.T) {
@@ -407,6 +407,7 @@ func TestBuildSquashedTable(t *testing.T) {
 	refsCommitFilesSchema := append(gitbase.RefsSchema, gitbase.CommitFilesSchema...)
 	commitsCommitFilesSchema := append(gitbase.CommitsSchema, gitbase.CommitFilesSchema...)
 	commitFilesFilesSchema := append(gitbase.CommitFilesSchema, gitbase.FilesSchema...)
+	commitFilesBlobsSchema := append(gitbase.CommitFilesSchema, gitbase.BlobsSchema...)
 
 	repoFilter := eq(
 		col(0, gitbase.RepositoriesTableName, "repository_id"),
@@ -713,6 +714,16 @@ func TestBuildSquashedTable(t *testing.T) {
 		col(0, gitbase.FilesTableName, "file_path"),
 	)
 
+	commitFilesBlobsRedundantFilter := eq(
+		col(0, gitbase.CommitFilesTableName, "blob_hash"),
+		col(0, gitbase.BlobsTableName, "blob_hash"),
+	)
+
+	commitFilesBlobsFilter := eq(
+		col(0, gitbase.CommitFilesTableName, "commit_hash"),
+		col(0, gitbase.BlobsTableName, "blob_size"),
+	)
+
 	idx1, idx2 := &dummyLookup{1}, &dummyLookup{2}
 
 	testCases := []struct {
@@ -2085,6 +2096,41 @@ func TestBuildSquashedTable(t *testing.T) {
 				gitbase.FilesTableName,
 			)),
 		},
+		{
+			"commit_files with blobs",
+			[]sql.Table{commitFiles, blobs},
+			[]sql.Expression{
+				blobFilter,
+				commitFilesFilter,
+				commitFilesBlobsFilter,
+				commitFilesBlobsRedundantFilter,
+			},
+			nil,
+			nil,
+			nil,
+			plan.NewResolvedTable(gitbase.NewSquashedTable(
+				gitbase.NewCommitFileBlobsIter(
+					gitbase.NewAllCommitFilesIter(
+						fixIdx(t, commitFilesFilter, gitbase.CommitFilesSchema),
+					),
+					and(
+						fixIdx(t, blobFilter, commitFilesBlobsSchema),
+						fixIdx(t, commitFilesBlobsFilter, commitFilesBlobsSchema),
+					),
+					false,
+				),
+				nil,
+				[]sql.Expression{
+					blobFilter,
+					commitFilesFilter,
+					commitFilesBlobsFilter,
+					commitFilesBlobsRedundantFilter,
+				},
+				nil,
+				gitbase.CommitFilesTableName,
+				gitbase.BlobsTableName,
+			)),
+		},
 		{
 			"commit_files with indexes",
 			[]sql.Table{commitFiles, files},

squash_iterator.go

@@ -4,13 +4,13 @@ import (
 	"fmt"
 	"io"
 
+	"github.com/src-d/go-mysql-server/sql"
 	errors "gopkg.in/src-d/go-errors.v1"
 	git "gopkg.in/src-d/go-git.v4"
 	"gopkg.in/src-d/go-git.v4/plumbing"
 	"gopkg.in/src-d/go-git.v4/plumbing/filemode"
 	"gopkg.in/src-d/go-git.v4/plumbing/object"
 	"gopkg.in/src-d/go-git.v4/plumbing/storer"
-	"github.com/src-d/go-mysql-server/sql"
 
 	"github.com/sirupsen/logrus"
 )
@@ -3410,6 +3410,81 @@ func (i *squashCommitFileFilesIter) Close() error {
 	return i.files.Close()
 }
 
+type squashCommitFileBlobsIter struct {
+	files       FilesIter
+	readContent bool
+	row         sql.Row
+	filters     sql.Expression
+	ctx         *sql.Context
+}
+
+// NewCommitFileBlobsIter returns all blobs for the commit files in the given
+// iterator.
+func NewCommitFileBlobsIter(
+	files FilesIter,
+	filters sql.Expression,
+	readContent bool,
+) ChainableIter {
+	return &squashCommitFileBlobsIter{
+		files:       files,
+		filters:     filters,
+		readContent: readContent,
+	}
+}
+
+func (i *squashCommitFileBlobsIter) New(ctx *sql.Context, repo *Repository) (ChainableIter, error) {
+	iter, err := i.files.New(ctx, repo)
+	if err != nil {
+		return nil, err
+	}
+
+	return &squashCommitFileBlobsIter{
+		files:       iter.(FilesIter),
+		ctx:         ctx,
+		filters:     i.filters,
+		readContent: i.readContent,
+	}, nil
+}
+
+func (i *squashCommitFileBlobsIter) Advance() error {
+	for {
+		err := i.files.Advance()
+		if err != nil {
+			return err
+		}
+
+		f := i.files.File()
+		row, err := blobToRow(i.Repository().ID, &f.Blob, i.readContent)
+		if err != nil {
+			return err
+		}
+
+		i.row = append(i.files.Row(), row...)
+
+		if i.filters != nil {
+			ok, err := evalFilters(i.ctx, i.row, i.filters)
+			if err != nil {
+				return err
+			}
+
+			if !ok {
+				continue
+			}
+		}
+
+		return nil
+	}
+}
+
+func (i *squashCommitFileBlobsIter) Repository() *Repository { return i.files.Repository() }
+func (i *squashCommitFileBlobsIter) Row() sql.Row            { return i.row }
+func (i *squashCommitFileBlobsIter) Schema() sql.Schema {
+	return append(i.files.Schema(), BlobsSchema...)
+}
+func (i *squashCommitFileBlobsIter) Close() error {
+	return i.files.Close()
+}
+
 func evalFilters(ctx *sql.Context, row sql.Row, filters sql.Expression) (bool, error) {
 	v, err := filters.Eval(ctx, row)
 	if err != nil {

squash_iterator_test.go

@@ -699,6 +699,45 @@ func TestCommitBlobsIter(t *testing.T) {
 	require.Len(rows, 52)
 }
 
+func TestCommitFileBlobsIter(t *testing.T) {
+	require := require.New(t)
+	ctx, cleanup := setupIter(t)
+	defer cleanup()
+
+	rows := chainableIterRows(
+		t, ctx,
+		NewCommitFileBlobsIter(
+			NewAllCommitFilesIter(nil).(FilesIter),
+			nil,
+			true,
+		),
+	)
+
+	expected := chainableIterRows(
+		t, ctx,
+		NewCommitFileFilesIter(
+			NewAllCommitFilesIter(nil).(FilesIter),
+			nil,
+			true,
+		),
+	)
+
+	// transform the result of the files table into the expected for blobs
+	offset := len(CommitFilesSchema)
+	size := offset + len(BlobsSchema)
+	for i, e := range expected {
+		var newRow = make(sql.Row, size)
+		copy(newRow[:offset], e[:offset])
+		newRow[offset] = e[offset]     // repository_id
+		newRow[offset+1] = e[offset+2] // blob_hash
+		newRow[offset+2] = e[offset+6] // blob_size
+		newRow[offset+3] = e[offset+5] // blob_content
+		expected[i] = newRow
+	}
+
+	require.ElementsMatch(expected, rows)
+}
+
 func chainableIterRowsError(t *testing.T, ctx *sql.Context, iter ChainableIter) {
 	table := newSquashTable(iter)
 	_, err := tableToRows(ctx, table)