*: tests for squash vs no squash correctness

This commit implements a test suite to ensure results using the squash optimization are the exact same as the ones not using it.
It also fixes two of these differences:
- Without squashing, remote iterator did not return a string but another type for the refspecs.
- Joining blobs with refs or commits could lead to extra rows because blobs could appear more than once per commit or ref. Now they only appear once.

Signed-off-by: Miguel Molina <[email protected]>

Author: erizocosmico

.gitignore

@@ -22,6 +22,7 @@ _testmain.go
 *.exe
 *.test
 *.prof
+gitbase
 
 # CI
 .ci/

integration_test.go

@@ -189,6 +189,95 @@ func TestUastQueries(t *testing.T) {
 	require.Len(rows, 3)
 }
 
+func TestSquashCorrectness(t *testing.T) {
+	engine := sqle.New()
+	squashEngine := sqle.New()
+	require.NoError(t, fixtures.Init())
+	defer func() {
+		require.NoError(t, fixtures.Clean())
+	}()
+
+	pool := gitbase.NewRepositoryPool()
+	for _, f := range fixtures.ByTag("worktree") {
+		pool.AddGit(f.Worktree().Root())
+	}
+
+	engine.AddDatabase(gitbase.NewDatabase("foo"))
+	engine.Catalog.RegisterFunctions(function.Functions)
+
+	squashEngine.AddDatabase(gitbase.NewDatabase("foo"))
+	squashEngine.Catalog.RegisterFunctions(function.Functions)
+	squashEngine.Analyzer.AddRule(rule.SquashJoinsRule, rule.SquashJoins)
+
+	queries := []string{
+		`SELECT * FROM repositories`,
+		`SELECT * FROM refs`,
+		`SELECT * FROM remotes`,
+		`SELECT * FROM commits`,
+		`SELECT * FROM tree_entries`,
+		`SELECT * FROM blobs`,
+		`SELECT * FROM repositories r INNER JOIN refs ON r.id = refs.repository_id`,
+		`SELECT * FROM repositories r INNER JOIN remotes ON r.id = remotes.repository_id`,
+		`SELECT * FROM refs r INNER JOIN remotes re ON r.repository_id = re.repository_id`,
+		`SELECT * FROM refs r INNER JOIN commits c ON r.hash = c.hash`,
+		`SELECT * FROM refs r INNER JOIN commits c ON history_idx(r.hash, c.hash) >= 0`,
+		`SELECT * FROM refs r INNER JOIN tree_entries te ON commit_has_tree(r.hash, te.tree_hash)`,
+		`SELECT * FROM refs r INNER JOIN blobs b ON commit_has_blob(r.hash, b.hash)`,
+		`SELECT * FROM commits c INNER JOIN tree_entries te ON commit_has_tree(c.hash, te.tree_hash)`,
+		`SELECT * FROM commits c INNER JOIN tree_entries te ON c.tree_hash = te.tree_hash`,
+		`SELECT * FROM commits c INNER JOIN blobs b ON commit_has_blob(c.hash, b.hash)`,
+		`SELECT * FROM tree_entries te INNER JOIN blobs b ON te.entry_hash = b.hash`,
+
+		`SELECT * FROM repositories r
+		INNER JOIN refs re 
+			ON r.id = re.repository_id
+		INNER JOIN commits c 
+			ON re.hash = c.hash 
+		WHERE re.name = 'HEAD'`,
+
+		`SELECT * FROM commits c
+		INNER JOIN tree_entries te
+			ON c.tree_hash = te.tree_hash
+		INNER JOIN blobs b
+			ON te.entry_hash = b.hash
+		WHERE te.name = 'LICENSE'`,
+
+		`SELECT * FROM repositories,
+		commits c INNER JOIN tree_entries te
+			ON c.tree_hash = te.tree_hash`,
+	}
+
+	for _, q := range queries {
+		t.Run(q, func(t *testing.T) {
+			expected := queryResults(t, engine, &pool, q)
+			result := queryResults(t, squashEngine, &pool, q)
+			require.ElementsMatch(
+				t,
+				expected,
+				result,
+			)
+		})
+	}
+}
+
+func queryResults(
+	t *testing.T,
+	e *sqle.Engine,
+	pool *gitbase.RepositoryPool,
+	q string,
+) []sql.Row {
+	session := gitbase.NewSession(pool)
+	ctx := sql.NewContext(context.TODO(), sql.WithSession(session))
+
+	_, iter, err := e.Query(ctx, q)
+	require.NoError(t, err)
+
+	rows, err := sql.RowIterToRows(iter)
+	require.NoError(t, err)
+
+	return rows
+}
+
 func BenchmarkQueries(b *testing.B) {
 	queries := []struct {
 		name  string

internal/rule/squashjoins.go

@@ -322,16 +322,8 @@ func buildSquashedTable(
 					return nil, err
 				}
 
-				iter = gitbase.NewTreeEntryBlobsIter(
-					gitbase.NewCommitTreeEntriesIter(
-						gitbase.NewRefHEADCommitsIter(
-							it,
-							nil,
-							true,
-						),
-						nil,
-						true,
-					),
+				iter = gitbase.NewCommitBlobsIter(
+					gitbase.NewRefHEADCommitsIter(it, nil, true),
 					f,
 				)
 			case gitbase.CommitsIter:

internal/rule/squashjoins_test.go

@@ -653,14 +653,10 @@ func TestBuildSquashedTable(t *testing.T) {
 			},
 			nil,
 			newSquashedTable(
-				gitbase.NewTreeEntryBlobsIter(
-					gitbase.NewCommitTreeEntriesIter(
-						gitbase.NewRefHEADCommitsIter(
-							gitbase.NewAllRefsIter(
-								fixIdx(t, refFilter, refsBlobsSchema),
-							),
-							nil,
-							true,
+				gitbase.NewCommitBlobsIter(
+					gitbase.NewRefHEADCommitsIter(
+						gitbase.NewAllRefsIter(
+							fixIdx(t, refFilter, refsBlobsSchema),
 						),
 						nil,
 						true,

iterator.go

@@ -1442,6 +1442,115 @@ func (i *treeEntryBlobsIter) Schema() sql.Schema {
 	return append(i.treeEntries.Schema(), BlobsSchema...)
 }
 
+type commitBlobsIter struct {
+	ctx     *sql.Context
+	repo    *git.Repository
+	filters sql.Expression
+	commits CommitsIter
+	files   *object.FileIter
+	row     sql.Row
+	seen    map[plumbing.Hash]struct{}
+}
+
+// NewCommitBlobsIter returns an iterator that will return all blobs
+// for the commit in the given iter that match the given filters.
+func NewCommitBlobsIter(
+	commits CommitsIter,
+	filters sql.Expression,
+) BlobsIter {
+	return &commitBlobsIter{commits: commits, filters: filters}
+}
+
+func (i *commitBlobsIter) Close() error {
+	if i.commits != nil {
+		return i.commits.Close()
+	}
+
+	return nil
+}
+func (i *commitBlobsIter) New(ctx *sql.Context, repo *Repository) (ChainableIter, error) {
+	iter, err := i.commits.New(ctx, repo)
+	if err != nil {
+		return nil, err
+	}
+
+	return &commitBlobsIter{
+		ctx:     ctx,
+		repo:    repo.Repo,
+		commits: iter.(CommitsIter),
+		filters: i.filters,
+		seen:    make(map[plumbing.Hash]struct{}),
+	}, nil
+}
+func (i *commitBlobsIter) Row() sql.Row { return i.row }
+func (i *commitBlobsIter) Advance() error {
+	for {
+		if i.commits == nil {
+			return io.EOF
+		}
+
+		if i.files == nil {
+			err := i.commits.Advance()
+			if err == io.EOF {
+				i.commits = nil
+				return io.EOF
+			}
+
+			if err != nil {
+				return err
+			}
+
+			tree, err := i.repo.TreeObject(i.commits.Commit().TreeHash)
+			if err != nil {
+				return err
+			}
+
+			i.files = tree.Files()
+			// uniqueness of blob is per commit, so we need to reset the seen map
+			i.seen = make(map[plumbing.Hash]struct{})
+		}
+
+		file, err := i.files.Next()
+		if err == io.EOF {
+			i.files = nil
+			continue
+		}
+
+		if _, ok := i.seen[file.Hash]; ok {
+			continue
+		}
+
+		i.seen[file.Hash] = struct{}{}
+		blob, err := i.repo.BlobObject(file.Hash)
+		if err != nil {
+			return err
+		}
+
+		row, err := blobToRow(blob)
+		if err != nil {
+			return err
+		}
+
+		i.row = append(i.commits.Row(), row...)
+
+		if i.filters != nil {
+			ok, err := evalFilters(i.ctx, i.row, i.filters)
+			if err != nil {
+				return err
+			}
+
+			if !ok {
+				continue
+			}
+		}
+
+		return nil
+	}
+}
+func (i *commitBlobsIter) Schema() sql.Schema {
+	return append(i.commits.Schema(), BlobsSchema...)
+}
+
 // NewChainableRowRepoIter creates a new RowRepoIter from a ChainableIter.
 func NewChainableRowRepoIter(ctx *sql.Context, iter ChainableIter) RowRepoIter {
 	return &chainableRowRepoIter{iter, ctx}

iterator_test.go

@@ -419,6 +419,26 @@ func TestRecursiveTreeFileIter(t *testing.T) {
 	require.Equal(expected, result)
 }
 
+func TestCommitBlobsIter(t *testing.T) {
+	require := require.New(t)
+	ctx, cleanup := setupIter(t)
+	defer cleanup()
+
+	rows := chainableIterRows(
+		t, ctx,
+		NewCommitBlobsIter(
+			NewRefHEADCommitsIter(
+				NewAllRefsIter(nil),
+				nil,
+				true,
+			),
+			nil,
+		),
+	)
+
+	require.Len(rows, 42)
+}
+
 func chainableIterRows(t *testing.T, ctx *sql.Context, iter ChainableIter) []sql.Row {
 	it, err := NewRowRepoIter(ctx, NewChainableRowRepoIter(ctx, iter))
 	require.NoError(t, err)

remotes.go

@@ -130,8 +130,8 @@ func (i *remotesIter) Next() (sql.Row, error) {
 		config.Name,
 		config.URLs[i.urlPos],
 		config.URLs[i.urlPos],
-		config.Fetch[i.urlPos],
-		config.Fetch[i.urlPos],
+		config.Fetch[i.urlPos].String(),
+		config.Fetch[i.urlPos].String(),
 	)
 
 	i.urlPos++

remotes_test.go

@@ -81,8 +81,8 @@ func TestRemotesTable_RowIter(t *testing.T) {
 			require.Equal(url, row[3]) // fetch
 
 			ref := fmt.Sprintf("refs/heads/*:refs/remotes/fetch%v/*", num)
-			require.Equal(gitconfig.RefSpec(ref), row[4]) // push
-			require.Equal(gitconfig.RefSpec(ref), row[5]) // fetch
+			require.Equal(gitconfig.RefSpec(ref).String(), row[4]) // push
+			require.Equal(gitconfig.RefSpec(ref).String(), row[5]) // fetch
 		} else {
 			require.Equal("origin", row[1])
 		}