gitbase: add support for siva files

erizocosmico · erizocosmico · commit fb4f97c04e9b · 2018-04-13T15:28:35.000+02:00
Add support in RepositoryPool for handling siva files as repositories.

Added a new `AddSivaDir` method to `RepositoryPool`, which adds all the siva files inside that directory (a siva file is considered to be any file whose extension is `.siva`) and the direct child directories to allow bucketing.

Also some minor changes related to the RepositoryPool:
- NewRepositoryFromPath returns now *Repository for convenience.
- NewRepositoryPool returns now *RepositoryPool, as it always needs to be used with a pointer.
- Fixed golint warning about returning an unexported type in an exported method, which may be annoying by replacing the type with `sql.RowIter`.

Signed-off-by: Miguel Molina &lt;miguel@erizocosmi.co&gt;
diff --git a/_testdata/05893125684f2d3943cd84a7ab2b75e53668fba1.siva b/_testdata/05893125684f2d3943cd84a7ab2b75e53668fba1.siva
diff --git a/_testdata/ff/fff840f8784ef162dc83a1465fc5763d890b68ba.siva b/_testdata/ff/fff840f8784ef162dc83a1465fc5763d890b68ba.siva
diff --git a/_testdata/fff7062de8474d10a67d417ccea87ba6f58ca81d.siva b/_testdata/fff7062de8474d10a67d417ccea87ba6f58ca81d.siva
diff --git a/_testdata/not-siva.txt b/_testdata/not-siva.txt
diff --git a/cmd/gitbase/server.go b/cmd/gitbase/server.go
@@ -24,7 +24,8 @@ var enableUnstableSquash = os.Getenv("UNSTABLE_SQUASH_ENABLE") != ""
 type cmdServer struct {
 	Verbose bool `short:"v" description:"Activates the verbose mode"`
 
-	Git      string `short:"g" long:"git" description:"Path where the git repositories are located, one per dir"`
+	Git      string `short:"g" long:"git" description:"Path where the git repositories are located"`
+	Siva     string `long:"siva" description:"Path where the siva repositories are located"`
 	Host     string `short:"h" long:"host" default:"localhost" description:"Host where the server is going to listen"`
 	Port     int    `short:"p" long:"port" default:"3306" description:"Port where the server is going to listen"`
 	User     string `short:"u" long:"user" default:"root" description:"User name used for connection"`
@@ -44,12 +45,12 @@ func (c *cmdServer) buildDatabase() error {
 		logrus.WithField("dir", c.Git).Debug("added folder containing git repositories")
 	}
 
-	var err error
+	c.pool = gitbase.NewRepositoryPool()
+	if err := c.pool.AddDir(c.Git); err != nil {
+		return err
+	}
 
-	pool := gitbase.NewRepositoryPool()
-	c.pool = &pool
-	err = c.pool.AddDir(c.Git)
-	if err != nil {
+	if err := c.pool.AddSivaDir(c.Siva); err != nil {
 		return err
 	}
 
diff --git a/common_test.go b/common_test.go
@@ -30,7 +30,7 @@ func setup(t *testing.T) (ctx *sql.Context, path string, cleanup CleanupFunc) {
 		require.NoError(fixtures.Clean())
 	}
 
-	session := NewSession(&pool)
+	session := NewSession(pool)
 	ctx = sql.NewContext(context.TODO(), sql.WithSession(session))
 
 	return ctx, path, cleanup
diff --git a/integration_test.go b/integration_test.go
@@ -139,7 +139,7 @@ func TestIntegration(t *testing.T) {
 			t.Run(tt.query, func(t *testing.T) {
 				require := require.New(t)
 
-				session := gitbase.NewSession(&pool)
+				session := gitbase.NewSession(pool)
 				ctx := sql.NewContext(context.TODO(), sql.WithSession(session))
 
 				_, iter, err := engine.Query(ctx, tt.query)
@@ -173,7 +173,7 @@ func TestUastQueries(t *testing.T) {
 	engine.AddDatabase(gitbase.NewDatabase("foo"))
 	engine.Catalog.RegisterFunctions(function.Functions)
 
-	session := gitbase.NewSession(&pool)
+	session := gitbase.NewSession(pool)
 	ctx := sql.NewContext(context.TODO(), sql.WithSession(session))
 	_, iter, err := engine.Query(ctx, `
 		SELECT uast_xpath(uast(content, 'php'), '//*[@roleIdentifier]') as uast, name 
@@ -249,8 +249,8 @@ func TestSquashCorrectness(t *testing.T) {
 
 	for _, q := range queries {
 		t.Run(q, func(t *testing.T) {
-			expected := queryResults(t, engine, &pool, q)
-			result := queryResults(t, squashEngine, &pool, q)
+			expected := queryResults(t, engine, pool, q)
+			result := queryResults(t, squashEngine, pool, q)
 			require.ElementsMatch(
 				t,
 				expected,
@@ -374,7 +374,7 @@ func benchmarkQuery(b *testing.B, query string) {
 	pool := gitbase.NewRepositoryPool()
 	_, err := pool.AddGit(path)
 	require.NoError(b, err)
-	session := gitbase.NewSession(&pool)
+	session := gitbase.NewSession(pool)
 	ctx := sql.NewContext(context.TODO(), sql.WithSession(session))
 
 	run := func(b *testing.B) {
diff --git a/internal/function/commit_has_blob_test.go b/internal/function/commit_has_blob_test.go
@@ -27,7 +27,7 @@ func TestCommitHasBlob(t *testing.T) {
 		pool.AddGit(f.Worktree().Root())
 	}
 
-	session := gitbase.NewSession(&pool)
+	session := gitbase.NewSession(pool)
 	ctx := sql.NewContext(context.TODO(), sql.WithSession(session))
 
 	testCases := []struct {
@@ -72,7 +72,7 @@ func BenchmarkCommitHasBlob(b *testing.B) {
 		pool.AddGit(f.Worktree().Root())
 	}
 
-	session := gitbase.NewSession(&pool)
+	session := gitbase.NewSession(pool)
 	ctx := sql.NewContext(context.TODO(), sql.WithSession(session))
 
 	rows := []sql.Row{
diff --git a/internal/function/commit_has_tree_test.go b/internal/function/commit_has_tree_test.go
@@ -27,7 +27,7 @@ func TestCommitHasTree(t *testing.T) {
 		pool.AddGit(f.Worktree().Root())
 	}
 
-	session := gitbase.NewSession(&pool)
+	session := gitbase.NewSession(pool)
 	ctx := sql.NewContext(context.TODO(), sql.WithSession(session))
 
 	testCases := []struct {
@@ -73,7 +73,7 @@ func BenchmarkCommitHasTree(b *testing.B) {
 		pool.AddGit(f.Worktree().Root())
 	}
 
-	session := gitbase.NewSession(&pool)
+	session := gitbase.NewSession(pool)
 	ctx := sql.NewContext(context.TODO(), sql.WithSession(session))
 
 	rows := []sql.Row{
diff --git a/internal/function/history_idx_test.go b/internal/function/history_idx_test.go
@@ -27,7 +27,7 @@ func TestHistoryIdx(t *testing.T) {
 		pool.AddGit(f.Worktree().Root())
 	}
 
-	session := gitbase.NewSession(&pool)
+	session := gitbase.NewSession(pool)
 	ctx := sql.NewContext(context.TODO(), sql.WithSession(session))
 
 	testCases := []struct {
@@ -80,7 +80,7 @@ func BenchmarkHistoryIdx(b *testing.B) {
 		pool.AddGit(f.Worktree().Root())
 	}
 
-	session := gitbase.NewSession(&pool)
+	session := gitbase.NewSession(pool)
 	ctx := sql.NewContext(context.TODO(), sql.WithSession(session))
 
 	cases := []struct {
diff --git a/internal/function/uast_test.go b/internal/function/uast_test.go
@@ -191,7 +191,7 @@ func setup(t *testing.T) (*sql.Context, func()) {
 		pool.AddGit(f.Worktree().Root())
 	}
 
-	session := gitbase.NewSession(&pool)
+	session := gitbase.NewSession(pool)
 	ctx := sql.NewContext(context.TODO(), sql.WithSession(session))
 
 	return ctx, func() {
diff --git a/iterator_test.go b/iterator_test.go
@@ -458,7 +458,7 @@ func setupIter(t *testing.T) (*sql.Context, func()) {
 		pool.AddGit(f.Worktree().Root())
 	}
 
-	session := NewSession(&pool)
+	session := NewSession(pool)
 	ctx := sql.NewContext(context.TODO(), sql.WithSession(session))
 	cleanup := func() {
 		require.NoError(t, fixtures.Clean())
diff --git a/repositories_test.go b/repositories_test.go
@@ -39,10 +39,10 @@ func TestRepositoriesTable_RowIter(t *testing.T) {
 	pool := NewRepositoryPool()
 
 	for _, id := range repoIDs {
-		pool.Add(id, "")
+		pool.Add(id, "", gitRepo)
 	}
 
-	session := NewSession(&pool)
+	session := NewSession(pool)
 	ctx := sql.NewContext(context.TODO(), sql.WithSession(session))
 
 	db := NewDatabase(RepositoriesTableName)
diff --git a/repository_pool.go b/repository_pool.go
@@ -3,12 +3,18 @@ package gitbase
 import (
 	"io"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"sync"
 
 	"github.com/sirupsen/logrus"
+	"gopkg.in/src-d/go-billy-siva.v4"
+	"gopkg.in/src-d/go-billy.v4/osfs"
+	errors "gopkg.in/src-d/go-errors.v1"
 	"gopkg.in/src-d/go-git.v4"
+	"gopkg.in/src-d/go-git.v4/storage/filesystem"
 	"gopkg.in/src-d/go-mysql-server.v0/sql"
 )
 
@@ -19,46 +25,87 @@ type Repository struct {
 }
 
 // NewRepository creates and initializes a new Repository structure
-func NewRepository(id string, repo *git.Repository) Repository {
-	return Repository{
+func NewRepository(id string, repo *git.Repository) *Repository {
+	return &Repository{
 		ID:   id,
 		Repo: repo,
 	}
 }
 
 // NewRepositoryFromPath creates and initializes a new Repository structure
 // and initializes a go-git repository
-func NewRepositoryFromPath(id, path string) (Repository, error) {
+func NewRepositoryFromPath(id, path string) (*Repository, error) {
 	repo, err := git.PlainOpen(path)
 	if err != nil {
-		return Repository{}, err
+		return nil, err
+	}
+
+	return NewRepository(id, repo), nil
+}
+
+// NewSivaRepositoryFromPath creates and initializes a new Repository structure
+// and initializes a go-git repository backed by a siva file.
+func NewSivaRepositoryFromPath(id, path string) (*Repository, error) {
+	localfs := osfs.New(filepath.Dir(path))
+
+	tmpDir, err := ioutil.TempDir(os.TempDir(), "gitbase-siva")
+	if err != nil {
+		return nil, err
+	}
+
+	tmpfs := osfs.New(tmpDir)
+
+	fs, err := sivafs.NewFilesystem(localfs, filepath.Base(path), tmpfs)
+	if err != nil {
+		return nil, err
+	}
+
+	sto, err := filesystem.NewStorage(fs)
+	if err != nil {
+		return nil, err
+	}
+
+	repo, err := git.Open(sto, nil)
+	if err != nil {
+		return nil, err
 	}
 
 	return NewRepository(id, repo), nil
 }
 
+type repository struct {
+	kind repoKind
+	path string
+}
+
+type repoKind byte
+
+const (
+	gitRepo repoKind = iota
+	sivaRepo
+)
+
 // RepositoryPool holds a pool git repository paths and
 // functionality to open and iterate them.
 type RepositoryPool struct {
-	repositories map[string]string
+	repositories map[string]repository
 	idOrder      []string
 }
 
 // NewRepositoryPool initializes a new RepositoryPool
-func NewRepositoryPool() RepositoryPool {
-	return RepositoryPool{
-		repositories: make(map[string]string),
+func NewRepositoryPool() *RepositoryPool {
+	return &RepositoryPool{
+		repositories: make(map[string]repository),
 	}
 }
 
 // Add inserts a new repository in the pool
-func (p *RepositoryPool) Add(id, path string) {
-	_, ok := p.repositories[id]
-	if !ok {
+func (p *RepositoryPool) Add(id, path string, kind repoKind) {
+	if _, ok := p.repositories[id]; !ok {
 		p.idOrder = append(p.idOrder, id)
 	}
 
-	p.repositories[id] = path
+	p.repositories[id] = repository{kind, path}
 }
 
 // AddGit checks if a git repository can be opened and adds it to the pool. It
@@ -69,12 +116,12 @@ func (p *RepositoryPool) AddGit(path string) (string, error) {
 		return "", err
 	}
 
-	p.Add(path, path)
+	p.Add(path, path, gitRepo)
 
 	return path, nil
 }
 
-// AddDir adds all direct subdirectories from path as repos
+// AddDir adds all direct subdirectories from path as git repos.
 func (p *RepositoryPool) AddDir(path string) error {
 	dirs, err := ioutil.ReadDir(path)
 	if err != nil {
@@ -95,8 +142,57 @@ func (p *RepositoryPool) AddDir(path string) error {
 	return nil
 }
 
+// AddSivaDir adds to the repository pool all siva files found inside the given
+// directory and in its children directories, but not the children of those
+// directories.
+func (p *RepositoryPool) AddSivaDir(path string) error {
+	return p.addSivaDir(path, path, true)
+}
+
+func (p *RepositoryPool) addSivaDir(root, path string, recursive bool) error {
+	dirs, err := ioutil.ReadDir(path)
+	if err != nil {
+		return err
+	}
+
+	for _, f := range dirs {
+		if f.IsDir() && recursive {
+			dirPath := filepath.Join(path, f.Name())
+			if err := p.addSivaDir(root, dirPath, false); err != nil {
+				return err
+			}
+		} else {
+			p.addSivaFile(root, path, f)
+		}
+	}
+
+	return nil
+}
+
+// addSivaFile adds to the pool the given file if it's a siva repository,
+// that is, has the .siva extension.
+func (p *RepositoryPool) addSivaFile(root, path string, f os.FileInfo) {
+	var relativeFileName string
+	if root == path {
+		relativeFileName = f.Name()
+	} else {
+		relPath := strings.TrimPrefix(strings.Replace(path, root, "", -1), "/\\")
+		relativeFileName = filepath.Join(relPath, f.Name())
+	}
+
+	if strings.HasSuffix(f.Name(), ".siva") {
+		path := filepath.Join(path, f.Name())
+		p.Add(path, path, sivaRepo)
+		logrus.WithField("file", relativeFileName).Debug("repository added")
+	} else {
+		logrus.WithField("file", relativeFileName).Warn("found a non-siva file, skipping")
+	}
+}
+
+var errInvalidRepoKind = errors.NewKind("invalid repo kind: %d")
+
 // GetPos retrieves a repository at a given position. If the position is
-// out of bounds it returns io.EOF
+// out of bounds it returns io.EOF.
 func (p *RepositoryPool) GetPos(pos int) (*Repository, error) {
 	if pos >= len(p.repositories) {
 		return nil, io.EOF
@@ -107,13 +203,23 @@ func (p *RepositoryPool) GetPos(pos int) (*Repository, error) {
 		return nil, io.EOF
 	}
 
-	path := p.repositories[id]
-	repo, err := NewRepositoryFromPath(id, path)
+	r := p.repositories[id]
+	var repo *Repository
+	var err error
+	switch r.kind {
+	case gitRepo:
+		repo, err = NewRepositoryFromPath(id, r.path)
+	case sivaRepo:
+		repo, err = NewSivaRepositoryFromPath(id, r.path)
+	default:
+		err = errInvalidRepoKind.New(r.kind)
+	}
+
 	if err != nil {
 		return nil, err
 	}
 
-	return &repo, nil
+	return repo, nil
 }
 
 // RepoIter creates a new Repository iterator
@@ -189,7 +295,7 @@ type rowRepoIter struct {
 func NewRowRepoIter(
 	ctx *sql.Context,
 	iter RowRepoIter,
-) (*rowRepoIter, error) {
+) (sql.RowIter, error) {
 	s, ok := ctx.Session.(*Session)
 	if !ok || s == nil {
 		return nil, ErrInvalidGitbaseSession.New(ctx.Session)
diff --git a/repository_pool_test.go b/repository_pool_test.go

Original file line number	Diff line number	Diff line change
`@@ -30,7 +30,7 @@ func setup(t testing.T) (ctx sql.Context, path string, cleanup CleanupFunc) {`
`30`	`30`	`require.NoError(fixtures.Clean())`
`31`	`31`	`}`
`32`	`32`
`33`		`- session := NewSession(&pool)`
	`33`	`+ session := NewSession(pool)`
`34`	`34`	`ctx = sql.NewContext(context.TODO(), sql.WithSession(session))`
`35`	`35`
`36`	`36`	`return ctx, path, cleanup`
Original file line number	Diff line number	Diff line change
`@@ -27,7 +27,7 @@ func TestCommitHasBlob(t *testing.T) {`
`27`	`27`	`pool.AddGit(f.Worktree().Root())`
`28`	`28`	`}`
`29`	`29`
`30`		`- session := gitbase.NewSession(&pool)`
	`30`	`+ session := gitbase.NewSession(pool)`
`31`	`31`	`ctx := sql.NewContext(context.TODO(), sql.WithSession(session))`
`32`	`32`
`33`	`33`	`testCases := []struct {`
`@@ -72,7 +72,7 @@ func BenchmarkCommitHasBlob(b *testing.B) {`
`72`	`72`	`pool.AddGit(f.Worktree().Root())`
`73`	`73`	`}`
`74`	`74`
`75`		`- session := gitbase.NewSession(&pool)`
	`75`	`+ session := gitbase.NewSession(pool)`
`76`	`76`	`ctx := sql.NewContext(context.TODO(), sql.WithSession(session))`
`77`	`77`
`78`	`78`	`rows := []sql.Row{`
Original file line number	Diff line number	Diff line change
`@@ -27,7 +27,7 @@ func TestCommitHasTree(t *testing.T) {`
`27`	`27`	`pool.AddGit(f.Worktree().Root())`
`28`	`28`	`}`
`29`	`29`
`30`		`- session := gitbase.NewSession(&pool)`
	`30`	`+ session := gitbase.NewSession(pool)`
`31`	`31`	`ctx := sql.NewContext(context.TODO(), sql.WithSession(session))`
`32`	`32`
`33`	`33`	`testCases := []struct {`
`@@ -73,7 +73,7 @@ func BenchmarkCommitHasTree(b *testing.B) {`
`73`	`73`	`pool.AddGit(f.Worktree().Root())`
`74`	`74`	`}`
`75`	`75`
`76`		`- session := gitbase.NewSession(&pool)`
	`76`	`+ session := gitbase.NewSession(pool)`
`77`	`77`	`ctx := sql.NewContext(context.TODO(), sql.WithSession(session))`
`78`	`78`
`79`	`79`	`rows := []sql.Row{`
Original file line number	Diff line number	Diff line change
`@@ -27,7 +27,7 @@ func TestHistoryIdx(t *testing.T) {`
`27`	`27`	`pool.AddGit(f.Worktree().Root())`
`28`	`28`	`}`
`29`	`29`
`30`		`- session := gitbase.NewSession(&pool)`
	`30`	`+ session := gitbase.NewSession(pool)`
`31`	`31`	`ctx := sql.NewContext(context.TODO(), sql.WithSession(session))`
`32`	`32`
`33`	`33`	`testCases := []struct {`
`@@ -80,7 +80,7 @@ func BenchmarkHistoryIdx(b *testing.B) {`
`80`	`80`	`pool.AddGit(f.Worktree().Root())`
`81`	`81`	`}`
`82`	`82`
`83`		`- session := gitbase.NewSession(&pool)`
	`83`	`+ session := gitbase.NewSession(pool)`
`84`	`84`	`ctx := sql.NewContext(context.TODO(), sql.WithSession(session))`
`85`	`85`
`86`	`86`	`cases := []struct {`