Merge pull request #380 from jfontan/improvement/test-filesystem-errors

Test filesystem errors

Author: ajnavarro

files.go

@@ -249,6 +249,8 @@ func (i *filesIter) Next() (sql.Row, error) {
 				i.files = nil
 				continue
 			}
+
+			return nil, err
 		}
 
 		if !i.shouldVisitFile(f) {

fs_error_test.go

@@ -0,0 +1,241 @@
+package gitbase
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
+	billy "gopkg.in/src-d/go-billy.v4"
+	fixtures "gopkg.in/src-d/go-git-fixtures.v3"
+	git "gopkg.in/src-d/go-git.v4"
+	"gopkg.in/src-d/go-git.v4/storage/filesystem"
+	"gopkg.in/src-d/go-mysql-server.v0/sql"
+)
+
+func TestFSErrorTables(t *testing.T) {
+	logrus.SetLevel(logrus.FatalLevel)
+
+	tests := []struct {
+		table string
+		rows  int
+	}{
+		{BlobsTableName, 14},
+		{CommitBlobsTableName, 88},
+		{CommitFilesTableName, 88},
+		{CommitTreesTableName, 40},
+		{CommitsTableName, 9},
+		{FilesTableName, 82},
+		{RefCommitsTableName, 64},
+		{ReferencesTableName, 8},
+		{RepositoriesTableName, 3},
+		{TreeEntriesTableName, 45},
+	}
+
+	for _, test := range tests {
+		t.Run(test.table, func(t *testing.T) {
+			testTable(t, test.table, test.rows)
+		})
+	}
+}
+
+// setupErrorRepos creates a pool with three repos. One with read error in
+// packfile, another with an index file missing (and ghost packfile) and
+// finally a correct repository.
+func setupErrorRepos(t *testing.T) (*sql.Context, CleanupFunc) {
+	require := require.New(t)
+
+	t.Helper()
+
+	require.NoError(fixtures.Init())
+
+	fixture := fixtures.ByTag("worktree").One()
+	baseFS := fixture.Worktree()
+
+	pool := NewRepositoryPool()
+
+	repo, err := brokenRepo(brokenPackfile, baseFS)
+	require.NoError(err)
+	pool.AddInitialized("packfile", repo)
+
+	repo, err = brokenRepo(brokenIndex, baseFS)
+	require.NoError(err)
+	pool.AddInitialized("index", repo)
+
+	repo, err = brokenRepo(0, baseFS)
+	require.NoError(err)
+	pool.AddInitialized("ok", repo)
+
+	session := NewSession(pool, WithSkipGitErrors(true))
+	ctx := sql.NewContext(context.TODO(), sql.WithSession(session))
+
+	cleanup := func() {
+		t.Helper()
+		require.NoError(fixtures.Clean())
+	}
+
+	return ctx, cleanup
+}
+
+func brokenRepo(
+	brokenType brokenType,
+	fs billy.Filesystem,
+) (*git.Repository, error) {
+	dotFS, err := fs.Chroot(".git")
+	if err != nil {
+		return nil, err
+	}
+
+	var brokenFS billy.Filesystem
+	if brokenType == 0 {
+		brokenFS = dotFS
+	} else {
+		brokenFS = NewBrokenFS(brokenType, dotFS)
+	}
+
+	storage, err := filesystem.NewStorage(brokenFS)
+	if err != nil {
+		return nil, err
+	}
+
+	return git.Open(storage, fs)
+}
+
+func testTable(t *testing.T, tableName string, number int) {
+	require := require.New(t)
+
+	ctx, cleanup := setupErrorRepos(t)
+	defer cleanup()
+
+	table := getTable(require, tableName)
+	rows, err := sql.NodeToRows(ctx, table)
+	require.NoError(err)
+
+	if len(rows) < number {
+		t.Errorf("table %s returned %v rows and it should be at least %v",
+			tableName, len(rows), number)
+		t.FailNow()
+	}
+}
+
+type brokenType uint64
+
+const (
+	// packfile has read errors
+	brokenPackfile brokenType = 1 << iota
+	// there's no index for one packfile
+	brokenIndex
+
+	packFileGlob   = "objects/pack/pack-*.pack"
+	packBrokenName = "pack-ffffffffffffffffffffffffffffffffffffffff.pack"
+)
+
+func NewBrokenFS(b brokenType, fs billy.Filesystem) billy.Filesystem {
+	return &BrokenFS{
+		Filesystem: fs,
+		brokenType: b,
+	}
+}
+
+type BrokenFS struct {
+	billy.Filesystem
+	brokenType brokenType
+}
+
+func (fs *BrokenFS) Open(filename string) (billy.File, error) {
+	return fs.OpenFile(filename, os.O_RDONLY, 0)
+}
+
+func (fs *BrokenFS) OpenFile(
+	name string,
+	flag int,
+	perm os.FileMode,
+) (billy.File, error) {
+	file, err := fs.Filesystem.OpenFile(name, flag, perm)
+	if err != nil {
+		return nil, err
+	}
+
+	if fs.brokenType&brokenPackfile == 0 {
+		return file, err
+	}
+
+	match, err := filepath.Match(packFileGlob, name)
+	if err != nil {
+		return nil, err
+	}
+
+	if !match {
+		return file, nil
+	}
+
+	return &BrokenFile{
+		File: file,
+	}, nil
+}
+
+func (fs *BrokenFS) ReadDir(path string) ([]os.FileInfo, error) {
+	files, err := fs.Filesystem.ReadDir(path)
+	if err != nil {
+		return nil, err
+	}
+
+	if fs.brokenType&brokenIndex != 0 {
+		dummyPack := &brokenFileInfo{packBrokenName}
+		files = append(files, dummyPack)
+	}
+
+	return files, err
+}
+
+type BrokenFile struct {
+	billy.File
+	count int
+}
+
+func (fs *BrokenFile) Read(p []byte) (int, error) {
+	_, err := fs.Seek(0, os.SEEK_CUR)
+	if err != nil {
+		return 0, err
+	}
+
+	fs.count++
+
+	if fs.count == 10 {
+		return 0, fmt.Errorf("could not read from broken file")
+	}
+
+	return fs.File.Read(p)
+}
+
+type brokenFileInfo struct {
+	name string
+}
+
+func (b *brokenFileInfo) Name() string {
+	return b.name
+}
+
+func (b *brokenFileInfo) Size() int64 {
+	return 1024 * 1024
+}
+
+func (b *brokenFileInfo) Mode() os.FileMode {
+	return 0600
+}
+
+func (b *brokenFileInfo) ModTime() time.Time {
+	return time.Now()
+}
+
+func (b *brokenFileInfo) IsDir() bool {
+	return false
+}
+
+func (b *brokenFileInfo) Sys() interface{} {
+	return nil
+}

repository_pool.go

@@ -82,13 +82,15 @@ func NewSivaRepositoryFromPath(id, path string) (*Repository, error) {
 type repository struct {
 	kind repoKind
 	path string
+	repo *git.Repository
 }
 
 type repoKind byte
 
 const (
 	gitRepo repoKind = iota
 	sivaRepo
+	initializedrepo
 )
 
 // RepositoryPool holds a pool git repository paths and
@@ -105,14 +107,18 @@ func NewRepositoryPool() *RepositoryPool {
 	}
 }
 
-// Add inserts a new repository in the pool
+// Add inserts a new repository in the pool.
 func (p *RepositoryPool) Add(id, path string, kind repoKind) error {
+	return p.add(id, repository{kind, path, nil})
+}
+
+func (p *RepositoryPool) add(id string, repo repository) error {
 	if r, ok := p.repositories[id]; ok {
 		return errRepoAlreadyRegistered.New(r.path)
 	}
 
 	p.idOrder = append(p.idOrder, id)
-	p.repositories[id] = repository{kind, path}
+	p.repositories[id] = repo
 
 	return nil
 }
@@ -213,6 +219,11 @@ func (p *RepositoryPool) addSivaFile(root, path string, f os.FileInfo) {
 	}
 }
 
+// AddInitialized inserts an already initialized repository to the pool.
+func (p *RepositoryPool) AddInitialized(id string, repo *git.Repository) error {
+	return p.add(id, repository{initializedrepo, "", repo})
+}
+
 // GetPos retrieves a repository at a given position. If the position is
 // out of bounds it returns io.EOF.
 func (p *RepositoryPool) GetPos(pos int) (*Repository, error) {
@@ -245,6 +256,8 @@ func (p *RepositoryPool) GetRepo(id string) (*Repository, error) {
 		repo, err = NewRepositoryFromPath(id, r.path)
 	case sivaRepo:
 		repo, err = NewSivaRepositoryFromPath(id, r.path)
+	case initializedrepo:
+		repo = NewRepository(id, r.repo)
 	default:
 		err = errInvalidRepoKind.New(r.kind)
 	}
@@ -369,6 +382,10 @@ func (i *rowRepoIter) Next() (sql.Row, error) {
 
 				i.currRepoIter, err = i.iter.NewIterator(repo)
 				if err != nil {
+					if i.session.SkipGitErrors {
+						continue
+					}
+
 					return nil, err
 				}
 			}