Add env var to skip go-git errors

Signed-off-by: Javi Fontan <[email protected]>

Author: jfontan

cmd/gitbase/server.go

@@ -21,6 +21,11 @@ import (
 // not be empty.
 var enableUnstableSquash = os.Getenv("UNSTABLE_SQUASH_ENABLE") != ""
 
+// By default when gitbase encounters and error in a repository it stops
+// the query. With this parameter it won't complain and just skip those
+// rows or repositories.
+var skipGitErrors = os.Getenv("GITBASE_SKIP_GIT_ERRORS") != ""
+
 type cmdServer struct {
 	Verbose bool `short:"v" description:"Activates the verbose mode"`
 
@@ -84,7 +89,9 @@ func (c *cmdServer) Execute(args []string) error {
 			Auth:     auth,
 		},
 		c.engine,
-		gitbase.NewSessionBuilder(c.pool),
+		gitbase.NewSessionBuilder(c.pool,
+			gitbase.WithSkipGitErrors(skipGitErrors),
+		),
 	)
 	if err != nil {
 		return err

repository_pool.go

@@ -242,13 +242,9 @@ type RepositoryIter struct {
 // when there are no more Repositories to retrieve.
 func (i *RepositoryIter) Next() (*Repository, error) {
 	r, err := i.pool.GetPos(i.pos)
-	if err != nil {
-		return nil, err
-	}
-
 	i.pos++
 
-	return r, nil
+	return r, err
 }
 
 // Close finished iterator. It's no-op.
@@ -330,6 +326,7 @@ func NewRowRepoIter(
 	go func() {
 		repoIter.wg.Wait()
 		close(repoIter.rows)
+		closeIter(&repoIter)
 	}()
 
 	return &repoIter, nil
@@ -387,9 +384,11 @@ func (i *rowRepoIter) fillRepoChannel() {
 				return
 
 			default:
-				closeIter(i)
-				i.setError(err)
-				return
+				if !i.session.SkipGitErrors {
+					closeIter(i)
+					i.setError(err)
+					return
+				}
 			}
 		}
 	}
@@ -439,10 +438,14 @@ func (i *rowRepoIter) rowReader(num int) {
 					break loop
 
 				default:
-					iter.Close()
-					i.setError(err)
-					closeIter(i)
-					return
+					if !i.session.SkipGitErrors {
+						iter.Close()
+						i.setError(err)
+						closeIter(i)
+						return
+					} else {
+						break loop
+					}
 				}
 			}
 		}

repository_pool_test.go

@@ -266,83 +266,177 @@ func TestRepositoryPoolAddDir(t *testing.T) {
 	require.ElementsMatch(arrayExpected, arrayID)
 }
 
+func TestRepositoryPoolSiva(t *testing.T) {
+	require := require.New(t)
+
+	expectedRepos := 3
+
+	pool := NewRepositoryPool()
+	path := filepath.Join(
+		os.Getenv("GOPATH"),
+		"src", "github.com", "src-d", "gitbase",
+		"_testdata",
+	)
+
+	require.NoError(pool.AddSivaDir(path))
+	require.Equal(expectedRepos, len(pool.repositories))
+
+	expected := []int{606, 452, 75}
+	result := make([]int, expectedRepos)
+
+	for i := 0; i < expectedRepos; i++ {
+		repo, err := pool.GetPos(i)
+		require.NoError(err)
+
+		iter, err := repo.Repo.CommitObjects()
+		require.NoError(err)
+
+		require.NoError(iter.ForEach(func(c *object.Commit) error {
+			result[i]++
+			return nil
+		}))
+	}
+
+	require.Equal(expected, result)
+}
+
 var errIter = fmt.Errorf("Error iter")
 
-type testErrorIter struct{}
+type newIteratorFunc func(*Repository) (RowRepoIter, error)
+type nextFunc func() (sql.Row, error)
+
+type testErrorIter struct {
+	newIterator newIteratorFunc
+	next        nextFunc
+}
 
 func (d *testErrorIter) NewIterator(
 	repo *Repository,
 ) (RowRepoIter, error) {
+	if d.newIterator != nil {
+		return d.newIterator(repo)
+	}
+
 	return nil, errIter
-	// return &testErrorIter{}, nil
 }
 
 func (d *testErrorIter) Next() (sql.Row, error) {
+	if d.next != nil {
+		return d.next()
+	}
+
 	return nil, io.EOF
 }
 
 func (d *testErrorIter) Close() error {
 	return nil
 }
 
-func TestRepositoryErrorIter(t *testing.T) {
+func testCaseRepositoryErrorIter(
+	t *testing.T,
+	pool *RepositoryPool,
+	iter RowRepoIter,
+	retError error,
+	skipGitErrors bool,
+) {
 	require := require.New(t)
 
-	path := fixtures.Basic().ByTag("worktree").One().Worktree().Root()
-	pool := NewRepositoryPool()
-	pool.Add("one", path, gitRepo)
-
 	timeout, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	ctx := sql.NewContext(timeout,
+		sql.WithSession(NewSession(pool, WithSkipGitErrors(skipGitErrors))),
+	)
 
-	ctx := sql.NewContext(timeout, sql.WithSession(NewSession(pool)))
-	eIter := &testErrorIter{}
-
-	iter, err := NewRowRepoIter(ctx, eIter)
-	repoIter := iter.(*rowRepoIter)
+	r, err := NewRowRepoIter(ctx, iter)
 	require.NoError(err)
 
+	repoIter, ok := r.(*rowRepoIter)
+	require.True(ok)
+
 	go func() {
-		repoIter.Next()
+		for {
+			_, err := repoIter.Next()
+			if err != nil {
+				break
+			}
+		}
 	}()
 
 	select {
 	case <-repoIter.done:
-		require.Equal(errIter, repoIter.err)
+		require.Equal(retError, repoIter.err)
 	}
 
 	cancel()
 }
 
-func TestRepositoryPoolSiva(t *testing.T) {
-	require := require.New(t)
+func TestRepositoryErrorIter(t *testing.T) {
+	path := fixtures.Basic().ByTag("worktree").One().Worktree().Root()
+	pool := NewRepositoryPool()
+	pool.Add("one", path, gitRepo)
 
-	expectedRepos := 3
+	iter := &testErrorIter{}
+	testCaseRepositoryErrorIter(t, pool, iter, errIter, false)
+}
 
+func TestRepositoryErrorBadRepository(t *testing.T) {
 	pool := NewRepositoryPool()
-	path := filepath.Join(
-		os.Getenv("GOPATH"),
-		"src", "github.com", "src-d", "gitbase",
-		"_testdata",
-	)
+	pool.Add("one", "badpath", gitRepo)
 
-	require.NoError(pool.AddSivaDir(path))
-	require.Equal(expectedRepos, len(pool.repositories))
+	iter := &testErrorIter{}
 
-	expected := []int{606, 452, 75}
-	result := make([]int, expectedRepos)
+	newIterator := func(*Repository) (RowRepoIter, error) {
+		return iter, nil
+	}
 
-	for i := 0; i < expectedRepos; i++ {
-		repo, err := pool.GetPos(i)
-		require.NoError(err)
+	count := 0
+	next := func() (sql.Row, error) {
+		if count >= 10 {
+			return nil, io.EOF
+		}
 
-		iter, err := repo.Repo.CommitObjects()
-		require.NoError(err)
+		count++
 
-		require.NoError(iter.ForEach(func(c *object.Commit) error {
-			result[i]++
-			return nil
-		}))
+		return sql.NewRow("test"), nil
 	}
 
-	require.Equal(expected, result)
+	iter.newIterator = newIterator
+	iter.next = next
+
+	testCaseRepositoryErrorIter(t, pool, iter, git.ErrRepositoryNotExists, false)
+	testCaseRepositoryErrorIter(t, pool, iter, io.EOF, true)
+}
+
+func TestRepositoryErrorBadRow(t *testing.T) {
+	path := fixtures.Basic().ByTag("worktree").One().Worktree().Root()
+	pool := NewRepositoryPool()
+	pool.Add("one", path, gitRepo)
+
+	iter := &testErrorIter{}
+
+	newIterator := func(*Repository) (RowRepoIter, error) {
+		return iter, nil
+	}
+
+	errRow := fmt.Errorf("bad row")
+
+	count := 0
+	next := func() (sql.Row, error) {
+		if count == 5 {
+			return nil, errRow
+		}
+
+		if count >= 10 {
+			return nil, io.EOF
+		}
+
+		count++
+
+		return sql.NewRow("test"), nil
+	}
+
+	iter.newIterator = newIterator
+	iter.next = next
+
+	testCaseRepositoryErrorIter(t, pool, iter, errRow, false)
+	testCaseRepositoryErrorIter(t, pool, iter, io.EOF, true)
 }

session.go

@@ -21,6 +21,8 @@ type Session struct {
 	bblfshMu       sync.Mutex
 	bblfshEndpoint string
 	bblfshClient   *bblfsh.Client
+
+	SkipGitErrors bool
 }
 
 const (
@@ -38,6 +40,13 @@ func WithBblfshEndpoint(endpoint string) SessionOption {
 	}
 }
 
+// WithSkipGitErrors changes the behavior with go-git error.
+func WithSkipGitErrors(enabled bool) SessionOption {
+	return func(s *Session) {
+		s.SkipGitErrors = enabled
+	}
+}
+
 // NewSession creates a new Session. It requires a repository pool and any
 // number of session options can be passed to configure the session.
 func NewSession(pool *RepositoryPool, opts ...SessionOption) *Session {