Merge pull request #230 from jfontan/improvement/skip-git-errors

Add env var to skip go-git errors

Author: jfontan

README.md

@@ -53,12 +53,13 @@ SELECT hash, author_email, author_name FROM commits LIMIT 2;
 
 ### Environment variables
 
-| Name                         | Description                                       |
-|:-----------------------------|:--------------------------------------------------|
-| `BBLFSH_ENDPOINT`            | bblfshd endpoint, default "127.0.0.1:9432"        |
-| `GITBASE_BLOBS_MAX_SIZE`     | maximum blob size to return in MiB, default 5 MiB |
-| `GITBASE_BLOBS_ALLOW_BINARY` | enable retrieval of binary blobs, default `false` |
-| `UNSTABLE_SQUASH_ENABLE`     | **UNSTABLE** check *Unstable features*            |
+| Name                         | Description                                         |
+|:-----------------------------|:----------------------------------------------------|
+| `BBLFSH_ENDPOINT`            | bblfshd endpoint, default "127.0.0.1:9432"          |
+| `GITBASE_BLOBS_MAX_SIZE`     | maximum blob size to return in MiB, default 5 MiB   |
+| `GITBASE_BLOBS_ALLOW_BINARY` | enable retrieval of binary blobs, default `false`   |
+| `UNSTABLE_SQUASH_ENABLE`     | **UNSTABLE** check *Unstable features*              |
+| `GITBASE_SKIP_GIT_ERRORS`    | do not stop queries on git errors, default disabled |
 
 ## Tables
 

cmd/gitbase/server.go

@@ -21,6 +21,11 @@ import (
 // not be empty.
 var enableUnstableSquash = os.Getenv("UNSTABLE_SQUASH_ENABLE") != ""
 
+// By default when gitbase encounters and error in a repository it stops
+// the query. With this parameter it won't complain and just skip those
+// rows or repositories.
+var skipGitErrors = os.Getenv("GITBASE_SKIP_GIT_ERRORS") != ""
+
 type cmdServer struct {
 	Verbose bool `short:"v" description:"Activates the verbose mode"`
 
@@ -84,7 +89,9 @@ func (c *cmdServer) Execute(args []string) error {
 			Auth:     auth,
 		},
 		c.engine,
-		gitbase.NewSessionBuilder(c.pool),
+		gitbase.NewSessionBuilder(c.pool,
+			gitbase.WithSkipGitErrors(skipGitErrors),
+		),
 	)
 	if err != nil {
 		return err

internal/function/commit_has_blob.go

@@ -5,6 +5,7 @@ import (
 	"io"
 
 	"github.com/hashicorp/golang-lru"
+	"github.com/sirupsen/logrus"
 
 	"github.com/src-d/gitbase"
 	"gopkg.in/src-d/go-git.v4/plumbing"
@@ -40,10 +41,6 @@ func (CommitHasBlob) Type() sql.Type {
 
 // Eval implements the Expression interface.
 func (f *CommitHasBlob) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) {
-	s, ok := ctx.Session.(*gitbase.Session)
-	if !ok {
-		return nil, gitbase.ErrInvalidGitbaseSession.New(ctx.Session)
-	}
 
 	commitHash, err := f.Left.Eval(ctx, row)
 	if err != nil {
@@ -74,7 +71,7 @@ func (f *CommitHasBlob) Eval(ctx *sql.Context, row sql.Row) (interface{}, error)
 	}
 
 	return f.commitHasBlob(
-		s.Pool,
+		ctx,
 		plumbing.NewHash(commitHash.(string)),
 		plumbing.NewHash(blob.(string)),
 	)
@@ -85,25 +82,49 @@ type commitBlobKey struct {
 }
 
 func (f *CommitHasBlob) commitHasBlob(
-	pool *gitbase.RepositoryPool,
+	ctx *sql.Context,
 	commitHash, blob plumbing.Hash,
 ) (bool, error) {
 	if val, ok := f.cache.Get(commitBlobKey{commitHash, blob}); ok {
 		return val.(bool), nil
 	}
 
-	iter, err := pool.RepoIter()
+	s, ok := ctx.Session.(*gitbase.Session)
+	if !ok {
+		return false, gitbase.ErrInvalidGitbaseSession.New(ctx.Session)
+	}
+
+	log := logrus.WithFields(logrus.Fields{
+		"function":    "commit_hash_blob",
+		"commit_hash": commitHash.String(),
+		"blob":        blob.String(),
+	})
+
+	iter, err := s.Pool.RepoIter()
 	if err != nil {
+		log.WithField("error", err).Error("cannot create repository iterator")
 		return false, err
 	}
 
 	for {
+		select {
+		case <-ctx.Done():
+			log.Debug("query canceled")
+			return false, gitbase.ErrSessionCanceled.New()
+		default:
+		}
+
 		repository, err := iter.Next()
 		if err == io.EOF {
 			break
 		}
 
 		if err != nil {
+			log.WithField("error", err).Error("could not get repository")
+
+			if s.SkipGitErrors {
+				continue
+			}
 			return false, err
 		}
 
@@ -113,17 +134,36 @@ func (f *CommitHasBlob) commitHasBlob(
 			continue
 		}
 
+		log = log.WithFields(logrus.Fields{
+			"repo": repository.ID,
+		})
+
 		if err != nil {
+			logrus.WithField("error", err).Error("could not get commit")
+
+			if s.SkipGitErrors {
+				continue
+			}
 			return false, err
 		}
 
 		tree, err := commit.Tree()
 		if err != nil {
+			logrus.WithField("error", err).Error("could not get tree")
+
+			if s.SkipGitErrors {
+				continue
+			}
 			return false, err
 		}
 
 		contained, err := f.hashInTree(blob, commitHash, tree)
 		if err != nil {
+			logrus.WithField("error", err).Error("error searching hash in tree")
+
+			if s.SkipGitErrors {
+				continue
+			}
 			return false, err
 		}
 

internal/function/commit_has_tree.go

@@ -5,6 +5,7 @@ import (
 	"io"
 
 	"github.com/hashicorp/golang-lru"
+	"github.com/sirupsen/logrus"
 
 	"gopkg.in/src-d/go-git.v4/plumbing/filemode"
 	"gopkg.in/src-d/go-git.v4/plumbing/object"
@@ -49,13 +50,21 @@ func (f *CommitHasTree) Eval(ctx *sql.Context, row sql.Row) (interface{}, error)
 	span, ctx := ctx.Span("gitbase.CommitHasTree")
 	defer span.Finish()
 
+	log := logrus.WithFields(logrus.Fields{
+		"function": "commit_hash_tree",
+		"row":      row,
+		"left":     f.Left.String(),
+		"right":    f.Right.String(),
+	})
+
 	s, ok := ctx.Session.(*gitbase.Session)
 	if !ok {
 		return nil, gitbase.ErrInvalidGitbaseSession.New(ctx.Session)
 	}
 
 	left, err := f.Left.Eval(ctx, row)
 	if err != nil {
+		log.WithField("error", err).Error("cannot eval left side")
 		return nil, err
 	}
 
@@ -65,11 +74,13 @@ func (f *CommitHasTree) Eval(ctx *sql.Context, row sql.Row) (interface{}, error)
 
 	left, err = sql.Text.Convert(left)
 	if err != nil {
+		log.WithField("error", err).Error("cannot convert left side")
 		return nil, err
 	}
 
 	right, err := f.Right.Eval(ctx, row)
 	if err != nil {
+		log.WithField("error", err).Error("cannot eval right side")
 		return nil, err
 	}
 
@@ -79,18 +90,25 @@ func (f *CommitHasTree) Eval(ctx *sql.Context, row sql.Row) (interface{}, error)
 
 	right, err = sql.Text.Convert(right)
 	if err != nil {
+		log.WithField("error", err).Error("cannot convert right side")
 		return nil, err
 	}
 
 	commitHash := plumbing.NewHash(left.(string))
 	treeHash := plumbing.NewHash(right.(string))
 
+	log = log.WithFields(logrus.Fields{
+		"commit_hash": commitHash.String(),
+		"tree_hash":   treeHash.String(),
+	})
+
 	if val, ok := f.cache.Get(commitTreeKey{commitHash, treeHash}); ok {
 		return val.(bool), nil
 	}
 
 	iter, err := s.Pool.RepoIter()
 	if err != nil {
+		log.WithField("error", err).Error("cannot create repository iterator")
 		return nil, err
 	}
 
@@ -101,6 +119,11 @@ func (f *CommitHasTree) Eval(ctx *sql.Context, row sql.Row) (interface{}, error)
 		}
 
 		if err != nil {
+			log.WithField("error", err).Error("could not get next repository")
+
+			if s.SkipGitErrors {
+				continue
+			}
 			return nil, err
 		}
 
@@ -109,6 +132,18 @@ func (f *CommitHasTree) Eval(ctx *sql.Context, row sql.Row) (interface{}, error)
 			continue
 		}
 
+		if err != nil {
+			log.WithFields(logrus.Fields{
+				"repo":  repo.ID,
+				"error": err,
+			}).Error("error searching commit in tree")
+
+			if s.SkipGitErrors {
+				continue
+			}
+			return nil, err
+		}
+
 		return ok, nil
 	}
 }