Merge branch 'master' into feature/config-bblfsh-size

Author: ajnavarro

Gopkg.lock

@@ -54,15 +54,15 @@
 
 [[constraint]]
   name = "gopkg.in/bblfsh/client-go.v3"
-  version = "3.0.0"
+  version = "3.1.0"
 
 [[override]]
   name = "gopkg.in/bblfsh/sdk.v1"
   version = "1.16.0"
 
 [[constraint]]
   name = "gopkg.in/bblfsh/sdk.v2"
-  version = "2.3.0"
+  version = "2.7.0"
 
 [[constraint]]
   name = "github.com/uber/jaeger-client-go"

Gopkg.toml

@@ -2,6 +2,10 @@
 
 **gitbase**, is a SQL database interface to Git repositories.
 
+This project is now part of [source{d} Engine](https://sourced.tech/engine),
+which provides the simplest way to get started with a single command.
+Visit [sourced.tech/engine](https://sourced.tech/engine) for more information.
+
 It can be used to perform SQL queries about the Git history and
 about the [Universal AST](https://doc.bblf.sh/) of the code itself. gitbase is being built to work on top of any number of git repositories.
 

README.md

@@ -202,3 +202,41 @@ The UDF `uast_children` will return a flattened array of the children nodes from
 ```sql
 SELECT file_path, uast_children(uast(blob_content, language(file_path), "//uast:Alias")) FROM files;
 ```
+
+## Monitor the progress of a query
+
+You can monitor the progress of a gitbase query (either a regular query or an index creation query using `SHOW PROCESSLIST`).
+
+Let's say we do the following query over a huge repository:
+
+```sql
+SELECT language(file_path, blob_content) FROM files
+```
+
+With this query we can monitor its progress:
+
+```sql
+SHOW PROCESSLIST
+```
+
+We'll get the following output:
+
+```
++------+------+----------------+---------+---------+------+------------+-----------------------------------------------------+
+| Id   | User | Host           | db      | Command | Time | State      | Info                                                |
++------+------+----------------+---------+---------+------+------------+-----------------------------------------------------+
+|    2 | root | 127.0.0.1:3306 | gitbase | query   |   36 | files(1/3) | select language(file_path, blob_content) from files |
+|   12 | root | 127.0.0.1:3306 | gitbase | query   |    0 | running    | show processlist                                    |
++------+------+----------------+---------+---------+------+------------+-----------------------------------------------------+
+2 rows in set (0,00 sec)
+```
+
+From this output, we can obtain some information about our query:
+- It's been running for 36 seconds.
+- It's only querying files table and has processed 1 out of 3 partitions.
+
+To kill a query that's currently running you can use the value in `Id`. If we were to kill the previous query, we would need to use the following query:
+
+```sql
+KILL QUERY 2
+```

docs/using-gitbase/examples.md

@@ -12,6 +12,7 @@ import (
 	lru "github.com/hashicorp/golang-lru"
 	"github.com/sirupsen/logrus"
 	bblfsh "gopkg.in/bblfsh/client-go.v3"
+	derrors "gopkg.in/bblfsh/sdk.v2/driver/errors"
 	"gopkg.in/bblfsh/sdk.v2/uast"
 	"gopkg.in/bblfsh/sdk.v2/uast/nodes"
 
@@ -224,12 +225,11 @@ func (u *uastFunc) getUAST(
 		var err error
 		node, err = getUASTFromBblfsh(ctx, blob, lang, xpath, mode)
 		if err != nil {
-			if ErrParseBlob.Is(err) {
+			if ErrParseBlob.Is(err) || derrors.ErrSyntax.Is(err) {
 				return nil, nil
 			}
 
-			logrus.WithField("err", err).Error("unable to get UAST from bblfsh")
-			return nil, nil
+			return nil, err
 		}
 
 		uastCache.Add(key, node)

internal/function/uast.go

@@ -177,7 +177,7 @@ func (s *Session) BblfshClient() (*BblfshClient, error) {
 	var attempts, totalAttempts int
 	for {
 		if attempts > bblfshMaxAttempts || totalAttempts > 3*bblfshMaxAttempts {
-			return nil, ErrBblfshConnection.New()
+			return nil, ErrBblfshConnection.New("max attempts exceeded")
 		}
 
 		switch s.bblfshClient.GetState() {
@@ -190,7 +190,7 @@ func (s *Session) BblfshClient() (*BblfshClient, error) {
 			time.Sleep(100 * time.Millisecond)
 		default:
 			if err := s.bblfshClient.Close(); err != nil {
-				return nil, err
+				return nil, ErrBblfshConnection.New(err)
 			}
 
 			logrus.Debug("bblfsh connection is closed, opening a new one")
@@ -223,10 +223,10 @@ func connectToBblfsh(endpoint string) (*bblfsh.Client, error) {
 	client, err := bblfsh.NewClient(endpoint)
 	if err != nil {
 		if err == context.DeadlineExceeded {
-			return nil, ErrBblfshConnection.New()
+			return nil, ErrBblfshConnection.New(err)
 		}
 
-		return nil, err
+		return nil, ErrBblfshConnection.New(err)
 	}
 
 	return client, nil
@@ -252,7 +252,7 @@ var ErrInvalidGitbaseSession = errors.NewKind("expecting gitbase session, but re
 var ErrInvalidContext = errors.NewKind("invalid context received: %v")
 
 // ErrBblfshConnection is returned when it's impossible to connect to bblfsh.
-var ErrBblfshConnection = errors.NewKind("unable to establish a connection with the bblfsh server")
+var ErrBblfshConnection = errors.NewKind("unable to establish a connection with the bblfsh server: %s")
 
 func shouldSkipErrors(ctx *sql.Context) bool {
 	s, err := getSession(ctx)