Merge branch 'master' into patch-1

Author: ajnavarro

docs/using-gitbase/examples.md

@@ -18,34 +18,30 @@ FROM refs
 WHERE ref_name = 'HEAD';
 ```
 
-## First commit on HEAD history for all repositories
+## Files in the first commit on HEAD history for all repositories
 
 ```sql
 SELECT file_path,
        ref_commits.repository_id
 FROM commit_files
 NATURAL JOIN ref_commits
-WHERE ref_commits.ref_name = 'HEAD'
-  AND ref_commits.history_index = 0;
+WHERE ref_name = 'HEAD'
+  AND history_index = 0;
 ```
 
 ## Commits that appear in more than one reference
 
 ```sql
-SELECT *
-FROM
-  (SELECT COUNT(c.commit_hash) AS num,
-          c.commit_hash
-   FROM ref_commits r
-   NATURAL JOIN commits c
-   GROUP BY c.commit_hash) t
-WHERE num > 1;
+SELECT COUNT(commit_hash) AS num, commit_hash
+FROM ref_commits
+GROUP BY commit_hash
+HAVING num > 1;
 ```
 
 ## Get the number of blobs per HEAD commit
 
 ```sql
-SELECT COUNT(commit_hash),
+SELECT COUNT(commit_blob),
        commit_hash
 FROM ref_commits
 NATURAL JOIN commits
@@ -85,8 +81,8 @@ SELECT
     SUM(JSON_EXTRACT(LOC(file_path, blob_content), '$.Comments')) as comments,
     SUM(JSON_EXTRACT(LOC(file_path, blob_content), '$.Blanks')) as blanks,
     COUNT(1) as files
-FROM commit_files
-NATURAL JOIN refs
+FROM refs
+NATURAL JOIN commit_files
 NATURAL JOIN blobs
 WHERE ref_name='HEAD'
 GROUP BY lang;
@@ -98,13 +94,13 @@ GROUP BY lang;
 SELECT file_path,
        repository_id,
        blob_content
-FROM files
+FROM ref_commits
 NATURAL JOIN commit_files
-NATURAL JOIN ref_commits
+NATURAL JOIN files
 WHERE ref_name = 'HEAD'
-  AND ref_commits.history_index BETWEEN 0 AND 5
-  AND is_binary(blob_content) = FALSE
-  AND files.file_path NOT REGEXP '^vendor.*'
+  AND history_index BETWEEN 0 AND 5
+  AND NOT IS_BINARY(blob_content)
+  AND NOT IS_VENDOR(file_path)
   AND (blob_content REGEXP '(?i)facebook.*[\'\\"][0-9a-f]{32}[\'\\"]'
        OR blob_content REGEXP '(?i)twitter.*[\'\\"][0-9a-zA-Z]{35,44}[\'\\"]'
        OR blob_content REGEXP '(?i)github.*[\'\\"][0-9a-zA-Z]{35,40}[\'\\"]'
@@ -145,7 +141,7 @@ DROP INDEX files_lang_idx ON files;
 
 This query will report how many lines of actual code (only code, not comments, blank lines or text) changed in the last commit of each repository.
 
-```
+```sql
 SELECT
     repo,
     JSON_EXTRACT(stats, '$.Code.Additions') AS code_lines_added,
@@ -175,7 +171,7 @@ The output will be similar to this:
 This query will report how many lines of actual code (only code, not comments, blank lines or text) changed in each file of the last commit of each repository. It's similar to the previous example. `COMMIT_STATS` is an aggregation over the result of `COMMIT_FILE_STATS` so to speak.
 We will only report those files that whose language has been identified.
 
-```
+```sql
 SELECT
     repo,
     JSON_UNQUOTE(JSON_EXTRACT(stats, '$.Path')) AS file_path,
@@ -210,29 +206,15 @@ First of all, you should check out the [bblfsh documentation](https://docs.sourc
 
 Also, you can take a look to all the UDFs and their signatures in the [functions section](/docs/using-gitbase/functions.md)
 
-## Extract all import paths for every *Go* file on *HEAD* reference
-
-```sql
-SELECT repository_id,
-       file_path,
-       uast_extract(uast(blob_content, LANGUAGE(file_path), '//uast:Import/Path'), "Value") AS imports
-FROM commit_files
-NATURAL JOIN refs
-NATURAL JOIN blobs
-WHERE ref_name = 'HEAD'
-  AND LANGUAGE(file_path) = 'Go'
-  AND ARRAY_LENGTH(imports) > 0;
-```
-
 ## Extracting all identifier names
 
 ```sql
 SELECT file_path,
        uast_extract(uast(blob_content, LANGUAGE(file_path), '//uast:Identifier'), "Name") name
-FROM commit_files
-NATURAL JOIN refs
+FROM refs
+NATURAL JOIN commit_files
 NATURAL JOIN blobs
-WHERE ref_name='HEAD' AND LANGUAGE(file_path) = 'Go';
+WHERE ref_name = 'HEAD' AND LANGUAGE(file_path) = 'Go';
 ```
 
 As result, you will get an array showing a list of the retrieved information. Each element in the list matches a node in the given sequence of nodes having a value for that property. It means that the length of the properties list may not be equal to the length of the given sequence of nodes:

docs/using-gitbase/functions.md

@@ -169,6 +169,26 @@ Also, if you want to retrieve values from a non common property, you can pass it
 
 > uast_extract(nodes_column, 'some-property')
 
+## How to use `loc`
+
+`loc` will return statistics about the lines of code in a file, such as the code lines, comment lines, etc.
+
+It requires a file path and a file content.
+
+> loc(file_path, blob_content)
+
+The result of this function is a JSON document with the following shape:
+
+```
+{
+	"Code": code lines,
+	"Comment": comment lines,
+	"Blank": blank lines,
+	"Name": file name,
+	"Lang": language
+}
+```
+
 ## How to use `commit_file_stats`
 
 `commit_file_stats` will return statistics about the line changes in all files in the given range of commits classifying them in 4 categories: code, comments, blank lines and other.

internal/function/loc.go

@@ -52,6 +52,15 @@ func (f *LOC) WithChildren(children ...sql.Expression) (sql.Expression, error) {
 	return NewLOC(children...)
 }
 
+// LocFile is the result of the LOC function for each file.
+type LocFile struct {
+	Code     int32  `json:"Code"`
+	Comments int32  `json:"Comment"`
+	Blanks   int32  `json:"Blank"`
+	Name     string `json:"Name"`
+	Lang     string `json:"Language"`
+}
+
 // Eval implements the Expression interface.
 func (f *LOC) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) {
 	span, ctx := ctx.Span("gitbase.LOC")
@@ -70,11 +79,19 @@ func (f *LOC) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) {
 		return nil, nil
 	}
 
-	return gocloc.AnalyzeReader(
+	file := gocloc.AnalyzeReader(
 		path,
 		languages.Langs[lang],
 		bytes.NewReader(blob), &gocloc.ClocOptions{},
-	), nil
+	)
+
+	return LocFile{
+		Code:     file.Code,
+		Comments: file.Comments,
+		Blanks:   file.Blanks,
+		Name:     file.Name,
+		Lang:     file.Lang,
+	}, nil
 }
 
 func (f *LOC) getInputValues(ctx *sql.Context, row sql.Row) (string, []byte, error) {

internal/function/loc_test.go

@@ -3,11 +3,10 @@ package function
 import (
 	"testing"
 
-	"github.com/hhatto/gocloc"
-	"github.com/stretchr/testify/require"
-	"gopkg.in/src-d/go-errors.v1"
 	"github.com/src-d/go-mysql-server/sql"
 	"github.com/src-d/go-mysql-server/sql/expression"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/src-d/go-errors.v1"
 )
 
 func TestLoc(t *testing.T) {
@@ -22,7 +21,7 @@ func TestLoc(t *testing.T) {
 		{"too few args given", sql.NewRow("foo.foobar"), nil, nil},
 		{"too many args given", sql.NewRow("foo.rb", "bar", "baz"), nil, sql.ErrInvalidArgumentNumber},
 		{"invalid blob type given", sql.NewRow("foo", 5), nil, sql.ErrInvalidType},
-		{"path and blob are given", sql.NewRow("foo", "#!/usr/bin/env python\n\nprint 'foo'"), &gocloc.ClocFile{
+		{"path and blob are given", sql.NewRow("foo", "#!/usr/bin/env python\n\nprint 'foo'"), LocFile{
 			Code: 2, Comments: 0, Blanks: 1, Name: "foo", Lang: "Python",
 		}, nil},
 	}