Merge branch 'master' into feature/index-checksum

Author: ajnavarro

Jenkinsfile

@@ -0,0 +1,54 @@
+pipeline {
+  agent {
+    kubernetes {
+      label 'regression-gitbase'
+      inheritFrom 'performance'
+      defaultContainer 'regression-gitbase'
+      containerTemplate {
+        name 'regression-gitbase'
+        image 'srcd/regression-gitbase:v0.2.0'
+        ttyEnabled true
+        command 'cat'
+      }
+    }
+  }
+  environment {
+    GOPATH = "/go"
+    GO_IMPORT_PATH = "github.com/src-d/regression-gibase"
+    GO_IMPORT_FULL_PATH = "${env.GOPATH}/src/${env.GO_IMPORT_PATH}"
+  }
+  triggers { pollSCM('0 0,12 * * *') }
+  stages {
+    stage('Run') {
+      when { branch 'master' }
+      steps {
+        sh '/bin/regression --complexity=2 --csv local:HEAD'
+      }
+    }
+    stage('Plot') {
+      when { branch 'master' }
+      steps {
+        script {
+          plotFiles = findFiles(glob: "plot_*.csv")
+          plotFiles.each {
+            echo "plot ${it.getName()}"
+            sh "cat ${it.getName()}"
+            plot(
+              group: 'performance',
+              csvFileName: it.getName(),
+              title: it.getName(),
+              numBuilds: '100',
+              style: 'line',
+              csvSeries: [[
+                displayTableFlag: false,
+                exclusionValues: '',
+                file: it.getName(),
+                inclusionFlag: 'OFF',
+              ]]
+            )
+          }
+        }
+      }
+    }
+  }
+}

cmd/gitbase/main.go

@@ -19,9 +19,10 @@ var (
 func main() {
 	parser := flags.NewNamedParser(name, flags.Default)
 	parser.UnknownOptionHandler = func(option string, arg flags.SplitArgument, args []string) ([]string, error) {
-		if "g" != option {
+		if option != "g" {
 			return nil, fmt.Errorf("unknown flag `%s'", option)
 		}
+
 		if len(args) == 0 {
 			return nil, fmt.Errorf("unknown flag `%s'", option)
 		}

commit_files.go

@@ -513,8 +513,6 @@ func (i *commitFilesKeyValueIter) Close() error {
 type commitFilesIndexIter struct {
 	index   sql.IndexValueIter
 	decoder *objectDecoder
-
-	file *object.File // holds the last obtained key
 }
 
 func newCommitFilesIndexIter(

commits.go

@@ -347,40 +347,38 @@ func newCommitsKeyValueIter(
 }
 
 func (i *commitsKeyValueIter) Next() ([]interface{}, []byte, error) {
-	for {
-		commit, err := i.commits.Next()
-		if err != nil {
-			return nil, nil, err
-		}
-
-		offset, packfile, err := i.idx.find(commit.Hash)
-		if err != nil {
-			return nil, nil, err
-		}
+	commit, err := i.commits.Next()
+	if err != nil {
+		return nil, nil, err
+	}
 
-		var hash string
-		if offset < 0 {
-			hash = commit.Hash.String()
-		}
+	offset, packfile, err := i.idx.find(commit.Hash)
+	if err != nil {
+		return nil, nil, err
+	}
 
-		key, err := encodeIndexKey(&packOffsetIndexKey{
-			Repository: i.repo.ID,
-			Packfile:   packfile.String(),
-			Offset:     offset,
-			Hash:       hash,
-		})
-		if err != nil {
-			return nil, nil, err
-		}
+	var hash string
+	if offset < 0 {
+		hash = commit.Hash.String()
+	}
 
-		row := commitToRow(i.repo.ID, commit)
-		values, err := rowIndexValues(row, i.columns, CommitsSchema)
-		if err != nil {
-			return nil, nil, err
-		}
+	key, err := encodeIndexKey(&packOffsetIndexKey{
+		Repository: i.repo.ID,
+		Packfile:   packfile.String(),
+		Offset:     offset,
+		Hash:       hash,
+	})
+	if err != nil {
+		return nil, nil, err
+	}
 
-		return values, key, nil
+	row := commitToRow(i.repo.ID, commit)
+	values, err := rowIndexValues(row, i.columns, CommitsSchema)
+	if err != nil {
+		return nil, nil, err
 	}
+
+	return values, key, nil
 }
 
 func (i *commitsKeyValueIter) Close() error {

filters.go

@@ -66,23 +66,6 @@ func (s selectors) textValues(key string) ([]string, error) {
 	return result, nil
 }
 
-// filtersToExpression concatenates all filters and turns them into an
-// expression using the AND expression.
-func filtersToExpression(filters []sql.Expression) sql.Expression {
-	switch len(filters) {
-	case 0:
-		return nil
-	case 1:
-		return filters[0]
-	default:
-		exp := expression.NewAnd(filters[0], filters[1])
-		for _, f := range filters[2:] {
-			exp = expression.NewAnd(exp, f)
-		}
-		return exp
-	}
-}
-
 // canHandleEquals returns whether the given equals expression can be handled
 // as a selector. For that to happen one of the sides must be a GetField expr
 // that exists in the given schema and the other must be a literal.
@@ -246,21 +229,22 @@ func classifyFilters(
 			// check all unfolded exprs can be handled, if not we have to
 			// resort to treating them as conditions
 			valid := true
+		Loop:
 			for _, e := range exprs {
 				switch e := e.(type) {
 				case *expression.Equals:
 					if !canHandleEquals(schema, table, e) {
 						valid = false
-						break
+						break Loop
 					}
 				case *expression.In:
 					if !canHandleIn(schema, table, e) {
 						valid = false
-						break
+						break Loop
 					}
 				default:
 					valid = false
-					break
+					break Loop
 				}
 			}
 

index.go

@@ -101,34 +101,6 @@ type rowKeyMapper interface {
 	fromRow(sql.Row) ([]byte, error)
 }
 
-type rowKeyValueIter struct {
-	mapper  rowKeyMapper
-	iter    sql.RowIter
-	columns []string
-	schema  sql.Schema
-}
-
-func (i *rowKeyValueIter) Next() ([]interface{}, []byte, error) {
-	row, err := i.iter.Next()
-	if err != nil {
-		return nil, nil, err
-	}
-
-	key, err := i.mapper.fromRow(row)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	values, err := rowIndexValues(row, i.columns, i.schema)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	return values, key, nil
-}
-
-func (i *rowKeyValueIter) Close() error { return i.iter.Close() }
-
 var (
 	errRowKeyMapperRowLength = errors.NewKind("row should have %d columns, has: %d")
 	errRowKeyMapperColType   = errors.NewKind("row column %d should have type %T, has: %T")

integration_test.go

@@ -21,7 +21,6 @@ import (
 	"gopkg.in/src-d/go-mysql-server.v0/auth"
 	"gopkg.in/src-d/go-mysql-server.v0/sql"
 	"gopkg.in/src-d/go-mysql-server.v0/sql/analyzer"
-	"gopkg.in/src-d/go-mysql-server.v0/sql/expression"
 	sqlfunction "gopkg.in/src-d/go-mysql-server.v0/sql/expression/function"
 	"gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosa"
 )
@@ -752,17 +751,6 @@ func TestIndexes(t *testing.T) {
 	}
 }
 
-func col(t testing.TB, schema sql.Schema, name string) sql.Expression {
-	for i, col := range schema {
-		if col.Name == name {
-			return expression.NewGetFieldWithTable(i, col.Type, col.Source, col.Name, col.Nullable)
-		}
-	}
-
-	t.Fatalf("unknown column %s in schema", name)
-	return nil
-}
-
 type indexData struct {
 	id    string
 	table string

internal/rule/squashjoins.go

@@ -27,7 +27,7 @@ func SquashJoins(
 		return n, nil
 	}
 
-	span, ctx := ctx.Span("gitbase.SquashJoins")
+	span, _ := ctx.Span("gitbase.SquashJoins")
 	defer span.Finish()
 
 	a.Log("squashing joins, node of type %T", n)
@@ -123,10 +123,6 @@ func squashProjects(parent, child *plan.Project) (sql.Node, bool) {
 	return plan.NewProject(projections, child.Child), true
 }
 
-func referenceSameColumn(parent, child *expression.GetField) bool {
-	return parent.Name() == child.Name() && parent.Table() == child.Table()
-}
-
 func squashJoin(join *plan.InnerJoin) (sql.Node, error) {
 	if !isJoinSquashable(join) {
 		return join, nil
@@ -675,6 +671,9 @@ func buildSquashedTable(
 					filters,
 					append(it.Schema(), gitbase.TreeEntriesSchema...),
 				)
+				if err != nil {
+					return nil, err
+				}
 
 				iter = gitbase.NewTreeTreeEntriesIter(it, f, false)
 			case nil:
@@ -1545,53 +1544,6 @@ func isCol(table, name string) validator {
 	}
 }
 
-func isGte(left, right validator) validator {
-	return func(e sql.Expression) bool {
-		switch e := e.(type) {
-		case *expression.GreaterThanOrEqual:
-			return left(e.Left()) && right(e.Right())
-		case *expression.LessThanOrEqual:
-			return left(e.Right()) && right(e.Left())
-		default:
-			return false
-		}
-	}
-}
-
-func isNum(n int64) validator {
-	return func(e sql.Expression) bool {
-		lit, ok := e.(*expression.Literal)
-		if !ok {
-			return false
-		}
-
-		result, err := lit.Eval(nil, nil)
-		if err != nil {
-			return false
-		}
-
-		num, ok := result.(int64)
-		if !ok {
-			return false
-		}
-
-		return num == n
-	}
-}
-
-func containsField(e sql.Expression, table, name string) bool {
-	var found bool
-	expression.Inspect(e, func(e sql.Expression) bool {
-		gf, ok := e.(*expression.GetField)
-		if ok && gf.Table() == table && gf.Name() == name {
-			found = true
-			return false
-		}
-		return true
-	})
-	return found
-}
-
 func fixFieldIndexes(e sql.Expression, schema sql.Schema) (sql.Expression, error) {
 	if e == nil {
 		return nil, nil

internal/rule/squashjoins_test.go

@@ -2758,14 +2758,6 @@ func lit(v interface{}) sql.Expression {
 	return expression.NewLiteral(v, sql.Int64)
 }
 
-func gte(left, right sql.Expression) sql.Expression {
-	return expression.NewGreaterThanOrEqual(left, right)
-}
-
-func lte(left, right sql.Expression) sql.Expression {
-	return expression.NewLessThanOrEqual(left, right)
-}
-
 func and(exprs ...sql.Expression) sql.Expression {
 	return expression.JoinAnd(exprs...)
 }

packfiles.go

@@ -19,10 +19,6 @@ import (
 	"gopkg.in/src-d/go-git.v4/plumbing/format/packfile"
 )
 
-type packRepository struct {
-	packs map[plumbing.Hash]idxfile.Index
-}
-
 func repositoryPackfiles(fs billy.Filesystem) (*dotgit.DotGit, []plumbing.Hash, error) {
 	fs, err := findDotGit(fs)
 	if err != nil {