Update go-git and go-mysql-server dependencies

Some refactor was needed to be able to use the new packfile decoder API from go-git.

Speed performance on full scans and generating indexes of 40%-60%, depending of the dataset.

Signed-off-by: Antonio Jesus Navarro Perez <[email protected]>

Author: ajnavarro

Gopkg.lock

@@ -1,6 +1,6 @@
 [[constraint]]
   name = "gopkg.in/src-d/go-mysql-server.v0"
-  revision = "5da99c3ba25f3c637ea5dd041cf5ba73e24ac1f2"
+  revision = "12ca4f8c777a8436a71015f531d840393e3c36e1"
 
 [[constraint]]
   name = "github.com/jessevdk/go-flags"
@@ -17,7 +17,7 @@
 [[constraint]]
   name = "gopkg.in/src-d/go-git.v4"
   source = "github.com/src-d/go-git"
-  revision = "0710c6cb710a0cdab04ab7f61cc62e23cfcacbee"
+  revision = "a28c2ce44695f13ddf28748958f236afd8e0b544"
 
 [[constraint]]
   name = "gopkg.in/src-d/go-git-fixtures.v3"

Gopkg.toml

@@ -14,4 +14,4 @@ To make some common tasks easier for the user, there are some functions to inter
 
 ## Standard functions
 
-You can check standard functions in [`go-mysql-server` documentation](https://github.com/src-d/go-mysql-server/tree/5da99c3ba25f3c637ea5dd041cf5ba73e24ac1f2#custom-functions).
+You can check standard functions in [`go-mysql-server` documentation](https://github.com/src-d/go-mysql-server/tree/12ca4f8c777a8436a71015f531d840393e3c36e1#custom-functions).

docs/using-gitbase/functions.md

@@ -10,4 +10,4 @@ Note that you can create an index either **on one or more columns** or **on a si
 
 You can find some more examples in the [examples](./examples.md#create-an-index-for-columns-on-a-table) section.
 
-See [go-mysql-server](https://github.com/src-d/go-mysql-server/tree/5da99c3ba25f3c637ea5dd041cf5ba73e24ac1f2#indexes) documentation for more details
+See [go-mysql-server](https://github.com/src-d/go-mysql-server/tree/12ca4f8c777a8436a71015f531d840393e3c36e1#indexes) documentation for more details

docs/using-gitbase/indexes.md

@@ -1,3 +1,3 @@
 ## Supported syntax
 
-To see the SQL subset currently supported take a look at [this list](https://github.com/src-d/go-mysql-server/blob/5da99c3ba25f3c637ea5dd041cf5ba73e24ac1f2/SUPPORTED.md) from [src-d/go-mysql-server](https://github.com/src-d/go-mysql-server).
+To see the SQL subset currently supported take a look at [this list](https://github.com/src-d/go-mysql-server/blob/12ca4f8c777a8436a71015f531d840393e3c36e1/SUPPORTED.md) from [src-d/go-mysql-server](https://github.com/src-d/go-mysql-server).

docs/using-gitbase/supported-syntax.md

@@ -19,7 +19,7 @@ import (
 )
 
 type packRepository struct {
-	packs map[plumbing.Hash]packfile.Index
+	packs map[plumbing.Hash]idxfile.Index
 }
 
 func repositoryPackfiles(repo repository) (*dotgit.DotGit, []plumbing.Hash, error) {
@@ -44,7 +44,7 @@ func repositoryPackfiles(repo repository) (*dotgit.DotGit, []plumbing.Hash, erro
 
 type packfileIndex struct {
 	packfile plumbing.Hash
-	idx      *packfile.Index
+	idx      idxfile.Index
 }
 
 type repositoryIndex struct {
@@ -74,28 +74,35 @@ func newRepositoryIndex(repo repository) (*repositoryIndex, error) {
 func openPackfileIndex(
 	dotGit *dotgit.DotGit,
 	hash plumbing.Hash,
-) (*packfile.Index, error) {
+) (*idxfile.MemoryIndex, error) {
 	f, err := dotGit.ObjectPackIdx(hash)
 	if err != nil {
 		return nil, err
 	}
 	defer f.Close()
 
-	idx := idxfile.NewIdxfile()
+	idx := idxfile.NewMemoryIndex()
 	if err := idxfile.NewDecoder(f).Decode(idx); err != nil {
 		return nil, err
 	}
 
-	return packfile.NewIndexFromIdxFile(idx), nil
+	return idx, nil
 }
 
 var errHashNotInIndex = errors.NewKind("object hash %s is not in repository")
 
 func (i *repositoryIndex) find(hash plumbing.Hash) (int64, plumbing.Hash, error) {
 	for _, idx := range i.indexes {
-		if entry, ok := idx.idx.LookupHash(hash); ok {
-			return int64(entry.Offset), idx.packfile, nil
+		ofs, err := idx.idx.FindOffset(hash)
+		if err == plumbing.ErrObjectNotFound {
+			continue
 		}
+
+		if err != nil {
+			return 0, plumbing.ZeroHash, err
+		}
+
+		return ofs, idx.packfile, nil
 	}
 
 	ok, err := i.isUnpacked(hash)
@@ -192,8 +199,8 @@ func getUnpackedObject(repo repository, hash plumbing.Hash) (o object.Object, er
 
 type repoObjectDecoder struct {
 	repo     string
-	packfile plumbing.Hash
-	decoder  *packfile.Decoder
+	hash     plumbing.Hash
+	packfile *packfile.Packfile
 	storage  storer.EncodedObjectStorer
 }
 
@@ -223,26 +230,31 @@ func newRepoObjectDecoder(
 		return nil, err
 	}
 
-	decoder, err := packfile.NewDecoder(packfile.NewScanner(packf), storage)
+	idx, err := openPackfileIndex(dot, hash)
+	if err != nil {
+		return nil, err
+	}
+
+	packfile := packfile.NewPackfile(idx, fs, packf)
 	if err != nil {
 		_ = packf.Close()
 		return nil, err
 	}
 
 	return &repoObjectDecoder{
 		repo:     repo.ID(),
-		packfile: hash,
-		decoder:  decoder,
+		hash:     hash,
+		packfile: packfile,
 		storage:  storage,
 	}, nil
 }
 
-func (d *repoObjectDecoder) equals(repo string, packfile plumbing.Hash) bool {
-	return d.repo == repo && d.packfile == packfile
+func (d *repoObjectDecoder) equals(repo string, hash plumbing.Hash) bool {
+	return d.repo == repo && d.hash == hash
 }
 
 func (d *repoObjectDecoder) get(offset int64) (object.Object, error) {
-	encodedObj, err := d.decoder.DecodeObjectAt(offset)
+	encodedObj, err := d.packfile.GetByOffset(offset)
 	if err != nil {
 		return nil, err
 	}
@@ -251,7 +263,7 @@ func (d *repoObjectDecoder) get(offset int64) (object.Object, error) {
 }
 
 func (d *repoObjectDecoder) Close() error {
-	return d.decoder.Close()
+	return d.packfile.Close()
 }
 
 type objectDecoder struct {