[go-borges] Use go-borges to access repositories (#888)

[go-borges] Use go-borges to access repositories

Author: ajnavarro

CHANGELOG.md

@@ -6,6 +6,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+- Now gitbase uses [go-borges](https://github.com/src-d/go-borges) to access repositories
+  - The type of files in each directory has to be specified ([#867](https://github.com/src-d/gitbase/pull/867))
+  - Supports new rooted repository format and separates references and objects from each repo (https://github.com/src-d/borges/issues/389)
+- Changed cli to be able to specify different formats ([#866](https://github.com/src-d/gitbase/issues/866))
+
 ## [0.21.0-beta3] - 2019-06-19
 
 ### Fixed

_testdata/05893125684f2d3943cd84a7ab2b75e53668fba1.siva

@@ -209,7 +209,7 @@ func (i *blobRowIter) nextByHash() (sql.Row, error) {
 			return nil, err
 		}
 
-		return blobToRow(i.repo.ID, blob, i.readContent)
+		return blobToRow(i.repo.ID(), blob, i.readContent)
 	}
 }
 
@@ -236,7 +236,7 @@ func (i *blobRowIter) next() (sql.Row, error) {
 			return nil, err
 		}
 
-		return blobToRow(i.repo.ID, o, i.readContent)
+		return blobToRow(i.repo.ID(), o, i.readContent)
 	}
 }
 
@@ -347,8 +347,7 @@ func newBlobsKeyValueIter(
 		return nil, err
 	}
 
-	r := pool.repositories[repo.ID]
-	idx, err := newRepositoryIndex(r)
+	idx, err := newRepositoryIndex(repo)
 	if err != nil {
 		return nil, err
 	}
@@ -379,7 +378,7 @@ func (i *blobsKeyValueIter) Next() ([]interface{}, []byte, error) {
 	}
 
 	key, err := encodeIndexKey(&packOffsetIndexKey{
-		Repository: i.repo.ID,
+		Repository: i.repo.ID(),
 		Packfile:   packfile.String(),
 		Offset:     offset,
 		Hash:       hash,
@@ -388,7 +387,7 @@ func (i *blobsKeyValueIter) Next() ([]interface{}, []byte, error) {
 		return nil, nil, err
 	}
 
-	row, err := blobToRow(i.repo.ID, blob, stringContains(i.columns, "blob_content"))
+	row, err := blobToRow(i.repo.ID(), blob, stringContains(i.columns, "blob_content"))
 	if err != nil {
 		return nil, nil, err
 	}

_testdata/fff7062de8474d10a67d417ccea87ba6f58ca81d.siva

@@ -3,9 +3,9 @@ package gitbase
 import (
 	"testing"
 
-	"github.com/stretchr/testify/require"
 	"github.com/src-d/go-mysql-server/sql"
 	"github.com/src-d/go-mysql-server/sql/expression"
+	"github.com/stretchr/testify/require"
 )
 
 func TestBlobsTable(t *testing.T) {

_testdata/ff/fff840f8784ef162dc83a1465fc5763d890b68ba.siva renamed to _testdata/fff840f8784ef162dc83a1465fc5763d890b68ba.siva

@@ -8,7 +8,6 @@ import (
 	"sort"
 	"strings"
 
-	git "gopkg.in/src-d/go-git.v4"
 	"gopkg.in/src-d/go-git.v4/plumbing"
 )
 
@@ -18,9 +17,23 @@ type checksumable struct {
 
 func (c *checksumable) Checksum() (string, error) {
 	hash := sha1.New()
-	for _, id := range c.pool.idOrder {
-		repo := c.pool.repositories[id]
-		hash.Write([]byte(id))
+	iter, err := c.pool.RepoIter()
+	if err != nil {
+		return "", err
+	}
+	defer iter.Close()
+
+	var checksums checksums
+	for {
+		hash.Reset()
+
+		repo, err := iter.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return "", err
+		}
 
 		bytes, err := readChecksum(repo)
 		if err != nil {
@@ -39,12 +52,28 @@ func (c *checksumable) Checksum() (string, error) {
 		if _, err = hash.Write(bytes); err != nil {
 			return "", err
 		}
+
+		c := checksum{
+			name: repo.ID(),
+			hash: hash.Sum(nil),
+		}
+
+		checksums = append(checksums, c)
+	}
+
+	sort.Stable(checksums)
+	hash.Reset()
+
+	for _, c := range checksums {
+		if _, err = hash.Write(c.hash); err != nil {
+			return "", err
+		}
 	}
 
 	return base64.StdEncoding.EncodeToString(hash.Sum(nil)), nil
 }
 
-func readChecksum(r repository) ([]byte, error) {
+func readChecksum(r *Repository) ([]byte, error) {
 	fs, err := r.FS()
 	if err != nil {
 		return nil, err
@@ -99,15 +128,23 @@ func (b byHashAndName) Less(i, j int) bool {
 	return strings.Compare(b[i].name, b[j].name) < 0
 }
 
-func readRefs(r repository) ([]byte, error) {
-	repo, err := r.Repo()
-	if err != nil {
-		if err == git.ErrRepositoryNotExists {
-			return nil, nil
-		}
-		return nil, err
+type checksum struct {
+	name string
+	hash []byte
+}
+
+type checksums []checksum
+
+func (b checksums) Len() int      { return len(b) }
+func (b checksums) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
+func (b checksums) Less(i, j int) bool {
+	if cmp := bytes.Compare(b[i].hash, b[j].hash); cmp != 0 {
+		return cmp < 0
 	}
+	return strings.Compare(b[i].name, b[j].name) < 0
+}
 
+func readRefs(repo *Repository) ([]byte, error) {
 	buf := bytes.NewBuffer(nil)
 
 	refs, err := repo.References()

blobs.go

@@ -8,7 +8,13 @@ import (
 
 	fixtures "github.com/src-d/go-git-fixtures"
 	"github.com/stretchr/testify/require"
-	"gopkg.in/src-d/go-git.v4/plumbing/cache"
+	"gopkg.in/src-d/go-billy.v4/osfs"
+)
+
+const (
+	checksumMulti  = "W/lxpR0jZ6O6BqVANTYTDMlAS/4="
+	checksumSingle = "zqLF31JlrtJ57XNC+cQ+2hSkBkw="
+	checksumSiva   = "X27U+Lww5UOk1+/21bVFgI4uJyM="
 )
 
 func TestChecksum(t *testing.T) {
@@ -18,40 +24,48 @@ func TestChecksum(t *testing.T) {
 		require.NoError(fixtures.Clean())
 	}()
 
-	pool := NewRepositoryPool(cache.DefaultMaxSize)
+	lib, pool, err := newMultiPool()
+	require.NoError(err)
 
 	for i, f := range fixtures.ByTag("worktree") {
 		path := f.Worktree().Root()
-		require.NoError(pool.AddGitWithID(fmt.Sprintf("repo_%d", i), path))
+		require.NoError(lib.AddPlain(fmt.Sprintf("repo_%d", i), path, nil))
 	}
 
 	c := &checksumable{pool}
 	checksum, err := c.Checksum()
 	require.NoError(err)
-	require.Equal("mGPoKCyOIkXX4reGe1vTBPIOg2E=", checksum)
+	require.Equal(checksumMulti, checksum)
 
-	pool = NewRepositoryPool(cache.DefaultMaxSize)
+	lib, pool, err = newMultiPool()
+	require.NoError(err)
 	path := fixtures.ByTag("worktree").One().Worktree().Root()
-	require.NoError(pool.AddGitWithID("worktree", path))
+	require.NoError(lib.AddPlain("worktree", path, nil))
 
 	c = &checksumable{pool}
 	checksum, err = c.Checksum()
 	require.NoError(err)
-	require.Equal("rwQnBj7HRazv9wuU//nQ+nuf0WY=", checksum)
+	require.Equal(checksumSingle, checksum)
 }
 
 func TestChecksumSiva(t *testing.T) {
 	require := require.New(t)
 
-	pool := NewRepositoryPool(cache.DefaultMaxSize)
+	lib, pool, err := newMultiPool()
+	require.NoError(err)
+
+	cwd, err := os.Getwd()
+	require.NoError(err)
+	cwdFS := osfs.New(cwd)
+
 	require.NoError(
 		filepath.Walk("_testdata", func(path string, info os.FileInfo, err error) error {
 			if err != nil {
 				return err
 			}
 
 			if IsSivaFile(path) {
-				require.NoError(pool.AddSivaFile(path))
+				require.NoError(lib.AddSiva(path, cwdFS))
 			}
 
 			return nil
@@ -61,7 +75,7 @@ func TestChecksumSiva(t *testing.T) {
 	c := &checksumable{pool}
 	checksum, err := c.Checksum()
 	require.NoError(err)
-	require.Equal("wJEvZNAc7QRszsf9KhGu+UeKto0=", checksum)
+	require.Equal(checksumSiva, checksum)
 }
 
 func TestChecksumStable(t *testing.T) {
@@ -71,18 +85,19 @@ func TestChecksumStable(t *testing.T) {
 		require.NoError(fixtures.Clean())
 	}()
 
-	pool := NewRepositoryPool(cache.DefaultMaxSize)
+	lib, pool, err := newMultiPool()
+	require.NoError(err)
 
 	for i, f := range fixtures.ByTag("worktree") {
 		path := f.Worktree().Root()
-		require.NoError(pool.AddGitWithID(fmt.Sprintf("repo_%d", i), path))
+		require.NoError(lib.AddPlain(fmt.Sprintf("repo_%d", i), path, nil))
 	}
 
 	c := &checksumable{pool}
 
 	for i := 0; i < 100; i++ {
 		checksum, err := c.Checksum()
 		require.NoError(err)
-		require.Equal("mGPoKCyOIkXX4reGe1vTBPIOg2E=", checksum)
+		require.Equal(checksumMulti, checksum)
 	}
 }