Skip to content

Commit b1c8628

Browse files
author
kuba--
committed
One cache for repository pool.
Signed-off-by: kuba-- <[email protected]>
1 parent 0876aa6 commit b1c8628

File tree

11 files changed

+96
-81
lines changed

11 files changed

+96
-81
lines changed

‎cmd/gitbase/command/server.go

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
gopilosa "github.com/pilosa/go-pilosa"
1616
"github.com/sirupsen/logrus"
1717
"github.com/uber/jaeger-client-go/config"
18+
"gopkg.in/src-d/go-git.v4/plumbing/cache"
1819
sqle "gopkg.in/src-d/go-mysql-server.v0"
1920
"gopkg.in/src-d/go-mysql-server.v0/server"
2021
"gopkg.in/src-d/go-mysql-server.v0/sql"
@@ -39,18 +40,19 @@ type Server struct {
3940
pool *gitbase.RepositoryPool
4041
name string
4142

42-
Version string // Version of the application.
43-
Directories []string `short:"d" long:"directories" description:"Path where the git repositories are located (standard and siva), multiple directories can be defined. Accepts globs."`
44-
Depth int `long:"depth" default:"1000" description:"load repositories looking at less than <depth> nested subdirectories."`
45-
Host string `long:"host" default:"localhost" description:"Host where the server is going to listen"`
46-
Port int `short:"p" long:"port" default:"3306" description:"Port where the server is going to listen"`
47-
User string `short:"u" long:"user" default:"root" description:"User name used for connection"`
48-
Password string `short:"P" long:"password" default:"" description:"Password used for connection"`
49-
PilosaURL string `long:"pilosa" default:"http://localhost:10101" description:"URL to your pilosa server" env:"PILOSA_ENDPOINT"`
50-
IndexDir string `short:"i" long:"index" default:"/var/lib/gitbase/index" description:"Directory where the gitbase indexes information will be persisted." env:"GITBASE_INDEX_DIR"`
51-
DisableSquash bool `long:"no-squash" description:"Disables the table squashing."`
52-
TraceEnabled bool `long:"trace" env:"GITBASE_TRACE" description:"Enables jaeger tracing"`
53-
ReadOnly bool `short:"r" long:"readonly" description:"Only allow read queries. This disables creating and deleting indexes as well." env:"GITBASE_READONLY"`
43+
Version string // Version of the application.
44+
Directories []string `short:"d" long:"directories" description:"Path where the git repositories are located (standard and siva), multiple directories can be defined. Accepts globs."`
45+
Depth int `long:"depth" default:"1000" description:"load repositories looking at less than <depth> nested subdirectories."`
46+
Host string `long:"host" default:"localhost" description:"Host where the server is going to listen."`
47+
Port int `short:"p" long:"port" default:"3306" description:"Port where the server is going to listen."`
48+
User string `short:"u" long:"user" default:"root" description:"User name used for connection."`
49+
Password string `short:"P" long:"password" default:"" description:"Password used for connection."`
50+
PilosaURL string `long:"pilosa" default:"http://localhost:10101" description:"URL to your pilosa server." env:"PILOSA_ENDPOINT"`
51+
IndexDir string `short:"i" long:"index" default:"/var/lib/gitbase/index" description:"Directory where the gitbase indexes information will be persisted." env:"GITBASE_INDEX_DIR"`
52+
CacheSize cache.FileSize `long:"cache" default:"536870912" description:"Object cache size" env:"GITBASE_CACHE_SIZE"`
53+
DisableSquash bool `long:"no-squash" description:"Disables the table squashing."`
54+
TraceEnabled bool `long:"trace" env:"GITBASE_TRACE" description:"Enables jaeger tracing"`
55+
ReadOnly bool `short:"r" long:"readonly" description:"Only allow read queries. This disables creating and deleting indexes as well." env:"GITBASE_READONLY"`
5456

5557
SkipGitErrors bool // SkipGitErrors disables failing when Git errors are found.
5658
DisableGit bool `long:"no-git" description:"disable the load of git standard repositories."`
@@ -159,7 +161,7 @@ func (c *Server) buildDatabase() error {
159161
c.engine = NewDatabaseEngine(c.ReadOnly, c.Version)
160162
}
161163

162-
c.pool = gitbase.NewRepositoryPool()
164+
c.pool = gitbase.NewRepositoryPool(c.CacheSize)
163165

164166
if err := c.addDirectories(); err != nil {
165167
return err

‎common_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66

77
"github.com/stretchr/testify/require"
88
fixtures "gopkg.in/src-d/go-git-fixtures.v3"
9+
"gopkg.in/src-d/go-git.v4/plumbing/cache"
910
"gopkg.in/src-d/go-mysql-server.v0/sql"
1011
)
1112

@@ -35,7 +36,7 @@ func buildSession(t *testing.T, repos fixtures.Fixtures,
3536

3637
require.NoError(fixtures.Init())
3738

38-
pool := NewRepositoryPool()
39+
pool := NewRepositoryPool(cache.DefaultMaxSize)
3940
for _, fixture := range repos {
4041
path := fixture.Worktree().Root()
4142
ok, err := IsGitRepo(path)

‎fs_error_test.go

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
billy "gopkg.in/src-d/go-billy.v4"
1414
fixtures "gopkg.in/src-d/go-git-fixtures.v3"
1515
git "gopkg.in/src-d/go-git.v4"
16+
"gopkg.in/src-d/go-git.v4/plumbing/cache"
1617
"gopkg.in/src-d/go-git.v4/storage/filesystem"
1718
"gopkg.in/src-d/go-mysql-server.v0/sql"
1819
)
@@ -56,7 +57,7 @@ func setupErrorRepos(t *testing.T) (*sql.Context, CleanupFunc) {
5657
fixture := fixtures.ByTag("worktree").One()
5758
baseFS := fixture.Worktree()
5859

59-
pool := NewRepositoryPool()
60+
pool := NewRepositoryPool(cache.DefaultMaxSize)
6061

6162
fs, err := brokenFS(brokenPackfile, baseFS)
6263
require.NoError(err)
@@ -118,23 +119,21 @@ func testTable(t *testing.T, tableName string, number int) {
118119
}
119120

120121
type billyRepository struct {
121-
id string
122-
fs billy.Filesystem
122+
id string
123+
fs billy.Filesystem
124+
cache cache.Object
123125
}
124126

125127
func billyRepo(id string, fs billy.Filesystem) repository {
126-
return &billyRepository{id, fs}
128+
return &billyRepository{id, fs, cache.NewObjectLRUDefault()}
127129
}
128130

129131
func (r *billyRepository) ID() string {
130132
return r.id
131133
}
132134

133135
func (r *billyRepository) Repo() (*Repository, error) {
134-
storage, err := filesystem.NewStorage(r.fs)
135-
if err != nil {
136-
return nil, err
137-
}
136+
storage := filesystem.NewStorage(r.fs, r.cache)
138137

139138
repo, err := git.Open(storage, r.fs)
140139
if err != nil {
@@ -152,6 +151,10 @@ func (r *billyRepository) Path() string {
152151
return r.id
153152
}
154153

154+
func (r *billyRepository) Cache() cache.Object {
155+
return r.cache
156+
}
157+
155158
type brokenType uint64
156159

157160
const (

‎integration_test.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
"github.com/src-d/gitbase/internal/function"
1717
"github.com/stretchr/testify/require"
1818
fixtures "gopkg.in/src-d/go-git-fixtures.v3"
19+
"gopkg.in/src-d/go-git.v4/plumbing/cache"
1920
sqle "gopkg.in/src-d/go-mysql-server.v0"
2021
"gopkg.in/src-d/go-mysql-server.v0/sql"
2122
"gopkg.in/src-d/go-mysql-server.v0/sql/analyzer"
@@ -33,7 +34,7 @@ func TestIntegration(t *testing.T) {
3334

3435
path := fixtures.ByTag("worktree").One().Worktree().Root()
3536

36-
pool := gitbase.NewRepositoryPool()
37+
pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize)
3738
require.NoError(t, pool.AddGitWithID("worktree", path))
3839

3940
testCases := []struct {
@@ -443,7 +444,7 @@ func TestMissingHeadRefs(t *testing.T) {
443444
"_testdata",
444445
)
445446

446-
pool := gitbase.NewRepositoryPool()
447+
pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize)
447448
require.NoError(
448449
filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
449450
if err != nil {
@@ -829,7 +830,7 @@ func setup(t testing.TB) (*sqle.Engine, *gitbase.RepositoryPool, func()) {
829830
require.NoError(t, fixtures.Clean())
830831
}
831832

832-
pool := gitbase.NewRepositoryPool()
833+
pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize)
833834
for _, f := range fixtures.ByTag("worktree") {
834835
pool.AddGitWithID("worktree", f.Worktree().Root())
835836
}

‎internal/function/uast_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"gopkg.in/bblfsh/sdk.v1/protocol"
1212
"gopkg.in/bblfsh/sdk.v1/uast"
1313
fixtures "gopkg.in/src-d/go-git-fixtures.v3"
14+
"gopkg.in/src-d/go-git.v4/plumbing/cache"
1415
"gopkg.in/src-d/go-mysql-server.v0/sql"
1516
"gopkg.in/src-d/go-mysql-server.v0/sql/expression"
1617
)
@@ -418,7 +419,7 @@ func setup(t *testing.T) (*sql.Context, func()) {
418419
t.Helper()
419420
require.NoError(t, fixtures.Init())
420421

421-
pool := gitbase.NewRepositoryPool()
422+
pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize)
422423
for _, f := range fixtures.ByTag("worktree") {
423424
pool.AddGit(f.Worktree().Root())
424425
}

‎packfiles.go

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -166,10 +166,7 @@ func getUnpackedObject(repo repository, hash plumbing.Hash) (o object.Object, er
166166

167167
defer ioutil.CheckClose(f, &err)
168168

169-
storage, err := filesystem.NewStorage(fs)
170-
if err != nil {
171-
return nil, err
172-
}
169+
storage := filesystem.NewStorage(fs, repo.Cache())
173170

174171
obj := storage.NewEncodedObject()
175172
r, err := objfile.NewReader(f)
@@ -224,11 +221,7 @@ func newRepoObjectDecoder(
224221
return nil, err
225222
}
226223

227-
storage, err := filesystem.NewStorage(fs)
228-
if err != nil {
229-
_ = packf.Close()
230-
return nil, err
231-
}
224+
storage := filesystem.NewStorage(fs, repo.Cache())
232225

233226
idx, err := openPackfileIndex(dot, hash)
234227
if err != nil {

‎packfiles_test.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,15 @@ import (
66

77
"github.com/stretchr/testify/require"
88
"gopkg.in/src-d/go-git.v4/plumbing"
9+
"gopkg.in/src-d/go-git.v4/plumbing/cache"
910
)
1011

1112
var testSivaFilePath = filepath.Join("_testdata", "fff7062de8474d10a67d417ccea87ba6f58ca81d.siva")
1213

1314
func TestRepositoryPackfiles(t *testing.T) {
1415
require := require.New(t)
1516

16-
fs, packfiles, err := repositoryPackfiles(sivaRepo("siva", testSivaFilePath))
17+
fs, packfiles, err := repositoryPackfiles(sivaRepo("siva", testSivaFilePath, cache.NewObjectLRUDefault()))
1718

1819
require.NoError(err)
1920
require.Equal([]plumbing.Hash{
@@ -24,7 +25,7 @@ func TestRepositoryPackfiles(t *testing.T) {
2425
}
2526

2627
func TestRepositoryIndex(t *testing.T) {
27-
idx, err := newRepositoryIndex(sivaRepo("siva", testSivaFilePath))
28+
idx, err := newRepositoryIndex(sivaRepo("siva", testSivaFilePath, cache.NewObjectLRUDefault()))
2829
require.NoError(t, err)
2930

3031
testCases := []struct {

‎repositories_test.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"testing"
66

77
"github.com/stretchr/testify/require"
8+
"gopkg.in/src-d/go-git.v4/plumbing/cache"
89
"gopkg.in/src-d/go-mysql-server.v0/sql"
910
"gopkg.in/src-d/go-mysql-server.v0/sql/expression"
1011
)
@@ -36,10 +37,10 @@ func TestRepositoriesTable_RowIter(t *testing.T) {
3637
"seven", "eight", "nine",
3738
}
3839

39-
pool := NewRepositoryPool()
40+
pool := NewRepositoryPool(cache.DefaultMaxSize)
4041

4142
for _, id := range repoIDs {
42-
pool.Add(gitRepo(id, ""))
43+
pool.Add(gitRepo(id, "", pool.cache))
4344
}
4445

4546
session := NewSession(pool)

‎repository_pool.go

Lines changed: 30 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"gopkg.in/src-d/go-billy.v4/osfs"
1414
errors "gopkg.in/src-d/go-errors.v1"
1515
"gopkg.in/src-d/go-git.v4"
16+
"gopkg.in/src-d/go-git.v4/plumbing/cache"
1617
"gopkg.in/src-d/go-git.v4/storage/filesystem"
1718
"gopkg.in/src-d/go-mysql-server.v0/sql"
1819
)
@@ -50,7 +51,7 @@ func NewRepositoryFromPath(id, path string) (*Repository, error) {
5051

5152
// NewSivaRepositoryFromPath creates and initializes a new Repository structure
5253
// and initializes a go-git repository backed by a siva file.
53-
func NewSivaRepositoryFromPath(id, path string) (*Repository, error) {
54+
func NewSivaRepositoryFromPath(id, path string, cache cache.Object) (*Repository, error) {
5455
localfs := osfs.New(filepath.Dir(path))
5556

5657
tmpDir, err := ioutil.TempDir(os.TempDir(), "gitbase-siva")
@@ -65,11 +66,7 @@ func NewSivaRepositoryFromPath(id, path string) (*Repository, error) {
6566
return nil, err
6667
}
6768

68-
sto, err := filesystem.NewStorage(fs)
69-
if err != nil {
70-
return nil, err
71-
}
72-
69+
sto := filesystem.NewStorage(fs, cache)
7370
repo, err := git.Open(sto, nil)
7471
if err != nil {
7572
return nil, err
@@ -83,15 +80,17 @@ type repository interface {
8380
Repo() (*Repository, error)
8481
FS() (billy.Filesystem, error)
8582
Path() string
83+
Cache() cache.Object
8684
}
8785

8886
type gitRepository struct {
89-
id string
90-
path string
87+
id string
88+
path string
89+
cache cache.Object
9190
}
9291

93-
func gitRepo(id, path string) repository {
94-
return &gitRepository{id, path}
92+
func gitRepo(id, path string, cache cache.Object) repository {
93+
return &gitRepository{id, path, cache}
9594
}
9695

9796
func (r *gitRepository) ID() string {
@@ -110,21 +109,26 @@ func (r *gitRepository) Path() string {
110109
return r.path
111110
}
112111

112+
func (r *gitRepository) Cache() cache.Object {
113+
return r.cache
114+
}
115+
113116
type sivaRepository struct {
114-
id string
115-
path string
117+
id string
118+
path string
119+
cache cache.Object
116120
}
117121

118-
func sivaRepo(id, path string) repository {
119-
return &sivaRepository{id, path}
122+
func sivaRepo(id, path string, cache cache.Object) repository {
123+
return &sivaRepository{id, path, cache}
120124
}
121125

122126
func (r *sivaRepository) ID() string {
123127
return r.id
124128
}
125129

126130
func (r *sivaRepository) Repo() (*Repository, error) {
127-
return NewSivaRepositoryFromPath(r.id, r.path)
131+
return NewSivaRepositoryFromPath(r.id, r.path, r.cache)
128132
}
129133

130134
func (r *sivaRepository) FS() (billy.Filesystem, error) {
@@ -144,17 +148,23 @@ func (r *sivaRepository) Path() string {
144148
return r.path
145149
}
146150

151+
func (r *sivaRepository) Cache() cache.Object {
152+
return r.cache
153+
}
154+
147155
// RepositoryPool holds a pool git repository paths and
148156
// functionality to open and iterate them.
149157
type RepositoryPool struct {
150158
repositories map[string]repository
151159
idOrder []string
160+
cache cache.Object
152161
}
153162

154-
// NewRepositoryPool initializes a new RepositoryPool
155-
func NewRepositoryPool() *RepositoryPool {
163+
// NewRepositoryPool initializes a new RepositoryPool with LRU cache.
164+
func NewRepositoryPool(maxCacheSize cache.FileSize) *RepositoryPool {
156165
return &RepositoryPool{
157166
repositories: make(map[string]repository),
167+
cache: cache.NewObjectLRU(maxCacheSize),
158168
}
159169
}
160170

@@ -178,17 +188,17 @@ func (p *RepositoryPool) AddGit(path string) error {
178188

179189
// AddGitWithID adds a git repository to the pool. ID should be specified.
180190
func (p *RepositoryPool) AddGitWithID(id, path string) error {
181-
return p.Add(gitRepo(id, path))
191+
return p.Add(gitRepo(id, path, p.cache))
182192
}
183193

184194
// AddSivaFile adds a siva file to the pool. It also sets its path as ID.
185195
func (p *RepositoryPool) AddSivaFile(path string) error {
186-
return p.Add(sivaRepo(path, path))
196+
return p.Add(sivaRepo(path, path, p.cache))
187197
}
188198

189199
// AddSivaFileWithID adds a siva file to the pool. ID should be specified.
190200
func (p *RepositoryPool) AddSivaFileWithID(id, path string) error {
191-
return p.Add(sivaRepo(id, path))
201+
return p.Add(sivaRepo(id, path, p.cache))
192202
}
193203

194204
// GetPos retrieves a repository at a given position. If the position is

0 commit comments

Comments
 (0)