storage/filesystem: reuse delta cache

Reuse delta base object cache for packfile decoders
across multiple instances.

Author: smola

plumbing/format/packfile/decoder.go

@@ -52,6 +52,8 @@ var (
 // is destroyed. The Offsets and CRCs are calculated whether an
 // ObjectStorer was provided or not.
 type Decoder struct {
+	DeltaBaseCache cache.Object
+
 	s  *Scanner
 	o  storer.EncodedObjectStorer
 	tx storer.Transaction
@@ -65,8 +67,6 @@ type Decoder struct {
 
 	offsetToType map[int64]plumbing.ObjectType
 	decoderType  plumbing.ObjectType
-
-	cache cache.Object
 }
 
 // NewDecoder returns a new Decoder that decodes a Packfile using the given
@@ -107,8 +107,6 @@ func NewDecoderForType(s *Scanner, o storer.EncodedObjectStorer,
 		idx:          NewIndex(0),
 		offsetToType: make(map[int64]plumbing.ObjectType, 0),
 		decoderType:  t,
-
-		cache: cache.NewObjectFIFO(cache.MaxSize),
 	}, nil
 }
 
@@ -355,7 +353,7 @@ func (d *Decoder) fillREFDeltaObjectContent(obj plumbing.EncodedObject, ref plum
 		return 0, err
 	}
 
-	base, ok := d.cache.Get(ref)
+	base, ok := d.cacheGet(ref)
 	if !ok {
 		base, err = d.recallByHash(ref)
 		if err != nil {
@@ -365,7 +363,7 @@ func (d *Decoder) fillREFDeltaObjectContent(obj plumbing.EncodedObject, ref plum
 
 	obj.SetType(base.Type())
 	err = ApplyDelta(obj, base, buf.Bytes())
-	d.cache.Put(obj)
+	d.cachePut(obj)
 
 	return crc, err
 }
@@ -380,7 +378,7 @@ func (d *Decoder) fillOFSDeltaObjectContent(obj plumbing.EncodedObject, offset i
 	e, ok := d.idx.LookupOffset(uint64(offset))
 	var base plumbing.EncodedObject
 	if ok {
-		base, ok = d.cache.Get(e.Hash)
+		base, ok = d.cacheGet(e.Hash)
 	}
 
 	if !ok {
@@ -392,11 +390,27 @@ func (d *Decoder) fillOFSDeltaObjectContent(obj plumbing.EncodedObject, offset i
 
 	obj.SetType(base.Type())
 	err = ApplyDelta(obj, base, buf.Bytes())
-	d.cache.Put(obj)
+	d.cachePut(obj)
 
 	return crc, err
 }
 
+func (d *Decoder) cacheGet(h plumbing.Hash) (plumbing.EncodedObject, bool) {
+	if d.DeltaBaseCache == nil {
+		return nil, false
+	}
+
+	return d.DeltaBaseCache.Get(h)
+}
+
+func (d *Decoder) cachePut(obj plumbing.EncodedObject) {
+	if d.DeltaBaseCache == nil {
+		return
+	}
+
+	d.DeltaBaseCache.Put(obj)
+}
+
 func (d *Decoder) recallByOffset(o int64) (plumbing.EncodedObject, error) {
 	if d.s.IsSeekable {
 		return d.DecodeObjectAt(o)
@@ -454,7 +468,5 @@ func (d *Decoder) Index() *Index {
 // Close closes the Scanner. usually this mean that the whole reader is read and
 // discarded
 func (d *Decoder) Close() error {
-	d.cache.Clear()
-
 	return d.s.Close()
 }

storage/filesystem/object.go

@@ -5,6 +5,7 @@ import (
 	"os"
 
 	"gopkg.in/src-d/go-git.v4/plumbing"
+	"gopkg.in/src-d/go-git.v4/plumbing/cache"
 	"gopkg.in/src-d/go-git.v4/plumbing/format/idxfile"
 	"gopkg.in/src-d/go-git.v4/plumbing/format/objfile"
 	"gopkg.in/src-d/go-git.v4/plumbing/format/packfile"
@@ -16,14 +17,20 @@ import (
 	"gopkg.in/src-d/go-billy.v3"
 )
 
+const DefaultMaxDeltaBaseCacheSize = 92 * cache.MiByte
+
 type ObjectStorage struct {
+	// DeltaBaseCache is an object cache uses to cache delta's bases when
+	DeltaBaseCache cache.Object
+
 	dir   *dotgit.DotGit
 	index map[plumbing.Hash]*packfile.Index
 }
 
 func newObjectStorage(dir *dotgit.DotGit) (ObjectStorage, error) {
 	s := ObjectStorage{
-		dir: dir,
+		DeltaBaseCache: cache.NewObjectFIFO(DefaultMaxDeltaBaseCacheSize),
+		dir:            dir,
 	}
 
 	return s, nil
@@ -198,6 +205,7 @@ func (s *ObjectStorage) getFromPackfile(h plumbing.Hash) (plumbing.EncodedObject
 		return nil, err
 	}
 
+	d.DeltaBaseCache = s.DeltaBaseCache
 	d.SetIndex(s.index[pack])
 	obj, err := d.DecodeObjectAt(offset)
 	return obj, err