Merge pull request #563 from erizocosmico/perf/dotgit-ref-cache

dotgit: rewrite the way references are looked up

Author: mcuadros

storage/filesystem/internal/dotgit/dotgit.go

@@ -8,6 +8,7 @@ import (
 	stdioutil "io/ioutil"
 	"os"
 	"strings"
+	"time"
 
 	"gopkg.in/src-d/go-git.v4/plumbing"
 	"gopkg.in/src-d/go-git.v4/utils/ioutil"
@@ -56,14 +57,16 @@ var (
 // The DotGit type represents a local git repository on disk. This
 // type is not zero-value-safe, use the New function to initialize it.
 type DotGit struct {
-	fs billy.Filesystem
+	fs                billy.Filesystem
+	cachedPackedRefs  refCache
+	packedRefsLastMod time.Time
 }
 
 // New returns a DotGit value ready to be used. The path argument must
 // be the absolute path of a git repository directory (e.g.
 // "/foo/bar/.git").
 func New(fs billy.Filesystem) *DotGit {
-	return &DotGit{fs: fs}
+	return &DotGit{fs: fs, cachedPackedRefs: make(refCache)}
 }
 
 // Initialize creates all the folder scaffolding.
@@ -285,15 +288,57 @@ func (d *DotGit) Ref(name plumbing.ReferenceName) (*plumbing.Reference, error) {
 		return ref, nil
 	}
 
-	refs, err := d.Refs()
+	return d.packedRef(name)
+}
+
+func (d *DotGit) syncPackedRefs() error {
+	fi, err := d.fs.Stat(packedRefsPath)
+	if os.IsNotExist(err) {
+		return nil
+	}
+
 	if err != nil {
-		return nil, err
+		return err
 	}
 
-	for _, ref := range refs {
-		if ref.Name() == name {
-			return ref, nil
+	if d.packedRefsLastMod.Before(fi.ModTime()) {
+		d.cachedPackedRefs = make(refCache)
+		f, err := d.fs.Open(packedRefsPath)
+		if err != nil {
+			if os.IsNotExist(err) {
+				return nil
+			}
+			return err
 		}
+		defer ioutil.CheckClose(f, &err)
+
+		s := bufio.NewScanner(f)
+		for s.Scan() {
+			ref, err := d.processLine(s.Text())
+			if err != nil {
+				return err
+			}
+
+			if ref != nil {
+				d.cachedPackedRefs[ref.Name()] = ref
+			}
+		}
+
+		d.packedRefsLastMod = fi.ModTime()
+
+		return s.Err()
+	}
+
+	return nil
+}
+
+func (d *DotGit) packedRef(name plumbing.ReferenceName) (*plumbing.Reference, error) {
+	if err := d.syncPackedRefs(); err != nil {
+		return nil, err
+	}
+
+	if ref, ok := d.cachedPackedRefs[name]; ok {
+		return ref, nil
 	}
 
 	return nil, plumbing.ErrReferenceNotFound
@@ -315,28 +360,15 @@ func (d *DotGit) RemoveRef(name plumbing.ReferenceName) error {
 }
 
 func (d *DotGit) addRefsFromPackedRefs(refs *[]*plumbing.Reference) (err error) {
-	f, err := d.fs.Open(packedRefsPath)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
+	if err := d.syncPackedRefs(); err != nil {
 		return err
 	}
-	defer ioutil.CheckClose(f, &err)
 
-	s := bufio.NewScanner(f)
-	for s.Scan() {
-		ref, err := d.processLine(s.Text())
-		if err != nil {
-			return err
-		}
-
-		if ref != nil {
-			*refs = append(*refs, ref)
-		}
+	for _, ref := range d.cachedPackedRefs {
+		*refs = append(*refs, ref)
 	}
 
-	return s.Err()
+	return nil
 }
 
 func (d *DotGit) rewritePackedRefsWithoutRef(name plumbing.ReferenceName) (err error) {
@@ -511,3 +543,5 @@ func isNum(b byte) bool {
 func isHexAlpha(b byte) bool {
 	return b >= 'a' && b <= 'f' || b >= 'A' && b <= 'F'
 }
+
+type refCache map[plumbing.ReferenceName]*plumbing.Reference

storage/filesystem/internal/dotgit/dotgit_test.go

@@ -134,6 +134,24 @@ func (s *SuiteDotGit) TestRefsFromReferenceFile(c *C) {
 
 }
 
+func BenchmarkRefMultipleTimes(b *testing.B) {
+	fs := fixtures.Basic().ByTag(".git").One().DotGit()
+	refname := plumbing.ReferenceName("refs/remotes/origin/branch")
+
+	dir := New(fs)
+	_, err := dir.Ref(refname)
+	if err != nil {
+		b.Fatalf("unexpected error: %s", err)
+	}
+
+	for i := 0; i < b.N; i++ {
+		_, err := dir.Ref(refname)
+		if err != nil {
+			b.Fatalf("unexpected error: %s", err)
+		}
+	}
+}
+
 func (s *SuiteDotGit) TestRemoveRefFromReferenceFile(c *C) {
 	fs := fixtures.Basic().ByTag(".git").One().DotGit()
 	dir := New(fs)