config: support pack.window

Author: strib

config/config.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"sort"
+	"strconv"
 
 	format "gopkg.in/src-d/go-git.v4/plumbing/format/config"
 )
@@ -40,6 +41,14 @@ type Config struct {
 		// Worktree is the path to the root of the working tree.
 		Worktree string
 	}
+
+	Pack struct {
+		// Window controls the size of the sliding window for delta
+		// compression.  The default is 10.  A value of 0 turns off
+		// delta compression entirely.
+		Window uint
+	}
+
 	// Remotes list of repository remotes, the key of the map is the name
 	// of the remote, should equal to RemoteConfig.Name.
 	Remotes map[string]*RemoteConfig
@@ -81,10 +90,14 @@ const (
 	remoteSection    = "remote"
 	submoduleSection = "submodule"
 	coreSection      = "core"
+	packSection      = "pack"
 	fetchKey         = "fetch"
 	urlKey           = "url"
 	bareKey          = "bare"
 	worktreeKey      = "worktree"
+	windowKey        = "window"
+
+	defaultPackWindow = uint(10)
 )
 
 // Unmarshal parses a git-config file and stores it.
@@ -98,6 +111,9 @@ func (c *Config) Unmarshal(b []byte) error {
 	}
 
 	c.unmarshalCore()
+	if err := c.unmarshalPack(); err != nil {
+		return err
+	}
 	c.unmarshalSubmodules()
 	return c.unmarshalRemotes()
 }
@@ -111,6 +127,21 @@ func (c *Config) unmarshalCore() {
 	c.Core.Worktree = s.Options.Get(worktreeKey)
 }
 
+func (c *Config) unmarshalPack() error {
+	s := c.Raw.Section(packSection)
+	window := s.Options.Get(windowKey)
+	if window == "" {
+		c.Pack.Window = defaultPackWindow
+	} else {
+		winUint, err := strconv.ParseUint(window, 10, 32)
+		if err != nil {
+			return err
+		}
+		c.Pack.Window = uint(winUint)
+	}
+	return nil
+}
+
 func (c *Config) unmarshalRemotes() error {
 	s := c.Raw.Section(remoteSection)
 	for _, sub := range s.Subsections {
@@ -138,6 +169,7 @@ func (c *Config) unmarshalSubmodules() {
 // Marshal returns Config encoded as a git-config file.
 func (c *Config) Marshal() ([]byte, error) {
 	c.marshalCore()
+	c.marshalPack()
 	c.marshalRemotes()
 	c.marshalSubmodules()
 
@@ -158,6 +190,13 @@ func (c *Config) marshalCore() {
 	}
 }
 
+func (c *Config) marshalPack() {
+	s := c.Raw.Section(packSection)
+	if c.Pack.Window != defaultPackWindow {
+		s.SetOption(windowKey, fmt.Sprintf("%d", c.Pack.Window))
+	}
+}
+
 func (c *Config) marshalRemotes() {
 	s := c.Raw.Section(remoteSection)
 	newSubsections := make(format.Subsections, 0, len(c.Remotes))

config/config_test.go

@@ -10,6 +10,8 @@ func (s *ConfigSuite) TestUnmarshall(c *C) {
 	input := []byte(`[core]
         bare = true
 		worktree = foo
+[pack]
+		window = 20
 [remote "origin"]
         url = [email protected]:mcuadros/go-git.git
         fetch = +refs/heads/*:refs/remotes/origin/*
@@ -33,6 +35,7 @@ func (s *ConfigSuite) TestUnmarshall(c *C) {
 
 	c.Assert(cfg.Core.IsBare, Equals, true)
 	c.Assert(cfg.Core.Worktree, Equals, "foo")
+	c.Assert(cfg.Pack.Window, Equals, uint(20))
 	c.Assert(cfg.Remotes, HasLen, 2)
 	c.Assert(cfg.Remotes["origin"].Name, Equals, "origin")
 	c.Assert(cfg.Remotes["origin"].URLs, DeepEquals, []string{"[email protected]:mcuadros/go-git.git"})
@@ -51,6 +54,8 @@ func (s *ConfigSuite) TestMarshall(c *C) {
 	output := []byte(`[core]
 	bare = true
 	worktree = bar
+[pack]
+	window = 20
 [remote "alt"]
 	url = [email protected]:mcuadros/go-git.git
 	url = [email protected]:src-d/go-git.git
@@ -65,6 +70,7 @@ func (s *ConfigSuite) TestMarshall(c *C) {
 	cfg := NewConfig()
 	cfg.Core.IsBare = true
 	cfg.Core.Worktree = "bar"
+	cfg.Pack.Window = 20
 	cfg.Remotes["origin"] = &RemoteConfig{
 		Name: "origin",
 		URLs: []string{"[email protected]:mcuadros/go-git.git"},
@@ -92,6 +98,8 @@ func (s *ConfigSuite) TestUnmarshallMarshall(c *C) {
 	bare = true
 	worktree = foo
 	custom = ignored
+[pack]
+	window = 20
 [remote "origin"]
 	url = [email protected]:mcuadros/go-git.git
 	fetch = +refs/heads/*:refs/remotes/origin/*

options.go

@@ -175,15 +175,8 @@ type PushOptions struct {
 	Auth transport.AuthMethod
 	// Progress is where the human readable information sent by the server is
 	// stored, if nil nothing is stored.
-	Progress sideband.Progress
-	// SkipCompression can be true if the caller doesn't need
-	// delta-compressed objects pushed to the remote.  This can be useful to
-	// avoid CPU overhead when the objects don't need to be
-	// transferred across a network.  If the remote wants a compressed
-	// repo after the transfer, it can run `git gc --aggressive` or
-	// `git repack -a -d -f --depth=250 --window=250` as needed.
-	SkipCompression bool
-	StatusChan      plumbing.StatusChan
+	Progress   sideband.Progress
+	StatusChan plumbing.StatusChan
 }
 
 // Validate validates the fields and sets the default values.

plumbing/format/packfile/delta_selector.go

@@ -9,9 +9,6 @@ import (
 )
 
 const (
-	// How far back in the sorted list to search for deltas.  10 is
-	// the default in command line git.
-	deltaWindowSize = 10
 	// deltas based on deltas, how many steps we can do.
 	// 50 is the default value used in JGit
 	maxDepth = int64(50)
@@ -35,7 +32,7 @@ func newDeltaSelector(s storer.EncodedObjectStorer) *deltaSelector {
 // creating deltas if it's suitable, using an specific internal logic
 func (dw *deltaSelector) ObjectsToPack(
 	hashes []plumbing.Hash,
-	skipCompression bool,
+	packWindow uint,
 	statusChan plumbing.StatusChan,
 ) ([]*ObjectToPack, error) {
 	update := plumbing.StatusUpdate{
@@ -44,12 +41,12 @@ func (dw *deltaSelector) ObjectsToPack(
 	}
 	statusChan.SendUpdate(update)
 
-	otp, err := dw.objectsToPack(hashes, skipCompression, statusChan, update)
+	otp, err := dw.objectsToPack(hashes, packWindow, statusChan, update)
 	if err != nil {
 		return nil, err
 	}
 
-	if skipCompression {
+	if packWindow == 0 {
 		return otp, nil
 	}
 
@@ -87,7 +84,7 @@ func (dw *deltaSelector) ObjectsToPack(
 		objs := objs
 		wg.Add(1)
 		go func() {
-			if walkErr := dw.walk(objs, statusChan, &update, &updateMutex); walkErr != nil {
+			if walkErr := dw.walk(objs, packWindow, statusChan, &update, &updateMutex); walkErr != nil {
 				once.Do(func() {
 					err = walkErr
 				})
@@ -106,15 +103,15 @@ func (dw *deltaSelector) ObjectsToPack(
 
 func (dw *deltaSelector) objectsToPack(
 	hashes []plumbing.Hash,
-	skipCompression bool,
+	packWindow uint,
 	statusChan plumbing.StatusChan,
 	update plumbing.StatusUpdate,
 ) ([]*ObjectToPack, error) {
 	var objectsToPack []*ObjectToPack
 	for _, h := range hashes {
 		var o plumbing.EncodedObject
 		var err error
-		if skipCompression {
+		if packWindow == 0 {
 			o, err = dw.encodedObject(h)
 		} else {
 			o, err = dw.encodedDeltaObject(h)
@@ -134,7 +131,7 @@ func (dw *deltaSelector) objectsToPack(
 		statusChan.SendUpdateIfPossible(update)
 	}
 
-	if skipCompression {
+	if packWindow == 0 {
 		return objectsToPack, nil
 	}
 
@@ -259,6 +256,7 @@ func (dw *deltaSelector) sort(objectsToPack []*ObjectToPack) {
 
 func (dw *deltaSelector) walk(
 	objectsToPack []*ObjectToPack,
+	packWindow uint,
 	statusChan plumbing.StatusChan,
 	update *plumbing.StatusUpdate,
 	updateMutex *sync.Mutex,
@@ -288,7 +286,7 @@ func (dw *deltaSelector) walk(
 			continue
 		}
 
-		for j := i - 1; j >= 0 && i-j < deltaWindowSize; j-- {
+		for j := i - 1; j >= 0 && i-j < int(packWindow); j-- {
 			base := objectsToPack[j]
 			// Objects must use only the same type as their delta base.
 			// Since objectsToPack is sorted by type and size, once we find

plumbing/format/packfile/encoder.go

@@ -49,11 +49,11 @@ func NewEncoder(w io.Writer, s storer.EncodedObjectStorer, useRefDeltas bool) *E
 // and writes it to the writer in the Encoder.
 func (e *Encoder) Encode(
 	hashes []plumbing.Hash,
-	skipCompression bool,
+	packWindow uint,
 	statusChan plumbing.StatusChan,
 ) (plumbing.Hash, error) {
 	objects, err := e.selector.ObjectsToPack(
-		hashes, skipCompression, statusChan)
+		hashes, packWindow, statusChan)
 	if err != nil {
 		return plumbing.ZeroHash, err
 	}

plumbing/transport/server/server.go

@@ -165,7 +165,8 @@ func (s *upSession) UploadPack(ctx context.Context, req *packp.UploadPackRequest
 	pr, pw := io.Pipe()
 	e := packfile.NewEncoder(pw, s.storer, false)
 	go func() {
-		_, err := e.Encode(objs, false, nil)
+		// TODO: plumb through a pack window.
+		_, err := e.Encode(objs, 10, nil)
 		pw.CloseWithError(err)
 	}()
 

plumbing/transport/test/receive_pack.go

@@ -348,7 +348,7 @@ func (s *ReceivePackSuite) testSendPackDeleteReference(c *C) {
 func (s *ReceivePackSuite) emptyPackfile() io.ReadCloser {
 	var buf bytes.Buffer
 	e := packfile.NewEncoder(&buf, memory.NewStorage(), false)
-	_, err := e.Encode(nil, false, nil)
+	_, err := e.Encode(nil, 10, nil)
 	if err != nil {
 		panic(err)
 	}

plumbing/transport/test/upload_pack.go

@@ -262,7 +262,7 @@ func (s *UploadPackSuite) checkObjectNumber(c *C, r io.Reader, n int) {
 	storage := memory.NewStorage()
 	d, err := packfile.NewDecoder(scanner, storage)
 	c.Assert(err, IsNil)
-	_, err = d.Decode()
+	_, err = d.Decode(nil)
 	c.Assert(err, IsNil)
 	c.Assert(len(storage.Objects), Equals, n)
 }

remote.go

@@ -149,8 +149,7 @@ func (r *Remote) PushContext(ctx context.Context, o *PushOptions) error {
 		}
 	}
 
-	rs, err := pushHashes(
-		ctx, s, r.s, req, hashesToPush, o.SkipCompression, o.StatusChan)
+	rs, err := pushHashes(ctx, s, r.s, req, hashesToPush, o.StatusChan)
 	if err != nil {
 		return err
 	}
@@ -814,19 +813,22 @@ func referencesToHashes(refs storer.ReferenceStorer) ([]plumbing.Hash, error) {
 func pushHashes(
 	ctx context.Context,
 	sess transport.ReceivePackSession,
-	sto storer.EncodedObjectStorer,
+	s storage.Storer,
 	req *packp.ReferenceUpdateRequest,
 	hs []plumbing.Hash,
-	skipCompression bool,
 	statusChan plumbing.StatusChan,
 ) (*packp.ReportStatus, error) {
 
 	rd, wr := io.Pipe()
 	req.Packfile = rd
+	config, err := s.Config()
+	if err != nil {
+		return nil, err
+	}
 	done := make(chan error)
 	go func() {
-		e := packfile.NewEncoder(wr, sto, false)
-		if _, err := e.Encode(hs, skipCompression, statusChan); err != nil {
+		e := packfile.NewEncoder(wr, s, false)
+		if _, err := e.Encode(hs, config.Pack.Window, statusChan); err != nil {
 			done <- wr.CloseWithError(err)
 			return
 		}

storage/test/storage_suite.go

@@ -158,7 +158,7 @@ func (s *BaseStorageSuite) TestPackfileWriter(c *C) {
 		c.Skip("not a storer.PackWriter")
 	}
 
-	pw, err := pwr.PackfileWriter()
+	pw, err := pwr.PackfileWriter(nil)
 	c.Assert(err, IsNil)
 
 	f := fixtures.Basic().One()
@@ -414,7 +414,7 @@ func (s *BaseStorageSuite) TestDeltaObjectStorer(c *C) {
 		c.Skip("not a storer.PackWriter")
 	}
 
-	pw, err := pwr.PackfileWriter()
+	pw, err := pwr.PackfileWriter(nil)
 	c.Assert(err, IsNil)
 
 	f := fixtures.Basic().One()