Skip to content

Commit b67a323

Browse files
committed
Refactor CatFile batch implementation and introduce batch-command for git 2.36
1 parent a9a705f commit b67a323

16 files changed

+222
-129
lines changed

modules/git/batch.go

Lines changed: 132 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -8,40 +8,160 @@ import (
88
"context"
99
)
1010

11-
type Batch struct {
11+
type batchCatFile struct {
1212
cancel context.CancelFunc
1313
Reader *bufio.Reader
1414
Writer WriteCloserError
1515
}
1616

17+
func (b *batchCatFile) Close() {
18+
if b.cancel != nil {
19+
b.cancel()
20+
b.Reader = nil
21+
b.Writer = nil
22+
b.cancel = nil
23+
}
24+
}
25+
1726
// NewBatch creates a new batch for the given repository, the Close must be invoked before release the batch
18-
func NewBatch(ctx context.Context, repoPath string) (*Batch, error) {
27+
func newBatch(ctx context.Context, repoPath string) (*batchCatFile, error) {
1928
// Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first!
2029
if err := ensureValidGitRepository(ctx, repoPath); err != nil {
2130
return nil, err
2231
}
2332

24-
var batch Batch
25-
batch.Writer, batch.Reader, batch.cancel = catFileBatch(ctx, repoPath)
33+
var batch batchCatFile
34+
batch.Writer, batch.Reader, batch.cancel = catFileBatch(ctx, repoPath, "--batch")
2635
return &batch, nil
2736
}
2837

29-
func NewBatchCheck(ctx context.Context, repoPath string) (*Batch, error) {
38+
func newBatchCheck(ctx context.Context, repoPath string) (*batchCatFile, error) {
3039
// Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first!
3140
if err := ensureValidGitRepository(ctx, repoPath); err != nil {
3241
return nil, err
3342
}
3443

35-
var check Batch
44+
var check batchCatFile
3645
check.Writer, check.Reader, check.cancel = catFileBatchCheck(ctx, repoPath)
3746
return &check, nil
3847
}
3948

40-
func (b *Batch) Close() {
41-
if b.cancel != nil {
42-
b.cancel()
43-
b.Reader = nil
44-
b.Writer = nil
45-
b.cancel = nil
49+
func newBatchCommand(ctx context.Context, repoPath string) (*batchCatFile, error) {
50+
// Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first!
51+
if err := ensureValidGitRepository(ctx, repoPath); err != nil {
52+
return nil, err
53+
}
54+
55+
var check batchCatFile
56+
check.Writer, check.Reader, check.cancel = catFileBatch(ctx, repoPath, "--batch-command")
57+
return &check, nil
58+
}
59+
60+
type Batch interface {
61+
Write([]byte) (int, error)
62+
WriteCheck([]byte) (int, error)
63+
Reader() *bufio.Reader
64+
CheckReader() *bufio.Reader
65+
Close()
66+
}
67+
68+
// batchCatFileWithCheck implements the Batch interface using the "cat-file --batch" command and "cat-file --batch-check" command
69+
// ref: https://git-scm.com/docs/git-cat-file#Documentation/git-cat-file.txt---batch
70+
// To align with --batch-command, we creates the two commands both at the same time if git version is lower than 2.36
71+
type batchCatFileWithCheck struct {
72+
batch *batchCatFile
73+
batchCheck *batchCatFile
74+
}
75+
76+
var _ Batch = &batchCatFileWithCheck{}
77+
78+
func newBatchCatFileWithCheck(ctx context.Context, repoPath string) (*batchCatFileWithCheck, error) {
79+
batch, err := newBatch(ctx, repoPath)
80+
if err != nil {
81+
return nil, err
82+
}
83+
84+
batchCheck, err := newBatchCheck(ctx, repoPath)
85+
if err != nil {
86+
return nil, err
87+
}
88+
89+
return &batchCatFileWithCheck{
90+
batch: batch,
91+
batchCheck: batchCheck,
92+
}, nil
93+
}
94+
95+
func (b *batchCatFileWithCheck) Write(bs []byte) (int, error) {
96+
return b.batch.Writer.Write(bs)
97+
}
98+
99+
func (b *batchCatFileWithCheck) WriteCheck(bs []byte) (int, error) {
100+
return b.batchCheck.Writer.Write(bs)
101+
}
102+
103+
func (b *batchCatFileWithCheck) Reader() *bufio.Reader {
104+
return b.batch.Reader
105+
}
106+
107+
func (b *batchCatFileWithCheck) CheckReader() *bufio.Reader {
108+
return b.batchCheck.Reader
109+
}
110+
111+
func (b *batchCatFileWithCheck) Close() {
112+
if b.batch != nil {
113+
b.batch.Close()
114+
b.batch = nil
115+
}
116+
if b.batchCheck != nil {
117+
b.batchCheck.Close()
118+
b.batchCheck = nil
119+
}
120+
}
121+
122+
// batchCommandCatFile implements the Batch interface using the "cat-file --batch-command" command
123+
// ref: https://git-scm.com/docs/git-cat-file#Documentation/git-cat-file.txt---batch-command
124+
type batchCommandCatFile struct {
125+
batch *batchCatFile
126+
}
127+
128+
func newBatchCommandCatFile(ctx context.Context, repoPath string) (*batchCommandCatFile, error) {
129+
batch, err := newBatchCommand(ctx, repoPath)
130+
if err != nil {
131+
return nil, err
132+
}
133+
134+
return &batchCommandCatFile{
135+
batch: batch,
136+
}, nil
137+
}
138+
139+
func (b *batchCommandCatFile) Write(bs []byte) (int, error) {
140+
return b.batch.Writer.Write(append([]byte("contents "), bs...))
141+
}
142+
143+
func (b *batchCommandCatFile) WriteCheck(bs []byte) (int, error) {
144+
return b.batch.Writer.Write(append([]byte("info "), bs...))
145+
}
146+
147+
func (b *batchCommandCatFile) Reader() *bufio.Reader {
148+
return b.batch.Reader
149+
}
150+
151+
func (b *batchCommandCatFile) CheckReader() *bufio.Reader {
152+
return b.batch.Reader
153+
}
154+
155+
func (b *batchCommandCatFile) Close() {
156+
if b.batch != nil {
157+
b.batch.Close()
158+
b.batch = nil
159+
}
160+
}
161+
162+
func NewBatch(ctx context.Context, repoPath string) (Batch, error) {
163+
if DefaultFeatures().SupportCatFileBatchCommand {
164+
return newBatchCommandCatFile(ctx, repoPath)
46165
}
166+
return newBatchCatFileWithCheck(ctx, repoPath)
47167
}

modules/git/batch_reader.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,8 @@ func catFileBatchCheck(ctx context.Context, repoPath string) (WriteCloserError,
8787
}
8888

8989
// catFileBatch opens git cat-file --batch in the provided repo and returns a stdin pipe, a stdout reader and cancel function
90-
func catFileBatch(ctx context.Context, repoPath string) (WriteCloserError, *bufio.Reader, func()) {
90+
// batchArg is the argument to pass to cat-file --batch, e.g. "--batch" or "--batch-command"
91+
func catFileBatch(ctx context.Context, repoPath, batchArg string) (WriteCloserError, *bufio.Reader, func()) {
9192
// We often want to feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary.
9293
// so let's create a batch stdin and stdout
9394
batchStdinReader, batchStdinWriter := io.Pipe()
@@ -109,7 +110,8 @@ func catFileBatch(ctx context.Context, repoPath string) (WriteCloserError, *bufi
109110

110111
go func() {
111112
stderr := strings.Builder{}
112-
err := NewCommand("cat-file", "--batch").
113+
err := NewCommand("cat-file").
114+
AddDynamicArguments(batchArg).
113115
Run(ctx, &RunOpts{
114116
Dir: repoPath,
115117
Stdin: batchStdinReader,

modules/git/blob_nogogit.go

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,16 +26,17 @@ type Blob struct {
2626
// DataAsync gets a ReadCloser for the contents of a blob without reading it all.
2727
// Calling the Close function on the result will discard all unread output.
2828
func (b *Blob) DataAsync() (io.ReadCloser, error) {
29-
wr, rd, cancel, err := b.repo.CatFileBatch(b.repo.Ctx)
29+
batch, cancel, err := b.repo.CatFileBatch(b.repo.Ctx)
3030
if err != nil {
3131
return nil, err
3232
}
3333

34-
_, err = wr.Write([]byte(b.ID.String() + "\n"))
34+
_, err = batch.Write([]byte(b.ID.String() + "\n"))
3535
if err != nil {
3636
cancel()
3737
return nil, err
3838
}
39+
rd := batch.Reader()
3940
_, _, size, err := ReadBatchLine(rd)
4041
if err != nil {
4142
cancel()
@@ -67,18 +68,18 @@ func (b *Blob) Size() int64 {
6768
return b.size
6869
}
6970

70-
wr, rd, cancel, err := b.repo.CatFileBatchCheck(b.repo.Ctx)
71+
batch, cancel, err := b.repo.CatFileBatch(b.repo.Ctx)
7172
if err != nil {
7273
log.Debug("error whilst reading size for %s in %s. Error: %v", b.ID.String(), b.repo.Path, err)
7374
return 0
7475
}
7576
defer cancel()
76-
_, err = wr.Write([]byte(b.ID.String() + "\n"))
77+
_, err = batch.WriteCheck([]byte(b.ID.String() + "\n"))
7778
if err != nil {
7879
log.Debug("error whilst reading size for %s in %s. Error: %v", b.ID.String(), b.repo.Path, err)
7980
return 0
8081
}
81-
_, _, b.size, err = ReadBatchLine(rd)
82+
_, _, b.size, err = ReadBatchLine(batch.CheckReader())
8283
if err != nil {
8384
log.Debug("error whilst reading size for %s in %s. Error: %v", b.ID.String(), b.repo.Path, err)
8485
return 0

modules/git/commit_info_nogogit.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -124,14 +124,15 @@ func GetLastCommitForPaths(ctx context.Context, commit *Commit, treePath string,
124124
return nil, err
125125
}
126126

127-
batchStdinWriter, batchReader, cancel, err := commit.repo.CatFileBatch(ctx)
127+
batch, cancel, err := commit.repo.CatFileBatch(ctx)
128128
if err != nil {
129129
return nil, err
130130
}
131131
defer cancel()
132132

133133
commitsMap := map[string]*Commit{}
134134
commitsMap[commit.ID.String()] = commit
135+
batchReader := batch.Reader()
135136

136137
commitCommits := map[string]*Commit{}
137138
for path, commitID := range revs {
@@ -145,7 +146,7 @@ func GetLastCommitForPaths(ctx context.Context, commit *Commit, treePath string,
145146
continue
146147
}
147148

148-
_, err := batchStdinWriter.Write([]byte(commitID + "\n"))
149+
_, err := batch.Write([]byte(commitID + "\n"))
149150
if err != nil {
150151
return nil, err
151152
}

modules/git/git.go

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,12 @@ const RequiredVersion = "2.0.0" // the minimum Git version required
2626
type Features struct {
2727
gitVersion *version.Version
2828

29-
UsingGogit bool
30-
SupportProcReceive bool // >= 2.29
31-
SupportHashSha256 bool // >= 2.42, SHA-256 repositories no longer an ‘experimental curiosity’
32-
SupportedObjectFormats []ObjectFormat // sha1, sha256
33-
SupportCheckAttrOnBare bool // >= 2.40
29+
UsingGogit bool
30+
SupportProcReceive bool // >= 2.29
31+
SupportHashSha256 bool // >= 2.42, SHA-256 repositories no longer an ‘experimental curiosity’
32+
SupportedObjectFormats []ObjectFormat // sha1, sha256
33+
SupportCheckAttrOnBare bool // >= 2.40
34+
SupportCatFileBatchCommand bool // >= 2.36, support `git cat-file --batch-command`
3435
}
3536

3637
var (
@@ -79,6 +80,7 @@ func loadGitVersionFeatures() (*Features, error) {
7980
features.SupportedObjectFormats = append(features.SupportedObjectFormats, Sha256ObjectFormat)
8081
}
8182
features.SupportCheckAttrOnBare = features.CheckVersionAtLeast("2.40")
83+
features.SupportCatFileBatchCommand = features.CheckVersionAtLeast("2.36")
8284
return features, nil
8385
}
8486

modules/git/languagestats/language_stats_nogogit.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,20 +22,21 @@ import (
2222
func GetLanguageStats(repo *git.Repository, commitID string) (map[string]int64, error) {
2323
// We will feed the commit IDs in order into cat-file --batch, followed by blobs as necessary.
2424
// so let's create a batch stdin and stdout
25-
batchStdinWriter, batchReader, cancel, err := repo.CatFileBatch(repo.Ctx)
25+
batch, cancel, err := repo.CatFileBatch(repo.Ctx)
2626
if err != nil {
2727
return nil, err
2828
}
2929
defer cancel()
3030

3131
writeID := func(id string) error {
32-
_, err := batchStdinWriter.Write([]byte(id + "\n"))
32+
_, err := batch.Write([]byte(id + "\n"))
3333
return err
3434
}
3535

3636
if err := writeID(commitID); err != nil {
3737
return nil, err
3838
}
39+
batchReader := batch.Reader()
3940
shaBytes, typ, size, err := git.ReadBatchLine(batchReader)
4041
if typ != "commit" {
4142
log.Debug("Unable to get commit for: %s. Err: %v", commitID, err)

modules/git/pipeline/lfs_nogogit.go

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err
4646

4747
// Next feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary.
4848
// so let's create a batch stdin and stdout
49-
batchStdinWriter, batchReader, cancel, err := repo.CatFileBatch(repo.Ctx)
49+
batch, cancel, err := repo.CatFileBatch(repo.Ctx)
5050
if err != nil {
5151
return nil, err
5252
}
@@ -60,17 +60,14 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err
6060
fnameBuf := make([]byte, 4096)
6161
modeBuf := make([]byte, 40)
6262
workingShaBuf := make([]byte, objectID.Type().FullLength()/2)
63+
batchReader := batch.Reader()
6364

6465
for scan.Scan() {
6566
// Get the next commit ID
6667
commitID := scan.Bytes()
6768

6869
// push the commit to the cat-file --batch process
69-
_, err := batchStdinWriter.Write(commitID)
70-
if err != nil {
71-
return nil, err
72-
}
73-
_, err = batchStdinWriter.Write([]byte{'\n'})
70+
_, err := batch.Write(append(commitID, '\n'))
7471
if err != nil {
7572
return nil, err
7673
}
@@ -92,7 +89,7 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err
9289
if err != nil {
9390
return nil, err
9491
}
95-
_, err = batchStdinWriter.Write([]byte(id + "\n"))
92+
_, err = batch.Write([]byte(id + "\n"))
9693
if err != nil {
9794
return nil, err
9895
}
@@ -107,7 +104,7 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err
107104
return nil, err
108105
}
109106

110-
if _, err := batchStdinWriter.Write([]byte(curCommit.Tree.ID.String() + "\n")); err != nil {
107+
if _, err := batch.Write([]byte(curCommit.Tree.ID.String() + "\n")); err != nil {
111108
return nil, err
112109
}
113110
curPath = ""
@@ -139,11 +136,7 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err
139136
return nil, err
140137
}
141138
if len(trees) > 0 {
142-
_, err := batchStdinWriter.Write(trees[len(trees)-1])
143-
if err != nil {
144-
return nil, err
145-
}
146-
_, err = batchStdinWriter.Write([]byte("\n"))
139+
_, err := batch.Write(append(trees[len(trees)-1], '\n'))
147140
if err != nil {
148141
return nil, err
149142
}

0 commit comments

Comments
 (0)