Skip to content

Commit 9b81193

Browse files
lunnytechknowlogicklafriks
authored
Cache last commit when pushing for big repository (#10109)
* Cache last commit when pushing for big repository * Fix bug * detect force push * Refactor cache push * Finish cache last commit info when push * Some improvements * Fix lint * Remove unused changes * Move pull request test before cache * Fix test mysql Co-authored-by: techknowlogick <[email protected]> Co-authored-by: Lauris BH <[email protected]>
1 parent f06ee37 commit 9b81193

File tree

4 files changed

+148
-11
lines changed

4 files changed

+148
-11
lines changed

‎modules/git/commit_info.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache LastCom
4040
return nil, nil, err
4141
}
4242
if len(unHitPaths) > 0 {
43-
revs2, err := getLastCommitForPaths(c, treePath, unHitPaths)
43+
revs2, err := GetLastCommitForPaths(c, treePath, unHitPaths)
4444
if err != nil {
4545
return nil, nil, err
4646
}
@@ -53,7 +53,7 @@ func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache LastCom
5353
}
5454
}
5555
} else {
56-
revs, err = getLastCommitForPaths(c, treePath, entryPaths)
56+
revs, err = GetLastCommitForPaths(c, treePath, entryPaths)
5757
}
5858
if err != nil {
5959
return nil, nil, err
@@ -170,7 +170,8 @@ func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cac
170170
return results, unHitEntryPaths, nil
171171
}
172172

173-
func getLastCommitForPaths(c cgobject.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) {
173+
// GetLastCommitForPaths returns last commit information
174+
func GetLastCommitForPaths(c cgobject.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) {
174175
// We do a tree traversal with nodes sorted by commit time
175176
heap := binaryheap.NewWith(func(a, b interface{}) int {
176177
if a.(*commitAndPaths).commit.CommitTime().Before(b.(*commitAndPaths).commit.CommitTime()) {

‎modules/git/notes.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ func GetNote(repo *Repository, commitID string, note *Note) error {
7070
return err
7171
}
7272

73-
lastCommits, err := getLastCommitForPaths(commitNode, "", []string{path})
73+
lastCommits, err := GetLastCommitForPaths(commitNode, "", []string{path})
7474
if err != nil {
7575
return err
7676
}

‎modules/repository/cache.go

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
// Copyright 2020 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package repository
6+
7+
import (
8+
"path"
9+
"strings"
10+
11+
"code.gitea.io/gitea/models"
12+
"code.gitea.io/gitea/modules/cache"
13+
"code.gitea.io/gitea/modules/git"
14+
"code.gitea.io/gitea/modules/setting"
15+
16+
cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph"
17+
)
18+
19+
func recusiveCache(gitRepo *git.Repository, c cgobject.CommitNode, tree *git.Tree, treePath string, ca *cache.LastCommitCache, level int) error {
20+
if level == 0 {
21+
return nil
22+
}
23+
24+
entries, err := tree.ListEntries()
25+
if err != nil {
26+
return err
27+
}
28+
29+
entryPaths := make([]string, len(entries))
30+
entryMap := make(map[string]*git.TreeEntry)
31+
for i, entry := range entries {
32+
entryPaths[i] = entry.Name()
33+
entryMap[entry.Name()] = entry
34+
}
35+
36+
commits, err := git.GetLastCommitForPaths(c, treePath, entryPaths)
37+
if err != nil {
38+
return err
39+
}
40+
41+
for entry, cm := range commits {
42+
if err := ca.Put(c.ID().String(), path.Join(treePath, entry), cm.ID().String()); err != nil {
43+
return err
44+
}
45+
if entryMap[entry].IsDir() {
46+
subTree, err := tree.SubTree(entry)
47+
if err != nil {
48+
return err
49+
}
50+
if err := recusiveCache(gitRepo, c, subTree, entry, ca, level-1); err != nil {
51+
return err
52+
}
53+
}
54+
}
55+
56+
return nil
57+
}
58+
59+
func getRefName(fullRefName string) string {
60+
if strings.HasPrefix(fullRefName, git.TagPrefix) {
61+
return fullRefName[len(git.TagPrefix):]
62+
} else if strings.HasPrefix(fullRefName, git.BranchPrefix) {
63+
return fullRefName[len(git.BranchPrefix):]
64+
}
65+
return ""
66+
}
67+
68+
// CacheRef cachhe last commit information of the branch or the tag
69+
func CacheRef(repo *models.Repository, gitRepo *git.Repository, fullRefName string) error {
70+
if !setting.CacheService.LastCommit.Enabled {
71+
return nil
72+
}
73+
74+
commit, err := gitRepo.GetCommit(fullRefName)
75+
if err != nil {
76+
return err
77+
}
78+
79+
commitsCount, err := cache.GetInt64(repo.GetCommitsCountCacheKey(getRefName(fullRefName), true), commit.CommitsCount)
80+
if err != nil {
81+
return err
82+
}
83+
if commitsCount < setting.CacheService.LastCommit.CommitsCount {
84+
return nil
85+
}
86+
87+
commitNodeIndex, _ := gitRepo.CommitNodeIndex()
88+
89+
c, err := commitNodeIndex.Get(commit.ID)
90+
if err != nil {
91+
return err
92+
}
93+
94+
ca := cache.NewLastCommitCache(repo.FullName(), gitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds()))
95+
96+
return recusiveCache(gitRepo, c, &commit.Tree, "", ca, 1)
97+
}

‎services/repository/push.go

Lines changed: 46 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ type PushUpdateOptions struct {
3030
PusherName string
3131
RepoUserName string
3232
RepoName string
33-
RefFullName string
33+
RefFullName string // branch, tag or other name to push
3434
OldCommitID string
3535
NewCommitID string
3636
}
@@ -95,11 +95,36 @@ func (opts PushUpdateOptions) BranchName() string {
9595
return opts.RefFullName[len(git.BranchPrefix):]
9696
}
9797

98+
// RefName returns simple name for ref
99+
func (opts PushUpdateOptions) RefName() string {
100+
if strings.HasPrefix(opts.RefFullName, git.TagPrefix) {
101+
return opts.RefFullName[len(git.TagPrefix):]
102+
} else if strings.HasPrefix(opts.RefFullName, git.BranchPrefix) {
103+
return opts.RefFullName[len(git.BranchPrefix):]
104+
}
105+
return ""
106+
}
107+
98108
// RepoFullName returns repo full name
99109
func (opts PushUpdateOptions) RepoFullName() string {
100110
return opts.RepoUserName + "/" + opts.RepoName
101111
}
102112

113+
// isForcePush detect if a push is a force push
114+
func isForcePush(repoPath string, opts *PushUpdateOptions) (bool, error) {
115+
if !opts.IsUpdateBranch() {
116+
return false, nil
117+
}
118+
119+
output, err := git.NewCommand("rev-list", "--max-count=1", opts.OldCommitID, "^"+opts.NewCommitID).RunInDir(repoPath)
120+
if err != nil {
121+
return false, err
122+
} else if len(output) > 0 {
123+
return true, nil
124+
}
125+
return false, nil
126+
}
127+
103128
// pushQueue represents a queue to handle update pull request tests
104129
var pushQueue queue.Queue
105130

@@ -184,7 +209,6 @@ func pushUpdates(optsList []*PushUpdateOptions) error {
184209
if opts.IsDelRef() {
185210
delTags = append(delTags, tagName)
186211
} else { // is new tag
187-
cache.Remove(repo.GetCommitsCountCacheKey(tagName, true))
188212
addTags = append(addTags, tagName)
189213
}
190214
} else if opts.IsBranch() { // If is branch reference
@@ -197,8 +221,8 @@ func pushUpdates(optsList []*PushUpdateOptions) error {
197221

198222
branch := opts.BranchName()
199223
if !opts.IsDelRef() {
200-
// Clear cache for branch commit count
201-
cache.Remove(repo.GetCommitsCountCacheKey(opts.BranchName(), true))
224+
log.Trace("TriggerTask '%s/%s' by %s", repo.Name, branch, pusher.Name)
225+
go pull_service.AddTestPullRequestTask(pusher, repo.ID, branch, true, opts.OldCommitID, opts.NewCommitID)
202226

203227
newCommit, err := gitRepo.GetCommit(opts.NewCommitID)
204228
if err != nil {
@@ -217,6 +241,20 @@ func pushUpdates(optsList []*PushUpdateOptions) error {
217241
if err != nil {
218242
return fmt.Errorf("newCommit.CommitsBeforeUntil: %v", err)
219243
}
244+
245+
isForce, err := isForcePush(repo.RepoPath(), opts)
246+
if err != nil {
247+
log.Error("isForcePush %s/%s failed: %v", repo.ID, branch, err)
248+
}
249+
250+
if isForce {
251+
log.Trace("Push %s is a force push", opts.NewCommitID)
252+
253+
cache.Remove(repo.GetCommitsCountCacheKey(opts.RefName(), true))
254+
} else {
255+
// TODO: increment update the commit count cache but not remove
256+
cache.Remove(repo.GetCommitsCountCacheKey(opts.RefName(), true))
257+
}
220258
}
221259

222260
commits = repo_module.ListToPushCommits(l)
@@ -225,9 +263,10 @@ func pushUpdates(optsList []*PushUpdateOptions) error {
225263
log.Error("models.RemoveDeletedBranch %s/%s failed: %v", repo.ID, branch, err)
226264
}
227265

228-
log.Trace("TriggerTask '%s/%s' by %s", repo.Name, branch, pusher.Name)
229-
230-
go pull_service.AddTestPullRequestTask(pusher, repo.ID, branch, true, opts.OldCommitID, opts.NewCommitID)
266+
// Cache for big repository
267+
if err := repo_module.CacheRef(repo, gitRepo, opts.RefFullName); err != nil {
268+
log.Error("repo_module.CacheRef %s/%s failed: %v", repo.ID, branch, err)
269+
}
231270
} else if err = pull_service.CloseBranchPulls(pusher, repo.ID, branch); err != nil {
232271
// close all related pulls
233272
log.Error("close related pull request failed: %v", err)

0 commit comments

Comments
 (0)