Skip to content

Commit d98f05c

Browse files
committed
pkg/repo: speedup remote KEP loading
I tried running the PRR kepctl query command suggested in the docs and found it to be... slow. Could have been my internet connection? kepctl query --sig '.*' --prr '(at)johnbelamaric' --include-prs --gh-token-path ~/.secrets/github.token --status implementable,provisional While I waited for it to finish running, I looked at what the code was doing: - cloning the kubernetes/enhancements repo for each sig - listing all PRs for kubernetes/enhancements for each sig I opted to add two non-exported fields to hold a cached copy of: - a clone of kubernetes/enhancements - a list of all kubernetes/enhacements PRs This provides two benefits: - The kepctl query command is ~30X faster, down from O(minutes) to ~15s - The query is based off of a point-in-time snapshot of the repo and PRs, rather than potentially picking up new results as PRs are opened / merged
1 parent 3d72dc6 commit d98f05c

File tree

1 file changed

+46
-33
lines changed

1 file changed

+46
-33
lines changed

pkg/repo/repo.go

Lines changed: 46 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,12 @@ type Repo struct {
7070

7171
// Templates
7272
ProposalTemplate []byte
73+
74+
// Temporary caches
75+
// a local git clone of remoteOrg/remoteRepo
76+
gitRepo *git.Repo
77+
// all open pull requests for remoteOrg/remoteRepo
78+
allPRs []*github.PullRequest
7379
}
7480

7581
// New returns a new repo client configured to use the the normal os.Stdxxx and Filesystem
@@ -284,42 +290,47 @@ func (r *Repo) LoadLocalKEPs(sig string) ([]*api.Proposal, error) {
284290
}
285291

286292
func (r *Repo) loadKEPPullRequests(sig string) ([]*api.Proposal, error) {
293+
// Initialize github client
287294
var auth *http.Client
288295
ctx := context.Background()
289296
if r.Token != "" {
290297
ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: r.Token})
291298
auth = oauth2.NewClient(ctx, ts)
292299
}
293-
294300
gh := github.NewClient(auth)
295-
allPulls := []*github.PullRequest{}
296-
opt := &github.PullRequestListOptions{
297-
ListOptions: github.ListOptions{
298-
PerPage: 100,
299-
},
300-
}
301301

302-
for {
303-
pulls, resp, err := gh.PullRequests.List(
304-
ctx,
305-
remoteOrg,
306-
remoteRepo,
307-
opt,
308-
)
309-
if err != nil {
310-
return nil, err
311-
}
302+
// Fetch list of all PRs if none exists
303+
if r.allPRs == nil {
304+
r.allPRs = []*github.PullRequest{}
312305

313-
allPulls = append(allPulls, pulls...)
314-
if resp.NextPage == 0 {
315-
break
306+
opt := &github.PullRequestListOptions{
307+
ListOptions: github.ListOptions{
308+
PerPage: 100,
309+
},
316310
}
317311

318-
opt.Page = resp.NextPage
312+
for {
313+
pulls, resp, err := gh.PullRequests.List(
314+
ctx,
315+
remoteOrg,
316+
remoteRepo,
317+
opt,
318+
)
319+
if err != nil {
320+
return nil, err
321+
}
322+
323+
r.allPRs = append(r.allPRs, pulls...)
324+
if resp.NextPage == 0 {
325+
break
326+
}
327+
328+
opt.Page = resp.NextPage
329+
}
319330
}
320331

321332
kepPRs := make([]*github.PullRequest, 10)
322-
for _, pr := range allPulls {
333+
for _, pr := range r.allPRs {
323334
foundKind, foundSIG := false, false
324335
sigLabel := strings.Replace(sig, "-", "/", 1)
325336

@@ -344,18 +355,20 @@ func (r *Repo) loadKEPPullRequests(sig string) ([]*api.Proposal, error) {
344355
return nil, nil
345356
}
346357

347-
// Pull a temporary clone of the repo
348-
g, err := git.NewClient()
349-
if err != nil {
350-
return nil, err
351-
}
358+
// Pull a temporary clone of the repo if none already exists
359+
if r.gitRepo == nil {
360+
g, err := git.NewClient()
361+
if err != nil {
362+
return nil, err
363+
}
352364

353-
g.SetCredentials("", func() []byte { return []byte{} })
354-
g.SetRemote(krgh.GitHubURL)
365+
g.SetCredentials("", func() []byte { return []byte{} })
366+
g.SetRemote(krgh.GitHubURL)
355367

356-
repo, err := g.Clone(remoteOrg, remoteRepo)
357-
if err != nil {
358-
return nil, err
368+
r.gitRepo, err = g.Clone(remoteOrg, remoteRepo)
369+
if err != nil {
370+
return nil, err
371+
}
359372
}
360373

361374
// read out each PR, and create a Proposal for each KEP that is
@@ -395,7 +408,7 @@ func (r *Repo) loadKEPPullRequests(sig string) ([]*api.Proposal, error) {
395408
continue
396409
}
397410

398-
err = repo.CheckoutPullRequest(pr.GetNumber())
411+
err = r.gitRepo.CheckoutPullRequest(pr.GetNumber())
399412
if err != nil {
400413
return nil, err
401414
}

0 commit comments

Comments
 (0)