Skip to content

Commit b824298

Browse files
sgoedeckesgoedecke
authored
Merge pull request #4 from AntonioMIN/main
Implement pagination for GitHub commits and issues
2 parents 14b5448 + c53b8f6 commit b824298

File tree

1 file changed

+180
-75
lines changed

1 file changed

+180
-75
lines changed

‎internal/github/client.go‎

Lines changed: 180 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -88,16 +88,19 @@ func (c *Client) CollectActivity(username, repo string, startDate, endDate time.
8888
func (c *Client) getCommits(username, repo string, startDate, endDate time.Time) ([]types.GitHubActivity, error) {
8989
var activities []types.GitHubActivity
9090

91-
// Search for commits by author
92-
query := fmt.Sprintf("author:%s committer-date:%s..%s",
91+
// Base query for commits search
92+
baseQuery := fmt.Sprintf("author:%s committer-date:%s..%s",
9393
username, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
9494

9595
if repo != "" {
96-
query += fmt.Sprintf(" repo:%s", repo)
96+
baseQuery += fmt.Sprintf(" repo:%s", repo)
9797
}
9898

99+
escapedQuery := strings.ReplaceAll(baseQuery, " ", "%20")
100+
99101
var searchResult struct {
100-
Items []struct {
102+
TotalCount int `json:"total_count"`
103+
Items []struct {
101104
SHA string `json:"sha"`
102105
Repository struct {
103106
FullName string `json:"full_name"`
@@ -112,24 +115,48 @@ func (c *Client) getCommits(username, repo string, startDate, endDate time.Time)
112115
} `json:"items"`
113116
}
114117

115-
// The commits search API often fails or is slow due to GitHub's rate limiting
116-
// If it fails, we'll return an error so the caller can handle it gracefully
117-
escapedQuery := strings.ReplaceAll(query, " ", "%20")
118-
err := c.client.Get(fmt.Sprintf("search/commits?q=%s&sort=committer-date&order=desc", escapedQuery), &searchResult)
119-
if err != nil {
120-
// Return error so caller knows commits search failed
121-
return activities, fmt.Errorf("commits search failed (this is common due to GitHub API restrictions): %w", err)
122-
}
123-
124-
for _, item := range searchResult.Items {
125-
activities = append(activities, types.GitHubActivity{
126-
Type: "commit",
127-
Repository: item.Repository.FullName,
128-
Title: strings.Split(item.Commit.Message, "\n")[0],
129-
Description: item.Commit.Message,
130-
URL: item.HTMLURL,
131-
CreatedAt: item.Commit.Author.Date,
132-
})
118+
// Pagination to get all commits
119+
page := 1
120+
perPage := 100
121+
122+
for {
123+
// Build query with pagination
124+
125+
err := c.client.Get(fmt.Sprintf("search/commits?q=%s&per_page=%d&page=%d&sort=committer-date&order=desc", escapedQuery, perPage, page), &searchResult)
126+
if err != nil {
127+
// Return error so caller knows commits search failed
128+
return activities, fmt.Errorf("commits search failed (this is common due to GitHub API restrictions): %w", err)
129+
}
130+
131+
// If no items returned, we've reached the end
132+
if len(searchResult.Items) == 0 {
133+
break
134+
}
135+
136+
// Add items from current page
137+
for _, item := range searchResult.Items {
138+
activities = append(activities, types.GitHubActivity{
139+
Type: "commit",
140+
Repository: item.Repository.FullName,
141+
Title: strings.Split(item.Commit.Message, "\n")[0],
142+
Description: item.Commit.Message,
143+
URL: item.HTMLURL,
144+
CreatedAt: item.Commit.Author.Date,
145+
})
146+
}
147+
148+
// If we got less than perPage items, we've reached the end
149+
if len(searchResult.Items) < perPage {
150+
break
151+
}
152+
153+
// Move to next page
154+
page++
155+
156+
// Safety check to prevent infinite loops (GitHub API has limits)
157+
if page > 10 { // Max 1000 commits (100 * 10 pages)
158+
break
159+
}
133160
}
134161

135162
return activities, nil
@@ -138,14 +165,16 @@ func (c *Client) getCommits(username, repo string, startDate, endDate time.Time)
138165
func (c *Client) getPullRequests(username, repo string, startDate, endDate time.Time) ([]types.GitHubActivity, error) {
139166
var activities []types.GitHubActivity
140167

141-
// Search for pull requests
142-
query := fmt.Sprintf("author:%s created:%s..%s",
168+
// Base query for pull requests search
169+
baseQuery := fmt.Sprintf("author:%s created:%s..%s",
143170
username, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
144171

145172
if repo != "" {
146-
query += fmt.Sprintf(" repo:%s", repo)
173+
baseQuery += fmt.Sprintf(" repo:%s", repo)
147174
}
148175

176+
escapedQuery := strings.ReplaceAll(baseQuery, " ", "%20")
177+
149178
var searchResult struct {
150179
Items []struct {
151180
Number int `json:"number"`
@@ -160,21 +189,45 @@ func (c *Client) getPullRequests(username, repo string, startDate, endDate time.
160189
} `json:"items"`
161190
}
162191

163-
escapedQuery := strings.ReplaceAll(query, " ", "%20")
164-
err := c.client.Get(fmt.Sprintf("search/issues?q=%s+type:pr&sort=created&order=desc", escapedQuery), &searchResult)
165-
if err != nil {
166-
return activities, nil
167-
}
168-
169-
for _, item := range searchResult.Items {
170-
activities = append(activities, types.GitHubActivity{
171-
Type: "pull_request",
172-
Repository: item.Repository.FullName,
173-
Title: fmt.Sprintf("PR #%d: %s", item.Number, item.Title),
174-
Description: item.Body,
175-
URL: item.HTMLURL,
176-
CreatedAt: item.CreatedAt,
177-
})
192+
// Pagination to get all pull requests
193+
page := 1
194+
perPage := 100
195+
196+
for {
197+
err := c.client.Get(fmt.Sprintf("search/issues?q=%s+type:pr&per_page=%d&page=%d&sort=created&order=desc", escapedQuery, perPage, page), &searchResult)
198+
if err != nil {
199+
return activities, err
200+
}
201+
202+
// If no items returned, we've reached the end
203+
if len(searchResult.Items) == 0 {
204+
break
205+
}
206+
207+
// Add items from current page
208+
for _, item := range searchResult.Items {
209+
activities = append(activities, types.GitHubActivity{
210+
Type: "pull_request",
211+
Repository: item.Repository.FullName,
212+
Title: fmt.Sprintf("PR #%d: %s", item.Number, item.Title),
213+
Description: item.Body,
214+
URL: item.HTMLURL,
215+
CreatedAt: item.CreatedAt,
216+
})
217+
}
218+
219+
// If we got less than perPage items, we've reached the end
220+
if len(searchResult.Items) < perPage {
221+
break
222+
}
223+
224+
// Move to next page
225+
page++
226+
227+
// Safety check to prevent infinite loops
228+
if page > 10 { // Max 1000 pull requests (100 * 10 pages)
229+
break
230+
}
178231
}
179232

180233
return activities, nil
@@ -183,14 +236,16 @@ func (c *Client) getPullRequests(username, repo string, startDate, endDate time.
183236
func (c *Client) getIssues(username, repo string, startDate, endDate time.Time) ([]types.GitHubActivity, error) {
184237
var activities []types.GitHubActivity
185238

186-
// Search for issues created by user
187-
query := fmt.Sprintf("author:%s created:%s..%s",
239+
// Base query for issues search
240+
baseQuery := fmt.Sprintf("author:%s created:%s..%s",
188241
username, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
189242

190243
if repo != "" {
191-
query += fmt.Sprintf(" repo:%s", repo)
244+
baseQuery += fmt.Sprintf(" repo:%s", repo)
192245
}
193246

247+
escapedQuery := strings.ReplaceAll(baseQuery, " ", "%20")
248+
194249
var searchResult struct {
195250
Items []struct {
196251
Number int `json:"number"`
@@ -205,21 +260,45 @@ func (c *Client) getIssues(username, repo string, startDate, endDate time.Time)
205260
} `json:"items"`
206261
}
207262

208-
escapedQuery := strings.ReplaceAll(query, " ", "%20")
209-
err := c.client.Get(fmt.Sprintf("search/issues?q=%s+type:issue&sort=created&order=desc", escapedQuery), &searchResult)
210-
if err != nil {
211-
return activities, nil
212-
}
213-
214-
for _, item := range searchResult.Items {
215-
activities = append(activities, types.GitHubActivity{
216-
Type: "issue",
217-
Repository: item.Repository.FullName,
218-
Title: fmt.Sprintf("Issue #%d: %s", item.Number, item.Title),
219-
Description: item.Body,
220-
URL: item.HTMLURL,
221-
CreatedAt: item.CreatedAt,
222-
})
263+
// Pagination to get all issues
264+
page := 1
265+
perPage := 100
266+
267+
for {
268+
err := c.client.Get(fmt.Sprintf("search/issues?q=%s+type:issue&per_page=%d&page=%d&sort=created&order=desc", escapedQuery, perPage, page), &searchResult)
269+
if err != nil {
270+
return activities, err
271+
}
272+
273+
// If no items returned, we've reached the end
274+
if len(searchResult.Items) == 0 {
275+
break
276+
}
277+
278+
// Add items from current page
279+
for _, item := range searchResult.Items {
280+
activities = append(activities, types.GitHubActivity{
281+
Type: "issue",
282+
Repository: item.Repository.FullName,
283+
Title: fmt.Sprintf("Issue #%d: %s", item.Number, item.Title),
284+
Description: item.Body,
285+
URL: item.HTMLURL,
286+
CreatedAt: item.CreatedAt,
287+
})
288+
}
289+
290+
// If we got less than perPage items, we've reached the end
291+
if len(searchResult.Items) < perPage {
292+
break
293+
}
294+
295+
// Move to next page
296+
page++
297+
298+
// Safety check to prevent infinite loops
299+
if page > 10 { // Max 1000 issues (100 * 10 pages)
300+
break
301+
}
223302
}
224303

225304
return activities, nil
@@ -228,10 +307,12 @@ func (c *Client) getIssues(username, repo string, startDate, endDate time.Time)
228307
func (c *Client) getReviews(username string, startDate, endDate time.Time) ([]types.GitHubActivity, error) {
229308
var activities []types.GitHubActivity
230309

231-
// Search for pull requests reviewed by user
232-
query := fmt.Sprintf("reviewed-by:%s created:%s..%s",
310+
// Base query for pull requests reviewed by user
311+
baseQuery := fmt.Sprintf("reviewed-by:%s created:%s..%s",
233312
username, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
234313

314+
escapedQuery := strings.ReplaceAll(baseQuery, " ", "%20")
315+
235316
var searchResult struct {
236317
Items []struct {
237318
Number int `json:"number"`
@@ -244,21 +325,45 @@ func (c *Client) getReviews(username string, startDate, endDate time.Time) ([]ty
244325
} `json:"items"`
245326
}
246327

247-
escapedQuery := strings.ReplaceAll(query, " ", "%20")
248-
err := c.client.Get(fmt.Sprintf("search/issues?q=%s+type:pr&sort=created&order=desc", escapedQuery), &searchResult)
249-
if err != nil {
250-
return activities, nil
251-
}
252-
253-
for _, item := range searchResult.Items {
254-
activities = append(activities, types.GitHubActivity{
255-
Type: "review",
256-
Repository: item.Repository.FullName,
257-
Title: fmt.Sprintf("Reviewed PR #%d: %s", item.Number, item.Title),
258-
Description: fmt.Sprintf("Reviewed pull request: %s", item.Title),
259-
URL: item.HTMLURL,
260-
CreatedAt: item.CreatedAt,
261-
})
328+
// Pagination to get all reviews
329+
page := 1
330+
perPage := 100
331+
332+
for {
333+
err := c.client.Get(fmt.Sprintf("search/issues?q=%s+type:pr&per_page=%d&page=%d&sort=created&order=desc", escapedQuery, perPage, page), &searchResult)
334+
if err != nil {
335+
return activities, err
336+
}
337+
338+
// If no items returned, we've reached the end
339+
if len(searchResult.Items) == 0 {
340+
break
341+
}
342+
343+
// Add items from current page
344+
for _, item := range searchResult.Items {
345+
activities = append(activities, types.GitHubActivity{
346+
Type: "review",
347+
Repository: item.Repository.FullName,
348+
Title: fmt.Sprintf("Reviewed PR #%d: %s", item.Number, item.Title),
349+
Description: fmt.Sprintf("Reviewed pull request: %s", item.Title),
350+
URL: item.HTMLURL,
351+
CreatedAt: item.CreatedAt,
352+
})
353+
}
354+
355+
// If we got less than perPage items, we've reached the end
356+
if len(searchResult.Items) < perPage {
357+
break
358+
}
359+
360+
// Move to next page
361+
page++
362+
363+
// Safety check to prevent infinite loops
364+
if page > 10 { // Max 1000 reviews (100 * 10 pages)
365+
break
366+
}
262367
}
263368

264369
return activities, nil

0 commit comments

Comments
 (0)