paginate repo list, bug fix for issue #1941 (#2093)

* paginate repo list

Author: jamesboi951

backend/controllers/github.go

@@ -6,8 +6,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/diggerhq/digger/libs/digger_config/terragrunt/tac"
-	"github.com/diggerhq/digger/libs/git_utils"
 	"log/slog"
 	"math/rand"
 	"net/http"
@@ -21,6 +19,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/diggerhq/digger/libs/digger_config/terragrunt/tac"
+	"github.com/diggerhq/digger/libs/git_utils"
+
 	"github.com/diggerhq/digger/backend/ci_backends"
 	config2 "github.com/diggerhq/digger/backend/config"
 	"github.com/diggerhq/digger/backend/locking"
@@ -2599,8 +2600,7 @@ func (d DiggerController) GithubAppCallbackPage(c *gin.Context) {
 	// we get repos accessible to this installation
 	slog.Debug("Listing repositories for installation", "installationId", installationId64)
 
-	opt := &github.ListOptions{Page: 1, PerPage: 100}
-	listRepos, _, err := client.Apps.ListRepos(context.Background(), opt)
+	repos, err := dg_github.ListGithubRepos(client)
 	if err != nil {
 		slog.Error("Failed to list existing repositories",
 			"installationId", installationId64,
@@ -2609,12 +2609,6 @@ func (d DiggerController) GithubAppCallbackPage(c *gin.Context) {
 		c.String(http.StatusInternalServerError, "Failed to list existing repos: %v", err)
 		return
 	}
-	repos := listRepos.Repositories
-
-	slog.Info("Retrieved repositories for installation",
-		"installationId", installationId64,
-		"repoCount", len(repos),
-	)
 
 	// resets all existing installations (soft delete)
 	slog.Debug("Resetting existing GitHub installations",

libs/ci/github/github_utils.go

@@ -0,0 +1,46 @@
+package github
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+
+	"github.com/google/go-github/v61/github"
+)
+
+func ListGithubRepos(client *github.Client) ([]*github.Repository, error) {
+	allRepos := make([]*github.Repository, 0)
+	//err := c.BindJSON(&request)
+	opts := &github.ListOptions{PerPage: 100}
+
+	pageLimit := 0
+	for {
+		listRepos, resp, err := client.Apps.ListRepos(context.Background(), opts)
+		if err != nil {
+			// Check specifically for rate limit errors
+			if _, ok := err.(*github.RateLimitError); ok {
+				slog.Error("GitHub API rate limit exceeded",
+					"error", err,
+				)
+				// Wait and retry after a delay or return a specific error
+				// For now, we'll just return with the rate limit error
+				return nil, err
+			}
+			slog.Error("Failed to list existing repositories",
+				"error", err,
+			)
+			return nil, err
+		}
+		allRepos = append(allRepos, listRepos.Repositories...)
+		pageLimit++
+		if pageLimit == 20 {
+			slog.Error("Exceeded maximum number of pages (20) when listing repositories, results may be incomplete")
+			return nil, fmt.Errorf("exceeded maximum number of pages (20) when listing repositories, results may be incomplete")
+		}
+		if resp.NextPage == 0 {
+			break
+		}
+		opts.Page = resp.NextPage
+	}
+	return allRepos, nil
+}

libs/ci/github/github_utils_test.go

@@ -0,0 +1,55 @@
+package github
+
+import (
+	"encoding/base64"
+	"fmt"
+	net "net/http"
+	"os"
+	"strconv"
+	"testing"
+
+	"github.com/bradleyfalzon/ghinstallation/v2"
+	"github.com/google/go-github/v61/github"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestListRepositoriesReturnsAllReposities(t *testing.T) {
+	if os.Getenv("CI") != "" {
+		t.Skip("skipping in CI")
+	}
+	githubAppPrivateKeyB64 := os.Getenv("GITHUB_APP_PRIVATE_KEY_BASE64")
+	decodedBytes, err := base64.StdEncoding.DecodeString(githubAppPrivateKeyB64)
+	if err != nil {
+		fmt.Printf("Failed to decode GITHUB_APP_PRIVATE_KEY_BASE64, error: %v\n", err)
+		t.Error(err)
+	}
+	githubAppPrivateKey := string(decodedBytes)
+	tr := net.DefaultTransport
+	var githubAppId = os.Getenv("GITHUB_APP_ID")
+	var installationId = os.Getenv("INSTALLATION_ID")
+
+	githubAppIdintValue, err := strconv.ParseInt(githubAppId, 10, 64)
+	if err != nil {
+		fmt.Printf("Error converting environment variable to int64: %v\n", err)
+		t.Fatalf("Failed to parse GITHUB_APP_ID: %v", err)
+	}
+	installationIdintValue, err := strconv.ParseInt(installationId, 10, 64)
+	if err != nil {
+		fmt.Printf("Error converting environment variable to int64: %v\n", err)
+		t.Fatalf("Failed to parse INSTALLATION_ID: %v", err)
+	}
+
+	itr, err := ghinstallation.New(tr, githubAppIdintValue, installationIdintValue, []byte(githubAppPrivateKey))
+	if err != nil {
+		fmt.Printf("Failed to initialize GitHub app installation: githubAppId=%s, installationId=%s, error=%v\n", githubAppId, installationId, err)
+		t.Error(err)
+	}
+
+	client := github.NewClient(&net.Client{Transport: itr})
+
+	allRepos, err := ListGithubRepos(client)
+	fmt.Println("err is", err)
+	assert.Nil(t, err)
+	// Currently Digger has 388 repositories, Update the hardcoded value to the expected number of repositories if the number changes in the future.
+	assert.Equal(t, 388, len(allRepos))
+}