Merge pull request #91 from ConductorOne/BB1027_max_page

[BB-1027] baton-github: use max_page_size to limit the api usage.

Author: Bencheng21

pkg/connector/connector.go

@@ -29,7 +29,10 @@ import (
 
 const githubDotCom = "https://github.com"
 
-var ValidAssetDomains = []string{"avatars.githubusercontent.com"}
+var (
+	ValidAssetDomains     = []string{"avatars.githubusercontent.com"}
+	maxPageSize       int = 100 // maximum page size github supported.
+)
 
 var (
 	resourceTypeOrg = &v2.ResourceType{
@@ -424,7 +427,7 @@ func getOrgs(ctx context.Context, client *github.Client, orgs []string) ([]strin
 		orgLogins []string
 	)
 	for {
-		orgs, resp, err := client.Organizations.List(ctx, "", &github.ListOptions{Page: page})
+		orgs, resp, err := client.Organizations.List(ctx, "", &github.ListOptions{Page: page, PerPage: maxPageSize})
 		if err != nil {
 			return nil, fmt.Errorf("github-connector: failed to retrieve org: %w", err)
 		}

pkg/connector/org.go

@@ -95,7 +95,7 @@ func (o *orgResourceType) List(
 
 	opts := &github.ListOptions{
 		Page:    page,
-		PerPage: pToken.Size,
+		PerPage: maxPageSize,
 	}
 
 	orgs, resp, err := o.client.Organizations.List(ctx, "", opts)
@@ -188,7 +188,7 @@ func (o *orgResourceType) Grants(
 	opts := github.ListMembersOptions{
 		ListOptions: github.ListOptions{
 			Page:    page,
-			PerPage: pToken.Size,
+			PerPage: maxPageSize,
 		},
 	}
 

pkg/connector/org_role.go

@@ -163,7 +163,8 @@ func (o *orgRoleResourceType) Grants(
 		})
 	case resourceTypeUser.Id:
 		opts := &github.ListOptions{
-			Page: page,
+			Page:    page,
+			PerPage: maxPageSize,
 		}
 		users, resp, err := o.client.Organizations.ListUsersAssignedToOrgRole(ctx, orgName, roleID, opts)
 		if err != nil {
@@ -207,7 +208,8 @@ func (o *orgRoleResourceType) Grants(
 		}
 	case resourceTypeTeam.Id:
 		opts := &github.ListOptions{
-			Page: page,
+			Page:    page,
+			PerPage: maxPageSize,
 		}
 		teams, resp, err := o.client.Organizations.ListTeamsAssignedToOrgRole(ctx, orgName, roleID, opts)
 		if err != nil {

pkg/connector/repository.go

@@ -82,7 +82,7 @@ func (o *repositoryResourceType) List(ctx context.Context, parentID *v2.Resource
 	opts := &github.RepositoryListByOrgOptions{
 		ListOptions: github.ListOptions{
 			Page:    page,
-			PerPage: pt.Size,
+			PerPage: maxPageSize,
 		},
 	}
 
@@ -161,7 +161,10 @@ func (o *repositoryResourceType) Grants(
 	case resourceTypeUser.Id:
 		opts := &github.ListCollaboratorsOptions{
 			Affiliation: "all",
-			ListOptions: github.ListOptions{Page: page},
+			ListOptions: github.ListOptions{
+				Page:    page,
+				PerPage: maxPageSize,
+			},
 		}
 		users, resp, err := o.client.Repositories.ListCollaborators(ctx, orgName, resource.DisplayName, opts)
 		if err != nil {
@@ -208,7 +211,8 @@ func (o *repositoryResourceType) Grants(
 
 	case resourceTypeTeam.Id:
 		opts := &github.ListOptions{
-			Page: page,
+			Page:    page,
+			PerPage: maxPageSize,
 		}
 		teams, resp, err := o.client.Repositories.ListTeams(ctx, orgName, resource.DisplayName, opts)
 		if err != nil {

pkg/connector/team.go

@@ -78,7 +78,7 @@ func (o *teamResourceType) List(ctx context.Context, parentID *v2.ResourceId, pt
 
 	opts := &github.ListOptions{
 		Page:    page,
-		PerPage: pt.Size,
+		PerPage: maxPageSize,
 	}
 
 	orgID, err := parseResourceToGitHub(parentID)
@@ -175,7 +175,10 @@ func (o *teamResourceType) Grants(ctx context.Context, resource *v2.Resource, pT
 	}
 
 	opts := github.TeamListTeamMembersOptions{
-		ListOptions: github.ListOptions{Page: page},
+		ListOptions: github.ListOptions{
+			Page:    page,
+			PerPage: maxPageSize,
+		},
 	}
 
 	users, resp, err := o.client.Teams.ListTeamMembersByID(ctx, org.GetID(), githubID, &opts)

pkg/connector/user.go

@@ -123,7 +123,10 @@ func (o *userResourceType) List(ctx context.Context, parentID *v2.ResourceId, pt
 	var restApiRateLimit *v2.RateLimitDescription
 
 	opts := github.ListMembersOptions{
-		ListOptions: github.ListOptions{Page: page, PerPage: pt.Size},
+		ListOptions: github.ListOptions{
+			Page:    page,
+			PerPage: maxPageSize,
+		},
 	}
 
 	users, resp, err := o.client.Organizations.ListMembers(ctx, orgName, &opts)