diff --git a/pkg/connector/api_token.go b/pkg/connector/api_token.go index 601d50ca..713e5e8b 100644 --- a/pkg/connector/api_token.go +++ b/pkg/connector/api_token.go @@ -8,7 +8,9 @@ import ( "github.com/conductorone/baton-sdk/pkg/annotations" "github.com/conductorone/baton-sdk/pkg/pagination" resourceSdk "github.com/conductorone/baton-sdk/pkg/types/resource" + "github.com/conductorone/baton-sdk/pkg/uhttp" "github.com/google/go-github/v69/github" + "google.golang.org/grpc/codes" ) func apiTokenResource(ctx context.Context, token *github.PersonalAccessToken) (*v2.Resource, error) { @@ -92,6 +94,9 @@ func (o *apiTokenResourceType) List( }, }) if err != nil { + if isRatelimited(resp) { + return nil, "", nil, uhttp.WrapErrors(codes.Unavailable, "too many requests", err) + } return nil, "", nil, err } diff --git a/pkg/connector/connector.go b/pkg/connector/connector.go index 9f131628..efe6f68e 100644 --- a/pkg/connector/connector.go +++ b/pkg/connector/connector.go @@ -180,7 +180,7 @@ func (gh *GitHub) Validate(ctx context.Context) (annotations.Annotations, error) membership, _, err := gh.client.Organizations.GetOrgMembership(ctx, "", o) if err != nil { if filterOrgs { - return nil, fmt.Errorf("access token must be an admin on the %s organization", o) + return nil, fmt.Errorf("access token must be an admin on the %s organization: %w", o, err) } continue } @@ -450,6 +450,9 @@ func getOrgs(ctx context.Context, client *github.Client, orgs []string) ([]strin for { orgs, resp, err := client.Organizations.List(ctx, "", &github.ListOptions{Page: page, PerPage: maxPageSize}) if err != nil { + if isRatelimited(resp) { + return nil, uhttp.WrapErrors(codes.Unavailable, "too many requests", err) + } return nil, fmt.Errorf("github-connector: failed to retrieve org: %w", err) } if resp.StatusCode == http.StatusUnauthorized { diff --git a/pkg/connector/enterprise_role.go b/pkg/connector/enterprise_role.go index dff737d6..7da06449 100644 --- a/pkg/connector/enterprise_role.go +++ b/pkg/connector/enterprise_role.go @@ -13,7 +13,9 @@ import ( "github.com/conductorone/baton-sdk/pkg/types/entitlement" "github.com/conductorone/baton-sdk/pkg/types/grant" resourceSdk "github.com/conductorone/baton-sdk/pkg/types/resource" + "github.com/conductorone/baton-sdk/pkg/uhttp" "github.com/google/go-github/v69/github" + "google.golang.org/grpc/codes" ) type enterpriseRoleResourceType struct { @@ -137,8 +139,11 @@ func (o *enterpriseRoleResourceType) Grants( ret := []*v2.Grant{} for _, userLogin := range cache[resource.Id.Resource] { - user, _, err := o.client.Users.Get(ctx, userLogin) + user, resp, err := o.client.Users.Get(ctx, userLogin) if err != nil { + if isRatelimited(resp) { + return nil, "", nil, uhttp.WrapErrors(codes.Unavailable, "too many requests", err) + } return nil, "", nil, fmt.Errorf("baton-github: error getting user %s: %w", userLogin, err) } diff --git a/pkg/connector/org.go b/pkg/connector/org.go index c3614b10..cc2eec7e 100644 --- a/pkg/connector/org.go +++ b/pkg/connector/org.go @@ -102,6 +102,9 @@ func (o *orgResourceType) List( orgs, resp, err := o.client.Organizations.List(ctx, "", opts) if err != nil { + if isRatelimited(resp) { + return nil, "", nil, uhttp.WrapErrors(codes.Unavailable, "too many requests", err) + } return nil, "", nil, fmt.Errorf("github-connector: failed to fetch org: %w", err) } @@ -126,6 +129,10 @@ func (o *orgResourceType) List( l.Warn("insufficient access to list org membership, skipping org", zap.String("org", org.GetLogin())) continue } + + if isRatelimited(resp) { + return nil, "", nil, uhttp.WrapErrors(codes.Unavailable, "too many requests", err) + } return nil, "", nil, err } diff --git a/pkg/connector/org_role.go b/pkg/connector/org_role.go index 629d9f68..f6450c0b 100644 --- a/pkg/connector/org_role.go +++ b/pkg/connector/org_role.go @@ -13,9 +13,11 @@ import ( "github.com/conductorone/baton-sdk/pkg/types/entitlement" "github.com/conductorone/baton-sdk/pkg/types/grant" "github.com/conductorone/baton-sdk/pkg/types/resource" + "github.com/conductorone/baton-sdk/pkg/uhttp" "github.com/google/go-github/v69/github" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" + "google.golang.org/grpc/codes" ) type OrganizationRole struct { @@ -88,6 +90,9 @@ func (o *orgRoleResourceType) List( // Return empty list with no error to indicate we skipped this resource return nil, "", nil, nil } + if isRatelimited(resp) { + return nil, "", nil, uhttp.WrapErrors(codes.Unavailable, "too many requests", err) + } return nil, "", nil, fmt.Errorf("failed to list organization roles: %w", err) } @@ -222,6 +227,9 @@ func (o *orgRoleResourceType) Grants( } return nil, pageToken, nil, nil } + if isRatelimited(resp) { + return nil, "", nil, uhttp.WrapErrors(codes.Unavailable, "too many requests", err) + } return nil, "", nil, fmt.Errorf("failed to list role teams: %w", err) } diff --git a/pkg/connector/repository.go b/pkg/connector/repository.go index 3bae644f..12aeaed4 100644 --- a/pkg/connector/repository.go +++ b/pkg/connector/repository.go @@ -181,6 +181,9 @@ func (o *repositoryResourceType) Grants( if isNotFoundError(resp) { return nil, "", nil, uhttp.WrapErrors(codes.NotFound, fmt.Sprintf("repo: %s not found", resource.DisplayName)) } + if isRatelimited(resp) { + return nil, "", nil, uhttp.WrapErrors(codes.Unavailable, "too many requests", err) + } return nil, "", nil, fmt.Errorf("github-connector: failed to list repos: %w", err) } @@ -233,6 +236,10 @@ func (o *repositoryResourceType) Grants( if isNotFoundError(resp) { return nil, "", nil, uhttp.WrapErrors(codes.NotFound, fmt.Sprintf("repo: %s not found", resource.DisplayName)) } + + if isRatelimited(resp) { + return nil, "", nil, uhttp.WrapErrors(codes.Unavailable, "too many requests", err) + } return nil, "", nil, fmt.Errorf("github-connector: failed to list repos: %w", err) } diff --git a/pkg/connector/team.go b/pkg/connector/team.go index 6775bae6..88a13b03 100644 --- a/pkg/connector/team.go +++ b/pkg/connector/team.go @@ -95,6 +95,9 @@ func (o *teamResourceType) List(ctx context.Context, parentID *v2.ResourceId, pt teams, resp, err := o.client.Teams.ListTeams(ctx, orgName, opts) if err != nil { + if isRatelimited(resp) { + return nil, "", nil, uhttp.WrapErrors(codes.Unavailable, "too many requests", err) + } return nil, "", nil, fmt.Errorf("github-connector: failed to list teams: %w", err) } @@ -104,8 +107,11 @@ func (o *teamResourceType) List(ctx context.Context, parentID *v2.ResourceId, pt } for _, team := range teams { - fullTeam, _, err := o.client.Teams.GetTeamByID(ctx, orgID, team.GetID()) //nolint:staticcheck // TODO: migrate to GetTeamBySlug + fullTeam, resp, err := o.client.Teams.GetTeamByID(ctx, orgID, team.GetID()) //nolint:staticcheck // TODO: migrate to GetTeamBySlug if err != nil { + if isRatelimited(resp) { + return nil, "", nil, uhttp.WrapErrors(codes.Unavailable, "too many requests", err) + } return nil, "", nil, err } @@ -164,8 +170,11 @@ func (o *teamResourceType) Grants(ctx context.Context, resource *v2.Resource, pT return nil, "", nil, fmt.Errorf("error fetching orgID from team profile") } - org, _, err := o.client.Organizations.GetByID(ctx, orgID) + org, resp, err := o.client.Organizations.GetByID(ctx, orgID) if err != nil { + if isRatelimited(resp) { + return nil, "", nil, uhttp.WrapErrors(codes.Unavailable, "too many requests", err) + } return nil, "", nil, err } diff --git a/pkg/connector/user.go b/pkg/connector/user.go index e698c0b2..98d9f887 100644 --- a/pkg/connector/user.go +++ b/pkg/connector/user.go @@ -12,10 +12,12 @@ import ( "github.com/conductorone/baton-sdk/pkg/annotations" "github.com/conductorone/baton-sdk/pkg/pagination" "github.com/conductorone/baton-sdk/pkg/types/resource" + "github.com/conductorone/baton-sdk/pkg/uhttp" "github.com/google/go-github/v69/github" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "github.com/shurcooL/githubv4" "go.uber.org/zap" + "google.golang.org/grpc/codes" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -131,6 +133,9 @@ func (o *userResourceType) List(ctx context.Context, parentID *v2.ResourceId, pt users, resp, err := o.client.Organizations.ListMembers(ctx, orgName, &opts) if err != nil { + if isRatelimited(resp) { + return nil, "", nil, uhttp.WrapErrors(codes.Unavailable, "too many requests", err) + } return nil, "", nil, fmt.Errorf("github-connector: ListMembers failed: %w", err) } @@ -154,6 +159,9 @@ func (o *userResourceType) List(ctx context.Context, parentID *v2.ResourceId, pt for _, user := range users { u, res, err := o.client.Users.GetByID(ctx, user.GetID()) if err != nil { + if isRatelimited(res) { + return nil, "", nil, uhttp.WrapErrors(codes.Unavailable, "too many requests", err) + } // This undocumented API can return 404 for some users. If this fails it means we won't get some of their details like email if res == nil || res.StatusCode != http.StatusNotFound { return nil, "", nil, err