diff --git a/go.mod b/go.mod index d4986fa8..94b0f470 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.22.3 require ( github.com/cloudflare/cloudflare-go v0.102.0 - github.com/conductorone/baton-sdk v0.2.26 + github.com/conductorone/baton-sdk v0.2.31 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go.uber.org/zap v1.27.0 ) diff --git a/go.sum b/go.sum index f2833b5e..b25d0c58 100644 --- a/go.sum +++ b/go.sum @@ -54,8 +54,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/cloudflare-go v0.102.0 h1:+0MGbkirM/yzVLOYpWMgW7CDdKzesSbdwA2Y+rABrWI= github.com/cloudflare/cloudflare-go v0.102.0/go.mod h1:BOB41tXf31ti/qtBO9paYhyapotQbGRDbQoLOAF7pSg= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/conductorone/baton-sdk v0.2.26 h1:nU/GinAhY8OvxrWuOIFKVsQ4QkcDI0b42+bSAiHPJtw= -github.com/conductorone/baton-sdk v0.2.26/go.mod h1:hmd/Oz3DPIKD+9QmkusZaA18ZoiinnTDdrxh2skcdUc= +github.com/conductorone/baton-sdk v0.2.31 h1:yBFZP0F+1Qu0BoRWUXAB91qMst2SQZ676bDabEGIWZE= +github.com/conductorone/baton-sdk v0.2.31/go.mod h1:hmd/Oz3DPIKD+9QmkusZaA18ZoiinnTDdrxh2skcdUc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/connectorbuilder.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/connectorbuilder.go index 2e4d6abc..8bd090d7 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/connectorbuilder.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/connectorbuilder.go @@ -373,21 +373,22 @@ func (b *builderImpl) ListResources(ctx context.Context, request *v2.ResourcesSe Size: int(request.PageSize), Token: request.PageToken, }) + resp := &v2.ResourcesServiceListResourcesResponse{ + List: out, + NextPageToken: nextPageToken, + Annotations: annos, + } if err != nil { b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start)) - return nil, fmt.Errorf("error: listing resources failed: %w", err) + return resp, fmt.Errorf("error: listing resources failed: %w", err) } if request.PageToken != "" && request.PageToken == nextPageToken { b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start)) - return nil, fmt.Errorf("error: listing resources failed: next page token is the same as the current page token. this is most likely a connector bug") + return resp, fmt.Errorf("error: listing resources failed: next page token is the same as the current page token. this is most likely a connector bug") } b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) - return &v2.ResourcesServiceListResourcesResponse{ - List: out, - NextPageToken: nextPageToken, - Annotations: annos, - }, nil + return resp, nil } // ListEntitlements returns all the entitlements for a given resource. @@ -404,21 +405,22 @@ func (b *builderImpl) ListEntitlements(ctx context.Context, request *v2.Entitlem Size: int(request.PageSize), Token: request.PageToken, }) + resp := &v2.EntitlementsServiceListEntitlementsResponse{ + List: out, + NextPageToken: nextPageToken, + Annotations: annos, + } if err != nil { b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start)) - return nil, fmt.Errorf("error: listing entitlements failed: %w", err) + return resp, fmt.Errorf("error: listing entitlements failed: %w", err) } if request.PageToken != "" && request.PageToken == nextPageToken { b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start)) - return nil, fmt.Errorf("error: listing entitlements failed: next page token is the same as the current page token. this is most likely a connector bug") + return resp, fmt.Errorf("error: listing entitlements failed: next page token is the same as the current page token. this is most likely a connector bug") } b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) - return &v2.EntitlementsServiceListEntitlementsResponse{ - List: out, - NextPageToken: nextPageToken, - Annotations: annos, - }, nil + return resp, nil } // ListGrants lists all the grants for a given resource. @@ -436,23 +438,24 @@ func (b *builderImpl) ListGrants(ctx context.Context, request *v2.GrantsServiceL Size: int(request.PageSize), Token: request.PageToken, }) + resp := &v2.GrantsServiceListGrantsResponse{ + List: out, + NextPageToken: nextPageToken, + Annotations: annos, + } if err != nil { b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start)) - return nil, fmt.Errorf("error: listing grants for resource %s/%s failed: %w", rid.ResourceType, rid.Resource, err) + return resp, fmt.Errorf("error: listing grants for resource %s/%s failed: %w", rid.ResourceType, rid.Resource, err) } if request.PageToken != "" && request.PageToken == nextPageToken { b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start)) - return nil, fmt.Errorf("error: listing grants for resource %s/%s failed: next page token is the same as the current page token. this is most likely a connector bug", + return resp, fmt.Errorf("error: listing grants for resource %s/%s failed: next page token is the same as the current page token. this is most likely a connector bug", rid.ResourceType, rid.Resource) } b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) - return &v2.GrantsServiceListGrantsResponse{ - List: out, - NextPageToken: nextPageToken, - Annotations: annos, - }, nil + return resp, nil } // GetMetadata gets all metadata for a connector. diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/decoder.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/decoder.go index dcef25d9..e2360c68 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/decoder.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/decoder.go @@ -167,6 +167,7 @@ func (d *decoder) Read(p []byte) (int, error) { // Do underlying read n, err := d.zd.Read(p) + //nolint:gosec // No risk of overflow/underflow because n is always >= 0. d.decodedBytes += uint64(n) if err != nil { // NOTE(morgabra) This happens if you set a small DecoderMaxMemory diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sql_helpers.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sql_helpers.go index 3bd3fb99..968e9277 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sql_helpers.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sql_helpers.go @@ -183,7 +183,7 @@ func (c *C1File) listConnectorObjects(ctx context.Context, tableName string, req } // Clamp the page size - pageSize := int(listReq.GetPageSize()) + pageSize := listReq.GetPageSize() if pageSize > maxPageSize || pageSize == 0 { pageSize = maxPageSize } @@ -206,7 +206,7 @@ func (c *C1File) listConnectorObjects(ctx context.Context, tableName string, req } defer rows.Close() - count := 0 + var count uint32 = 0 lastRow := 0 for rows.Next() { count++ diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_runs.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_runs.go index ed0792f7..9d1f7a79 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_runs.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_runs.go @@ -120,7 +120,7 @@ func (c *C1File) getFinishedSync(ctx context.Context, offset uint) (*syncRun, er return ret, nil } -func (c *C1File) ListSyncRuns(ctx context.Context, pageToken string, pageSize int) ([]*syncRun, string, error) { +func (c *C1File) ListSyncRuns(ctx context.Context, pageToken string, pageSize uint) ([]*syncRun, string, error) { err := c.validateDb(ctx) if err != nil { return nil, "", err @@ -138,7 +138,7 @@ func (c *C1File) ListSyncRuns(ctx context.Context, pageToken string, pageSize in } q = q.Order(goqu.C("id").Asc()) - q = q.Limit(uint(pageSize + 1)) + q = q.Limit(pageSize + 1) var ret []*syncRun @@ -153,7 +153,7 @@ func (c *C1File) ListSyncRuns(ctx context.Context, pageToken string, pageSize in } defer rows.Close() - count := 0 + var count uint = 0 lastRow := 0 for rows.Next() { count++ diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go b/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go index ca5b4368..1bc73ffc 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go @@ -1,3 +1,3 @@ package sdk -const Version = "v0.2.25" +const Version = "v0.2.30" diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go index dc97a73b..63ad39dd 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io" + "math" "os" "strconv" "time" @@ -92,16 +93,41 @@ func shouldWaitAndRetry(ctx context.Context, err error) bool { attempts = 0 return true } - if status.Code(err) != codes.Unavailable { + if status.Code(err) != codes.Unavailable && status.Code(err) != codes.DeadlineExceeded { return false } attempts++ l := ctxzap.Extract(ctx) + // use linear time by default var wait time.Duration = time.Duration(attempts) * time.Second - l.Error("retrying operation", zap.Error(err), zap.Duration("wait", wait)) + // If error contains rate limit data, use that instead + if st, ok := status.FromError(err); ok { + details := st.Details() + for _, detail := range details { + if rlData, ok := detail.(*v2.RateLimitDescription); ok { + waitResetAt := time.Until(rlData.ResetAt.AsTime()) + if waitResetAt <= 0 { + continue + } + duration := time.Duration(rlData.Limit) + if duration <= 0 { + continue + } + waitResetAt /= duration + // Round up to the nearest second to make sure we don't hit the rate limit again + waitResetAt = time.Duration(math.Ceil(waitResetAt.Seconds())) * time.Second + if waitResetAt > 0 { + wait = waitResetAt + break + } + } + } + } + + l.Warn("retrying operation", zap.Error(err), zap.Duration("wait", wait)) for { select { @@ -235,7 +261,7 @@ func (s *syncer) Sync(ctx context.Context) error { case SyncAssetsOp: err = s.SyncAssets(ctx) - if err != nil { + if !shouldWaitAndRetry(ctx, err) { return err } continue @@ -786,7 +812,7 @@ func (s *syncer) SyncGrantExpansion(ctx context.Context) error { pageToken := s.state.PageToken(ctx) if pageToken == "" { - ctxzap.Extract(ctx).Info("Expanding grants...") + l.Info("Expanding grants...") s.handleInitialActionForStep(ctx, *s.state.Current()) } @@ -823,7 +849,7 @@ func (s *syncer) SyncGrantExpansion(ctx context.Context) error { // FIXME(morgabra) Log and skip some of the error paths here? for _, srcEntitlementID := range expandable.EntitlementIds { - ctxzap.Extract(ctx).Debug( + l.Debug( "Expandable entitlement found", zap.String("src_entitlement_id", srcEntitlementID), zap.String("dst_entitlement_id", grant.GetEntitlement().GetId()), @@ -833,7 +859,12 @@ func (s *syncer) SyncGrantExpansion(ctx context.Context) error { EntitlementId: srcEntitlementID, }) if err != nil { - return err + l.Error("error fetching source entitlement", + zap.String("src_entitlement_id", srcEntitlementID), + zap.String("dst_entitlement_id", grant.GetEntitlement().GetId()), + zap.Error(err), + ) + continue } // The expand annotation points at entitlements by id. Those entitlements' resource should match @@ -1325,7 +1356,7 @@ func (s *syncer) runGrantExpandActions(ctx context.Context) (bool, error) { return false, nil } -func (s *syncer) newExpandedGrant(ctx context.Context, descEntitlement *v2.Entitlement, principal *v2.Resource) (*v2.Grant, error) { +func (s *syncer) newExpandedGrant(_ context.Context, descEntitlement *v2.Entitlement, principal *v2.Resource) (*v2.Grant, error) { enResource := descEntitlement.GetResource() if enResource == nil { return nil, fmt.Errorf("newExpandedGrant: entitlement has no resource") diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/types/grant/grant.go b/vendor/github.com/conductorone/baton-sdk/pkg/types/grant/grant.go index 25ff9ede..e59536c9 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/types/grant/grant.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/types/grant/grant.go @@ -17,6 +17,9 @@ type GrantPrincipal interface { GetBatonResource() bool } +// Sometimes C1 doesn't have the grant ID, but does have the principal and entitlement. +const UnknownGrantId string = "🧸_UNKNOWN_GRANT_ID" + func WithGrantMetadata(metadata map[string]interface{}) GrantOption { return func(g *v2.Grant) error { md, err := structpb.NewStruct(metadata) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/gocache.go b/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/gocache.go index 2cbbde41..c29d4296 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/gocache.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/gocache.go @@ -37,7 +37,17 @@ func NewGoCache(ctx context.Context, cfg CacheConfig) (GoCache, error) { return GoCache{}, err } - l.Debug("http cache config", zap.Any("config", config)) + l.Debug("http cache config", + zap.Dict("config", + zap.Int("Shards", config.Shards), + zap.Duration("LifeWindow", config.LifeWindow), + zap.Duration("CleanWindow", config.CleanWindow), + zap.Int("MaxEntriesInWindow", config.MaxEntriesInWindow), + zap.Int("MaxEntrySize", config.MaxEntrySize), + zap.Bool("StatsEnabled", config.StatsEnabled), + zap.Bool("Verbose", config.Verbose), + zap.Int("HardMaxCacheSize", config.HardMaxCacheSize), + )) gc := GoCache{ rootLibrary: cache, } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/transport.go b/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/transport.go index eb08761c..4fd56e47 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/transport.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/transport.go @@ -85,7 +85,7 @@ func (uat *userAgentTripper) RoundTrip(req *http.Request) (*http.Response, error return uat.next.RoundTrip(req) } -func (t *Transport) make(ctx context.Context) (http.RoundTripper, error) { +func (t *Transport) make(_ context.Context) (http.RoundTripper, error) { // based on http.DefaultTransport baseTransport := &http.Transport{ Proxy: http.ProxyFromEnvironment, diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/wrapper.go b/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/wrapper.go index c658210f..7c393f1d 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/wrapper.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/wrapper.go @@ -213,6 +213,23 @@ func WithResponse(response interface{}) DoOption { } } +func WrapErrorsWithRateLimitInfo(preferredCode codes.Code, resp *http.Response, errs ...error) error { + st := status.New(preferredCode, resp.Status) + + description, err := ratelimit.ExtractRateLimitData(resp.StatusCode, &resp.Header) + // Ignore any error extracting rate limit data + if err == nil { + st, _ = st.WithDetails(description) + } + + if len(errs) == 0 { + return st.Err() + } + + allErrs := append([]error{st.Err()}, errs...) + return errors.Join(allErrs...) +} + func (c *BaseHttpClient) Do(req *http.Request, options ...DoOption) (*http.Response, error) { var ( cacheKey string @@ -268,41 +285,46 @@ func (c *BaseHttpClient) Do(req *http.Request, options ...DoOption) (*http.Respo StatusCode: resp.StatusCode, Body: body, } + + var optErrs []error for _, option := range options { - err = option(&wresp) - if err != nil { - return resp, err + optErr := option(&wresp) + if optErr != nil { + optErrs = append(optErrs, optErr) } } switch resp.StatusCode { case http.StatusRequestTimeout: - return resp, status.Error(codes.DeadlineExceeded, resp.Status) - case http.StatusTooManyRequests: - return resp, status.Error(codes.Unavailable, resp.Status) + return resp, WrapErrorsWithRateLimitInfo(codes.DeadlineExceeded, resp, optErrs...) + case http.StatusTooManyRequests, http.StatusServiceUnavailable: + return resp, WrapErrorsWithRateLimitInfo(codes.Unavailable, resp, optErrs...) case http.StatusNotFound: - return resp, status.Error(codes.NotFound, resp.Status) + return resp, WrapErrorsWithRateLimitInfo(codes.NotFound, resp, optErrs...) case http.StatusUnauthorized: - return resp, status.Error(codes.Unauthenticated, resp.Status) + return resp, WrapErrorsWithRateLimitInfo(codes.Unauthenticated, resp, optErrs...) case http.StatusForbidden: - return resp, status.Error(codes.PermissionDenied, resp.Status) + return resp, WrapErrorsWithRateLimitInfo(codes.PermissionDenied, resp, optErrs...) case http.StatusNotImplemented: - return resp, status.Error(codes.Unimplemented, resp.Status) + return resp, WrapErrorsWithRateLimitInfo(codes.Unimplemented, resp, optErrs...) + } + + if resp.StatusCode >= 500 && resp.StatusCode <= 599 { + return resp, WrapErrorsWithRateLimitInfo(codes.Unavailable, resp, optErrs...) } if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return resp, status.Error(codes.Unknown, fmt.Sprintf("unexpected status code: %d", resp.StatusCode)) + return resp, WrapErrorsWithRateLimitInfo(codes.Unknown, resp, append(optErrs, fmt.Errorf("unexpected status code: %d", resp.StatusCode))...) } if req.Method == http.MethodGet && resp.StatusCode == http.StatusOK { - err := c.baseHttpCache.Set(cacheKey, resp) - if err != nil { - l.Debug("error setting cache", zap.String("cacheKey", cacheKey), zap.String("url", req.URL.String()), zap.Error(err)) - return resp, err + cacheErr := c.baseHttpCache.Set(cacheKey, resp) + if cacheErr != nil { + l.Warn("error setting cache", zap.String("cacheKey", cacheKey), zap.String("url", req.URL.String()), zap.Error(cacheErr)) } } - return resp, err + return resp, errors.Join(optErrs...) } func WithHeader(key, value string) RequestOption { diff --git a/vendor/modules.txt b/vendor/modules.txt index f8f3739e..7394066c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -147,7 +147,7 @@ github.com/benbjohnson/clock # github.com/cloudflare/cloudflare-go v0.102.0 ## explicit; go 1.19 github.com/cloudflare/cloudflare-go -# github.com/conductorone/baton-sdk v0.2.26 +# github.com/conductorone/baton-sdk v0.2.31 ## explicit; go 1.21 github.com/conductorone/baton-sdk/internal/connector github.com/conductorone/baton-sdk/pb/c1/c1z/v1