Merge pull request #19 from ConductorOne/jbernal/quickfix_limit

pagination fix

Author: aldevv

pkg/connector/client/auth0.go

@@ -97,7 +97,7 @@ func (c *Client) List(
 	path string,
 	target interface{},
 	limit int,
-	offset int,
+	page int,
 ) (
 	*v2.RateLimitDescription,
 	error,
@@ -108,7 +108,7 @@ func (c *Client) List(
 		map[string]interface{}{
 			// Note: `include_totals` changes the shape of the response
 			"include_totals": true,
-			"page":           offset,
+			"page":           page,
 			"per_page":       limit,
 		},
 		&target,
@@ -126,7 +126,7 @@ func (c *Client) List(
 func (c *Client) GetUsers(
 	ctx context.Context,
 	limit int,
-	offset int,
+	page int,
 ) (
 	[]User,
 	int,
@@ -139,7 +139,7 @@ func (c *Client) GetUsers(
 		apiPathGetUsers,
 		&target,
 		limit,
-		offset,
+		page,
 	)
 	if err != nil {
 		return nil, 0, rateLimitData, err
@@ -151,7 +151,7 @@ func (c *Client) GetUsers(
 func (c *Client) GetRoles(
 	ctx context.Context,
 	limit int,
-	offset int,
+	page int,
 ) (
 	[]Role,
 	int,
@@ -164,7 +164,7 @@ func (c *Client) GetRoles(
 		apiPathGetRoles,
 		&target,
 		limit,
-		offset,
+		page,
 	)
 	if err != nil {
 		return nil, 0, rateLimitData, err
@@ -176,7 +176,7 @@ func (c *Client) GetRoles(
 func (c *Client) GetOrganizations(
 	ctx context.Context,
 	limit int,
-	offset int,
+	page int,
 ) (
 	[]Organization,
 	int,
@@ -189,7 +189,7 @@ func (c *Client) GetOrganizations(
 		apiPathGetOrganizations,
 		&target,
 		limit,
-		offset,
+		page,
 	)
 	if err != nil {
 		return nil, 0, rateLimitData, err
@@ -202,7 +202,7 @@ func (c *Client) GetOrganizationMembers(
 	ctx context.Context,
 	organizationId string,
 	limit int,
-	offset int,
+	page int,
 ) (
 	[]User,
 	int,
@@ -215,7 +215,7 @@ func (c *Client) GetOrganizationMembers(
 		fmt.Sprintf(apiPathOrganizationMembers, organizationId),
 		&target,
 		limit,
-		offset,
+		page,
 	)
 	if err != nil {
 		return nil, 0, rateLimitData, err
@@ -228,7 +228,7 @@ func (c *Client) GetRoleUsers(
 	ctx context.Context,
 	roleId string,
 	limit int,
-	offset int,
+	page int,
 ) (
 	[]User,
 	int,
@@ -241,7 +241,7 @@ func (c *Client) GetRoleUsers(
 		fmt.Sprintf(apiPathUsersForRole, roleId),
 		&target,
 		limit,
-		offset,
+		page,
 	)
 	if err != nil {
 		return nil, 0, rateLimitData, err

pkg/connector/client/pagination.go

@@ -10,10 +10,10 @@ const PageSizeDefault = 100
 
 type Pagination struct {
 	PagingRequestId string `json:"pagingRequestId"`
-	Offset          int    `json:"offset"`
+	Page            int    `json:"page"`
 }
 
-// ParsePaginationToken - takes as pagination token and returns offset, limit,
+// ParsePaginationToken - takes as pagination token and returns page, limit,
 // and `pagingRequestId` in that order.
 func ParsePaginationToken(pToken *pagination.Token) (
 	int,
@@ -23,44 +23,44 @@ func ParsePaginationToken(pToken *pagination.Token) (
 ) {
 	var (
 		limit           = PageSizeDefault
-		offset          = 0
+		page            = 0
 		pagingRequestId = ""
 	)
 
-	if pToken != nil {
-		if pToken.Size > 0 {
-			limit = pToken.Size
-		}
+	if pToken == nil {
+		return page, limit, pagingRequestId, nil
+	}
+
+	if pToken.Size > 0 {
+		limit = pToken.Size
+	}
+
+	if pToken.Token == "" {
+		return page, limit, pagingRequestId, nil
+	}
 
-		if pToken.Token != "" {
-			var parsed Pagination
-			err := json.Unmarshal([]byte(pToken.Token), &parsed)
-			if err != nil {
-				return 0, 0, "", err
-			}
-			offset = parsed.Offset
-			pagingRequestId = parsed.PagingRequestId
-		}
+	var parsed Pagination
+	err := json.Unmarshal([]byte(pToken.Token), &parsed)
+	if err != nil {
+		return 0, 0, "", err
 	}
-	return offset, limit, pagingRequestId, nil
+
+	page = parsed.Page
+	pagingRequestId = parsed.PagingRequestId
+	return page, limit, pagingRequestId, nil
 }
 
-// GetNextToken given a limit and offset that were used to fetch _this_ page of
+// GetNextToken given a limit and page that were used to fetch _this_ page of
 // data, and total number of resources, return the next pagination token as a
 // string.
 func GetNextToken(
-	offset int,
+	page int,
 	limit int,
 	total int,
 ) string {
-	nextOffset := offset + limit
-	if nextOffset >= total {
-		return ""
-	}
-
 	bytes, err := json.Marshal(
 		Pagination{
-			Offset: nextOffset,
+			Page: page + 1,
 		},
 	)
 	if err != nil {

pkg/connector/organizations.go

@@ -64,17 +64,21 @@ func (o *organizationBuilder) List(
 	outputResources := make([]*v2.Resource, 0)
 	var outputAnnotations annotations.Annotations
 
-	offset, limit, _, err := client.ParsePaginationToken(pToken)
+	page, limit, _, err := client.ParsePaginationToken(pToken)
 	if err != nil {
 		return nil, "", nil, err
 	}
 
-	organizations, total, ratelimitData, err := o.client.GetOrganizations(ctx, limit, offset)
+	organizations, total, ratelimitData, err := o.client.GetOrganizations(ctx, limit, page)
 	outputAnnotations.WithRateLimiting(ratelimitData)
 	if err != nil {
 		return nil, "", outputAnnotations, err
 	}
 
+	if len(organizations) == 0 {
+		return outputResources, "", outputAnnotations, nil
+	}
+
 	for _, organization := range organizations {
 		organizationResource0, err := organizationResource(organization, parentResourceID)
 		if err != nil {
@@ -83,7 +87,7 @@ func (o *organizationBuilder) List(
 		outputResources = append(outputResources, organizationResource0)
 	}
 
-	nextToken := client.GetNextToken(offset, limit, total)
+	nextToken := client.GetNextToken(page, limit, total)
 
 	return outputResources, nextToken, outputAnnotations, nil
 }
@@ -124,7 +128,7 @@ func (o *organizationBuilder) Grants(
 	error,
 ) {
 	var outputAnnotations annotations.Annotations
-	offset, limit, _, err := client.ParsePaginationToken(token)
+	page, limit, _, err := client.ParsePaginationToken(token)
 	if err != nil {
 		return nil, "", nil, err
 	}
@@ -133,13 +137,17 @@ func (o *organizationBuilder) Grants(
 		ctx,
 		resource.Id.Resource,
 		limit,
-		offset,
+		page,
 	)
 	outputAnnotations.WithRateLimiting(ratelimitData)
 	if err != nil {
 		return nil, "", outputAnnotations, err
 	}
 
+	if len(members) == 0 {
+		return nil, "", outputAnnotations, nil
+	}
+
 	var grants []*v2.Grant
 	for _, member := range members {
 		principalId, err := resourceSdk.NewResourceID(userResourceType, member.UserId)
@@ -154,7 +162,7 @@ func (o *organizationBuilder) Grants(
 		grants = append(grants, nextGrant)
 	}
 
-	nextToken := client.GetNextToken(offset, limit, total)
+	nextToken := client.GetNextToken(page, limit, total)
 
 	return grants, nextToken, outputAnnotations, nil
 }

pkg/connector/roles.go

@@ -62,17 +62,21 @@ func (o *roleBuilder) List(
 	outputResources := make([]*v2.Resource, 0)
 	var outputAnnotations annotations.Annotations
 
-	offset, limit, _, err := client.ParsePaginationToken(pToken)
+	page, limit, _, err := client.ParsePaginationToken(pToken)
 	if err != nil {
 		return nil, "", nil, err
 	}
 
-	roles, total, ratelimitData, err := o.client.GetRoles(ctx, limit, offset)
+	roles, total, ratelimitData, err := o.client.GetRoles(ctx, limit, page)
 	outputAnnotations.WithRateLimiting(ratelimitData)
 	if err != nil {
 		return nil, "", outputAnnotations, err
 	}
 
+	if len(roles) == 0 {
+		return outputResources, "", outputAnnotations, nil
+	}
+
 	for _, role := range roles {
 		roleResource0, err := roleResource(role, parentResourceID)
 		if err != nil {
@@ -81,7 +85,7 @@ func (o *roleBuilder) List(
 		outputResources = append(outputResources, roleResource0)
 	}
 
-	nextToken := client.GetNextToken(offset, limit, total)
+	nextToken := client.GetNextToken(page, limit, total)
 
 	return outputResources, nextToken, outputAnnotations, nil
 }
@@ -124,7 +128,7 @@ func (o *roleBuilder) Grants(
 	error,
 ) {
 	var outputAnnotations annotations.Annotations
-	offset, limit, _, err := client.ParsePaginationToken(token)
+	page, limit, _, err := client.ParsePaginationToken(token)
 	if err != nil {
 		return nil, "", nil, err
 	}
@@ -133,13 +137,17 @@ func (o *roleBuilder) Grants(
 		ctx,
 		resource.Id.Resource,
 		limit,
-		offset,
+		page,
 	)
 	outputAnnotations.WithRateLimiting(ratelimitData)
 	if err != nil {
 		return nil, "", outputAnnotations, err
 	}
 
+	if len(users) == 0 {
+		return nil, "", outputAnnotations, nil
+	}
+
 	var grants []*v2.Grant
 	for _, user := range users {
 		principalId, err := resourceSdk.NewResourceID(userResourceType, user.UserId)
@@ -154,7 +162,7 @@ func (o *roleBuilder) Grants(
 		grants = append(grants, nextGrant)
 	}
 
-	nextToken := client.GetNextToken(offset, limit, total)
+	nextToken := client.GetNextToken(page, limit, total)
 
 	return grants, nextToken, outputAnnotations, nil
 }

pkg/connector/users.go

@@ -66,17 +66,21 @@ func (o *userBuilder) List(
 	outputResources := make([]*v2.Resource, 0)
 	var outputAnnotations annotations.Annotations
 
-	offset, limit, _, err := client.ParsePaginationToken(pToken)
+	page, limit, _, err := client.ParsePaginationToken(pToken)
 	if err != nil {
 		return nil, "", nil, err
 	}
 
-	users, total, ratelimitData, err := o.client.GetUsers(ctx, limit, offset)
+	users, total, ratelimitData, err := o.client.GetUsers(ctx, limit, page)
 	outputAnnotations.WithRateLimiting(ratelimitData)
 	if err != nil {
 		return nil, "", outputAnnotations, err
 	}
 
+	if len(users) == 0 {
+		return outputResources, "", outputAnnotations, nil
+	}
+
 	for _, user := range users {
 		userResource0, err := userResource(user, parentResourceID)
 		if err != nil {
@@ -87,7 +91,7 @@ func (o *userBuilder) List(
 
 	// TODO(marcos): it might never be possible to get a second page if we are limited to 1,000 results.
 	// See https://auth0.com/docs/users/search/v3/view-search-results-by-page#limitation.
-	nextToken := client.GetNextToken(offset, limit, total)
+	nextToken := client.GetNextToken(page, limit, total)
 
 	return outputResources, nextToken, outputAnnotations, nil
 }

pkg/connector/users_test.go

@@ -2,6 +2,7 @@ package connector
 
 import (
 	"context"
+	"encoding/json"
 	"testing"
 
 	"github.com/conductorone/baton-auth0/pkg/connector/client"
@@ -35,15 +36,17 @@ func TestUsersList(t *testing.T) {
 			Token: "",
 			Size:  1,
 		}
-		for {
+
+		for i := 0; i < 2; i++ {
 			nextResources, nextToken, listAnnotations, err := c.List(ctx, nil, &pToken)
 			resources = append(resources, nextResources...)
 
 			require.Nil(t, err)
 			test.AssertNoRatelimitAnnotations(t, listAnnotations)
-			if nextToken == "" {
-				break
-			}
+			var token client.Pagination
+			err = json.Unmarshal([]byte(nextToken), &token)
+			require.Nil(t, err)
+			require.Equal(t, token.Page, i+1)
 
 			pToken.Token = nextToken
 		}