Allow paging to be optionally disabled (ktrysmt#192)

- Added a new option, `DisableAutoPaging`. This defaults to false for backwards compatibility, but allows the behaviour of automatically requesting all pages of a paged response to be disabled.
- While doing this I split the `execute()` method into two versions - one that supports paging and another that doesn't. The reason I did this is that the previous version was adding the `pagelen` parameter for any paths containing `/repositories/`, but this breaks if you call `Repositories.ListForAccount()` with a role specified, in which case there's no trailing `/`. I figured it was better to be explicit about when to support paging or not.
- Moved the `max_depth` support out of the client and into `Repository.ListFiles()`, which is where it's actually supported. I noticed that `Repository.ListBranches()` also specifies the max depth. It doesn't look like this is actually supported by the API to me, but I didn't want to risk breaking it in case I'm wrong.
- Fixed a mistake in `GetCommitContents()` where it was using a `DELETE` instead of a `GET`.

Author: adamconnelly

branchrestrictions.go

@@ -17,7 +17,7 @@ type BranchRestrictions struct {
 
 func (b *BranchRestrictions) Gets(bo *BranchRestrictionsOptions) (interface{}, error) {
 	urlStr := b.c.requestUrl("/repositories/%s/%s/branch-restrictions", bo.Owner, bo.RepoSlug)
-	return b.c.execute("GET", urlStr, "")
+	return b.c.executePaginated("GET", urlStr, "")
 }
 
 func (b *BranchRestrictions) Create(bo *BranchRestrictionsOptions) (*BranchRestrictions, error) {

client.go

@@ -40,9 +40,12 @@ type Client struct {
 	Teams        teams
 	Repositories *Repositories
 	Workspaces   *Workspace
-	Pagelen      uint64
-	MaxDepth     uint64
-	apiBaseURL   *url.URL
+	Pagelen      int
+	MaxDepth     int
+	// DisableAutoPaging allows you to disable the default behavior of automatically requesting
+	// all the pages for a paginated response.
+	DisableAutoPaging bool
+	apiBaseURL        *url.URL
 
 	HttpClient *http.Client
 }
@@ -201,34 +204,37 @@ func (c *Client) executeRaw(method string, urlStr string, text string) (io.ReadC
 }
 
 func (c *Client) execute(method string, urlStr string, text string) (interface{}, error) {
-	// Use pagination if changed from default value
-	const DEC_RADIX = 10
-	if strings.Contains(urlStr, "/repositories/") {
-		if c.Pagelen != DEFAULT_PAGE_LENGTH {
-			urlObj, err := url.Parse(urlStr)
-			if err != nil {
-				return nil, err
-			}
-			q := urlObj.Query()
-			q.Set("pagelen", strconv.FormatUint(c.Pagelen, DEC_RADIX))
-			urlObj.RawQuery = q.Encode()
-			urlStr = urlObj.String()
-		}
+	body := strings.NewReader(text)
+	req, err := http.NewRequest(method, urlStr, body)
+	if err != nil {
+		return nil, err
+	}
+	if text != "" {
+		req.Header.Set("Content-Type", "application/json")
+	}
 
-		if c.MaxDepth != DEFAULT_MAX_DEPTH {
-			urlObj, err := url.Parse(urlStr)
-			if err != nil {
-				return nil, err
-			}
-			q := urlObj.Query()
-			q.Set("max_depth", strconv.FormatUint(c.MaxDepth, DEC_RADIX))
-			urlObj.RawQuery = q.Encode()
-			urlStr = urlObj.String()
+	c.authenticateRequest(req)
+	result, err := c.doRequest(req, false)
+	if err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}
+
+func (c *Client) executePaginated(method string, urlStr string, text string) (interface{}, error) {
+	if c.Pagelen != DEFAULT_PAGE_LENGTH {
+		urlObj, err := url.Parse(urlStr)
+		if err != nil {
+			return nil, err
 		}
+		q := urlObj.Query()
+		q.Set("pagelen", strconv.Itoa(c.Pagelen))
+		urlObj.RawQuery = q.Encode()
+		urlStr = urlObj.String()
 	}
 
 	body := strings.NewReader(text)
-
 	req, err := http.NewRequest(method, urlStr, body)
 	if err != nil {
 		return nil, err
@@ -238,7 +244,7 @@ func (c *Client) execute(method string, urlStr string, text string) (interface{}
 	}
 
 	c.authenticateRequest(req)
-	result, err := c.doRequest(req, false)
+	result, err := c.doPaginatedRequest(req, false)
 	if err != nil {
 		return nil, err
 	}
@@ -318,13 +324,37 @@ func (c *Client) doRequest(req *http.Request, emptyResponse bool) (interface{},
 		return resBody, err
 	}
 
+	var result interface{}
+	if err := json.Unmarshal(responseBytes, &result); err != nil {
+		log.Println("Could not unmarshal JSON payload, returning raw response")
+		return resBody, err
+	}
+	return result, nil
+}
+
+func (c *Client) doPaginatedRequest(req *http.Request, emptyResponse bool) (interface{}, error) {
+	resBody, err := c.doRawRequest(req, emptyResponse)
+	if err != nil {
+		return nil, err
+	}
+	if emptyResponse || resBody == nil {
+		return nil, nil
+	}
+
+	defer resBody.Close()
+
+	responseBytes, err := ioutil.ReadAll(resBody)
+	if err != nil {
+		return resBody, err
+	}
+
 	responsePaginated := &Response{}
 	err = json.Unmarshal(responseBytes, responsePaginated)
 	if err == nil && len(responsePaginated.Values) > 0 {
 		var values []interface{}
 		for {
 			values = append(values, responsePaginated.Values...)
-			if len(responsePaginated.Next) == 0 {
+			if c.DisableAutoPaging || len(responsePaginated.Next) == 0 {
 				break
 			}
 			newReq, err := http.NewRequest(req.Method, responsePaginated.Next, nil)
@@ -407,3 +437,14 @@ func (c *Client) requestUrl(template string, args ...interface{}) string {
 	}
 	return c.GetApiBaseURL() + fmt.Sprintf(template, args...)
 }
+
+func (c *Client) addMaxDepthParam(params *url.Values, customMaxDepth *int) {
+	maxDepth := c.MaxDepth
+	if customMaxDepth != nil && *customMaxDepth > 0 {
+		maxDepth = *customMaxDepth
+	}
+
+	if maxDepth != DEFAULT_MAX_DEPTH {
+		params.Set("max_depth", strconv.Itoa(maxDepth))
+	}
+}

commits.go

@@ -1,8 +1,8 @@
 package bitbucket
 
 import (
-	"net/url"
 	"encoding/json"
+	"net/url"
 )
 
 type Commits struct {
@@ -12,7 +12,7 @@ type Commits struct {
 func (cm *Commits) GetCommits(cmo *CommitsOptions) (interface{}, error) {
 	urlStr := cm.c.requestUrl("/repositories/%s/%s/commits/%s", cmo.Owner, cmo.RepoSlug, cmo.Branchortag)
 	urlStr += cm.buildCommitsQuery(cmo.Include, cmo.Exclude)
-	return cm.c.execute("GET", urlStr, "")
+	return cm.c.executePaginated("GET", urlStr, "")
 }
 
 func (cm *Commits) GetCommit(cmo *CommitsOptions) (interface{}, error) {
@@ -22,7 +22,7 @@ func (cm *Commits) GetCommit(cmo *CommitsOptions) (interface{}, error) {
 
 func (cm *Commits) GetCommitComments(cmo *CommitsOptions) (interface{}, error) {
 	urlStr := cm.c.requestUrl("/repositories/%s/%s/commit/%s/comments", cmo.Owner, cmo.RepoSlug, cmo.Revision)
-	return cm.c.execute("DELETE", urlStr, "")
+	return cm.c.executePaginated("GET", urlStr, "")
 }
 
 func (cm *Commits) GetCommitComment(cmo *CommitsOptions) (interface{}, error) {
@@ -32,7 +32,7 @@ func (cm *Commits) GetCommitComment(cmo *CommitsOptions) (interface{}, error) {
 
 func (cm *Commits) GetCommitStatuses(cmo *CommitsOptions) (interface{}, error) {
 	urlStr := cm.c.requestUrl("/repositories/%s/%s/commit/%s/statuses", cmo.Owner, cmo.RepoSlug, cmo.Revision)
-	return cm.c.execute("GET", urlStr, "")
+	return cm.c.executePaginated("GET", urlStr, "")
 }
 
 func (cm *Commits) GetCommitStatus(cmo *CommitsOptions, commitStatusKey string) (interface{}, error) {
@@ -50,7 +50,6 @@ func (cm *Commits) RemoveApprove(cmo *CommitsOptions) (interface{}, error) {
 	return cm.c.execute("DELETE", urlStr, "")
 }
 
-
 func (cm *Commits) CreateCommitStatus(cmo *CommitsOptions, cso *CommitStatusOptions) (interface{}, error) {
 	urlStr := cm.c.requestUrl("/repositories/%s/%s/commit/%s/statuses/build", cmo.Owner, cmo.RepoSlug, cmo.Revision)
 	data, err := json.Marshal(cso)
@@ -60,7 +59,6 @@ func (cm *Commits) CreateCommitStatus(cmo *CommitsOptions, cso *CommitStatusOpti
 	return cm.c.execute("POST", urlStr, string(data))
 }
 
-
 func (cm *Commits) buildCommitsQuery(include, exclude string) string {
 
 	p := url.Values{}

downloads.go

@@ -11,5 +11,5 @@ func (dl *Downloads) Create(do *DownloadsOptions) (interface{}, error) {
 
 func (dl *Downloads) List(do *DownloadsOptions) (interface{}, error) {
 	urlStr := dl.c.requestUrl("/repositories/%s/%s/downloads", do.Owner, do.RepoSlug)
-	return dl.c.execute("GET", urlStr, "")
+	return dl.c.executePaginated("GET", urlStr, "")
 }

issues.go

@@ -37,7 +37,7 @@ func (p *Issues) Gets(io *IssuesOptions) (interface{}, error) {
 		url.RawQuery = query.Encode()
 	}
 
-	return p.c.execute("GET", url.String(), "")
+	return p.c.executePaginated("GET", url.String(), "")
 }
 
 func (p *Issues) Get(io *IssuesOptions) (interface{}, error) {

pipelines.go

@@ -46,7 +46,7 @@ func (p *Pipelines) List(po *PipelinesOptions) (interface{}, error) {
 		urlStr = parsed.String()
 	}
 
-	return p.c.execute("GET", urlStr, "")
+	return p.c.executePaginated("GET", urlStr, "")
 }
 
 func (p *Pipelines) Get(po *PipelinesOptions) (interface{}, error) {
@@ -90,7 +90,7 @@ func (p *Pipelines) ListSteps(po *PipelinesOptions) (interface{}, error) {
 		urlStr = parsed.String()
 	}
 
-	return p.c.execute("GET", urlStr, "")
+	return p.c.executePaginated("GET", urlStr, "")
 }
 
 func (p *Pipelines) GetStep(po *PipelinesOptions) (interface{}, error) {

pullrequests.go

@@ -65,7 +65,7 @@ func (p *PullRequests) Gets(po *PullRequestsOptions) (interface{}, error) {
 		urlStr = parsed.String()
 	}
 
-	return p.c.execute("GET", urlStr, "")
+	return p.c.executePaginated("GET", urlStr, "")
 }
 
 func (p *PullRequests) Get(po *PullRequestsOptions) (interface{}, error) {
@@ -75,7 +75,7 @@ func (p *PullRequests) Get(po *PullRequestsOptions) (interface{}, error) {
 
 func (p *PullRequests) Activities(po *PullRequestsOptions) (interface{}, error) {
 	urlStr := p.c.GetApiBaseURL() + "/repositories/" + po.Owner + "/" + po.RepoSlug + "/pullrequests/activity"
-	return p.c.execute("GET", urlStr, "")
+	return p.c.executePaginated("GET", urlStr, "")
 }
 
 func (p *PullRequests) Activity(po *PullRequestsOptions) (interface{}, error) {
@@ -85,7 +85,7 @@ func (p *PullRequests) Activity(po *PullRequestsOptions) (interface{}, error) {
 
 func (p *PullRequests) Commits(po *PullRequestsOptions) (interface{}, error) {
 	urlStr := p.c.GetApiBaseURL() + "/repositories/" + po.Owner + "/" + po.RepoSlug + "/pullrequests/" + po.ID + "/commits"
-	return p.c.execute("GET", urlStr, "")
+	return p.c.executePaginated("GET", urlStr, "")
 }
 
 func (p *PullRequests) Patch(po *PullRequestsOptions) (interface{}, error) {
@@ -158,7 +158,7 @@ func (p *PullRequests) UpdateComment(co *PullRequestCommentOptions) (interface{}
 
 func (p *PullRequests) GetComments(po *PullRequestsOptions) (interface{}, error) {
 	urlStr := p.c.GetApiBaseURL() + "/repositories/" + po.Owner + "/" + po.RepoSlug + "/pullrequests/" + po.ID + "/comments/"
-	return p.c.execute("GET", urlStr, "")
+	return p.c.executePaginated("GET", urlStr, "")
 }
 
 func (p *PullRequests) GetComment(po *PullRequestsOptions) (interface{}, error) {
@@ -189,7 +189,7 @@ func (p *PullRequests) Statuses(po *PullRequestsOptions) (interface{}, error) {
 		parsed.RawQuery = query.Encode()
 		urlStr = parsed.String()
 	}
-	return p.c.execute("GET", urlStr, "")
+	return p.c.executePaginated("GET", urlStr, "")
 }
 
 func (p *PullRequests) buildPullRequestBody(po *PullRequestsOptions) (string, error) {

repositories.go

@@ -25,11 +25,10 @@ type Repositories struct {
 }
 
 type RepositoriesRes struct {
-	Page     int32
-	Pagelen  int32
-	MaxDepth int32
-	Size     int32
-	Items    []Repository
+	Page    int32
+	Pagelen int32
+	Size    int32
+	Items   []Repository
 }
 
 func (r *Repositories) ListForAccount(ro *RepositoriesOptions) (*RepositoriesRes, error) {
@@ -41,7 +40,7 @@ func (r *Repositories) ListForAccount(ro *RepositoriesOptions) (*RepositoriesRes
 	if ro.Role != "" {
 		urlStr += "?role=" + ro.Role
 	}
-	repos, err := r.c.execute("GET", urlStr, "")
+	repos, err := r.c.executePaginated("GET", urlStr, "")
 	if err != nil {
 		return nil, err
 	}
@@ -55,7 +54,7 @@ func (r *Repositories) ListForTeam(ro *RepositoriesOptions) (*RepositoriesRes, e
 
 func (r *Repositories) ListPublic() (*RepositoriesRes, error) {
 	urlStr := r.c.requestUrl("/repositories/")
-	repos, err := r.c.execute("GET", urlStr, "")
+	repos, err := r.c.executePaginated("GET", urlStr, "")
 	if err != nil {
 		return nil, err
 	}
@@ -87,21 +86,16 @@ func decodeRepositories(reposResponse interface{}) (*RepositoriesRes, error) {
 	if !ok {
 		pagelen = 0
 	}
-	max_depth, ok := reposResponseMap["max_width"].(float64)
-	if !ok {
-		max_depth = 0
-	}
 	size, ok := reposResponseMap["size"].(float64)
 	if !ok {
 		size = 0
 	}
 
 	repositories := RepositoriesRes{
-		Page:     int32(page),
-		Pagelen:  int32(pagelen),
-		MaxDepth: int32(max_depth),
-		Size:     int32(size),
-		Items:    repos,
+		Page:    int32(page),
+		Pagelen: int32(pagelen),
+		Size:    int32(size),
+		Items:   repos,
 	}
 	return &repositories, nil
 }

repository.go

@@ -243,8 +243,20 @@ func (r *Repository) Get(ro *RepositoryOptions) (*Repository, error) {
 
 func (r *Repository) ListFiles(ro *RepositoryFilesOptions) ([]RepositoryFile, error) {
 	filePath := path.Join("/repositories", ro.Owner, ro.RepoSlug, "src", ro.Ref, ro.Path) + "/"
+
 	urlStr := r.c.requestUrl(filePath)
-	response, err := r.c.execute("GET", urlStr, "")
+	url, err := url.Parse(urlStr)
+	if err != nil {
+		return nil, err
+	}
+
+	query := url.Query()
+	r.c.addMaxDepthParam(&query, nil)
+	url.RawQuery = query.Encode()
+
+	urlStr = url.String()
+
+	response, err := r.c.executePaginated("GET", urlStr, "")
 	if err != nil {
 		return nil, err
 	}
@@ -349,7 +361,7 @@ func (r *Repository) ListBranches(rbo *RepositoryBranchOptions) (*RepositoryBran
 	}
 
 	if rbo.MaxDepth > 0 {
-		params.Add("max_depth", strconv.Itoa(rbo.MaxDepth))
+		r.c.addMaxDepthParam(&params, &rbo.MaxDepth)
 	}
 
 	urlStr := r.c.requestUrl("/repositories/%s/%s/refs/branches?%s", rbo.Owner, rbo.RepoSlug, params.Encode())
@@ -503,18 +515,18 @@ func (r *Repository) Delete(ro *RepositoryOptions) (interface{}, error) {
 
 func (r *Repository) ListWatchers(ro *RepositoryOptions) (interface{}, error) {
 	urlStr := r.c.requestUrl("/repositories/%s/%s/watchers", ro.Owner, ro.RepoSlug)
-	return r.c.execute("GET", urlStr, "")
+	return r.c.executePaginated("GET", urlStr, "")
 }
 
 func (r *Repository) ListForks(ro *RepositoryOptions) (interface{}, error) {
 	urlStr := r.c.requestUrl("/repositories/%s/%s/forks", ro.Owner, ro.RepoSlug)
-	return r.c.execute("GET", urlStr, "")
+	return r.c.executePaginated("GET", urlStr, "")
 }
 
 func (r *Repository) ListDefaultReviewers(ro *RepositoryOptions) (*DefaultReviewers, error) {
 	urlStr := r.c.requestUrl("/repositories/%s/%s/default-reviewers?pagelen=1", ro.Owner, ro.RepoSlug)
 
-	res, err := r.c.execute("GET", urlStr, "")
+	res, err := r.c.executePaginated("GET", urlStr, "")
 	if err != nil {
 		return nil, err
 	}