Switch pagination method and add test

Author: denyskon

routers/api/v1/repo/file.go

@@ -25,9 +25,7 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/storage"
 	api "code.gitea.io/gitea/modules/structs"
-	"code.gitea.io/gitea/modules/util"
 	"code.gitea.io/gitea/modules/web"
-	"code.gitea.io/gitea/routers/api/v1/utils"
 	"code.gitea.io/gitea/routers/common"
 	"code.gitea.io/gitea/services/context"
 	pull_service "code.gitea.io/gitea/services/pull"
@@ -990,6 +988,11 @@ func GetFiles(ctx *context.APIContext) {
 	// swagger:operation POST /repos/{owner}/{repo}/files repository repoGetFiles
 	// ---
 	// summary: Get the metadata and contents of requested files
+	// description: Uses automatic pagination based on default page size and
+	// 							max response size and returns the maximum allowed number of files.
+	//							Files which could not be retrieved are null. Blobs which are too large
+	//							are being returned with `content = ""` and `size > 0`, they can be
+	// 							requested using the `download_url`.
 	// produces:
 	// - application/json
 	// parameters:
@@ -1008,14 +1011,6 @@ func GetFiles(ctx *context.APIContext) {
 	//   description: "The name of the commit/branch/tag. Default the repository’s default branch (usually master)"
 	//   type: string
 	//   required: false
-	// - name: page
-	//   in: query
-	//   description: page number of results to return (1-based)
-	//   type: integer
-	// - name: limit
-	//   in: query
-	//   description: page size of results
-	//   type: integer
 	// - name: body
 	//   in: body
 	//   required: true
@@ -1026,8 +1021,6 @@ func GetFiles(ctx *context.APIContext) {
 	//     "$ref": "#/responses/ContentsListResponse"
 	//   "404":
 	//     "$ref": "#/responses/notFound"
-	//   "413":
-	//		 "$ref": "#/responses/contentTooLarge"
 
 	apiOpts := web.GetForm(ctx).(*api.GetFilesOptions)
 
@@ -1042,16 +1035,7 @@ func GetFiles(ctx *context.APIContext) {
 	}
 
 	files := apiOpts.Files
+	filesResponse := files_service.GetContentsListFromTrees(ctx, ctx.Repo.Repository, ref, files)
 
-	count := len(files)
-	listOpts := utils.GetListOptions(ctx)
-	files = util.PaginateSlice(files, listOpts.Page, listOpts.PageSize).([]string)
-
-	filesResponse, err := files_service.GetContentsListFromTrees(ctx, ctx.Repo.Repository, ref, files)
-	if err != nil {
-		ctx.APIError(http.StatusRequestEntityTooLarge, err.Error())
-	}
-
-	ctx.SetTotalCountHeader(int64(count))
 	ctx.JSON(http.StatusOK, filesResponse)
 }

services/context/api.go

@@ -91,12 +91,6 @@ type APIForbiddenError struct {
 // swagger:response notFound
 type APINotFound struct{}
 
-// APIContentTooLarge is a content too large error response
-// swagger:response contentTooLarge
-type APIContentTooLarge struct {
-	APIError
-}
-
 // APIConflict is a conflict empty response
 // swagger:response conflict
 type APIConflict struct{}

services/repository/files/file.go

@@ -18,7 +18,7 @@ import (
 	"code.gitea.io/gitea/modules/util"
 )
 
-func GetContentsListFromTrees(ctx context.Context, repo *repo_model.Repository, branch string, treeNames []string) ([]*api.ContentsResponse, error) {
+func GetContentsListFromTrees(ctx context.Context, repo *repo_model.Repository, branch string, treeNames []string) []*api.ContentsResponse {
 	files := []*api.ContentsResponse{}
 	var size int64
 	for _, file := range treeNames {
@@ -27,18 +27,18 @@ func GetContentsListFromTrees(ctx context.Context, repo *repo_model.Repository,
 			size += fileContents.Size // if content isn't empty (e. g. due to the single blob being too large), add file size to response size
 		}
 		if size > setting.API.DefaultMaxResponseSize {
-			return nil, errors.New("the combined size of the requested blobs exceeds the per-request limit set by the server administrator")
+			return files // return if max page size would be exceeded
 		}
 		files = append(files, fileContents)
+		if len(files) == setting.API.DefaultPagingNum {
+			return files // return if paging num or max size reached
+		}
 	}
-	return files, nil
+	return files
 }
 
 func GetFilesResponseFromCommit(ctx context.Context, repo *repo_model.Repository, commit *git.Commit, branch string, treeNames []string) (*api.FilesResponse, error) {
-	files, err := GetContentsListFromTrees(ctx, repo, branch, treeNames)
-	if err != nil {
-		return nil, err
-	}
+	files := GetContentsListFromTrees(ctx, repo, branch, treeNames)
 	fileCommitResponse, _ := GetFileCommitResponse(repo, commit) // ok if fails, then will be nil
 	verification := GetPayloadCommitVerification(ctx, commit)
 	filesResponse := &api.FilesResponse{

templates/swagger/v1_json.tmpl

@@ -4,17 +4,21 @@
 package integration
 
 import (
+	"encoding/base64"
 	"fmt"
 	"net/http"
 	"net/url"
 	"testing"
+	"time"
 
 	auth_model "code.gitea.io/gitea/models/auth"
+	"code.gitea.io/gitea/models/db"
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/models/unittest"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/gitrepo"
+	"code.gitea.io/gitea/modules/setting"
 	api "code.gitea.io/gitea/modules/structs"
 	repo_service "code.gitea.io/gitea/services/repository"
 
@@ -142,4 +146,94 @@ func testAPIGetRequestedFiles(t *testing.T, u *url.URL) {
 	req = NewRequestWithJSON(t, "POST", fmt.Sprintf("/api/v1/repos/%s/%s/files", org3.Name, repo3.Name), &filesOptions).
 		AddTokenAuth(token2)
 	MakeRequest(t, req, http.StatusOK)
+
+	// Test pagination
+	for i := 0; i < 40; i++ {
+		filesOptions.Files = append(filesOptions.Files, filesOptions.Files[0])
+	}
+	req = NewRequestWithJSON(t, "POST", fmt.Sprintf("/api/v1/repos/%s/%s/files", user2.Name, repo1.Name), &filesOptions)
+	resp = MakeRequest(t, req, http.StatusOK)
+	DecodeJSON(t, resp, &contentsListResponse)
+	assert.NotNil(t, contentsListResponse)
+	assert.Equal(t, setting.API.DefaultPagingNum, len(contentsListResponse))
+
+	// create new repo for large file tests
+	baseRepo, err := repo_service.CreateRepository(db.DefaultContext, user2, user2, repo_service.CreateRepoOptions{
+		Name:          "repo-test-files-api",
+		Description:   "test files api",
+		AutoInit:      true,
+		Gitignores:    "Go",
+		License:       "MIT",
+		Readme:        "Default",
+		DefaultBranch: "main",
+		IsPrivate:     false,
+	})
+	assert.NoError(t, err)
+	assert.NotEmpty(t, baseRepo)
+
+	// Test file size limit
+	largeFile := make([]byte, 15728640) // 15 MiB -> over max blob size
+	for i := range largeFile {
+		largeFile[i] = byte(i % 256)
+	}
+	user2APICtx := NewAPITestContext(t, baseRepo.OwnerName, baseRepo.Name, auth_model.AccessTokenScopeWriteRepository)
+	doAPICreateFile(user2APICtx, "large-file.txt", &api.CreateFileOptions{
+		FileOptions: api.FileOptions{
+			Message: "create large-file.txt",
+			Author: api.Identity{
+				Name:  user2.Name,
+				Email: user2.Email,
+			},
+			Committer: api.Identity{
+				Name:  user2.Name,
+				Email: user2.Email,
+			},
+			Dates: api.CommitDateOptions{
+				Author:    time.Now(),
+				Committer: time.Now(),
+			},
+		},
+		ContentBase64: base64.StdEncoding.EncodeToString(largeFile),
+	})(t)
+
+	filesOptions.Files = []string{"large-file.txt"}
+	req = NewRequestWithJSON(t, "POST", fmt.Sprintf("/api/v1/repos/%s/%s/files", user2.Name, baseRepo.Name), &filesOptions)
+	resp = MakeRequest(t, req, http.StatusOK)
+	DecodeJSON(t, resp, &contentsListResponse)
+	assert.NotNil(t, contentsListResponse)
+	assert.Equal(t, int64(15728640), contentsListResponse[0].Size)
+	assert.Equal(t, "", *contentsListResponse[0].Content)
+
+	// Test response size limit
+	smallFile := make([]byte, 5242880) // 5 MiB -> under max blob size
+	for i := range smallFile {
+		smallFile[i] = byte(i % 256)
+	}
+	doAPICreateFile(user2APICtx, "small-file.txt", &api.CreateFileOptions{
+		FileOptions: api.FileOptions{
+			Message: "create small-file.txt",
+			Author: api.Identity{
+				Name:  user2.Name,
+				Email: user2.Email,
+			},
+			Committer: api.Identity{
+				Name:  user2.Name,
+				Email: user2.Email,
+			},
+			Dates: api.CommitDateOptions{
+				Author:    time.Now(),
+				Committer: time.Now(),
+			},
+		},
+		ContentBase64: base64.StdEncoding.EncodeToString(smallFile),
+	})(t)
+	filesOptions.Files = []string{"small-file.txt"}
+	for i := 0; i < 40; i++ {
+		filesOptions.Files = append(filesOptions.Files, filesOptions.Files[0])
+	}
+	req = NewRequestWithJSON(t, "POST", fmt.Sprintf("/api/v1/repos/%s/%s/files", user2.Name, baseRepo.Name), &filesOptions)
+	resp = MakeRequest(t, req, http.StatusOK)
+	DecodeJSON(t, resp, &contentsListResponse)
+	assert.NotNil(t, contentsListResponse)
+	assert.Equal(t, 20, len(contentsListResponse))
 }