Refactor repo contents API and add "contents-ext" API (#34822)

See the updated swagger document for details.

Author: wxiaoguang

modules/git/blob.go

@@ -9,6 +9,7 @@ import (
 	"encoding/base64"
 	"errors"
 	"io"
+	"strings"
 
 	"code.gitea.io/gitea/modules/typesniffer"
 	"code.gitea.io/gitea/modules/util"
@@ -63,33 +64,37 @@ func (b *Blob) GetBlobLineCount(w io.Writer) (int, error) {
 	}
 }
 
-// GetBlobContentBase64 Reads the content of the blob with a base64 encode and returns the encoded string
-func (b *Blob) GetBlobContentBase64() (string, error) {
+// GetBlobContentBase64 Reads the content of the blob with a base64 encoding and returns the encoded string
+func (b *Blob) GetBlobContentBase64(originContent *strings.Builder) (string, error) {
 	dataRc, err := b.DataAsync()
 	if err != nil {
 		return "", err
 	}
 	defer dataRc.Close()
 
-	pr, pw := io.Pipe()
-	encoder := base64.NewEncoder(base64.StdEncoding, pw)
-
-	go func() {
-		_, err := io.Copy(encoder, dataRc)
-		_ = encoder.Close()
-
-		if err != nil {
-			_ = pw.CloseWithError(err)
-		} else {
-			_ = pw.Close()
+	base64buf := &strings.Builder{}
+	encoder := base64.NewEncoder(base64.StdEncoding, base64buf)
+	buf := make([]byte, 32*1024)
+loop:
+	for {
+		n, err := dataRc.Read(buf)
+		if n > 0 {
+			if originContent != nil {
+				_, _ = originContent.Write(buf[:n])
+			}
+			if _, err := encoder.Write(buf[:n]); err != nil {
+				return "", err
+			}
+		}
+		switch {
+		case errors.Is(err, io.EOF):
+			break loop
+		case err != nil:
+			return "", err
 		}
-	}()
-
-	out, err := io.ReadAll(pr)
-	if err != nil {
-		return "", err
 	}
-	return string(out), nil
+	_ = encoder.Close()
+	return base64buf.String(), nil
 }
 
 // GuessContentType guesses the content type of the blob.

modules/git/tree_entry_nogogit.go

@@ -18,7 +18,7 @@ type TreeEntry struct {
 	sized     bool
 }
 
-// Name returns the name of the entry
+// Name returns the name of the entry (base name)
 func (te *TreeEntry) Name() string {
 	return te.name
 }

modules/lfs/pointer.go

@@ -15,15 +15,13 @@ import (
 	"strings"
 )
 
+// spec: https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md
 const (
-	blobSizeCutoff = 1024
+	MetaFileMaxSize = 1024 // spec says the maximum size of a pointer file must be smaller than 1024
 
-	// MetaFileIdentifier is the string appearing at the first line of LFS pointer files.
-	// https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md
-	MetaFileIdentifier = "version https://git-lfs.github.com/spec/v1"
+	MetaFileIdentifier = "version https://git-lfs.github.com/spec/v1" // the first line of a pointer file
 
-	// MetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash.
-	MetaFileOidPrefix = "oid sha256:"
+	MetaFileOidPrefix = "oid sha256:" // spec says the only supported hash is sha256 at the moment
 )
 
 var (
@@ -39,7 +37,7 @@ var (
 
 // ReadPointer tries to read LFS pointer data from the reader
 func ReadPointer(reader io.Reader) (Pointer, error) {
-	buf := make([]byte, blobSizeCutoff)
+	buf := make([]byte, MetaFileMaxSize)
 	n, err := io.ReadFull(reader, buf)
 	if err != nil && err != io.ErrUnexpectedEOF {
 		return Pointer{}, err
@@ -65,6 +63,7 @@ func ReadPointerFromBuffer(buf []byte) (Pointer, error) {
 		return p, ErrInvalidStructure
 	}
 
+	// spec says "key/value pairs MUST be sorted alphabetically in ascending order (version is exception and must be the first)"
 	oid := strings.TrimPrefix(splitLines[1], MetaFileOidPrefix)
 	if len(oid) != 64 || !oidPattern.MatchString(oid) {
 		return p, ErrInvalidOIDFormat

modules/lfs/pointer_scanner_gogit.go

@@ -31,7 +31,7 @@ func SearchPointerBlobs(ctx context.Context, repo *git.Repository, pointerChan c
 			default:
 			}
 
-			if blob.Size > blobSizeCutoff {
+			if blob.Size > MetaFileMaxSize {
 				return nil
 			}
 

modules/structs/git_blob.go

@@ -10,4 +10,7 @@ type GitBlobResponse struct {
 	URL      string  `json:"url"`
 	SHA      string  `json:"sha"`
 	Size     int64   `json:"size"`
+
+	LfsOid  *string `json:"lfs_oid,omitempty"`
+	LfsSize *int64  `json:"lfs_size,omitempty"`
 }

modules/structs/repo_file.go

@@ -119,6 +119,11 @@ type FileLinksResponse struct {
 	HTMLURL *string `json:"html"`
 }
 
+type ContentsExtResponse struct {
+	FileContents *ContentsResponse   `json:"file_contents,omitempty"`
+	DirContents  []*ContentsResponse `json:"dir_contents,omitempty"`
+}
+
 // ContentsResponse contains information about a repo's entry's (dir, file, symlink, submodule) metadata and content
 type ContentsResponse struct {
 	Name          string `json:"name"`
@@ -145,6 +150,9 @@ type ContentsResponse struct {
 	// `submodule_git_url` is populated when `type` is `submodule`, otherwise null
 	SubmoduleGitURL *string            `json:"submodule_git_url"`
 	Links           *FileLinksResponse `json:"_links"`
+
+	LfsOid  *string `json:"lfs_oid"`
+	LfsSize *int64  `json:"lfs_size"`
 }
 
 // FileCommitResponse contains information generated from a Git commit for a repo's file.

routers/api/v1/api.go

@@ -1435,6 +1435,10 @@ func Routes() *web.Router {
 						m.Delete("", bind(api.DeleteFileOptions{}), reqRepoBranchWriter, mustNotBeArchived, repo.DeleteFile)
 					}, reqToken())
 				}, reqRepoReader(unit.TypeCode), context.ReferencesGitRepo())
+				m.Group("/contents-ext", func() {
+					m.Get("", repo.GetContentsExt)
+					m.Get("/*", repo.GetContentsExt)
+				}, reqRepoReader(unit.TypeCode), context.ReferencesGitRepo())
 				m.Combo("/file-contents", reqRepoReader(unit.TypeCode), context.ReferencesGitRepo()).
 					Get(repo.GetFileContentsGet).
 					Post(bind(api.GetFilesOptions{}), repo.GetFileContentsPost) // POST method requires "write" permission, so we also support "GET" method above

routers/api/v1/repo/blob.go

@@ -47,7 +47,7 @@ func GetBlob(ctx *context.APIContext) {
 		return
 	}
 
-	if blob, err := files_service.GetBlobBySHA(ctx, ctx.Repo.Repository, ctx.Repo.GitRepo, sha); err != nil {
+	if blob, err := files_service.GetBlobBySHA(ctx.Repo.Repository, ctx.Repo.GitRepo, sha); err != nil {
 		ctx.APIError(http.StatusBadRequest, err)
 	} else {
 		ctx.JSON(http.StatusOK, blob)

routers/api/v1/repo/file.go

@@ -905,11 +905,71 @@ func resolveRefCommit(ctx *context.APIContext, ref string, minCommitIDLen ...int
 	return refCommit
 }
 
+func GetContentsExt(ctx *context.APIContext) {
+	// swagger:operation GET /repos/{owner}/{repo}/contents-ext/{filepath} repository repoGetContentsExt
+	// ---
+	// summary: The extended "contents" API, to get file metadata and/or content, or list a directory.
+	// description: It guarantees that only one of the response fields is set if the request succeeds.
+	//              Users can pass "includes=file_content" or "includes=lfs_metadata" to retrieve more fields.
+	// produces:
+	// - application/json
+	// parameters:
+	// - name: owner
+	//   in: path
+	//   description: owner of the repo
+	//   type: string
+	//   required: true
+	// - name: repo
+	//   in: path
+	//   description: name of the repo
+	//   type: string
+	//   required: true
+	// - name: filepath
+	//   in: path
+	//   description: path of the dir, file, symlink or submodule in the repo
+	//   type: string
+	//   required: true
+	// - name: ref
+	//   in: query
+	//   description: the name of the commit/branch/tag, default to the repository’s default branch.
+	//   type: string
+	//   required: false
+	// - name: includes
+	//   in: query
+	//   description: By default this API's response only contains file's metadata. Use comma-separated "includes" options to retrieve more fields.
+	//                Option "file_content" will try to retrieve the file content, option "lfs_metadata" will try to retrieve LFS metadata.
+	//   type: string
+	//   required: false
+	// responses:
+	//   "200":
+	//     "$ref": "#/responses/ContentsExtResponse"
+	//   "404":
+	//     "$ref": "#/responses/notFound"
+
+	opts := files_service.GetContentsOrListOptions{TreePath: ctx.PathParam("*")}
+	for includeOpt := range strings.SplitSeq(ctx.FormString("includes"), ",") {
+		if includeOpt == "" {
+			continue
+		}
+		switch includeOpt {
+		case "file_content":
+			opts.IncludeSingleFileContent = true
+		case "lfs_metadata":
+			opts.IncludeLfsMetadata = true
+		default:
+			ctx.APIError(http.StatusBadRequest, fmt.Sprintf("unknown include option %q", includeOpt))
+			return
+		}
+	}
+	ctx.JSON(http.StatusOK, getRepoContents(ctx, opts))
+}
+
 // GetContents Get the metadata and contents (if a file) of an entry in a repository, or a list of entries if a dir
 func GetContents(ctx *context.APIContext) {
 	// swagger:operation GET /repos/{owner}/{repo}/contents/{filepath} repository repoGetContents
 	// ---
-	// summary: Gets the metadata and contents (if a file) of an entry in a repository, or a list of entries if a dir
+	// summary: Gets the metadata and contents (if a file) of an entry in a repository, or a list of entries if a dir.
+	// description: This API follows GitHub's design, and it is not easy to use. Recommend to use our "contents-ext" API instead.
 	// produces:
 	// - application/json
 	// parameters:
@@ -938,29 +998,35 @@ func GetContents(ctx *context.APIContext) {
 	//     "$ref": "#/responses/ContentsResponse"
 	//   "404":
 	//     "$ref": "#/responses/notFound"
-
-	treePath := ctx.PathParam("*")
-	refCommit := resolveRefCommit(ctx, ctx.FormTrim("ref"))
+	ret := getRepoContents(ctx, files_service.GetContentsOrListOptions{TreePath: ctx.PathParam("*"), IncludeSingleFileContent: true})
 	if ctx.Written() {
 		return
 	}
+	ctx.JSON(http.StatusOK, util.Iif[any](ret.FileContents != nil, ret.FileContents, ret.DirContents))
+}
 
-	if fileList, err := files_service.GetContentsOrList(ctx, ctx.Repo.Repository, refCommit, treePath); err != nil {
+func getRepoContents(ctx *context.APIContext, opts files_service.GetContentsOrListOptions) *api.ContentsExtResponse {
+	refCommit := resolveRefCommit(ctx, ctx.FormTrim("ref"))
+	if ctx.Written() {
+		return nil
+	}
+	ret, err := files_service.GetContentsOrList(ctx, ctx.Repo.Repository, ctx.Repo.GitRepo, refCommit, opts)
+	if err != nil {
 		if git.IsErrNotExist(err) {
 			ctx.APIErrorNotFound("GetContentsOrList", err)
-			return
+			return nil
 		}
 		ctx.APIErrorInternal(err)
-	} else {
-		ctx.JSON(http.StatusOK, fileList)
 	}
+	return &ret
 }
 
 // GetContentsList Get the metadata of all the entries of the root dir
 func GetContentsList(ctx *context.APIContext) {
 	// swagger:operation GET /repos/{owner}/{repo}/contents repository repoGetContentsList
 	// ---
-	// summary: Gets the metadata of all the entries of the root dir
+	// summary: Gets the metadata of all the entries of the root dir.
+	// description: This API follows GitHub's design, and it is not easy to use. Recommend to use our "contents-ext" API instead.
 	// produces:
 	// - application/json
 	// parameters:
@@ -1084,6 +1150,6 @@ func handleGetFileContents(ctx *context.APIContext) {
 	if ctx.Written() {
 		return
 	}
-	filesResponse := files_service.GetContentsListFromTreePaths(ctx, ctx.Repo.Repository, refCommit, opts.Files)
+	filesResponse := files_service.GetContentsListFromTreePaths(ctx, ctx.Repo.Repository, ctx.Repo.GitRepo, refCommit, opts.Files)
 	ctx.JSON(http.StatusOK, util.SliceNilAsEmpty(filesResponse))
 }

routers/api/v1/repo/wiki.go

@@ -499,7 +499,7 @@ func wikiContentsByEntry(ctx *context.APIContext, entry *git.TreeEntry) string {
 	if blob.Size() > setting.API.DefaultMaxBlobSize {
 		return ""
 	}
-	content, err := blob.GetBlobContentBase64()
+	content, err := blob.GetBlobContentBase64(nil)
 	if err != nil {
 		ctx.APIErrorInternal(err)
 		return ""