Refactor iterate git tree

Author: lunny

modules/actions/workflows.go

@@ -5,6 +5,7 @@ package actions
 
 import (
 	"bytes"
+	"context"
 	"io"
 	"strings"
 
@@ -55,7 +56,7 @@ func ListWorkflows(commit *git.Commit) (git.Entries, error) {
 		return nil, err
 	}
 
-	entries, err := tree.ListEntriesRecursiveFast()
+	entries, err := tree.ListEntriesRecursiveFast(context.Background())
 	if err != nil {
 		return nil, err
 	}

modules/git/parse_nogogit.go

@@ -24,8 +24,15 @@ func ParseTreeEntries(data []byte) ([]*TreeEntry, error) {
 var sepSpace = []byte{' '}
 
 func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) {
-	var err error
 	entries := make([]*TreeEntry, 0, bytes.Count(data, []byte{'\n'})+1)
+	return entries, iterateTreeEntries(data, ptree, func(entry *TreeEntry) error {
+		entries = append(entries, entry)
+		return nil
+	})
+}
+
+func iterateTreeEntries(data []byte, ptree *Tree, f func(entry *TreeEntry) error) error {
+	var err error
 	for pos := 0; pos < len(data); {
 		// expect line to be of the form:
 		// <mode> <type> <sha> <space-padded-size>\t<filename>
@@ -39,7 +46,7 @@ func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) {
 		line := data[pos:posEnd]
 		posTab := bytes.IndexByte(line, '\t')
 		if posTab == -1 {
-			return nil, fmt.Errorf("invalid ls-tree output (no tab): %q", line)
+			return fmt.Errorf("invalid ls-tree output (no tab): %q", line)
 		}
 
 		entry := new(TreeEntry)
@@ -69,27 +76,29 @@ func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) {
 		case "040000", "040755": // git uses 040000 for tree object, but some users may get 040755 for unknown reasons
 			entry.entryMode = EntryModeTree
 		default:
-			return nil, fmt.Errorf("unknown type: %v", string(entryMode))
+			return fmt.Errorf("unknown type: %v", string(entryMode))
 		}
 
 		entry.ID, err = NewIDFromString(string(entryObjectID))
 		if err != nil {
-			return nil, fmt.Errorf("invalid ls-tree output (invalid object id): %q, err: %w", line, err)
+			return fmt.Errorf("invalid ls-tree output (invalid object id): %q, err: %w", line, err)
 		}
 
 		if len(entryName) > 0 && entryName[0] == '"' {
 			entry.name, err = strconv.Unquote(string(entryName))
 			if err != nil {
-				return nil, fmt.Errorf("invalid ls-tree output (invalid name): %q, err: %w", line, err)
+				return fmt.Errorf("invalid ls-tree output (invalid name): %q, err: %w", line, err)
 			}
 		} else {
 			entry.name = string(entryName)
 		}
 
 		pos = posEnd + 1
-		entries = append(entries, entry)
+		if err := f(entry); err != nil {
+			return err
+		}
 	}
-	return entries, nil
+	return nil
 }
 
 func catBatchParseTreeEntries(objectFormat ObjectFormat, ptree *Tree, rd *bufio.Reader, sz int64) ([]*TreeEntry, error) {

modules/git/repo_language_stats_nogogit.go

@@ -57,11 +57,6 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 
 	tree := commit.Tree
 
-	entries, err := tree.ListEntriesRecursiveWithSize()
-	if err != nil {
-		return nil, err
-	}
-
 	checker, deferable := repo.CheckAttributeReader(commitID)
 	defer deferable()
 
@@ -77,18 +72,18 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 	firstExcludedLanguage := ""
 	firstExcludedLanguageSize := int64(0)
 
-	for _, f := range entries {
+	if err := tree.IterateEntriesWithSize(func(f *TreeEntry) error {
 		select {
 		case <-repo.Ctx.Done():
-			return sizes, repo.Ctx.Err()
+			return repo.Ctx.Err()
 		default:
 		}
 
 		contentBuf.Reset()
 		content = contentBuf.Bytes()
 
 		if f.Size() == 0 {
-			continue
+			return nil
 		}
 
 		isVendored := optional.None[bool]()
@@ -101,22 +96,22 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 			if err == nil {
 				isVendored = AttributeToBool(attrs, AttributeLinguistVendored)
 				if isVendored.ValueOrDefault(false) {
-					continue
+					return nil
 				}
 
 				isGenerated = AttributeToBool(attrs, AttributeLinguistGenerated)
 				if isGenerated.ValueOrDefault(false) {
-					continue
+					return nil
 				}
 
 				isDocumentation = AttributeToBool(attrs, AttributeLinguistDocumentation)
 				if isDocumentation.ValueOrDefault(false) {
-					continue
+					return nil
 				}
 
 				isDetectable = AttributeToBool(attrs, AttributeLinguistDetectable)
 				if !isDetectable.ValueOrDefault(true) {
-					continue
+					return nil
 				}
 
 				hasLanguage := TryReadLanguageAttribute(attrs)
@@ -131,7 +126,7 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 
 					// this language will always be added to the size
 					sizes[language] += f.Size()
-					continue
+					return nil
 				}
 			}
 		}
@@ -140,19 +135,19 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 			enry.IsDotFile(f.Name()) ||
 			(!isDocumentation.Has() && enry.IsDocumentation(f.Name())) ||
 			enry.IsConfiguration(f.Name()) {
-			continue
+			return nil
 		}
 
 		// If content can not be read or file is too big just do detection by filename
 
 		if f.Size() <= bigFileSize {
 			if err := writeID(f.ID.String()); err != nil {
-				return nil, err
+				return err
 			}
 			_, _, size, err := ReadBatchLine(batchReader)
 			if err != nil {
 				log.Debug("Error reading blob: %s Err: %v", f.ID.String(), err)
-				return nil, err
+				return err
 			}
 
 			sizeToRead := size
@@ -164,22 +159,22 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 
 			_, err = contentBuf.ReadFrom(io.LimitReader(batchReader, sizeToRead))
 			if err != nil {
-				return nil, err
+				return err
 			}
 			content = contentBuf.Bytes()
 			if err := DiscardFull(batchReader, discard); err != nil {
-				return nil, err
+				return err
 			}
 		}
 		if !isGenerated.Has() && enry.IsGenerated(f.Name(), content) {
-			continue
+			return nil
 		}
 
 		// FIXME: Why can't we split this and the IsGenerated tests to avoid reading the blob unless absolutely necessary?
 		// - eg. do the all the detection tests using filename first before reading content.
 		language := analyze.GetCodeLanguage(f.Name(), content)
 		if language == "" {
-			continue
+			return nil
 		}
 
 		// group languages, such as Pug -> HTML; SCSS -> CSS
@@ -200,6 +195,9 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 			firstExcludedLanguage = language
 			firstExcludedLanguageSize += f.Size()
 		}
+		return nil
+	}); err != nil {
+		return sizes, err
 	}
 
 	// If there are no included languages add the first excluded language

modules/git/tree_nogogit.go

@@ -6,6 +6,8 @@
 package git
 
 import (
+	"bufio"
+	"context"
 	"io"
 	"strings"
 )
@@ -91,34 +93,83 @@ func (t *Tree) ListEntries() (Entries, error) {
 
 // listEntriesRecursive returns all entries of current tree recursively including all subtrees
 // extraArgs could be "-l" to get the size, which is slower
-func (t *Tree) listEntriesRecursive(extraArgs TrustedCmdArgs) (Entries, error) {
+func (t *Tree) listEntriesRecursive(ctx context.Context, extraArgs TrustedCmdArgs) (Entries, error) {
 	if t.entriesRecursiveParsed {
 		return t.entriesRecursive, nil
 	}
 
-	stdout, _, runErr := NewCommand(t.repo.Ctx, "ls-tree", "-t", "-r").
-		AddArguments(extraArgs...).
-		AddDynamicArguments(t.ID.String()).
-		RunStdBytes(&RunOpts{Dir: t.repo.Path})
-	if runErr != nil {
-		return nil, runErr
-	}
-
-	var err error
-	t.entriesRecursive, err = parseTreeEntries(stdout, t)
+	t.entriesRecursive = make([]*TreeEntry, 0)
+	err := t.iterateEntriesRecursive(func(entry *TreeEntry) error {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+		}
+		t.entriesRecursive = append(t.entriesRecursive, entry)
+		return nil
+	}, extraArgs)
 	if err == nil {
 		t.entriesRecursiveParsed = true
 	}
-
 	return t.entriesRecursive, err
 }
 
 // ListEntriesRecursiveFast returns all entries of current tree recursively including all subtrees, no size
-func (t *Tree) ListEntriesRecursiveFast() (Entries, error) {
-	return t.listEntriesRecursive(nil)
+func (t *Tree) ListEntriesRecursiveFast(ctx context.Context) (Entries, error) {
+	return t.listEntriesRecursive(ctx, nil)
 }
 
 // ListEntriesRecursiveWithSize returns all entries of current tree recursively including all subtrees, with size
-func (t *Tree) ListEntriesRecursiveWithSize() (Entries, error) {
-	return t.listEntriesRecursive(TrustedCmdArgs{"--long"})
+func (t *Tree) ListEntriesRecursiveWithSize(ctx context.Context) (Entries, error) {
+	return t.listEntriesRecursive(ctx, TrustedCmdArgs{"--long"})
+}
+
+// iterateEntriesRecursive returns iterate entries of current tree recursively including all subtrees
+// extraArgs could be "-l" to get the size, which is slower
+func (t *Tree) iterateEntriesRecursive(f func(entry *TreeEntry) error, extraArgs TrustedCmdArgs) error {
+	if t.entriesRecursiveParsed {
+		return nil
+	}
+
+	reader, writer := io.Pipe()
+	done := make(chan error)
+
+	go func(done chan error, writer *io.PipeWriter, reader *io.PipeReader) {
+		runErr := NewCommand(t.repo.Ctx, "ls-tree", "-t", "-r").
+			AddArguments(extraArgs...).
+			AddDynamicArguments(t.ID.String()).
+			Run(&RunOpts{
+				Dir:    t.repo.Path,
+				Stdout: writer,
+			})
+
+		_ = writer.Close()
+		_ = reader.Close()
+
+		done <- runErr
+	}(done, writer, reader)
+
+	scanner := bufio.NewScanner(reader)
+	for scanner.Scan() {
+		if err := scanner.Err(); err != nil {
+			return err
+		}
+		data := scanner.Bytes()
+		if err := iterateTreeEntries(data, t, func(entry *TreeEntry) error {
+			select {
+			case runErr := <-done:
+				return runErr
+			default:
+				return f(entry)
+			}
+		}); err != nil {
+			return err
+		}
+	}
+	t.entriesRecursiveParsed = true
+	return nil
+}
+
+func (t *Tree) IterateEntriesWithSize(f func(*TreeEntry) error) error {
+	return t.iterateEntriesRecursive(f, TrustedCmdArgs{"--long"})
 }

routers/web/repo/treelist.go

@@ -21,7 +21,7 @@ func TreeList(ctx *context.Context) {
 		return
 	}
 
-	entries, err := tree.ListEntriesRecursiveFast()
+	entries, err := tree.ListEntriesRecursiveFast(ctx)
 	if err != nil {
 		ctx.ServerError("ListEntriesRecursiveFast", err)
 		return

services/repository/files/tree.go

@@ -47,7 +47,7 @@ func GetTreeBySHA(ctx context.Context, repo *repo_model.Repository, gitRepo *git
 	tree.URL = repo.APIURL() + "/git/trees/" + url.PathEscape(tree.SHA)
 	var entries git.Entries
 	if recursive {
-		entries, err = gitTree.ListEntriesRecursiveWithSize()
+		entries, err = gitTree.ListEntriesRecursiveWithSize(ctx)
 	} else {
 		entries, err = gitTree.ListEntries()
 	}