Reduce memory consumption of graph (pipe sets) (#4498)

- **PR Description**

As a followup to #2533, reduce the memory consumption some more by
optimizing the storage of the pipe sets used for the commit graph.

Some coarse measurements (taken by opening the respective repo, typing
`4 >`, and waiting for all commits to be loaded, and then reading the
Memory column in Activity Monitor):

|               | master  | this PR |
| ------------- | ------- | ------- |
| git           | 2.5 GB  | 1.0 GB  |
| our work repo | 2.3 GB  | 1.3 GB  |
| linux kernel  | 94.8 GB | 38.0 GB |

It's still not really usable for the linux kernel, but for all other
repos that I come across in my daily use of lazygit, it works quite well
now.

Author: stefanhaller

go.mod

@@ -33,7 +33,7 @@ require (
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/afero v1.9.5
 	github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad
-	github.com/stefanhaller/git-todo-parser v0.0.7-0.20240406123903-fd957137b6e2
+	github.com/stefanhaller/git-todo-parser v0.0.7-0.20250429125209-dcf39e4641f5
 	github.com/stretchr/testify v1.10.0
 	github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56

go.sum

@@ -286,8 +286,8 @@ github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM=
 github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ=
 github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad h1:fiWzISvDn0Csy5H0iwgAuJGQTUpVfEMJJd4nRFXogbc=
 github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0=
-github.com/stefanhaller/git-todo-parser v0.0.7-0.20240406123903-fd957137b6e2 h1:RTNWemd/9z9A5L/AggEP3OdhRz5dXETB/wdAlYF0SuM=
-github.com/stefanhaller/git-todo-parser v0.0.7-0.20240406123903-fd957137b6e2/go.mod h1:HFt9hGqMzgQ+gVxMKcvTvGaFz4Y0yYycqqAp2V3wcJY=
+github.com/stefanhaller/git-todo-parser v0.0.7-0.20250429125209-dcf39e4641f5 h1:ZCI0NPs0xXd00Ej9lX+wwbHjQDkstJa3kUbX7WCOF8I=
+github.com/stefanhaller/git-todo-parser v0.0.7-0.20250429125209-dcf39e4641f5/go.mod h1:HFt9hGqMzgQ+gVxMKcvTvGaFz4Y0yYycqqAp2V3wcJY=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=

pkg/app/daemon/rebase.go

@@ -22,7 +22,7 @@ func (self *TodoLine) ToString() string {
 	if self.Action == "break" {
 		return self.Action + "\n"
 	} else {
-		return self.Action + " " + self.Commit.Hash + " " + self.Commit.Name + "\n"
+		return self.Action + " " + self.Commit.Hash() + " " + self.Commit.Name + "\n"
 	}
 }
 

pkg/commands/git_commands/commit_loader.go

@@ -66,6 +66,7 @@ type GetCommitsOptions struct {
 	// If non-empty, show divergence from this ref (left-right log)
 	RefToShowDivergenceFrom string
 	MainBranches            *MainBranches
+	HashPool                *utils.StringPool
 }
 
 // GetCommits obtains the commits of the current branch
@@ -74,7 +75,7 @@ func (self *CommitLoader) GetCommits(opts GetCommitsOptions) ([]*models.Commit,
 
 	if opts.IncludeRebaseCommits && opts.FilterPath == "" {
 		var err error
-		commits, err = self.MergeRebasingCommits(commits)
+		commits, err = self.MergeRebasingCommits(opts.HashPool, commits)
 		if err != nil {
 			return nil, err
 		}
@@ -89,7 +90,7 @@ func (self *CommitLoader) GetCommits(opts GetCommitsOptions) ([]*models.Commit,
 		defer wg.Done()
 
 		logErr = self.getLogCmd(opts).RunAndProcessLines(func(line string) (bool, error) {
-			commit := self.extractCommitFromLine(line, opts.RefToShowDivergenceFrom != "")
+			commit := self.extractCommitFromLine(opts.HashPool, line, opts.RefToShowDivergenceFrom != "")
 			commits = append(commits, commit)
 			return false, nil
 		})
@@ -121,7 +122,7 @@ func (self *CommitLoader) GetCommits(opts GetCommitsOptions) ([]*models.Commit,
 	}
 
 	for _, commit := range commits {
-		if commit.Hash == firstPushedCommit {
+		if commit.Hash() == firstPushedCommit {
 			passedFirstPushedCommit = true
 		}
 		if !commit.IsTODO() {
@@ -159,7 +160,7 @@ func (self *CommitLoader) GetCommits(opts GetCommitsOptions) ([]*models.Commit,
 	return commits, nil
 }
 
-func (self *CommitLoader) MergeRebasingCommits(commits []*models.Commit) ([]*models.Commit, error) {
+func (self *CommitLoader) MergeRebasingCommits(hashPool *utils.StringPool, commits []*models.Commit) ([]*models.Commit, error) {
 	// chances are we have as many commits as last time so we'll set the capacity to be the old length
 	result := make([]*models.Commit, 0, len(commits))
 	for i, commit := range commits {
@@ -172,7 +173,7 @@ func (self *CommitLoader) MergeRebasingCommits(commits []*models.Commit) ([]*mod
 	workingTreeState := self.getWorkingTreeState()
 	addConflictedRebasingCommit := true
 	if workingTreeState.CherryPicking || workingTreeState.Reverting {
-		sequencerCommits, err := self.getHydratedSequencerCommits(workingTreeState)
+		sequencerCommits, err := self.getHydratedSequencerCommits(hashPool, workingTreeState)
 		if err != nil {
 			return nil, err
 		}
@@ -181,7 +182,7 @@ func (self *CommitLoader) MergeRebasingCommits(commits []*models.Commit) ([]*mod
 	}
 
 	if workingTreeState.Rebasing {
-		rebasingCommits, err := self.getHydratedRebasingCommits(addConflictedRebasingCommit)
+		rebasingCommits, err := self.getHydratedRebasingCommits(hashPool, addConflictedRebasingCommit)
 		if err != nil {
 			return nil, err
 		}
@@ -196,7 +197,7 @@ func (self *CommitLoader) MergeRebasingCommits(commits []*models.Commit) ([]*mod
 // then puts them into a commit object
 // example input:
 // 8ad01fe32fcc20f07bc6693f87aa4977c327f1e1|10 hours ago|Jesse Duffield| (HEAD -> master, tag: v0.15.2)|refresh commits when adding a tag
-func (self *CommitLoader) extractCommitFromLine(line string, showDivergence bool) *models.Commit {
+func (self *CommitLoader) extractCommitFromLine(hashPool *utils.StringPool, line string, showDivergence bool) *models.Commit {
 	split := strings.SplitN(line, "\x00", 8)
 
 	hash := split[0]
@@ -234,7 +235,7 @@ func (self *CommitLoader) extractCommitFromLine(line string, showDivergence bool
 		parents = strings.Split(parentHashes, " ")
 	}
 
-	return &models.Commit{
+	return models.NewCommit(hashPool, models.NewCommitOpts{
 		Hash:          hash,
 		Name:          message,
 		Tags:          tags,
@@ -244,16 +245,16 @@ func (self *CommitLoader) extractCommitFromLine(line string, showDivergence bool
 		AuthorEmail:   authorEmail,
 		Parents:       parents,
 		Divergence:    divergence,
-	}
+	})
 }
 
-func (self *CommitLoader) getHydratedRebasingCommits(addConflictingCommit bool) ([]*models.Commit, error) {
+func (self *CommitLoader) getHydratedRebasingCommits(hashPool *utils.StringPool, addConflictingCommit bool) ([]*models.Commit, error) {
 	todoFileHasShortHashes := self.version.IsOlderThan(2, 25, 2)
-	return self.getHydratedTodoCommits(self.getRebasingCommits(addConflictingCommit), todoFileHasShortHashes)
+	return self.getHydratedTodoCommits(hashPool, self.getRebasingCommits(hashPool, addConflictingCommit), todoFileHasShortHashes)
 }
 
-func (self *CommitLoader) getHydratedSequencerCommits(workingTreeState models.WorkingTreeState) ([]*models.Commit, error) {
-	commits := self.getSequencerCommits()
+func (self *CommitLoader) getHydratedSequencerCommits(hashPool *utils.StringPool, workingTreeState models.WorkingTreeState) ([]*models.Commit, error) {
+	commits := self.getSequencerCommits(hashPool)
 	if len(commits) > 0 {
 		// If we have any commits in .git/sequencer/todo, then the last one of
 		// those is the conflicting one.
@@ -262,22 +263,22 @@ func (self *CommitLoader) getHydratedSequencerCommits(workingTreeState models.Wo
 		// For single-commit cherry-picks and reverts, git apparently doesn't
 		// use the sequencer; in that case, CHERRY_PICK_HEAD or REVERT_HEAD is
 		// our conflicting commit, so synthesize it here.
-		conflicedCommit := self.getConflictedSequencerCommit(workingTreeState)
+		conflicedCommit := self.getConflictedSequencerCommit(hashPool, workingTreeState)
 		if conflicedCommit != nil {
 			commits = append(commits, conflicedCommit)
 		}
 	}
 
-	return self.getHydratedTodoCommits(commits, true)
+	return self.getHydratedTodoCommits(hashPool, commits, true)
 }
 
-func (self *CommitLoader) getHydratedTodoCommits(todoCommits []*models.Commit, todoFileHasShortHashes bool) ([]*models.Commit, error) {
+func (self *CommitLoader) getHydratedTodoCommits(hashPool *utils.StringPool, todoCommits []*models.Commit, todoFileHasShortHashes bool) ([]*models.Commit, error) {
 	if len(todoCommits) == 0 {
 		return nil, nil
 	}
 
 	commitHashes := lo.FilterMap(todoCommits, func(commit *models.Commit, _ int) (string, bool) {
-		return commit.Hash, commit.Hash != ""
+		return commit.Hash(), commit.Hash() != ""
 	})
 
 	// note that we're not filtering these as we do non-rebasing commits just because
@@ -292,8 +293,8 @@ func (self *CommitLoader) getHydratedTodoCommits(todoCommits []*models.Commit, t
 
 	fullCommits := map[string]*models.Commit{}
 	err := cmdObj.RunAndProcessLines(func(line string) (bool, error) {
-		commit := self.extractCommitFromLine(line, false)
-		fullCommits[commit.Hash] = commit
+		commit := self.extractCommitFromLine(hashPool, line, false)
+		fullCommits[commit.Hash()] = commit
 		return false, nil
 	})
 	if err != nil {
@@ -315,9 +316,9 @@ func (self *CommitLoader) getHydratedTodoCommits(todoCommits []*models.Commit, t
 
 	hydratedCommits := make([]*models.Commit, 0, len(todoCommits))
 	for _, rebasingCommit := range todoCommits {
-		if rebasingCommit.Hash == "" {
+		if rebasingCommit.Hash() == "" {
 			hydratedCommits = append(hydratedCommits, rebasingCommit)
-		} else if commit := findFullCommit(rebasingCommit.Hash); commit != nil {
+		} else if commit := findFullCommit(rebasingCommit.Hash()); commit != nil {
 			commit.Action = rebasingCommit.Action
 			commit.Status = rebasingCommit.Status
 			hydratedCommits = append(hydratedCommits, commit)
@@ -331,7 +332,7 @@ func (self *CommitLoader) getHydratedTodoCommits(todoCommits []*models.Commit, t
 // git-rebase-todo example:
 // pick ac446ae94ee560bdb8d1d057278657b251aaef17 ac446ae
 // pick afb893148791a2fbd8091aeb81deba4930c73031 afb8931
-func (self *CommitLoader) getRebasingCommits(addConflictingCommit bool) []*models.Commit {
+func (self *CommitLoader) getRebasingCommits(hashPool *utils.StringPool, addConflictingCommit bool) []*models.Commit {
 	bytesContent, err := self.readFile(filepath.Join(self.repoPaths.WorktreeGitDirPath(), "rebase-merge/git-rebase-todo"))
 	if err != nil {
 		self.Log.Error(fmt.Sprintf("error occurred reading git-rebase-todo: %s", err.Error()))
@@ -350,7 +351,7 @@ func (self *CommitLoader) getRebasingCommits(addConflictingCommit bool) []*model
 	// See if the current commit couldn't be applied because it conflicted; if
 	// so, add a fake entry for it
 	if addConflictingCommit {
-		if conflictedCommit := self.getConflictedCommit(todos); conflictedCommit != nil {
+		if conflictedCommit := self.getConflictedCommit(hashPool, todos); conflictedCommit != nil {
 			commits = append(commits, conflictedCommit)
 		}
 	}
@@ -364,18 +365,18 @@ func (self *CommitLoader) getRebasingCommits(addConflictingCommit bool) []*model
 			// Command does not have a commit associated, skip
 			continue
 		}
-		commits = utils.Prepend(commits, &models.Commit{
+		commits = utils.Prepend(commits, models.NewCommit(hashPool, models.NewCommitOpts{
 			Hash:   t.Commit,
 			Name:   t.Msg,
 			Status: models.StatusRebasing,
 			Action: t.Command,
-		})
+		}))
 	}
 
 	return commits
 }
 
-func (self *CommitLoader) getConflictedCommit(todos []todo.Todo) *models.Commit {
+func (self *CommitLoader) getConflictedCommit(hashPool *utils.StringPool, todos []todo.Todo) *models.Commit {
 	bytesContent, err := self.readFile(filepath.Join(self.repoPaths.WorktreeGitDirPath(), "rebase-merge/done"))
 	if err != nil {
 		self.Log.Error(fmt.Sprintf("error occurred reading rebase-merge/done: %s", err.Error()))
@@ -391,10 +392,10 @@ func (self *CommitLoader) getConflictedCommit(todos []todo.Todo) *models.Commit
 	amendFileExists, _ := self.os.FileExists(filepath.Join(self.repoPaths.WorktreeGitDirPath(), "rebase-merge/amend"))
 	messageFileExists, _ := self.os.FileExists(filepath.Join(self.repoPaths.WorktreeGitDirPath(), "rebase-merge/message"))
 
-	return self.getConflictedCommitImpl(todos, doneTodos, amendFileExists, messageFileExists)
+	return self.getConflictedCommitImpl(hashPool, todos, doneTodos, amendFileExists, messageFileExists)
 }
 
-func (self *CommitLoader) getConflictedCommitImpl(todos []todo.Todo, doneTodos []todo.Todo, amendFileExists bool, messageFileExists bool) *models.Commit {
+func (self *CommitLoader) getConflictedCommitImpl(hashPool *utils.StringPool, todos []todo.Todo, doneTodos []todo.Todo, amendFileExists bool, messageFileExists bool) *models.Commit {
 	// Should never be possible, but just to be safe:
 	if len(doneTodos) == 0 {
 		self.Log.Error("no done entries in rebase-merge/done file")
@@ -465,14 +466,14 @@ func (self *CommitLoader) getConflictedCommitImpl(todos []todo.Todo, doneTodos [
 
 	// Any other todo that has a commit associated with it must have failed with
 	// a conflict, otherwise we wouldn't have stopped the rebase:
-	return &models.Commit{
+	return models.NewCommit(hashPool, models.NewCommitOpts{
 		Hash:   lastTodo.Commit,
 		Action: lastTodo.Command,
 		Status: models.StatusConflicted,
-	}
+	})
 }
 
-func (self *CommitLoader) getSequencerCommits() []*models.Commit {
+func (self *CommitLoader) getSequencerCommits(hashPool *utils.StringPool) []*models.Commit {
 	bytesContent, err := self.readFile(filepath.Join(self.repoPaths.WorktreeGitDirPath(), "sequencer/todo"))
 	if err != nil {
 		self.Log.Error(fmt.Sprintf("error occurred reading sequencer/todo: %s", err.Error()))
@@ -493,18 +494,18 @@ func (self *CommitLoader) getSequencerCommits() []*models.Commit {
 			// Command does not have a commit associated, skip
 			continue
 		}
-		commits = utils.Prepend(commits, &models.Commit{
+		commits = utils.Prepend(commits, models.NewCommit(hashPool, models.NewCommitOpts{
 			Hash:   t.Commit,
 			Name:   t.Msg,
 			Status: models.StatusCherryPickingOrReverting,
 			Action: t.Command,
-		})
+		}))
 	}
 
 	return commits
 }
 
-func (self *CommitLoader) getConflictedSequencerCommit(workingTreeState models.WorkingTreeState) *models.Commit {
+func (self *CommitLoader) getConflictedSequencerCommit(hashPool *utils.StringPool, workingTreeState models.WorkingTreeState) *models.Commit {
 	var shaFile string
 	var action todo.TodoCommand
 	if workingTreeState.CherryPicking {
@@ -526,11 +527,11 @@ func (self *CommitLoader) getConflictedSequencerCommit(workingTreeState models.W
 	if len(lines) == 0 {
 		return nil
 	}
-	return &models.Commit{
+	return models.NewCommit(hashPool, models.NewCommitOpts{
 		Hash:   lines[0],
 		Status: models.StatusConflicted,
 		Action: action,
-	}
+	})
 }
 
 func setCommitMergedStatuses(ancestor string, commits []*models.Commit) {
@@ -541,7 +542,7 @@ func setCommitMergedStatuses(ancestor string, commits []*models.Commit) {
 	passedAncestor := false
 	for i, commit := range commits {
 		// some commits aren't really commits and don't have hashes, such as the update-ref todo
-		if commit.Hash != "" && strings.HasPrefix(ancestor, commit.Hash) {
+		if commit.Hash() != "" && strings.HasPrefix(ancestor, commit.Hash()) {
 			passedAncestor = true
 		}
 		if commit.Status != models.StatusPushed && commit.Status != models.StatusUnpushed {
@@ -609,4 +610,4 @@ func (self *CommitLoader) getLogCmd(opts GetCommitsOptions) oscommands.ICmdObj {
 	return self.cmd.New(cmdArgs).DontLog()
 }
 
-const prettyFormat = `--pretty=format:%H%x00%at%x00%aN%x00%ae%x00%D%x00%p%x00%m%x00%s`
+const prettyFormat = `--pretty=format:%H%x00%at%x00%aN%x00%ae%x00%D%x00%P%x00%m%x00%s`