update: collapse list if it grows longer than 5 items

This constant of "5" will one day be replaced with a configurable value
whose default is probably "30". It has other issues when we add the
concept of "daily" and "hourly" bundles.

For now, 5 is sufficient. Convert the oldest bundles together into a
single bundle, now called "base-<token>.bundle". We need to be careful
when defining the refs to use for each of these tips, so create a new
'refs/base/' ref namespace. Some TODOs are marked to use 'git merge-base
--independent' to reduce the number of tips used by this base bundle.

We need to be careful, but it is possible that we could also "GC" the
base bundle if the tip commits are not reachable from any tips in the
later bundles. In such a case, we might need to rebuild all of those
bundles to ensure that the REF_DELTA bases exist as needed (perhaps this
is part of the prereq commits, so it might not be a real worry).

Author: derrickstolee

cmd/git-bundle-server/init.go

@@ -35,15 +35,15 @@ func (Init) run(args []string) error {
 	bundle := bundles.CreateInitialBundle(repo)
 	fmt.Printf("Constructing base bundle file at %s\n", bundle.Filename)
 
-	written, gitErr := git.CreateBundle(repo, bundle)
+	written, gitErr := git.CreateBundle(repo, bundle.Filename)
 	if gitErr != nil {
 		return fmt.Errorf("failed to create bundle: %w", gitErr)
 	}
 	if !written {
 		return fmt.Errorf("refused to write empty bundle. Is the repo empty?")
 	}
 
-	list := bundles.SingletonList(bundle)
+	list := bundles.CreateSingletonList(bundle)
 	listErr := bundles.WriteBundleList(list, repo)
 	if listErr != nil {
 		return fmt.Errorf("failed to write bundle list: %w", listErr)

cmd/git-bundle-server/update.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"git-bundle-server/internal/bundles"
 	"git-bundle-server/internal/core"
-	"git-bundle-server/internal/git"
 )
 
 type Update struct{}
@@ -28,24 +27,27 @@ func (Update) run(args []string) error {
 		return fmt.Errorf("failed to load bundle list: %w", err)
 	}
 
-	bundle := bundles.CreateDistinctBundle(repo, *list)
-
-	fmt.Printf("Constructing incremental bundle file at %s\n", bundle.Filename)
-
-	written, err := git.CreateIncrementalBundle(repo, bundle, *list)
+	fmt.Printf("Creating new incremental bundle\n")
+	bundle, err := bundles.CreateIncrementalBundle(repo, list)
 	if err != nil {
-		return fmt.Errorf("failed to create incremental bundle: %w", err)
+		return err
 	}
 
-	// Nothing to update
-	if !written {
+	// Nothing new!
+	if bundle == nil {
 		return nil
 	}
 
-	list.Bundles[bundle.CreationToken] = bundle
+	list.Bundles[bundle.CreationToken] = *bundle
+
+	fmt.Printf("Collapsing bundle list\n")
+	err = bundles.CollapseList(repo, list)
+	if err != nil {
+		return err
+	}
 
 	fmt.Printf("Writing updated bundle list\n")
-	listErr := bundles.WriteBundleList(*list, repo)
+	listErr := bundles.WriteBundleList(list, repo)
 	if listErr != nil {
 		return fmt.Errorf("failed to write bundle list: %w", listErr)
 	}

internal/bundles/bundles.go

@@ -5,7 +5,9 @@ import (
 	"encoding/json"
 	"fmt"
 	"git-bundle-server/internal/core"
+	"git-bundle-server/internal/git"
 	"os"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -34,7 +36,7 @@ type BundleList struct {
 	Bundles map[int64]Bundle
 }
 
-func addBundleToList(bundle Bundle, list BundleList) {
+func addBundleToList(bundle Bundle, list *BundleList) {
 	list.Bundles[bundle.CreationToken] = bundle
 }
 
@@ -51,14 +53,14 @@ func CreateInitialBundle(repo core.Repository) Bundle {
 	return bundle
 }
 
-func CreateDistinctBundle(repo core.Repository, list BundleList) Bundle {
+func CreateDistinctBundle(repo core.Repository, list *BundleList) Bundle {
 	timestamp := time.Now().UTC().Unix()
 
-	_, c := list.Bundles[timestamp]
+	keys := GetSortedCreationTokens(list)
 
-	for c {
-		timestamp++
-		_, c = list.Bundles[timestamp]
+	maxTimestamp := keys[len(keys)-1]
+	if timestamp <= maxTimestamp {
+		timestamp = maxTimestamp + 1
 	}
 
 	bundleName := "bundle-" + fmt.Sprint(timestamp) + ".bundle"
@@ -72,16 +74,16 @@ func CreateDistinctBundle(repo core.Repository, list BundleList) Bundle {
 	return bundle
 }
 
-func SingletonList(bundle Bundle) BundleList {
+func CreateSingletonList(bundle Bundle) *BundleList {
 	list := BundleList{1, "all", make(map[int64]Bundle)}
 
-	addBundleToList(bundle, list)
+	addBundleToList(bundle, &list)
 
-	return list
+	return &list
 }
 
 // Given a BundleList
-func WriteBundleList(list BundleList, repo core.Repository) error {
+func WriteBundleList(list *BundleList, repo core.Repository) error {
 	listFile := repo.WebDir + "/bundle-list"
 	jsonFile := repo.RepoDir + "/bundle-list.json"
 
@@ -97,7 +99,10 @@ func WriteBundleList(list BundleList, repo core.Repository) error {
 		out, "[bundle]\n\tversion = %d\n\tmode = %s\n\n",
 		list.Version, list.Mode)
 
-	for token, bundle := range list.Bundles {
+	keys := GetSortedCreationTokens(list)
+
+	for _, token := range keys {
+		bundle := list.Bundles[token]
 		fmt.Fprintf(
 			out, "[bundle \"%d\"]\n\turi = %s\n\tcreationToken = %d\n\n",
 			token, bundle.URI, token)
@@ -226,7 +231,7 @@ func GetBundleHeader(bundle Bundle) (*BundleHeader, error) {
 	return &header, nil
 }
 
-func GetAllPrereqsForIncrementalBundle(list BundleList) ([]string, error) {
+func GetAllPrereqsForIncrementalBundle(list *BundleList) ([]string, error) {
 	prereqs := []string{}
 
 	for _, bundle := range list.Bundles {
@@ -242,3 +247,93 @@ func GetAllPrereqsForIncrementalBundle(list BundleList) ([]string, error) {
 
 	return prereqs, nil
 }
+
+func CreateIncrementalBundle(repo core.Repository, list *BundleList) (*Bundle, error) {
+	bundle := CreateDistinctBundle(repo, list)
+
+	lines, err := GetAllPrereqsForIncrementalBundle(list)
+	if err != nil {
+		return nil, err
+	}
+
+	written, err := git.CreateIncrementalBundle(repo, bundle.Filename, lines)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create incremental bundle: %w", err)
+	}
+
+	if !written {
+		return nil, nil
+	}
+
+	return &bundle, nil
+}
+
+func CollapseList(repo core.Repository, list *BundleList) error {
+	maxBundles := 5
+
+	if len(list.Bundles) <= maxBundles {
+		return nil
+	}
+
+	keys := GetSortedCreationTokens(list)
+
+	refs := make(map[string]string)
+
+	maxTimestamp := int64(0)
+
+	for i := range keys[0 : len(keys)-maxBundles+1] {
+		bundle := list.Bundles[keys[i]]
+
+		if bundle.CreationToken > maxTimestamp {
+			maxTimestamp = bundle.CreationToken
+		}
+
+		header, err := GetBundleHeader(bundle)
+		if err != nil {
+			return fmt.Errorf("failed to parse bundle file %s: %w", bundle.Filename, err)
+		}
+
+		// Ignore the old ref name and instead use the OID
+		// to generate the ref name. This allows us to create new
+		// refs that point to exactly these objects without disturbing
+		// refs/heads/ which is tracking the remote refs.
+		for _, oid := range header.Refs {
+			refs["refs/base/"+oid] = oid
+		}
+
+		delete(list.Bundles, keys[i])
+	}
+
+	// TODO: Use Git to determine which OIDs are "maximal" in the set
+	// and which are not implied by the previous ones.
+
+	// TODO: Use Git to determine which OIDs are required as prerequisites
+	// of the remaining bundles and latest ref tips, so we can "GC" the
+	// branches that were never merged and may have been force-pushed or
+	// deleted.
+
+	bundle := Bundle{
+		CreationToken: maxTimestamp,
+		Filename:      fmt.Sprintf("%s/base-%d.bundle", repo.WebDir, maxTimestamp),
+		URI:           fmt.Sprintf("./base-%d.bundle", maxTimestamp),
+	}
+
+	err := git.CreateBundleFromRefs(repo, bundle.Filename, refs)
+	if err != nil {
+		return err
+	}
+
+	list.Bundles[maxTimestamp] = bundle
+	return nil
+}
+
+func GetSortedCreationTokens(list *BundleList) []int64 {
+	keys := make([]int64, 0, len(list.Bundles))
+	for timestamp := range list.Bundles {
+		keys = append(keys, timestamp)
+	}
+
+	sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] })
+
+	return keys
+}

internal/git/git.go

@@ -3,7 +3,6 @@ package git
 import (
 	"bytes"
 	"fmt"
-	"git-bundle-server/internal/bundles"
 	"git-bundle-server/internal/core"
 	"os"
 	"os/exec"
@@ -67,10 +66,10 @@ func GitCommandWithStdin(stdinLines []string, args ...string) error {
 	return err
 }
 
-func CreateBundle(repo core.Repository, bundle bundles.Bundle) (bool, error) {
+func CreateBundle(repo core.Repository, filename string) (bool, error) {
 	err := GitCommand(
 		"-C", repo.RepoDir, "bundle", "create",
-		bundle.Filename, "--all")
+		filename, "--all")
 	if err != nil {
 		if strings.Contains(err.Error(), "Refusing to create empty bundle") {
 			return false, nil
@@ -81,19 +80,33 @@ func CreateBundle(repo core.Repository, bundle bundles.Bundle) (bool, error) {
 	return true, nil
 }
 
-func CreateIncrementalBundle(repo core.Repository, bundle bundles.Bundle, list bundles.BundleList) (bool, error) {
-	lines, err := bundles.GetAllPrereqsForIncrementalBundle(list)
-	if err != nil {
-		return false, err
+func CreateBundleFromRefs(repo core.Repository, filename string, refs map[string]string) error {
+	refNames := []string{}
+
+	for ref, oid := range refs {
+		err := GitCommand("-C", repo.RepoDir, "branch", "-f", ref, oid)
+		if err != nil {
+			return fmt.Errorf("failed to create ref %s: %w", ref, err)
+		}
+
+		refNames = append(refNames, ref)
 	}
 
-	for _, line := range lines {
-		fmt.Printf("Sending prereq: %s\n", line)
+	err := GitCommandWithStdin(
+		refNames,
+		"-C", repo.RepoDir, "bundle", "create",
+		filename, "--stdin")
+	if err != nil {
+		return err
 	}
 
-	err = GitCommandWithStdin(
-		lines, "-C", repo.RepoDir, "bundle", "create",
-		bundle.Filename, "--stdin", "--all")
+	return nil
+}
+
+func CreateIncrementalBundle(repo core.Repository, filename string, prereqs []string) (bool, error) {
+	err := GitCommandWithStdin(
+		prereqs, "-C", repo.RepoDir, "bundle", "create",
+		filename, "--stdin", "--all")
 	if err != nil {
 		if strings.Contains(err.Error(), "Refusing to create empty bundle") {
 			return false, nil