Merge pull request #3: git-bundle-server CLI Part 2: collapse base bundle and the update-all subcommand

Based on #2.

The `update` subcommand now notices when there are more than 5 bundles and squashes the oldest ones into a new "base" bundle. The bundle is renamed but keeps the maximum creation token from that group of bundles.

Modified some of the parameters, especially those in the `git` package to avoid cyclic package dependencies. Also used struct pointers more often.

Repository routes are now stored in a new `routes` file (currently plaintext with line-separated list of routes).

The new `update-all` subcommand runs the `update` subcommand on all registered routes. It also passes any remaining arguments down to the subcommand, which will help when we add the `--daily` and `--hourly` options.

Author: derrickstolee

README.md

@@ -27,7 +27,7 @@ being managed by the bundle server.
   `https://github.com/git-for-windows/git` is assigned the route
   `/git-for-windows/git`. Run `git-bundle-server update` to initialize bundle
   information. Configure the web server to recognize this repository at that
-  route. Configure scheduler to run `git-bundle-server update --all` as
+  route. Configure scheduler to run `git-bundle-server update-all` as
   necessary.
 
 * `git-bundle-server update [--daily|--hourly] <route>`: For the

cmd/git-bundle-server/init.go

@@ -23,7 +23,10 @@ func (Init) run(args []string) error {
 	url := args[0]
 	route := args[1]
 
-	repo := core.GetRepository(route)
+	repo, err := core.CreateRepository(route)
+	if err != nil {
+		return err
+	}
 
 	fmt.Printf("Cloning repository from %s\n", url)
 	gitErr := git.GitCommand("clone", "--mirror", url, repo.RepoDir)
@@ -35,15 +38,15 @@ func (Init) run(args []string) error {
 	bundle := bundles.CreateInitialBundle(repo)
 	fmt.Printf("Constructing base bundle file at %s\n", bundle.Filename)
 
-	written, gitErr := git.CreateBundle(repo, bundle)
+	written, gitErr := git.CreateBundle(repo.RepoDir, bundle.Filename)
 	if gitErr != nil {
 		return fmt.Errorf("failed to create bundle: %w", gitErr)
 	}
 	if !written {
 		return fmt.Errorf("refused to write empty bundle. Is the repo empty?")
 	}
 
-	list := bundles.SingletonList(bundle)
+	list := bundles.CreateSingletonList(bundle)
 	listErr := bundles.WriteBundleList(list, repo)
 	if listErr != nil {
 		return fmt.Errorf("failed to write bundle list: %w", listErr)

cmd/git-bundle-server/subcommand.go

@@ -9,5 +9,6 @@ func all() []Subcommand {
 	return []Subcommand{
 		Init{},
 		Update{},
+		UpdateAll{},
 	}
 }

cmd/git-bundle-server/update-all.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"fmt"
+	"git-bundle-server/internal/core"
+	"os"
+	"os/exec"
+)
+
+type UpdateAll struct{}
+
+func (UpdateAll) subcommand() string {
+	return "update-all"
+}
+
+func (UpdateAll) run(args []string) error {
+	exe, err := os.Executable()
+	if err != nil {
+		return fmt.Errorf("failed to get path to execuable: %w", err)
+	}
+
+	repos, err := core.GetRepositories()
+	if err != nil {
+		return err
+	}
+
+	subargs := []string{"update", ""}
+	subargs = append(subargs, args...)
+
+	for route := range repos {
+		subargs[1] = route
+		cmd := exec.Command(exe, subargs...)
+		cmd.Stderr = os.Stderr
+		cmd.Stdout = os.Stdout
+
+		err := cmd.Start()
+		if err != nil {
+			return fmt.Errorf("git command failed to start: %w", err)
+		}
+
+		err = cmd.Wait()
+		if err != nil {
+			return fmt.Errorf("git command returned a failure: %w", err)
+		}
+	}
+
+	return nil
+}

cmd/git-bundle-server/update.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"git-bundle-server/internal/bundles"
 	"git-bundle-server/internal/core"
-	"git-bundle-server/internal/git"
 )
 
 type Update struct{}
@@ -21,31 +20,37 @@ func (Update) run(args []string) error {
 	}
 
 	route := args[0]
-	repo := core.GetRepository(route)
+	repo, err := core.CreateRepository(route)
+	if err != nil {
+		return err
+	}
 
 	list, err := bundles.GetBundleList(repo)
 	if err != nil {
 		return fmt.Errorf("failed to load bundle list: %w", err)
 	}
 
-	bundle := bundles.CreateDistinctBundle(repo, *list)
-
-	fmt.Printf("Constructing incremental bundle file at %s\n", bundle.Filename)
-
-	written, err := git.CreateIncrementalBundle(repo, bundle, *list)
+	fmt.Printf("Creating new incremental bundle\n")
+	bundle, err := bundles.CreateIncrementalBundle(repo, list)
 	if err != nil {
-		return fmt.Errorf("failed to create incremental bundle: %w", err)
+		return err
 	}
 
-	// Nothing to update
-	if !written {
+	// Nothing new!
+	if bundle == nil {
 		return nil
 	}
 
-	list.Bundles[bundle.CreationToken] = bundle
+	list.Bundles[bundle.CreationToken] = *bundle
+
+	fmt.Printf("Collapsing bundle list\n")
+	err = bundles.CollapseList(repo, list)
+	if err != nil {
+		return err
+	}
 
 	fmt.Printf("Writing updated bundle list\n")
-	listErr := bundles.WriteBundleList(*list, repo)
+	listErr := bundles.WriteBundleList(list, repo)
 	if listErr != nil {
 		return fmt.Errorf("failed to write bundle list: %w", listErr)
 	}

internal/bundles/bundles.go

@@ -5,7 +5,9 @@ import (
 	"encoding/json"
 	"fmt"
 	"git-bundle-server/internal/core"
+	"git-bundle-server/internal/git"
 	"os"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -34,11 +36,11 @@ type BundleList struct {
 	Bundles map[int64]Bundle
 }
 
-func addBundleToList(bundle Bundle, list BundleList) {
+func addBundleToList(bundle Bundle, list *BundleList) {
 	list.Bundles[bundle.CreationToken] = bundle
 }
 
-func CreateInitialBundle(repo core.Repository) Bundle {
+func CreateInitialBundle(repo *core.Repository) Bundle {
 	timestamp := time.Now().UTC().Unix()
 	bundleName := "bundle-" + fmt.Sprint(timestamp) + ".bundle"
 	bundleFile := repo.WebDir + "/" + bundleName
@@ -51,14 +53,14 @@ func CreateInitialBundle(repo core.Repository) Bundle {
 	return bundle
 }
 
-func CreateDistinctBundle(repo core.Repository, list BundleList) Bundle {
+func CreateDistinctBundle(repo *core.Repository, list *BundleList) Bundle {
 	timestamp := time.Now().UTC().Unix()
 
-	_, c := list.Bundles[timestamp]
+	keys := GetSortedCreationTokens(list)
 
-	for c {
-		timestamp++
-		_, c = list.Bundles[timestamp]
+	maxTimestamp := keys[len(keys)-1]
+	if timestamp <= maxTimestamp {
+		timestamp = maxTimestamp + 1
 	}
 
 	bundleName := "bundle-" + fmt.Sprint(timestamp) + ".bundle"
@@ -72,21 +74,21 @@ func CreateDistinctBundle(repo core.Repository, list BundleList) Bundle {
 	return bundle
 }
 
-func SingletonList(bundle Bundle) BundleList {
+func CreateSingletonList(bundle Bundle) *BundleList {
 	list := BundleList{1, "all", make(map[int64]Bundle)}
 
-	addBundleToList(bundle, list)
+	addBundleToList(bundle, &list)
 
-	return list
+	return &list
 }
 
 // Given a BundleList
-func WriteBundleList(list BundleList, repo core.Repository) error {
+func WriteBundleList(list *BundleList, repo *core.Repository) error {
 	listFile := repo.WebDir + "/bundle-list"
 	jsonFile := repo.RepoDir + "/bundle-list.json"
 
 	// TODO: Formalize lockfile concept.
-	f, err := os.OpenFile(listFile+".lock", os.O_WRONLY|os.O_CREATE, 0600)
+	f, err := os.OpenFile(listFile+".lock", os.O_WRONLY|os.O_CREATE, 0o600)
 	if err != nil {
 		return fmt.Errorf("failure to open file: %w", err)
 	}
@@ -97,7 +99,10 @@ func WriteBundleList(list BundleList, repo core.Repository) error {
 		out, "[bundle]\n\tversion = %d\n\tmode = %s\n\n",
 		list.Version, list.Mode)
 
-	for token, bundle := range list.Bundles {
+	keys := GetSortedCreationTokens(list)
+
+	for _, token := range keys {
+		bundle := list.Bundles[token]
 		fmt.Fprintf(
 			out, "[bundle \"%d\"]\n\turi = %s\n\tcreationToken = %d\n\n",
 			token, bundle.URI, token)
@@ -109,7 +114,7 @@ func WriteBundleList(list BundleList, repo core.Repository) error {
 		return fmt.Errorf("failed to close lock file: %w", err)
 	}
 
-	f, err = os.OpenFile(jsonFile+".lock", os.O_WRONLY|os.O_CREATE, 0600)
+	f, err = os.OpenFile(jsonFile+".lock", os.O_WRONLY|os.O_CREATE, 0o600)
 	if err != nil {
 		return fmt.Errorf("failed to open JSON file: %w", err)
 	}
@@ -139,7 +144,7 @@ func WriteBundleList(list BundleList, repo core.Repository) error {
 	return os.Rename(listFile+".lock", listFile)
 }
 
-func GetBundleList(repo core.Repository) (*BundleList, error) {
+func GetBundleList(repo *core.Repository) (*BundleList, error) {
 	jsonFile := repo.RepoDir + "/bundle-list.json"
 
 	reader, err := os.Open(jsonFile)
@@ -182,8 +187,8 @@ func GetBundleHeader(bundle Bundle) (*BundleHeader, error) {
 
 		if line[0] == '#' &&
 			strings.HasPrefix(line, "# v") &&
-			strings.HasSuffix(line, " git bundle\n") {
-			header.Version, err = strconv.ParseInt(line[3:len(line)-len(" git bundle\n")], 10, 64)
+			strings.HasSuffix(line, " git bundle") {
+			header.Version, err = strconv.ParseInt(line[3:len(line)-len(" git bundle")], 10, 64)
 			if err != nil {
 				return nil, fmt.Errorf("failed to parse bundle version: %s", err)
 			}
@@ -226,7 +231,7 @@ func GetBundleHeader(bundle Bundle) (*BundleHeader, error) {
 	return &header, nil
 }
 
-func GetAllPrereqsForIncrementalBundle(list BundleList) ([]string, error) {
+func GetAllPrereqsForIncrementalBundle(list *BundleList) ([]string, error) {
 	prereqs := []string{}
 
 	for _, bundle := range list.Bundles {
@@ -242,3 +247,93 @@ func GetAllPrereqsForIncrementalBundle(list BundleList) ([]string, error) {
 
 	return prereqs, nil
 }
+
+func CreateIncrementalBundle(repo *core.Repository, list *BundleList) (*Bundle, error) {
+	bundle := CreateDistinctBundle(repo, list)
+
+	lines, err := GetAllPrereqsForIncrementalBundle(list)
+	if err != nil {
+		return nil, err
+	}
+
+	written, err := git.CreateIncrementalBundle(repo.RepoDir, bundle.Filename, lines)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create incremental bundle: %w", err)
+	}
+
+	if !written {
+		return nil, nil
+	}
+
+	return &bundle, nil
+}
+
+func CollapseList(repo *core.Repository, list *BundleList) error {
+	maxBundles := 5
+
+	if len(list.Bundles) <= maxBundles {
+		return nil
+	}
+
+	keys := GetSortedCreationTokens(list)
+
+	refs := make(map[string]string)
+
+	maxTimestamp := int64(0)
+
+	for i := range keys[0 : len(keys)-maxBundles+1] {
+		bundle := list.Bundles[keys[i]]
+
+		if bundle.CreationToken > maxTimestamp {
+			maxTimestamp = bundle.CreationToken
+		}
+
+		header, err := GetBundleHeader(bundle)
+		if err != nil {
+			return fmt.Errorf("failed to parse bundle file %s: %w", bundle.Filename, err)
+		}
+
+		// Ignore the old ref name and instead use the OID
+		// to generate the ref name. This allows us to create new
+		// refs that point to exactly these objects without disturbing
+		// refs/heads/ which is tracking the remote refs.
+		for _, oid := range header.Refs {
+			refs["refs/base/"+oid] = oid
+		}
+
+		delete(list.Bundles, keys[i])
+	}
+
+	// TODO: Use Git to determine which OIDs are "maximal" in the set
+	// and which are not implied by the previous ones.
+
+	// TODO: Use Git to determine which OIDs are required as prerequisites
+	// of the remaining bundles and latest ref tips, so we can "GC" the
+	// branches that were never merged and may have been force-pushed or
+	// deleted.
+
+	bundle := Bundle{
+		CreationToken: maxTimestamp,
+		Filename:      fmt.Sprintf("%s/base-%d.bundle", repo.WebDir, maxTimestamp),
+		URI:           fmt.Sprintf("./base-%d.bundle", maxTimestamp),
+	}
+
+	err := git.CreateBundleFromRefs(repo.RepoDir, bundle.Filename, refs)
+	if err != nil {
+		return err
+	}
+
+	list.Bundles[maxTimestamp] = bundle
+	return nil
+}
+
+func GetSortedCreationTokens(list *BundleList) []int64 {
+	keys := make([]int64, 0, len(list.Bundles))
+	for timestamp := range list.Bundles {
+		keys = append(keys, timestamp)
+	}
+
+	sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] })
+
+	return keys
+}