Merge pull request #38 from github/vdye/list-repair

Add `list` and `repair` commands

Author: vdye

README.md

@@ -133,6 +133,13 @@ being managed by the bundle server.
 * `git-bundle-server delete <route>`: Remove the configuration for the given
   `<route>` and delete its repository data.
 
+* `git-bundle-server list [<options>]`: List each route and associated
+  information (e.g. Git remote URL) in the bundle server.
+
+* `git-bundle-server repair routes [<options>]`: Correct the contents of the
+  internal route registry by comparing to bundle server's internal repository
+  storage.
+
 ### Web server management
 
 Independent of the management of the individual repositories hosted by the

cmd/git-bundle-server/list.go

@@ -0,0 +1,65 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/github/git-bundle-server/cmd/utils"
+	"github.com/github/git-bundle-server/internal/argparse"
+	"github.com/github/git-bundle-server/internal/core"
+	"github.com/github/git-bundle-server/internal/git"
+	"github.com/github/git-bundle-server/internal/log"
+)
+
+type listCmd struct {
+	logger    log.TraceLogger
+	container *utils.DependencyContainer
+}
+
+func NewListCommand(logger log.TraceLogger, container *utils.DependencyContainer) argparse.Subcommand {
+	return &listCmd{
+		logger:    logger,
+		container: container,
+	}
+}
+
+func (listCmd) Name() string {
+	return "list"
+}
+
+func (listCmd) Description() string {
+	return `
+List the routes registered to the bundle server.`
+}
+
+func (l *listCmd) Run(ctx context.Context, args []string) error {
+	parser := argparse.NewArgParser(l.logger, "git-bundle-server list [--name-only]")
+	nameOnly := parser.Bool("name-only", false, "print only the names of configured routes")
+	parser.Parse(ctx, args)
+
+	repoProvider := utils.GetDependency[core.RepositoryProvider](ctx, l.container)
+	gitHelper := utils.GetDependency[git.GitHelper](ctx, l.container)
+
+	repos, err := repoProvider.GetRepositories(ctx)
+	if err != nil {
+		return l.logger.Error(ctx, err)
+	}
+
+	for _, repo := range repos {
+		info := []string{repo.Route}
+		if !*nameOnly {
+			remote, err := gitHelper.GetRemoteUrl(ctx, repo.RepoDir)
+			if err != nil {
+				return l.logger.Error(ctx, err)
+			}
+			info = append(info, remote)
+		}
+
+		// Join with space & tab to ensure each element of the info array is
+		// separated by at least two spaces (for better readability).
+		fmt.Println(strings.Join(info, " \t"))
+	}
+
+	return nil
+}

cmd/git-bundle-server/main.go

@@ -15,10 +15,12 @@ func all(logger log.TraceLogger) []argparse.Subcommand {
 	return []argparse.Subcommand{
 		NewDeleteCommand(logger, container),
 		NewInitCommand(logger, container),
+		NewRepairCommand(logger, container),
 		NewStartCommand(logger, container),
 		NewStopCommand(logger, container),
 		NewUpdateCommand(logger, container),
 		NewUpdateAllCommand(logger, container),
+		NewListCommand(logger, container),
 		NewWebServerCommand(logger, container),
 	}
 }

cmd/git-bundle-server/repair.go

@@ -0,0 +1,113 @@
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/github/git-bundle-server/cmd/utils"
+	"github.com/github/git-bundle-server/internal/argparse"
+	"github.com/github/git-bundle-server/internal/core"
+	"github.com/github/git-bundle-server/internal/log"
+	typeutils "github.com/github/git-bundle-server/internal/utils"
+)
+
+type repairCmd struct {
+	logger    log.TraceLogger
+	container *utils.DependencyContainer
+}
+
+func NewRepairCommand(logger log.TraceLogger, container *utils.DependencyContainer) argparse.Subcommand {
+	return &repairCmd{
+		logger:    logger,
+		container: container,
+	}
+}
+
+func (repairCmd) Name() string {
+	return "repair"
+}
+
+func (repairCmd) Description() string {
+	return `
+Scan and correct inconsistencies in the bundle server's internal registries and
+storage.`
+}
+
+func (r *repairCmd) repairRoutes(ctx context.Context, args []string) error {
+	parser := argparse.NewArgParser(r.logger, "git-bundle-server repair routes [--start-all] [--dry-run]")
+	enable := parser.Bool("start-all", false, "turn on bundle computation for all repositories found")
+	dryRun := parser.Bool("dry-run", false, "report the repairs needed, but do not perform them")
+	// TODO: add a '--cleanup' option to delete non-repo contents inside repo root
+	parser.Parse(ctx, args)
+
+	repoProvider := utils.GetDependency[core.RepositoryProvider](ctx, r.container)
+
+	// Read the routes file
+	repos, err := repoProvider.GetRepositories(ctx)
+	if err != nil {
+		// If routes file cannot be read, start over
+		fmt.Println("warning: cannot load routes file; rebuilding from scratch...")
+		repos = make(map[string]core.Repository)
+	}
+
+	// Read the repositories as represented by internal storage
+	storedRepos, err := repoProvider.ReadRepositoryStorage(ctx)
+	if err != nil {
+		return r.logger.Errorf(ctx, "could not read internal repository storage: %w", err)
+	}
+
+	_, missingOnDisk, notRegistered := typeutils.SegmentKeys(repos, storedRepos)
+
+	// Print the updates to be made
+	fmt.Print("\n")
+
+	if *enable && len(notRegistered) > 0 {
+		fmt.Println("Unregistered routes to add")
+		fmt.Println("--------------------------")
+		for _, route := range notRegistered {
+			fmt.Printf("* %s\n", route)
+			repos[route] = storedRepos[route]
+		}
+		fmt.Print("\n")
+	}
+
+	if len(missingOnDisk) > 0 {
+		fmt.Println("Missing or invalid routes to remove")
+		fmt.Println("-----------------------------------")
+		for _, route := range missingOnDisk {
+			fmt.Printf("* %s\n", route)
+			delete(repos, route)
+		}
+		fmt.Print("\n")
+	}
+
+	if (!*enable || len(notRegistered) == 0) && len(missingOnDisk) == 0 {
+		fmt.Println("No repairs needed.")
+		return nil
+	}
+
+	if *dryRun {
+		fmt.Println("Skipping updates (dry run)")
+	} else {
+		fmt.Println("Applying route repairs...")
+		err := repoProvider.WriteAllRoutes(ctx, repos)
+		if err != nil {
+			return err
+		}
+
+		// Start the global cron schedule (if it's not already running)
+		cron := utils.GetDependency[utils.CronHelper](ctx, r.container)
+		cron.SetCronSchedule(ctx)
+		fmt.Println("Done")
+	}
+
+	return nil
+}
+
+func (r *repairCmd) Run(ctx context.Context, args []string) error {
+	parser := argparse.NewArgParser(r.logger, "git-bundle-server repair <subcommand> [<options>]")
+	parser.Subcommand(argparse.NewSubcommand("routes", "Correct the contents of the internal route registry", r.repairRoutes))
+	parser.Parse(ctx, args)
+
+	return parser.InvokeSubcommand(ctx)
+}

cmd/git-bundle-web-server/bundle-server.go

@@ -13,8 +13,10 @@ import (
 	"time"
 
 	"github.com/github/git-bundle-server/internal/bundles"
+	"github.com/github/git-bundle-server/internal/cmd"
 	"github.com/github/git-bundle-server/internal/common"
 	"github.com/github/git-bundle-server/internal/core"
+	"github.com/github/git-bundle-server/internal/git"
 	"github.com/github/git-bundle-server/internal/log"
 )
 
@@ -84,7 +86,9 @@ func (b *bundleWebServer) serve(w http.ResponseWriter, r *http.Request) {
 
 	userProvider := common.NewUserProvider()
 	fileSystem := common.NewFileSystem()
-	repoProvider := core.NewRepositoryProvider(b.logger, userProvider, fileSystem)
+	commandExecutor := cmd.NewCommandExecutor(b.logger)
+	gitHelper := git.NewGitHelper(b.logger, commandExecutor)
+	repoProvider := core.NewRepositoryProvider(b.logger, userProvider, fileSystem, gitHelper)
 
 	repos, err := repoProvider.GetRepositories(ctx)
 	if err != nil {

cmd/utils/container-helpers.go

@@ -28,6 +28,7 @@ func BuildGitBundleServerContainer(logger log.TraceLogger) *DependencyContainer
 			logger,
 			GetDependency[common.UserProvider](ctx, container),
 			GetDependency[common.FileSystem](ctx, container),
+			GetDependency[git.GitHelper](ctx, container),
 		)
 	})
 	registerDependency(container, func(ctx context.Context) bundles.BundleProvider {

docs/man/git-bundle-server.adoc

@@ -84,6 +84,27 @@ fetching during scheduled bundle updates.
 *delete* _route_::
   Remove a repository configuration and delete its data on disk.
 
+*list* [*--name-only*]::
+  List the routes registered to the bundle server. Each line in the output
+  represents a unique route and includes (in order) the route name and the Git
+  remote URL associated with that route.
+
+  *--name-only*:::
+    Print only the route name on each line.
+
+*repair* *routes* [*--start-all*] [*--dry-run*]::
+  Correct the contents of the internal route registry by comparing to bundle
+  server's internal repository storage.
+
+  *--start-all*:::
+    If any valid repositories are found that are not registered to the bundle
+    server (for example, those deactivated with *git-bundle-server stop*),
+    enable them.
+
+  *--dry-run*:::
+    Collect and report the repairs that the command will perform, but do not
+    perform them.
+
 *web-server* *start* [*-f*|*--force*] [_server-options_]::
   Start a background process web server hosting bundle metadata and content. The
   web server daemon runs under the calling user's domain, and will continue

internal/common/filesystem.go

@@ -10,6 +10,8 @@ import (
 	"path"
 	"path/filepath"
 	"syscall"
+
+	"github.com/github/git-bundle-server/internal/utils"
 )
 
 const (
@@ -35,6 +37,29 @@ func (l *lockFile) Rollback() error {
 	return os.Remove(l.lockFilename)
 }
 
+type ReadDirEntry interface {
+	Path() string
+	fs.DirEntry
+}
+
+type fsEntry struct {
+	root string
+	fs.DirEntry
+}
+
+func (e *fsEntry) Path() string {
+	return filepath.Join(e.root, e.Name())
+}
+
+func mapDirEntry(root string) func(fs.DirEntry) ReadDirEntry {
+	return func(e fs.DirEntry) ReadDirEntry {
+		return &fsEntry{
+			root:     root,
+			DirEntry: e,
+		}
+	}
+}
+
 type FileSystem interface {
 	GetLocalExecutable(name string) (string, error)
 
@@ -43,6 +68,15 @@ type FileSystem interface {
 	WriteLockFileFunc(filename string, writeFunc func(io.Writer) error) (LockFile, error)
 	DeleteFile(filename string) (bool, error)
 	ReadFileLines(filename string) ([]string, error)
+
+	// ReadDirRecursive recurses into a given directory ('path') up to 'depth'
+	// levels deep. If 'strictDepth' is true, only the entries at *exactly* the
+	// given depth are returned (if any). If 'strictDepth' is false, though, the
+	// results will also include any files or empty directories for a depth <
+	// 'depth'.
+	//
+	// If 'depth' is <= 0, ReadDirRecursive returns an empty list.
+	ReadDirRecursive(path string, depth int, strictDepth bool) ([]ReadDirEntry, error)
 }
 
 type fileSystem struct{}
@@ -173,3 +207,41 @@ func (f *fileSystem) ReadFileLines(filename string) ([]string, error) {
 
 	return l, nil
 }
+
+func (f *fileSystem) ReadDirRecursive(path string, depth int, strictDepth bool) ([]ReadDirEntry, error) {
+	if depth <= 0 {
+		return []ReadDirEntry{}, nil
+	}
+
+	dirEntries, err := os.ReadDir(path)
+	if err != nil {
+		return nil, err
+	}
+
+	entries := utils.Map(dirEntries, mapDirEntry(path))
+	if depth == 1 {
+		return entries, nil
+	}
+
+	out := []ReadDirEntry{}
+	for _, entry := range entries {
+		if !entry.IsDir() {
+			if !strictDepth {
+				out = append(out, entry)
+			}
+			continue
+		}
+
+		subEntries, err := f.ReadDirRecursive(entry.Path(), depth-1, strictDepth)
+		if err != nil {
+			return nil, err
+		}
+		if !strictDepth && len(subEntries) == 0 {
+			out = append(out, entry)
+			continue
+		}
+		out = append(out, subEntries...)
+	}
+
+	return out, nil
+}

internal/core/paths.go

@@ -2,20 +2,21 @@ package core
 
 import (
 	"os/user"
+	"path/filepath"
 )
 
 func bundleroot(user *user.User) string {
-	return user.HomeDir + "/git-bundle-server/"
+	return filepath.Join(user.HomeDir, "git-bundle-server")
 }
 
 func webroot(user *user.User) string {
-	return bundleroot(user) + "www/"
+	return filepath.Join(bundleroot(user), "www")
 }
 
 func reporoot(user *user.User) string {
-	return bundleroot(user) + "git/"
+	return filepath.Join(bundleroot(user), "git")
 }
 
 func CrontabFile(user *user.User) string {
-	return bundleroot(user) + "cron-schedule"
+	return filepath.Join(bundleroot(user), "cron-schedule")
 }