Merge pull request #18 from jmhobbs/cache-all

Add command to download and cache lists.

Author: umutphp

internal/package/fetcher/all.go

@@ -0,0 +1,118 @@
+package fetcher
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/umutphp/awesome-cli/internal/package/node"
+	"github.com/umutphp/awesome-cli/internal/package/parser"
+)
+
+const DOWNLOAD_CONCURRENCY = 4
+
+type Progress struct {
+	Found uint64
+	Crawled uint64
+	Errors uint64
+}
+
+func FetchAllRepos(update chan Progress) (Progress, error) {
+	fetched, err := FetchAwesomeRootRepo()
+	if err != nil {
+		return Progress{1, 0, 1}, err
+	}
+
+	root := parser.ParseIndex(fetched)
+	urls := getAllChildrenURLs(root.GetChildren())
+
+	progress := Progress{
+		Found: uint64(1 + len(urls)),
+		Crawled: 1,
+		Errors: 0,
+	}
+	update <- progress
+
+	var wg sync.WaitGroup
+	queue := make(chan string)
+	errors := make(chan error)
+
+	for i := 0; i < 4; i++ {
+		go fetchWorker(queue, errors, &wg)
+	}
+
+	allErrors := MultiError{[]error{}}
+	go func () {
+		for err := range errors {
+			if err == nil {
+				progress.Crawled = progress.Crawled + 1
+			} else {
+				progress.Errors = progress.Errors + 1
+				allErrors.Errors = append(allErrors.Errors, err)
+			}
+			update <- progress
+		}
+		close(update)
+	}()
+
+	for _, url := range urls {
+		queue <- url
+	}
+
+	close(queue)
+	wg.Wait()
+	close(errors)
+
+	if len(allErrors.Errors) == 0 {
+		return progress, nil
+	}
+	return progress, allErrors
+}
+
+func getAllChildrenURLs(children []node.Node) []string {
+	urls := []string{}
+	for _, child := range children {
+		if child.GetURL() != "" {
+			urls = append(urls, child.GetURL())
+		}
+		urls = append(urls, getAllChildrenURLs(child.GetChildren())...)
+	}
+	return urls
+}
+
+type MultiError struct {
+	Errors []error
+}
+
+func (e MultiError) Error() string {
+	errStrings := []string{}
+	for _, err := range e.Errors {
+		errStrings = append(errStrings, err.Error())
+	}
+	return strings.Join(errStrings, "\n")
+}
+
+type FetchError struct {
+	URL string
+	Wrapped error
+}
+
+func (e FetchError) Unwrap() error {
+	return e.Wrapped
+}
+
+func (e FetchError) Error() string {
+	return fmt.Sprintf("failed to fetch %q: %v", e.URL, e.Wrapped)
+}
+
+func fetchWorker(queue chan string, errors chan error, wg *sync.WaitGroup) {
+	wg.Add(1)
+	defer wg.Done()
+	for url := range queue {
+		if _, err := FetchAwesomeRepo(url); err != nil {
+			errors <- FetchError{url, err}
+		} else {
+			errors <- nil
+		}
+	}
+}

internal/package/progress/progress.go

@@ -0,0 +1,24 @@
+package progress
+
+import (
+	"fmt"
+
+	"github.com/umutphp/awesome-cli/internal/package/fetcher"
+)
+
+func ProgressBar(done chan struct{}, updates chan fetcher.Progress) {
+	for progress := range updates {
+		percent := 100.0 * float64(progress.Crawled + progress.Errors) / float64(progress.Found)
+		fmt.Print("\n\033[1A\033[K")
+		fmt.Print("[")
+		for i := 0; i < 20; i++ {
+			if float64(i*5) <= percent {
+				fmt.Print("=")
+			} else {
+				fmt.Print(" ")
+			}
+		}
+		fmt.Printf("] % 3.0f%%", percent)
+	}
+	close(done)
+}

main.go

@@ -5,7 +5,9 @@ import (
 	"os"
 
 	"github.com/umutphp/awesome-cli/internal/package/favourite"
+	"github.com/umutphp/awesome-cli/internal/package/fetcher"
 	"github.com/umutphp/awesome-cli/internal/package/manager"
+	"github.com/umutphp/awesome-cli/internal/package/progress"
 	"github.com/umutphp/awesome-cli/internal/package/prompter"
 	"github.com/umutphp/awesome-cli/internal/package/selfupdate"
 )
@@ -43,6 +45,7 @@ func DisplayHelp() {
 	fmt.Printf("%-2v%-10v%-10v\n", "", "profile", "To see your previous choices.")
 	fmt.Printf("%-2v%-10v%-10v\n", "", "reset", "To clean your choices to start from the beginning.")
 	fmt.Printf("%-2v%-10v%-10v\n", "", "update", "Update awesome-cli to the latest version.")
+	fmt.Printf("%-2v%-10v%-10v\n", "", "cache", "Download all repository information to the cache.")
 
 	fmt.Println("")
 	fmt.Println("")
@@ -100,6 +103,23 @@ func DisplayRepoWithPath(url string, path []string) {
 	prompter.OpenInBrowser(url)
 }
 
+func CacheAll() {
+	fmt.Println("Downloading all reachable lists to cache.")
+	fmt.Println("This may take some time!")
+
+	updates := make(chan fetcher.Progress)
+	done := make(chan struct{})
+
+	go progress.ProgressBar(done, updates)
+	finalResult, errs := fetcher.FetchAllRepos(updates)
+	<-done
+	fmt.Print("\n")
+	fmt.Printf("Downloaded %d / Failed %d\n", finalResult.Crawled, finalResult.Errors)
+	if finalResult.Errors > 0 {
+		fmt.Println(errs)
+	}
+}
+
 func Argumented(param []string, man manager.Manager) {
 	if param[0] == "random" {
 		RandomRepo(man)
@@ -133,6 +153,11 @@ func Argumented(param []string, man manager.Manager) {
 
 		return
 	}
+
+	if param[0] == "cache" {
+		CacheAll()
+		return
+	}
 }
 
 func Walk(man manager.Manager) {