diff --git a/kadai3-2/pei/Makefile b/kadai3-2/pei/Makefile new file mode 100644 index 0000000..9aed06f --- /dev/null +++ b/kadai3-2/pei/Makefile @@ -0,0 +1,17 @@ +ROOT=github.com/gopherdojo/dojo6/kadai3-2/pei +BIN=split-download +MAIN=main.go +TEST=... + +.PHONY: build +build: ${MAIN} + go build -o ${BIN} ${GOPATH}/src/${ROOT}/$? + +.PHONY: test +test: + go test -v -cover ${ROOT}/${TEST} + +.PHONY: clean +clean: + rm ${BIN} + go clean diff --git a/kadai3-2/pei/README.md b/kadai3-2/pei/README.md new file mode 100644 index 0000000..b2df4f4 --- /dev/null +++ b/kadai3-2/pei/README.md @@ -0,0 +1,33 @@ +# split-download + +## Build + +``` +$ make build +``` + +## Usage + +``` +$ ./split-loadload -o [OutputPath] [URL] +``` + +## Development + +### Build + +``` +$ make build +``` + +### Test + +``` +$ make test +``` + +### Clean + +``` +$ make clean +``` diff --git a/kadai3-2/pei/main.go b/kadai3-2/pei/main.go new file mode 100644 index 0000000..92dd96e --- /dev/null +++ b/kadai3-2/pei/main.go @@ -0,0 +1,78 @@ +package main + +import ( + "flag" + "fmt" + "os" + "reflect" + + "github.com/gopherdojo/dojo6/kadai3-2/pei/pkg/download" +) + +const ( + exitCodeOk = 0 + exitCodeError = 1 + + splitNum = 4 +) + +type cliArgs struct { + url, outputPath string +} + +func (ca *cliArgs) validate() error { + if ca.url == "" { + return fmt.Errorf("No URL") + } + + if ca.outputPath == "" { + ca.outputPath = "./" + } + + return nil +} + +func main() { + os.Exit(Run()) +} + +// Run runs download +func Run() int { + ca := parseArgs() + if err := ca.validate(); err != nil { + fmt.Fprintln(os.Stderr, "Args error: ", err) + return exitCodeError + } + + downloader, err := download.NewDownloader(splitNum, ca.url, ca.outputPath) + if err != nil { + fmt.Fprintln(os.Stderr, "Create downloader error: ", err) + return exitCodeError + } + + outputPath, err := downloader.Do() + if err != nil { + fmt.Fprintln(os.Stderr, "Download error: ", err) + return exitCodeError + } + + var downloadType string + if reflect.TypeOf(downloader) == reflect.TypeOf(&download.RangeDownloader{}) { + downloadType = "Split Download" + } else { + downloadType = "Download" + } + fmt.Println("Download Type: ", downloadType) + fmt.Println("Download completed. Output: ", outputPath) + + return exitCodeOk +} + +func parseArgs() *cliArgs { + var ca cliArgs + flag.StringVar(&ca.outputPath, "o", "", "output path") + flag.Parse() + ca.url = flag.Arg(0) + + return &ca +} diff --git a/kadai3-2/pei/pkg/download/download.go b/kadai3-2/pei/pkg/download/download.go new file mode 100644 index 0000000..d0e8d64 --- /dev/null +++ b/kadai3-2/pei/pkg/download/download.go @@ -0,0 +1,171 @@ +package download + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + + "golang.org/x/sync/errgroup" +) + +// Downloader Interface +type Downloader interface { + Do() (string, error) +} + +// NonRangeDownloader has info for downloading +type NonRangeDownloader struct { + url, outputPath string +} + +// RangeDownloader has info for split downloading +type RangeDownloader struct { + splitNum int + ranges []*Range + url, outputPath string +} + +// Range has rangestart and rangeend +type Range struct { + start int64 + end int64 +} + +// NewDownloader creates Downloader +func NewDownloader(splitNum int, url, outputPath string) (Downloader, error) { + dir, fileName := parseDirAndFileName(outputPath) + if fileName == "" { + outputPath = filepath.Join(dir, parseFileName(url)) + } + + res, err := http.Head(url) + if err != nil { + return nil, err + } + defer res.Body.Close() + + if res.Header.Get("Accept-Ranges") != "bytes" { + return &NonRangeDownloader{url: url, outputPath: outputPath}, nil + } + + contentLength := res.ContentLength + unit := contentLength / int64(splitNum) + ranges := make([]*Range, splitNum) + + for i := range ranges { + var start, end int64 + if i != 0 { + start = int64(i)*unit + 1 + } + end = int64(i+1) * unit + if i == splitNum-1 { + end = contentLength + } + + ranges[i] = &Range{start: start, end: end} + } + + return &RangeDownloader{ + splitNum: splitNum, + ranges: ranges, + url: url, + outputPath: outputPath, + }, nil +} + +// Do download +func (d *NonRangeDownloader) Do() (string, error) { + req, err := http.NewRequest(http.MethodGet, d.url, nil) + if err != nil { + return "", err + } + + client := http.DefaultClient + res, err := client.Do(req) + if err != nil { + return "", err + } + defer res.Body.Close() + + return d.outputPath, saveResponseBody(d.outputPath, res) +} + +// Do split download +func (d *RangeDownloader) Do() (string, error) { + eg, ctx := errgroup.WithContext(context.TODO()) + + for i := range d.ranges { + i := i + eg.Go(func() error { + return d.do(ctx, i) + }) + } + + if err := eg.Wait(); err != nil { + return "", err + } + + return d.outputPath, d.mergeFiles() +} + +func (d *RangeDownloader) do(ctx context.Context, idx int) error { + req, err := http.NewRequest(http.MethodGet, d.url, nil) + if err != nil { + return err + } + req = req.WithContext(ctx) + + ran := d.ranges[idx] + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", ran.start, ran.end)) + + client := http.DefaultClient + res, err := client.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + + tmpFileName := fmt.Sprintf("%s.%d", d.outputPath, idx) + return saveResponseBody(tmpFileName, res) +} + +func (d *RangeDownloader) mergeFiles() error { + file, err := os.Create(d.outputPath) + if err != nil { + return err + } + defer file.Close() + + for i := range d.ranges { + tmpFileName := fmt.Sprintf("%s.%d", d.outputPath, i) + tmpFile, err := os.Open(tmpFileName) + if err != nil { + return err + } + + io.Copy(file, tmpFile) + tmpFile.Close() + if err := os.Remove(tmpFileName); err != nil { + return err + } + } + + return nil +} + +func saveResponseBody(fileName string, response *http.Response) error { + file, err := os.Create(fileName) + if err != nil { + return err + } + defer file.Close() + + if _, err := io.Copy(file, response.Body); err != nil { + return err + } + + return nil +} diff --git a/kadai3-2/pei/pkg/download/file.go b/kadai3-2/pei/pkg/download/file.go new file mode 100644 index 0000000..ccde7e1 --- /dev/null +++ b/kadai3-2/pei/pkg/download/file.go @@ -0,0 +1,20 @@ +package download + +import ( + "path/filepath" + "strings" +) + +func parseDirAndFileName(path string) (dir, file string) { + lastSlashIndex := strings.LastIndex(path, "/") + dir = path[:lastSlashIndex+1] + if len(dir) == len(path) { + return dir, "" + } + + return dir, path[(len(dir) + 1):] +} + +func parseFileName(url string) string { + return filepath.Base(url) +}