diff --git a/kadai3-2/hokita/README.md b/kadai3-2/hokita/README.md new file mode 100644 index 0000000..e409bf4 --- /dev/null +++ b/kadai3-2/hokita/README.md @@ -0,0 +1,45 @@ +# 課題3-2 +## 分割ダウンローダを作ろう +- 分割ダウンロードを行う + - Rangeアクセスを用いる + - いくつかのゴルーチンでダウンロードしてマージする + - エラー処理を工夫する + - golang.org/x/sync/errgourpパッケージなどを使ってみる + - キャンセルが発生した場合の実装を行う + +### 動作 +```shell +$ go build -o pdl cmd/pdl/main.go + +$ ./pdl -proc 10 https://blog.golang.org/gopher/header.jpg +start download worker: 1 +start download worker: 10 +start download worker: 4 +start download worker: 2 +start download worker: 5 +start download worker: 7 +start download worker: 9 +start download worker: 6 +start download worker: 3 +start download worker: 8 +finish download worker: 4 +finish download worker: 8 +finish download worker: 5 +finish download worker: 3 +finish download worker: 7 +finish download worker: 9 +finish download worker: 1 +finish download worker: 10 +finish download worker: 2 +finish download worker: 6 +finished + +$ ls testdata/header.jpg +testdata/header.jpg +``` + +## わからなかったこと、むずかしかったこと +- そもそもurlからダウンロードをどう実現するのかを考えるのに時間がかかった。 + - `pget`を参考にした。 + - cf. https://github.com/Code-Hex/pget + - 結局はurlでアクセスして読み込んだ情報(`io.Reader`)をファイルに書き込む(`io.writer`)だけだった。 diff --git a/kadai3-2/hokita/cmd/pdl/main.go b/kadai3-2/hokita/cmd/pdl/main.go new file mode 100644 index 0000000..576cddc --- /dev/null +++ b/kadai3-2/hokita/cmd/pdl/main.go @@ -0,0 +1,43 @@ +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/gopherdojo/dojo8/kadai3-2/hokita/pdl" +) + +const ( + ExitCode = 0 + ExitCodeErr = 1 +) + +var proc int +var dir string + +func init() { + flag.IntVar(&proc, "proc", 10, "split ratio to download file") + flag.StringVar(&dir, "dir", "testdata", "output file") +} + +func main() { + flag.Parse() + exitCode := run(flag.Arg(0)) + os.Exit(exitCode) +} + +func run(url string) int { + cli, err := pdl.New(proc, url, dir) + if err != nil { + fmt.Fprintf(os.Stderr, "Error:%v\n", err) + return ExitCodeErr + } + + if err := cli.Run(); err != nil { + fmt.Fprintf(os.Stderr, "Error:%v\n", err) + return ExitCodeErr + } + + return ExitCode +} diff --git a/kadai3-2/hokita/go.mod b/kadai3-2/hokita/go.mod new file mode 100644 index 0000000..14f0f61 --- /dev/null +++ b/kadai3-2/hokita/go.mod @@ -0,0 +1,8 @@ +module github.com/gopherdojo/dojo8/kadai3-2/hokita/pdl + +go 1.14 + +require ( + github.com/pkg/errors v0.9.1 + golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 +) diff --git a/kadai3-2/hokita/go.sum b/kadai3-2/hokita/go.sum new file mode 100644 index 0000000..e8a22ca --- /dev/null +++ b/kadai3-2/hokita/go.sum @@ -0,0 +1,4 @@ +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/kadai3-2/hokita/pdl.go b/kadai3-2/hokita/pdl.go new file mode 100644 index 0000000..b16e90a --- /dev/null +++ b/kadai3-2/hokita/pdl.go @@ -0,0 +1,216 @@ +package pdl + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" +) + +type PDL struct { + url string + proc uint + fileSize uint + split uint + dir string + filename string +} + +type Range struct { + start uint + end uint +} + +func New(proc int, url, dir string) (*PDL, error) { + if url == "" { + return nil, errors.New("no url specified") + } + + pdl := &PDL{ + proc: uint(proc), + url: url, + dir: dir, + } + + pdl.setSize() + pdl.setFilename() + + return pdl, nil +} + +func (p *PDL) Run() error { + if err := p.download(); err != nil { + return err + } + + if err := p.merge(); err != nil { + return err + } + + fmt.Println("finished") + return nil +} + +func (p *PDL) download() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + eg, egctx := errgroup.WithContext(ctx) + + for i := 0; i < int(p.proc); i++ { + i := i + eg.Go(func() error { + return p.partialDownload(egctx, i) + }) + } + if err := eg.Wait(); err != nil { + return err + } + return nil +} + +func (p *PDL) merge() (rerr error) { + out, err := os.Create(p.filePath()) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to Create %s in Merge", p.filePath())) + } + defer func() { + if err := out.Close(); err != nil { + rerr = err + } + }() + + for i := 0; i < int(p.proc); i++ { + worker := i + 1 + + tmpfile, err := os.Open(p.tmpFilePath(worker)) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to Open %s in Merge", p.tmpFilePath(worker))) + } + + _, err = io.Copy(out, tmpfile) + + // Not use defer + tmpfile.Close() + + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to Copy %s in Merge", p.tmpFilePath(worker))) + } + + // delete + if err := os.Remove(p.tmpFilePath(worker)); err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to Remove %s in Merge", p.tmpFilePath(worker))) + } + } + + return nil +} + +func (p *PDL) makeRange(i, proc uint) Range { + start := p.split * i + end := start + p.split - 1 + if i == proc-1 { + end = p.fileSize + } + + return Range{ + start: start, + end: end, + } +} + +func (p *PDL) setSize() error { + resp, err := http.Head(p.url) + if err != nil { + return errors.Wrap(err, "failed to get Head") + } + + p.fileSize = uint(resp.ContentLength) + p.split = p.fileSize / p.proc + + return nil +} + +func (p *PDL) setFilename() { + token := strings.Split(p.url, "/") + + var original string + for i := 1; original == ""; i++ { + original = token[len(token)-i] + } + + p.filename = original +} + +func (p *PDL) tmpFilename(worker int) string { + return fmt.Sprintf("%s.%d", p.filename, worker) +} + +func (p *PDL) partialDownload(ctx context.Context, index int) error { + worker := index + 1 + + fmt.Printf("start download worker: %d\n", worker) + + // request + req, err := http.NewRequest("GET", p.url, nil) + if err != nil { + return errors.Wrap(err, "failed to create NewRequest for GET") + } + + r := p.makeRange(uint(index), p.proc) + + // set header + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", r.start, r.end)) + + // do + resp, err := http.DefaultClient.Do(req) + if err != nil { + return errors.Wrap(err, "failed to access") + } + defer resp.Body.Close() + + // write + if err := p.writeTmpfile(resp.Body, int(worker)); err != nil { + return err + } + + select { + case <-ctx.Done(): + fmt.Printf("cancelled worker: %d\n", worker) + return ctx.Err() + default: + fmt.Printf("finish download worker: %d\n", worker) + return nil + } +} + +func (p *PDL) filePath() string { + return filepath.Join(p.dir, p.filename) +} + +func (p *PDL) tmpFilePath(worker int) string { + return filepath.Join(p.dir, p.tmpFilename(worker)) +} + +func (p *PDL) writeTmpfile(body io.Reader, worker int) (rerr error) { + out, err := os.Create(p.tmpFilePath(worker)) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to create file, worker: %d", worker)) + } + defer func() { + if err := out.Close(); err != nil { + rerr = errors.Wrap(err, fmt.Sprintf("failed to close file, worker: %d", worker)) + } + }() + + if _, err := io.Copy(out, body); err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to write file, worker: %d", worker)) + } + + return nil +} diff --git a/kadai3-2/hokita/pdl_test.go b/kadai3-2/hokita/pdl_test.go new file mode 100644 index 0000000..0d4c9fb --- /dev/null +++ b/kadai3-2/hokita/pdl_test.go @@ -0,0 +1,42 @@ +package pdl + +import ( + "os" + "path/filepath" + "testing" +) + +func TestRun(t *testing.T) { + tests := map[string]struct { + proc int + url string + dir string + want string + }{ + "download": { + proc: 3, + url: "https://blog.golang.org/gopher/header.jpg", + dir: "testdata", + want: filepath.Join("testdata", "header.jpg"), + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + pdl := New(test.proc, test.url, test.dir) + err := pdl.Run() + + if err != nil { + t.Fatal(err) + } + + if _, err := os.Stat(test.want); err != nil { + t.Errorf(`"%v" was not found`, test.want) + } + + if err := os.Remove(test.want); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/kadai3-2/hokita/testdata/.gitkeep b/kadai3-2/hokita/testdata/.gitkeep new file mode 100644 index 0000000..e69de29