From a459917c159bc7c14414bc9e86ca97f70231a2cc Mon Sep 17 00:00:00 2001 From: "asuke.yasukuni" Date: Mon, 23 Sep 2019 16:51:01 +0900 Subject: [PATCH 1/7] add normal download --- kadai3_2/go.mod | 3 +++ kadai3_2/main.go | 26 ++++++++++++++++++++++++ kadai3_2/muget/download.go | 41 ++++++++++++++++++++++++++++++++++++++ kadai3_2/muget/run.go | 16 +++++++++++++++ 4 files changed, 86 insertions(+) create mode 100644 kadai3_2/go.mod create mode 100644 kadai3_2/main.go create mode 100644 kadai3_2/muget/download.go create mode 100644 kadai3_2/muget/run.go diff --git a/kadai3_2/go.mod b/kadai3_2/go.mod new file mode 100644 index 0000000..69024f1 --- /dev/null +++ b/kadai3_2/go.mod @@ -0,0 +1,3 @@ +module github.com/gopherdojo/dojo7/kadai3_2/asuke-yasukuni + +go 1.12 diff --git a/kadai3_2/main.go b/kadai3_2/main.go new file mode 100644 index 0000000..9e586f9 --- /dev/null +++ b/kadai3_2/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "flag" + "log" + "os" + + "github.com/gopherdojo/dojo7/kadai3_2/asuke-yasukuni/muget" +) + +var get = flag.String("get", "", "ダウンロードパス") +var out = flag.String("out", "", "ファイルパス") + +func main() { + flag.Parse() + + log.Println("Download Start") + log.Println(*get) + log.Println(*out) + + if err := muget.Run(*get, *out); err != nil { + log.Fatal(err) + } + + os.Exit(0) +} diff --git a/kadai3_2/muget/download.go b/kadai3_2/muget/download.go new file mode 100644 index 0000000..99e11c1 --- /dev/null +++ b/kadai3_2/muget/download.go @@ -0,0 +1,41 @@ +package muget + +import ( + "io" + "log" + "net/http" + "os" + "path/filepath" +) + +func DownloadFile(url string, path string) error { + // Create the file + out, err := os.Create(path + filepath.Base(url)) + if err != nil { + return err + } + defer func() { + if err := out.Close(); err != nil { + log.Printf("\x1b[31m%s:%s\x1b[0m\n", "[encode error]", err) + } + }() + + // Get the data + resp, err := http.Get(url) + if err != nil { + return err + } + defer func() { + if err := resp.Body.Close(); err != nil { + log.Printf("\x1b[31m%s:%s\x1b[0m\n", "[encode error]", err) + } + }() + + // Write the body to file + _, err = io.Copy(out, resp.Body) + if err != nil { + return err + } + + return nil +} diff --git a/kadai3_2/muget/run.go b/kadai3_2/muget/run.go new file mode 100644 index 0000000..f948dc3 --- /dev/null +++ b/kadai3_2/muget/run.go @@ -0,0 +1,16 @@ +package muget + +type FileData struct { + Name string + Size uint + Dirname string + FullName string +} + +func Run(filePath, outPutPath string) error { + //TODO: 一旦普通のダウンロード + if err := DownloadFile(filePath, outPutPath); err != nil { + return err + } + return nil +} From c54d13b6d8d37212a62618c52ef4faef1e0c6a81 Mon Sep 17 00:00:00 2001 From: "asuke.yasukuni" Date: Tue, 24 Sep 2019 01:32:13 +0900 Subject: [PATCH 2/7] add start files --- kadai3_2/go.mod | 2 ++ kadai3_2/go.sum | 5 +++++ kadai3_2/muget/checker.go | 27 +++++++++++++++++++++++++++ kadai3_2/muget/download.go | 19 +++++++++++++++++-- kadai3_2/muget/run.go | 11 +++++++++-- 5 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 kadai3_2/go.sum create mode 100644 kadai3_2/muget/checker.go diff --git a/kadai3_2/go.mod b/kadai3_2/go.mod index 69024f1..cb60368 100644 --- a/kadai3_2/go.mod +++ b/kadai3_2/go.mod @@ -1,3 +1,5 @@ module github.com/gopherdojo/dojo7/kadai3_2/asuke-yasukuni go 1.12 + +require golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72 diff --git a/kadai3_2/go.sum b/kadai3_2/go.sum new file mode 100644 index 0000000..e63a29c --- /dev/null +++ b/kadai3_2/go.sum @@ -0,0 +1,5 @@ +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72 h1:PdU68SuVQNpTFEyGl0zoQOMysY+E0innv/QbAqV853w= +golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/kadai3_2/muget/checker.go b/kadai3_2/muget/checker.go new file mode 100644 index 0000000..d3a6720 --- /dev/null +++ b/kadai3_2/muget/checker.go @@ -0,0 +1,27 @@ +package muget + +import ( + "context" + "errors" + "fmt" + "net/http" + + "golang.org/x/net/context/ctxhttp" +) + +func CheckRanges(ctx context.Context, url string) (int, error) { + res, err := ctxhttp.Head(ctx, http.DefaultClient, url) + if err != nil { + return 0, err + } + + if res.Header.Get("Accept-Ranges") != "bytes" { + return 0, fmt.Errorf("not supported range access: %s", url) + } + + if res.ContentLength <= 0 { + return 0, errors.New("invalid content length") + } + + return int(res.ContentLength), nil +} diff --git a/kadai3_2/muget/download.go b/kadai3_2/muget/download.go index 99e11c1..7273fd6 100644 --- a/kadai3_2/muget/download.go +++ b/kadai3_2/muget/download.go @@ -1,6 +1,7 @@ package muget import ( + "fmt" "io" "log" "net/http" @@ -8,7 +9,7 @@ import ( "path/filepath" ) -func DownloadFile(url string, path string) error { +func DownloadFile(url string, path string, downloadStartSize, downloadEndSize int) error { // Create the file out, err := os.Create(path + filepath.Base(url)) if err != nil { @@ -21,7 +22,7 @@ func DownloadFile(url string, path string) error { }() // Get the data - resp, err := http.Get(url) + resp, err := RangeRequest(url, downloadStartSize, downloadEndSize) if err != nil { return err } @@ -39,3 +40,17 @@ func DownloadFile(url string, path string) error { return nil } + +// RangeRequest return *http.Response include context and range header +func RangeRequest(url string, low, high int) (*http.Response, error) { + // create get request + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + // set download ranges + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", low, high)) + + return http.DefaultClient.Do(req) +} diff --git a/kadai3_2/muget/run.go b/kadai3_2/muget/run.go index f948dc3..3652342 100644 --- a/kadai3_2/muget/run.go +++ b/kadai3_2/muget/run.go @@ -1,5 +1,7 @@ package muget +import "context" + type FileData struct { Name string Size uint @@ -7,9 +9,14 @@ type FileData struct { FullName string } -func Run(filePath, outPutPath string) error { +func Run(url, outPutPath string) error { //TODO: 一旦普通のダウンロード - if err := DownloadFile(filePath, outPutPath); err != nil { + size, err := CheckRanges(context.Background(), url) + if err != nil { + return err + } + + if err := DownloadFile(url, outPutPath, 0, size); err != nil { return err } return nil From 036ccaa3fc1e06ad2e4d1314618bd0124df96306 Mon Sep 17 00:00:00 2001 From: asuke-yasukuni Date: Tue, 24 Sep 2019 17:33:26 +0900 Subject: [PATCH 3/7] add multi download --- kadai3_2/muget/download.go | 15 +++++------ kadai3_2/muget/run.go | 51 +++++++++++++++++++++++++++++++++++--- 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/kadai3_2/muget/download.go b/kadai3_2/muget/download.go index 7273fd6..06ce920 100644 --- a/kadai3_2/muget/download.go +++ b/kadai3_2/muget/download.go @@ -3,21 +3,21 @@ package muget import ( "fmt" "io" - "log" "net/http" "os" "path/filepath" ) -func DownloadFile(url string, path string, downloadStartSize, downloadEndSize int) error { +func DownloadFile(url string, path string, downloadStartSize, downloadEndSize, downloadCount int) (err error) { // Create the file - out, err := os.Create(path + filepath.Base(url)) + out, err := os.Create(path + fmt.Sprint(downloadCount) + filepath.Ext(url)) if err != nil { return err } defer func() { - if err := out.Close(); err != nil { - log.Printf("\x1b[31m%s:%s\x1b[0m\n", "[encode error]", err) + err = out.Close() + if err != nil { + return } }() @@ -27,8 +27,9 @@ func DownloadFile(url string, path string, downloadStartSize, downloadEndSize in return err } defer func() { - if err := resp.Body.Close(); err != nil { - log.Printf("\x1b[31m%s:%s\x1b[0m\n", "[encode error]", err) + err = resp.Body.Close() + if err != nil { + return } }() diff --git a/kadai3_2/muget/run.go b/kadai3_2/muget/run.go index 3652342..b94b5d6 100644 --- a/kadai3_2/muget/run.go +++ b/kadai3_2/muget/run.go @@ -1,6 +1,10 @@ package muget -import "context" +import ( + "context" + "fmt" + "log" +) type FileData struct { Name string @@ -9,15 +13,54 @@ type FileData struct { FullName string } +type Range struct { + Start int + End int +} + func Run(url, outPutPath string) error { - //TODO: 一旦普通のダウンロード size, err := CheckRanges(context.Background(), url) if err != nil { return err } - if err := DownloadFile(url, outPutPath, 0, size); err != nil { - return err + start := 0 + end := 0 + var ranges []Range + for start <= size { + end = start+(size/10) + ranges = append(ranges, Range{ + Start: start, + End: end, + }) + start = end + } + + ch := make(chan int) + for i,r := range ranges { + go func(r Range, count int) { + fmt.Println(r.Start,"->",r.End) + if err := DownloadFile(url, outPutPath, r.Start, r.End, count); err != nil { + log.Fatal(err) + } + ch <- i + }(r,i) + } + + //ダウンロード完了まで待つ + var count int +D: + for { + select { + case <-ch: + count++ + fmt.Println(count) + if len(ranges) <= count { + close(ch) + break D + } + } } + return nil } From 89cbe2fd41f530a1559cddb72b31c2f4179a2957 Mon Sep 17 00:00:00 2001 From: asuke-yasukuni Date: Tue, 24 Sep 2019 17:54:34 +0900 Subject: [PATCH 4/7] add bind files --- kadai3_2/muget/merge.go | 41 +++++++++++++++++++++++++++++++++++++++++ kadai3_2/muget/run.go | 5 +++++ 2 files changed, 46 insertions(+) create mode 100644 kadai3_2/muget/merge.go diff --git a/kadai3_2/muget/merge.go b/kadai3_2/muget/merge.go new file mode 100644 index 0000000..b192bf3 --- /dev/null +++ b/kadai3_2/muget/merge.go @@ -0,0 +1,41 @@ +package muget + +import ( + "fmt" + "io" + "os" +) + +func BindwithFiles(count int, ext string) error { + + fmt.Println("\nbinding with files...") + + fh, err := os.Create("gogogo"+ ext) + if err != nil { + return err + } + defer fh.Close() + + var f string + for i := 0; i < count; i++ { + f = fmt.Sprintf("./%d%s", i,ext) + subfp, err := os.Open(f) + if err != nil { + return err + } + + io.Copy(fh, subfp) + + // Not use defer + subfp.Close() + + // remove a file in download location for join + if err := os.Remove(f); err != nil { + return err + } + } + + fmt.Println("Complete") + + return nil +} diff --git a/kadai3_2/muget/run.go b/kadai3_2/muget/run.go index b94b5d6..edbb750 100644 --- a/kadai3_2/muget/run.go +++ b/kadai3_2/muget/run.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log" + "path/filepath" ) type FileData struct { @@ -62,5 +63,9 @@ D: } } + if err := BindwithFiles(count, filepath.Ext(url)); err != nil { + return err + } + return nil } From 6c428559721ab5ef7609e992d621134e3256dc2c Mon Sep 17 00:00:00 2001 From: asuke-yasukuni Date: Tue, 24 Sep 2019 18:31:36 +0900 Subject: [PATCH 5/7] fix error handler --- kadai3_2/muget/merge.go | 35 +++++++++++++++++++++-------------- kadai3_2/muget/run.go | 30 +++++++++++++----------------- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/kadai3_2/muget/merge.go b/kadai3_2/muget/merge.go index b192bf3..eab6119 100644 --- a/kadai3_2/muget/merge.go +++ b/kadai3_2/muget/merge.go @@ -6,36 +6,43 @@ import ( "os" ) -func BindwithFiles(count int, ext string) error { +func MergeFiles(count int, fileName, ext string) (err error) { - fmt.Println("\nbinding with files...") - - fh, err := os.Create("gogogo"+ ext) + fh, err := os.Create(fileName) if err != nil { return err } - defer fh.Close() + defer func() { + err = fh.Close() + if err != nil { + return + } + }() var f string for i := 0; i < count; i++ { - f = fmt.Sprintf("./%d%s", i,ext) - subfp, err := os.Open(f) + f = fmt.Sprintf("./%d%s", i, ext) + openFile, err := os.Open(f) if err != nil { return err } - io.Copy(fh, subfp) + _, err = io.Copy(fh, openFile) + if err != nil { + return err + } - // Not use defer - subfp.Close() + err = openFile.Close() + if err != nil { + return err + } // remove a file in download location for join - if err := os.Remove(f); err != nil { + err = os.Remove(f) + if err != nil { return err } } - fmt.Println("Complete") - - return nil + return } diff --git a/kadai3_2/muget/run.go b/kadai3_2/muget/run.go index edbb750..69efecc 100644 --- a/kadai3_2/muget/run.go +++ b/kadai3_2/muget/run.go @@ -7,16 +7,9 @@ import ( "path/filepath" ) -type FileData struct { - Name string - Size uint - Dirname string - FullName string -} - type Range struct { Start int - End int + End int } func Run(url, outPutPath string) error { @@ -25,11 +18,13 @@ func Run(url, outPutPath string) error { return err } - start := 0 - end := 0 - var ranges []Range + var ( + start, end int + ranges []Range + ) + for start <= size { - end = start+(size/10) + end = start + (size / 10) ranges = append(ranges, Range{ Start: start, End: end, @@ -38,14 +33,13 @@ func Run(url, outPutPath string) error { } ch := make(chan int) - for i,r := range ranges { + for i, r := range ranges { go func(r Range, count int) { - fmt.Println(r.Start,"->",r.End) if err := DownloadFile(url, outPutPath, r.Start, r.End, count); err != nil { log.Fatal(err) } ch <- i - }(r,i) + }(r, i) } //ダウンロード完了まで待つ @@ -55,7 +49,6 @@ D: select { case <-ch: count++ - fmt.Println(count) if len(ranges) <= count { close(ch) break D @@ -63,7 +56,10 @@ D: } } - if err := BindwithFiles(count, filepath.Ext(url)); err != nil { + fmt.Println("\nbinding with files...") + + //ダウンロードファイルをマージ + if err := MergeFiles(count, filepath.Base(url), filepath.Ext(url)); err != nil { return err } From 47682808c052e937b79641d969cad617de710a5c Mon Sep 17 00:00:00 2001 From: asuke-yasukuni Date: Tue, 24 Sep 2019 19:16:27 +0900 Subject: [PATCH 6/7] add errgroup --- kadai3_2/asuke-yasukuni/go.mod | 8 +++++ kadai3_2/{ => asuke-yasukuni}/go.sum | 2 ++ kadai3_2/{ => asuke-yasukuni}/main.go | 0 .../{ => asuke-yasukuni}/muget/checker.go | 0 .../{ => asuke-yasukuni}/muget/download.go | 0 kadai3_2/{ => asuke-yasukuni}/muget/merge.go | 0 kadai3_2/{ => asuke-yasukuni}/muget/run.go | 31 +++++++------------ kadai3_2/go.mod | 5 --- 8 files changed, 21 insertions(+), 25 deletions(-) create mode 100644 kadai3_2/asuke-yasukuni/go.mod rename kadai3_2/{ => asuke-yasukuni}/go.sum (70%) rename kadai3_2/{ => asuke-yasukuni}/main.go (100%) rename kadai3_2/{ => asuke-yasukuni}/muget/checker.go (100%) rename kadai3_2/{ => asuke-yasukuni}/muget/download.go (100%) rename kadai3_2/{ => asuke-yasukuni}/muget/merge.go (100%) rename kadai3_2/{ => asuke-yasukuni}/muget/run.go (61%) delete mode 100644 kadai3_2/go.mod diff --git a/kadai3_2/asuke-yasukuni/go.mod b/kadai3_2/asuke-yasukuni/go.mod new file mode 100644 index 0000000..f6f988f --- /dev/null +++ b/kadai3_2/asuke-yasukuni/go.mod @@ -0,0 +1,8 @@ +module github.com/gopherdojo/dojo7/kadai3_2/asuke-yasukuni + +go 1.12 + +require ( + golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72 + golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e +) diff --git a/kadai3_2/go.sum b/kadai3_2/asuke-yasukuni/go.sum similarity index 70% rename from kadai3_2/go.sum rename to kadai3_2/asuke-yasukuni/go.sum index e63a29c..dd99482 100644 --- a/kadai3_2/go.sum +++ b/kadai3_2/asuke-yasukuni/go.sum @@ -1,5 +1,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72 h1:PdU68SuVQNpTFEyGl0zoQOMysY+E0innv/QbAqV853w= golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/kadai3_2/main.go b/kadai3_2/asuke-yasukuni/main.go similarity index 100% rename from kadai3_2/main.go rename to kadai3_2/asuke-yasukuni/main.go diff --git a/kadai3_2/muget/checker.go b/kadai3_2/asuke-yasukuni/muget/checker.go similarity index 100% rename from kadai3_2/muget/checker.go rename to kadai3_2/asuke-yasukuni/muget/checker.go diff --git a/kadai3_2/muget/download.go b/kadai3_2/asuke-yasukuni/muget/download.go similarity index 100% rename from kadai3_2/muget/download.go rename to kadai3_2/asuke-yasukuni/muget/download.go diff --git a/kadai3_2/muget/merge.go b/kadai3_2/asuke-yasukuni/muget/merge.go similarity index 100% rename from kadai3_2/muget/merge.go rename to kadai3_2/asuke-yasukuni/muget/merge.go diff --git a/kadai3_2/muget/run.go b/kadai3_2/asuke-yasukuni/muget/run.go similarity index 61% rename from kadai3_2/muget/run.go rename to kadai3_2/asuke-yasukuni/muget/run.go index 69efecc..92156da 100644 --- a/kadai3_2/muget/run.go +++ b/kadai3_2/asuke-yasukuni/muget/run.go @@ -3,8 +3,9 @@ package muget import ( "context" "fmt" - "log" "path/filepath" + + "golang.org/x/sync/errgroup" ) type Range struct { @@ -32,34 +33,24 @@ func Run(url, outPutPath string) error { start = end } - ch := make(chan int) + eg := errgroup.Group{} for i, r := range ranges { - go func(r Range, count int) { - if err := DownloadFile(url, outPutPath, r.Start, r.End, count); err != nil { - log.Fatal(err) - } - ch <- i - }(r, i) + i := i + r := r + eg.Go(func() error { + return DownloadFile(url, outPutPath, r.Start, r.End, i) + }) } //ダウンロード完了まで待つ - var count int -D: - for { - select { - case <-ch: - count++ - if len(ranges) <= count { - close(ch) - break D - } - } + if err := eg.Wait(); err != nil { + return err } fmt.Println("\nbinding with files...") //ダウンロードファイルをマージ - if err := MergeFiles(count, filepath.Base(url), filepath.Ext(url)); err != nil { + if err := MergeFiles(len(ranges), filepath.Base(url), filepath.Ext(url)); err != nil { return err } diff --git a/kadai3_2/go.mod b/kadai3_2/go.mod deleted file mode 100644 index cb60368..0000000 --- a/kadai3_2/go.mod +++ /dev/null @@ -1,5 +0,0 @@ -module github.com/gopherdojo/dojo7/kadai3_2/asuke-yasukuni - -go 1.12 - -require golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72 From dbaf2998b01c107425d3f90e4fa8414ffdbd0852 Mon Sep 17 00:00:00 2001 From: asuke-yasukuni Date: Tue, 24 Sep 2019 19:36:46 +0900 Subject: [PATCH 7/7] add comment --- kadai3_2/asuke-yasukuni/muget/run.go | 1 + 1 file changed, 1 insertion(+) diff --git a/kadai3_2/asuke-yasukuni/muget/run.go b/kadai3_2/asuke-yasukuni/muget/run.go index 92156da..c12ea0e 100644 --- a/kadai3_2/asuke-yasukuni/muget/run.go +++ b/kadai3_2/asuke-yasukuni/muget/run.go @@ -33,6 +33,7 @@ func Run(url, outPutPath string) error { start = end } + // TODO: contextとかつかってうまくキャンセルしてあげる eg := errgroup.Group{} for i, r := range ranges { i := i