From 9e626c3d2f7a752b29cae4727fafd73c5802c046 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=B1=E7=94=B0=E9=9B=84=E5=A4=AA?= Date: Sat, 5 May 2018 16:16:36 +0900 Subject: [PATCH 1/5] =?UTF-8?q?=E5=88=86=E5=89=B2=E3=83=80=E3=82=A6?= =?UTF-8?q?=E3=83=B3=E3=83=AD=E3=83=BC=E3=83=89=E3=82=92=E5=AE=9F=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3/yuuyamad/.gitignore | 1 + kadai3/yuuyamad/log.go | 13 ++++++ kadai3/yuuyamad/main.go | 91 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+) create mode 100644 kadai3/yuuyamad/.gitignore create mode 100644 kadai3/yuuyamad/log.go create mode 100644 kadai3/yuuyamad/main.go diff --git a/kadai3/yuuyamad/.gitignore b/kadai3/yuuyamad/.gitignore new file mode 100644 index 0000000..2464dd9 --- /dev/null +++ b/kadai3/yuuyamad/.gitignore @@ -0,0 +1 @@ +./idea diff --git a/kadai3/yuuyamad/log.go b/kadai3/yuuyamad/log.go new file mode 100644 index 0000000..9053696 --- /dev/null +++ b/kadai3/yuuyamad/log.go @@ -0,0 +1,13 @@ +package main + +import ( + "log" + "os" +) + +func logError(err error){ + if err != nil{ + log.Fatal(err) + os.Exit(1) + } +} \ No newline at end of file diff --git a/kadai3/yuuyamad/main.go b/kadai3/yuuyamad/main.go new file mode 100644 index 0000000..f15b4f8 --- /dev/null +++ b/kadai3/yuuyamad/main.go @@ -0,0 +1,91 @@ +package main + +import ( + "os" + "net/http" + "strconv" + "sync" + "io/ioutil" + "fmt" + "io" +) + +func main() { + + url := os.Args[1] + + // contents Headerを取得する + res, err := http.Head(url) + if err != nil { + logError(err) + } + + maps := res.Header + length, err := strconv.Atoi(maps["Content-Length"][0]) + + limit := 10 + + len_sub := length/limit + diff := length%limit + body := make([]string, 11) + + wg := &sync.WaitGroup{} + + for i:=0; i < limit; i++ { + wg.Add(1) + + min := len_sub * i + max := len_sub * (i + 1) + + if(i == limit - 1){ + max += diff + } + + go func(min int, max int, i int) { + client := &http.Client{} + req , err := http.NewRequest("GET", url, nil) + if err != nil { + logError(err) + } + range_header := "bytes=" + strconv.Itoa(min) + "-" + strconv.Itoa(max-1) + req.Header.Add("Range", range_header) + resp,err := client.Do(req) + if err != nil { + logError(err) + } + defer resp.Body.Close() + + + reader, err := ioutil.ReadAll(resp.Body) + body[i] = string(reader) + ioutil.WriteFile("hoge.png." + strconv.Itoa(i), []byte(string(body[i])), 0x777) + + wg.Done() + + + }(min, max, i) + } + wg.Wait() + + //分割ダウンロードしたファイルを結合する + fh, err := os.Create("hoge.png") + if err != nil { + logError(err) + } + defer fh.Close() + + for j:=0; j < limit; j++ { + f := fmt.Sprintf("%s.%d", "hoge.png", j) + subfp, err := os.Open(f) + if err != nil{ + logError(err) + } + + io.Copy(fh, subfp) + + subfp.Close() + if err := os.Remove(f); err != nil { + logError(err) + } + } +} From 709d55d1ee01ce6434f866c03e4c1168e9a82819 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=B1=E7=94=B0=E9=9B=84=E5=A4=AA?= Date: Sun, 13 May 2018 19:09:19 +0900 Subject: [PATCH 2/5] =?UTF-8?q?=E3=82=A8=E3=83=A9=E3=83=BC=E5=87=A6?= =?UTF-8?q?=E7=90=86=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3/yuuyamad/main.go | 129 +++++++++++++++++++++++++--------------- 1 file changed, 80 insertions(+), 49 deletions(-) diff --git a/kadai3/yuuyamad/main.go b/kadai3/yuuyamad/main.go index f15b4f8..a4df77a 100644 --- a/kadai3/yuuyamad/main.go +++ b/kadai3/yuuyamad/main.go @@ -4,14 +4,52 @@ import ( "os" "net/http" "strconv" - "sync" "io/ioutil" "fmt" "io" + "golang.org/x/sync/errgroup" + "context" + "time" ) +const LIMIT=10 + +type Range struct { + low int + high int + worker int +} + func main() { + err := download() + if err != nil { + logError(err) + } + + //分割ダウンロードしたファイルを結合する + fh, err := os.Create("hoge.png") + if err != nil { + logError(err) + } + defer fh.Close() + + for j:=0; j < LIMIT; j++ { + f := fmt.Sprintf("%s.%d", "hoge.png", j) + subfp, err := os.Open(f) + if err != nil{ + logError(err) + } + + io.Copy(fh, subfp) + + subfp.Close() + if err := os.Remove(f); err != nil { + logError(err) + } + } +} +func download() error{ url := os.Args[1] // contents Headerを取得する @@ -23,69 +61,62 @@ func main() { maps := res.Header length, err := strconv.Atoi(maps["Content-Length"][0]) - limit := 10 + len_sub := length/LIMIT + diff := length%LIMIT - len_sub := length/limit - diff := length%limit - body := make([]string, 11) + // errorGroup + grp, ctx := errgroup.WithContext(context.Background()) + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() - wg := &sync.WaitGroup{} - for i:=0; i < limit; i++ { - wg.Add(1) + for i:=0; i < LIMIT; i++ { min := len_sub * i max := len_sub * (i + 1) - if(i == limit - 1){ + r := Range{} + r.low = min + r.high = max + r.worker = i + + if(i == (LIMIT - 1)){ max += diff } + // execute get request + grp.Go(func() error { + return requests(ctx, url, r) + }) - go func(min int, max int, i int) { - client := &http.Client{} - req , err := http.NewRequest("GET", url, nil) - if err != nil { - logError(err) - } - range_header := "bytes=" + strconv.Itoa(min) + "-" + strconv.Itoa(max-1) - req.Header.Add("Range", range_header) - resp,err := client.Do(req) - if err != nil { - logError(err) - } - defer resp.Body.Close() - - - reader, err := ioutil.ReadAll(resp.Body) - body[i] = string(reader) - ioutil.WriteFile("hoge.png." + strconv.Itoa(i), []byte(string(body[i])), 0x777) - - wg.Done() - - - }(min, max, i) + fmt.Println(i) } - wg.Wait() + if err := grp.Wait(); err != nil { + return err + } + return nil +} - //分割ダウンロードしたファイルを結合する - fh, err := os.Create("hoge.png") +func requests(ctx context.Context, url string, r Range) error { + body := make([]string, 11) + client := &http.Client{} + req , err := http.NewRequest("GET", url, nil) if err != nil { - logError(err) + return err } - defer fh.Close() + req = req.WithContext(ctx) - for j:=0; j < limit; j++ { - f := fmt.Sprintf("%s.%d", "hoge.png", j) - subfp, err := os.Open(f) - if err != nil{ - logError(err) - } + range_header := "bytes=" + strconv.Itoa(r.low) + "-" + strconv.Itoa(r.high-1) + req.Header.Add("Range", range_header) + resp,err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() - io.Copy(fh, subfp) + reader, err := ioutil.ReadAll(resp.Body) + body[r.worker] = string(reader) + fmt.Println(r.worker) + ioutil.WriteFile("hoge.png." + strconv.Itoa(r.worker), []byte(string(body[r.worker])), 0x777) - subfp.Close() - if err := os.Remove(f); err != nil { - logError(err) - } - } + return nil } From 0699d79b109489544ad2c3b551eba20e101b806c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=B1=E7=94=B0=E9=9B=84=E5=A4=AA?= Date: Tue, 15 May 2018 08:11:13 +0900 Subject: [PATCH 3/5] =?UTF-8?q?context=E3=81=A7=E3=81=AE=E3=82=A8=E3=83=A9?= =?UTF-8?q?=E3=83=BC=E5=87=A6=E7=90=86=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3/yuuyamad/main.go | 94 +++++++++++++++++++++++++++-------------- 1 file changed, 63 insertions(+), 31 deletions(-) diff --git a/kadai3/yuuyamad/main.go b/kadai3/yuuyamad/main.go index a4df77a..404e0cf 100644 --- a/kadai3/yuuyamad/main.go +++ b/kadai3/yuuyamad/main.go @@ -10,32 +10,51 @@ import ( "golang.org/x/sync/errgroup" "context" "time" + "flag" ) -const LIMIT=10 - type Range struct { low int high int worker int } +type Downloader struct { + procs int + filename string + url string +} + func main() { - err := download() + var ( + procs = flag.Int("p", 10, "split ratio to download file") + output = flag.String("o", "", "output filename") + ) + + flag.Parse() + args := flag.Args() + + downloder := Downloader{} + downloder.procs = *procs + downloder.filename = *output + downloder.url = args[0] + + + err := downloder.download() if err != nil { logError(err) } //分割ダウンロードしたファイルを結合する - fh, err := os.Create("hoge.png") + fh, err := os.Create(downloder.filename) if err != nil { logError(err) } defer fh.Close() - for j:=0; j < LIMIT; j++ { - f := fmt.Sprintf("%s.%d", "hoge.png", j) + for j:=0; j < downloder.procs; j++ { + f := fmt.Sprintf("%s.%d", downloder.filename, j) subfp, err := os.Open(f) if err != nil{ logError(err) @@ -49,28 +68,27 @@ func main() { } } } -func download() error{ - url := os.Args[1] +func (d *Downloader)download() error{ // contents Headerを取得する - res, err := http.Head(url) + res, err := http.Head(d.url) if err != nil { - logError(err) + return err } maps := res.Header length, err := strconv.Atoi(maps["Content-Length"][0]) - len_sub := length/LIMIT - diff := length%LIMIT + len_sub := length/d.procs + diff := length%d.procs // errorGroup grp, ctx := errgroup.WithContext(context.Background()) - ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() - for i:=0; i < LIMIT; i++ { + for i:=0; i < d.procs; i++ { min := len_sub * i max := len_sub * (i + 1) @@ -80,15 +98,13 @@ func download() error{ r.high = max r.worker = i - if(i == (LIMIT - 1)){ + if(i == (d.procs - 1)){ max += diff } // execute get request grp.Go(func() error { - return requests(ctx, url, r) + return d.requests(ctx, r) }) - - fmt.Println(i) } if err := grp.Wait(); err != nil { return err @@ -96,27 +112,43 @@ func download() error{ return nil } -func requests(ctx context.Context, url string, r Range) error { - body := make([]string, 11) +func (d *Downloader)requests(ctx context.Context, r Range) error { + body := make([]string, 99) client := &http.Client{} - req , err := http.NewRequest("GET", url, nil) + req , err := http.NewRequest("GET", d.url, nil) if err != nil { return err } - req = req.WithContext(ctx) range_header := "bytes=" + strconv.Itoa(r.low) + "-" + strconv.Itoa(r.high-1) req.Header.Add("Range", range_header) - resp,err := client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - reader, err := ioutil.ReadAll(resp.Body) - body[r.worker] = string(reader) - fmt.Println(r.worker) - ioutil.WriteFile("hoge.png." + strconv.Itoa(r.worker), []byte(string(body[r.worker])), 0x777) + errCh := make(chan error, 1) + tmpfile := d.filename + "." + strconv.Itoa(r.worker) + + go func() { + resp, err := client.Do(req) + defer resp.Body.Close() + + reader, err := ioutil.ReadAll(resp.Body) + body[r.worker] = string(reader) + err = ioutil.WriteFile(tmpfile, []byte(string(body[r.worker])), 0x777) + errCh <- err + }() + + select { + case err := <-errCh: + fmt.Printf("requests: %s\n", err) + if err != nil { + return err + } + case <-ctx.Done(): + fmt.Printf("requests: %s\n", ctx.Err()) + os.Remove(tmpfile) + <-errCh + return ctx.Err() + + } return nil } From a9e32a0512e167a8dd5dd05b95da5e717a0cebce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=B1=E7=94=B0=E9=9B=84=E5=A4=AA?= Date: Wed, 16 May 2018 00:27:17 +0900 Subject: [PATCH 4/5] =?UTF-8?q?README=E3=81=AE=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3/yuuyamad/README.md | 20 ++++++++++++++++++ kadai3/yuuyamad/main.go | 43 ++++++++++++++++++--------------------- 2 files changed, 40 insertions(+), 23 deletions(-) create mode 100644 kadai3/yuuyamad/README.md diff --git a/kadai3/yuuyamad/README.md b/kadai3/yuuyamad/README.md new file mode 100644 index 0000000..7c4a1d2 --- /dev/null +++ b/kadai3/yuuyamad/README.md @@ -0,0 +1,20 @@ +# go concurrent downloader + +## Command Line Options + +### -p +split ratio to download file + +### -o +output file to + +## example +./yuuyamad -p 20 -o hoge.tar.xz "https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.6.3.tar.xz" + + +## 疑問点 +requestsメソッドをgoroutineで実行する時にcontextをわたしているのだが、１つのgoroutineがtimeoutしたりキャンセル処理された時に +同じcontextを渡して実行されているgoroutineにもキャンセルが伝播してcontextのDoneチャネルが返るのかなと考えて実装しました + +timeoutが他のgoroutineに伝播しているかというのは上手く検証できず、、自信がないのでもし認識間違いなどあれば教えて頂きたいです。 + diff --git a/kadai3/yuuyamad/main.go b/kadai3/yuuyamad/main.go index 404e0cf..e18e9a8 100644 --- a/kadai3/yuuyamad/main.go +++ b/kadai3/yuuyamad/main.go @@ -1,16 +1,16 @@ package main import ( - "os" - "net/http" - "strconv" - "io/ioutil" + "context" + "flag" "fmt" - "io" "golang.org/x/sync/errgroup" - "context" + "io" + "io/ioutil" + "net/http" + "os" + "strconv" "time" - "flag" ) type Range struct { @@ -20,15 +20,15 @@ type Range struct { } type Downloader struct { - procs int + procs int filename string - url string + url string } func main() { var ( - procs = flag.Int("p", 10, "split ratio to download file") + procs = flag.Int("p", 10, "split ratio to download file") output = flag.String("o", "", "output filename") ) @@ -40,7 +40,6 @@ func main() { downloder.filename = *output downloder.url = args[0] - err := downloder.download() if err != nil { logError(err) @@ -53,10 +52,10 @@ func main() { } defer fh.Close() - for j:=0; j < downloder.procs; j++ { + for j := 0; j < downloder.procs; j++ { f := fmt.Sprintf("%s.%d", downloder.filename, j) subfp, err := os.Open(f) - if err != nil{ + if err != nil { logError(err) } @@ -68,7 +67,7 @@ func main() { } } } -func (d *Downloader)download() error{ +func (d *Downloader) download() error { // contents Headerを取得する res, err := http.Head(d.url) @@ -79,16 +78,15 @@ func (d *Downloader)download() error{ maps := res.Header length, err := strconv.Atoi(maps["Content-Length"][0]) - len_sub := length/d.procs - diff := length%d.procs + len_sub := length / d.procs + diff := length % d.procs // errorGroup grp, ctx := errgroup.WithContext(context.Background()) - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - - for i:=0; i < d.procs; i++ { + for i := 0; i < d.procs; i++ { min := len_sub * i max := len_sub * (i + 1) @@ -98,7 +96,7 @@ func (d *Downloader)download() error{ r.high = max r.worker = i - if(i == (d.procs - 1)){ + if i == (d.procs - 1) { max += diff } // execute get request @@ -112,10 +110,10 @@ func (d *Downloader)download() error{ return nil } -func (d *Downloader)requests(ctx context.Context, r Range) error { +func (d *Downloader) requests(ctx context.Context, r Range) error { body := make([]string, 99) client := &http.Client{} - req , err := http.NewRequest("GET", d.url, nil) + req, err := http.NewRequest("GET", d.url, nil) if err != nil { return err } @@ -139,7 +137,6 @@ func (d *Downloader)requests(ctx context.Context, r Range) error { select { case err := <-errCh: - fmt.Printf("requests: %s\n", err) if err != nil { return err } From 3e25bc68d2b476b08509b12e86577e91253df673 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=B1=E7=94=B0=E9=9B=84=E5=A4=AA?= Date: Wed, 16 May 2018 00:29:31 +0900 Subject: [PATCH 5/5] =?UTF-8?q?README=E3=81=AE=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3/yuuyamad/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kadai3/yuuyamad/README.md b/kadai3/yuuyamad/README.md index 7c4a1d2..50de4b8 100644 --- a/kadai3/yuuyamad/README.md +++ b/kadai3/yuuyamad/README.md @@ -2,11 +2,11 @@ ## Command Line Options -### -p +### -p num split ratio to download file -### -o -output file to +### -o filename +output file to filename ## example ./yuuyamad -p 20 -o hoge.tar.xz "https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.6.3.tar.xz"