From 2ae740575a1b1456dc39bd48581fc2728fc79a3f Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Fri, 11 Jun 2021 12:10:34 +0900 Subject: [PATCH 01/51] =?UTF-8?q?=E8=AA=B2=E9=A1=8C2=E3=82=92=E6=8F=90?= =?UTF-8?q?=E5=87=BA=20=E8=AA=B2=E9=A1=8C1=E3=81=AE=E3=83=86=E3=82=B9?= =?UTF-8?q?=E3=83=88=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai2/Mizushima/.gitignore | 4 + kadai2/Mizushima/Makefile | 14 ++ kadai2/Mizushima/README.md | 49 +++++ kadai2/Mizushima/go.mod | 7 + kadai2/Mizushima/go.sum | 0 kadai2/Mizushima/main.go | 60 ++++++ kadai2/Mizushima/picconvert/go.mod | 5 + kadai2/Mizushima/picconvert/picconvert.go | 118 +++++++++++ .../Mizushima/picconvert/picconvert_test.go | 196 ++++++++++++++++++ kadai2/Mizushima/testdata/test01.gif | Bin 0 -> 1120 bytes kadai2/Mizushima/testdata/test01.jpg | Bin 0 -> 1641 bytes kadai2/Mizushima/testdata/test01.png | Bin 0 -> 415 bytes kadai2/Mizushima/testdata/testdata/test01.gif | Bin 0 -> 1120 bytes kadai2/Mizushima/testdata/testdata/test01.jpg | Bin 0 -> 1641 bytes kadai2/Mizushima/testdata/testdata/test01.png | Bin 0 -> 415 bytes 15 files changed, 453 insertions(+) create mode 100644 kadai2/Mizushima/.gitignore create mode 100644 kadai2/Mizushima/Makefile create mode 100644 kadai2/Mizushima/README.md create mode 100644 kadai2/Mizushima/go.mod create mode 100644 kadai2/Mizushima/go.sum create mode 100644 kadai2/Mizushima/main.go create mode 100644 kadai2/Mizushima/picconvert/go.mod create mode 100644 kadai2/Mizushima/picconvert/picconvert.go create mode 100644 kadai2/Mizushima/picconvert/picconvert_test.go create mode 100644 kadai2/Mizushima/testdata/test01.gif create mode 100644 kadai2/Mizushima/testdata/test01.jpg create mode 100644 kadai2/Mizushima/testdata/test01.png create mode 100644 kadai2/Mizushima/testdata/testdata/test01.gif create mode 100644 kadai2/Mizushima/testdata/testdata/test01.jpg create mode 100644 kadai2/Mizushima/testdata/testdata/test01.png diff --git a/kadai2/Mizushima/.gitignore b/kadai2/Mizushima/.gitignore new file mode 100644 index 00000000..e4ec1c79 --- /dev/null +++ b/kadai2/Mizushima/.gitignore @@ -0,0 +1,4 @@ +testdata/*_converted.* +testdata/*/*_converted.* +bin +*/profile \ No newline at end of file diff --git a/kadai2/Mizushima/Makefile b/kadai2/Mizushima/Makefile new file mode 100644 index 00000000..1f3cd5e1 --- /dev/null +++ b/kadai2/Mizushima/Makefile @@ -0,0 +1,14 @@ +NAME := bin/converter + +all: + go build -o $(NAME) + +test: clean all + cd ./picconvert; go test -v -cover + +clean: + rm -rf ./testdata/*_converted.* + rm -rf ./testdata/testdata/*_converted.* + rm -rf ./bin/* + +.PHONY: test clean diff --git a/kadai2/Mizushima/README.md b/kadai2/Mizushima/README.md new file mode 100644 index 00000000..cc3b042f --- /dev/null +++ b/kadai2/Mizushima/README.md @@ -0,0 +1,49 @@ +## 画像変換コマンド + +### 下記の仕様を満たす +- ディレクトリを指定 +- 指定したディレクトリ以下のJPGファイルをPNGに変換（デフォルト） +- ディレクトリ以下は再帰的に処理する +- 変換前と変換後の画像形式を指定できる（オプション） + + +### 下記を満たすように開発 +- mainパッケージと分離する +- 自作パッケージと標準パッケージと準標準パッケージのみ使う +- 標準標準パッケージ：golang.org/x以下のパッケージ +- ユーザ定義型を作ってみる +- GoDocを生成してみる +- Go Modulesを使ってみる + +### コマンドラインオプション + + | オプション | 説明 | デフォルト | + | --- | --- | --- | + | -pre | 変換前フォーマット | jpeg | + | -post | 変換後フォーマット | png | + + +### 対応している画像フォーマット +- png, jpeg(jpg), gif + + +### 使い方 +1. バイナリビルド（実行ファイル作成） +```bash +$ make +``` +2. ディレクトリを指定して実行 +```bash +$ ./bin/converter [directory] [options...] +``` + + +### テスト +- バイナリビルド & テスト +```bash +$ make test +``` +- テスト後の処理(掃除) +```bash +$ make clean +``` diff --git a/kadai2/Mizushima/go.mod b/kadai2/Mizushima/go.mod new file mode 100644 index 00000000..876315e1 --- /dev/null +++ b/kadai2/Mizushima/go.mod @@ -0,0 +1,7 @@ +module github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai2/Mizushima + +go 1.16 + +replace github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai2/Mizushima/picconvert => ./picconvert + +require github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai2/Mizushima/picconvert v0.0.0-00010101000000-000000000000 diff --git a/kadai2/Mizushima/go.sum b/kadai2/Mizushima/go.sum new file mode 100644 index 00000000..e69de29b diff --git a/kadai2/Mizushima/main.go b/kadai2/Mizushima/main.go new file mode 100644 index 00000000..8477e1e7 --- /dev/null +++ b/kadai2/Mizushima/main.go @@ -0,0 +1,60 @@ +package main + +import ( + "flag" + "log" + + picconvert "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai2/Mizushima/picconvert" +) + +var preFormat string +var afterFormat string + +func init() { + flag.StringVar(&preFormat, "pre", "jpeg", "the file format before conversion") + flag.StringVar(&afterFormat, "post", "png", "the file format after conversion") + flag.Parse() +} + +// isSupportedFormat returns true if "format" is supported, othrewise returns false. +func isSupportedFormat(format string) bool { + ext := []string{"jpg", "jpeg", "png", "gif"} + + for _, e := range ext { + if e == format { + return true + } + } + return false +} + +// validate occurs a error if there is something wrong with +// the entered parameters. +func validate() { + if (preFormat == afterFormat) || + (preFormat == "jpeg" && afterFormat == "jpg") || + (preFormat == "jpg" && afterFormat == "jpeg") { + log.Fatal("the parameter of -pre is same as that of -post.") + } + + if len(flag.Args()) == 0 { + log.Fatal("please input the path.") + } + + if !isSupportedFormat(preFormat) { + log.Fatal(preFormat, " is not supported.") + } else if !isSupportedFormat(afterFormat) { + log.Fatal(afterFormat, " is not supported.") + } +} + +func main() { + // fmt.Println("converting" ,preFormat, "to", afterFormat) + validate() + c := picconvert.NewPicConverter(flag.Args()[0], preFormat, afterFormat) + err := c.Conv() + + if err != nil { + log.Fatal(err) + } +} diff --git a/kadai2/Mizushima/picconvert/go.mod b/kadai2/Mizushima/picconvert/go.mod new file mode 100644 index 00000000..c16f9342 --- /dev/null +++ b/kadai2/Mizushima/picconvert/go.mod @@ -0,0 +1,5 @@ +module github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai2/Mizushima/picconvert + +go 1.16 + +replace "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai2/Mizushima/picconvert" => ./picconvert \ No newline at end of file diff --git a/kadai2/Mizushima/picconvert/picconvert.go b/kadai2/Mizushima/picconvert/picconvert.go new file mode 100644 index 00000000..d9d77800 --- /dev/null +++ b/kadai2/Mizushima/picconvert/picconvert.go @@ -0,0 +1,118 @@ +package picconvert + +import ( + "fmt" + "image" + "image/gif" + "image/jpeg" + "image/png" + "os" + "path/filepath" +) + +// PicConverter is user-defined type for converting +// a picture file format has root path, pre-conversion format, +// and after-conversion format. +type PicConverter struct { + Path string + PreFormat []string + AfterFormat string +} + +// Conv converts the picture file format. +func (p *PicConverter) Conv() error { + files, err := Glob(p.Path, p.PreFormat) + + if err != nil { + return err + } + + if files == nil { + return fmt.Errorf("there's no %s file", p.PreFormat) + } + + // prosessing for each file. + for _, file := range files { + // fmt.Println("from:", file) + f, err := os.Open(file) + if err != nil { + return err + } + defer f.Close() + + // reading the image file. + img, _, err := image.Decode(f) + if err != nil { + return err + } + + // creating filepath for output. + output, err := os.Create(fmt.Sprintf( + "%s_converted.%s", + baseName(file), p.AfterFormat)) + + if err != nil { + return err + } + + // converting the file. + switch p.AfterFormat { + case "png": + err = png.Encode(output, img) + case "jpg": + case "jpeg": + err = jpeg.Encode(output, img, nil) + case "gif": + err = gif.Encode(output, img, nil) + default: + err = fmt.Errorf("%s is not supported", p.AfterFormat) + } + + if err != nil { + return err + } + + // fmt.Printf("converted %s\n", output.Name()) + } + + return nil +} + +// NewPicConverter is the constructor for a PicConverter. +func NewPicConverter(Path string, PreFormat string, AfterFormat string) *PicConverter { + res := new(PicConverter) + res.Path = Path + + if PreFormat == "jpeg" || PreFormat == "jpg" { + res.PreFormat = []string{"jpeg", "jpg"} + } else { + res.PreFormat = []string{PreFormat} + } + + res.AfterFormat = AfterFormat + return res +} + +// Glob returns a slice of the file paths that meets the "format". +func Glob(path string, format []string) ([]string, error) { + var res []string + + var err error + for _, f := range format { + err = filepath.Walk(path, + func(path string, info os.FileInfo, err error) error { + if filepath.Ext(path) == "."+f && !info.IsDir() { + res = append(res, path) + } + return nil + }) + } + + return res, err +} + +// baseName returns the filepath without a extension. +func baseName(filePath string) string { + ext := filepath.Ext(filePath) + return filePath[:len(filePath)-len(ext)] +} diff --git a/kadai2/Mizushima/picconvert/picconvert_test.go b/kadai2/Mizushima/picconvert/picconvert_test.go new file mode 100644 index 00000000..c9233889 --- /dev/null +++ b/kadai2/Mizushima/picconvert/picconvert_test.go @@ -0,0 +1,196 @@ +package picconvert_test + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + picconvert "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai2/Mizushima/picconvert" +) + + +func TestGlob(t *testing.T) { + tmpTestDir := "../testdata" + + // make test cases + cases := []struct { + name string + path string + format []string + expected1 []string + expected2 error + }{ + {name: "jpeg files", + path: tmpTestDir, + format: []string{"jpg", "jpeg"}, + expected1: []string{tmpTestDir + "/test01.jpg", tmpTestDir + "/" + filepath.Base(tmpTestDir) + "/test01.jpg"}, + expected2: nil, + }, + {name: "png files", + path: tmpTestDir, + format: []string{"png"}, + expected1: []string{tmpTestDir + "/test01.png", tmpTestDir + "/" + filepath.Base(tmpTestDir) + "/test01.png"}, + expected2: nil, + }, + {name: "gif files", + path: tmpTestDir, + format: []string{"gif"}, + expected1: []string{tmpTestDir + "/test01.gif", tmpTestDir + "/" + filepath.Base(tmpTestDir) + "/test01.gif"}, + expected2: nil, + }, + } + + for _, c := range cases { + c := c + t.Run(c.name, func(t *testing.T) { + actual, err := picconvert.Glob(c.path, c.format) + // if actual != nil { fmt.Println(actual) } + if !reflect.DeepEqual(actual, c.expected1) && (err == nil && c.expected2 == nil) { + t.Errorf("want Glob(%s, %s) = %v, got %v", c.path, c.format, c.expected1, actual) + } + }) + } +} + +// TestPicConverter_Conv is the test for (p PicConverter)Conv() +func TestPicConverter_Conv(t *testing.T) { + tmpTestDir := "../testdata" + + // make test cases + cases := []struct { + name string + pc *picconvert.PicConverter + expected error + }{ + {name: "pre: jpeg, after: png", pc: picconvert.NewPicConverter(tmpTestDir, "jpeg", "png"), expected: nil}, + {name: "pre: jpeg, after: gif", pc: picconvert.NewPicConverter(tmpTestDir, "jpeg", "gif"), expected: nil}, + {name: "pre: jpg, after: png", pc: picconvert.NewPicConverter(tmpTestDir, "jpg", "png"), expected: nil}, + {name: "pre: jpg, after: gif", pc: picconvert.NewPicConverter(tmpTestDir, "jpg", "gif"), expected: nil}, + {name: "pre: png, after: jpg", pc: picconvert.NewPicConverter(tmpTestDir, "png", "jpg"), expected: nil}, + {name: "pre: png, after: gif", pc: picconvert.NewPicConverter(tmpTestDir, "png", "gif"), expected: nil}, + {name: "pre: gif, after: jpg", pc: picconvert.NewPicConverter(tmpTestDir, "gif", "jpg"), expected: nil}, + {name: "pre: gif, after: png", pc: picconvert.NewPicConverter(tmpTestDir, "gif", "png"), expected: nil}, + {name: "pre: jpg, after: xls", pc: picconvert.NewPicConverter(tmpTestDir, "jpg", "xls"), expected: fmt.Errorf("xls is not supported")}, + {name: "no file", pc: picconvert.NewPicConverter(tmpTestDir+"/test02.png", "gif", "png"), expected: fmt.Errorf("there's no [gif] file")}, + } + + for _, c := range cases { + c := c + t.Run(c.name, func(t *testing.T) { + actual := c.pc.Conv() + // if actual != nil { fmt.Println(actual) } + if actual != nil && actual.Error() != c.expected.Error() { + t.Errorf("want p.Conv() = %v, got %v", c.expected, actual) + } else if _, err := os.Stat(fmt.Sprintf("%s/test01_converted.%s", tmpTestDir, c.pc.AfterFormat)); err != nil && c.pc.AfterFormat != "xls" { + t.Errorf("%s file wasn't made", c.pc.AfterFormat) + } + }) + } + + testDeleteConveterd(t, tmpTestDir) +} + +// testDeleteConveterd returns a slice of the file paths that meets the "format". +func testDeleteConveterd(t *testing.T, path string) { + err := filepath.Walk(path, + func(path string, info os.FileInfo, err error) error { + if strings.Contains(path, "_converted") && !info.IsDir() { + os.Remove(path) + } + return nil + }) + + if err != nil { + t.Error(err) + } +} + +func TestNewPicConverter(t *testing.T) { + + cases := []struct { + name string + path string + pre string + post string + expected *picconvert.PicConverter + }{ + { + name: "pre: jpg, post: png", + path: "../testdata", + pre: "jpg", + post: "png", + expected: &picconvert.PicConverter{"../testdata", []string{"jpeg", "jpg"}, "png"}, + }, + { + name: "pre: jpeg, post: png", + path: "../testdata", + pre: "jpeg", + post: "png", + expected: &picconvert.PicConverter{"../testdata", []string{"jpeg", "jpg"}, "png"}, + }, + { + name: "pre: jpg, post: gif", + path: "../testdata", + pre: "jpg", + post: "gif", + expected: &picconvert.PicConverter{"../testdata", []string{"jpeg", "jpg"}, "gif"}, + }, + { + name: "pre: png, post: jpeg", + path: "../testdata", + pre: "png", + post: "jpeg", + expected: &picconvert.PicConverter{"../testdata", []string{"png"}, "jpeg"}, + }, + { + name: "pre: png, post: jpg", + path: "../testdata", + pre: "png", + post: "jpg", + expected: &picconvert.PicConverter{"../testdata", []string{"png"}, "jpg"}, + }, + { + name: "pre: png, post: gif", + path: "../testdata", + pre: "png", + post: "gif", + expected: &picconvert.PicConverter{"../testdata", []string{"png"}, "gif"}, + }, + { + name: "pre: gif, post: jpeg", + path: "../testdata", + pre: "gif", + post: "jpeg", + expected: &picconvert.PicConverter{"../testdata", []string{"gif"}, "jpeg"}, + }, + { + name: "pre: gif, post: jpg", + path: "../testdata", + pre: "gif", + post: "jpg", + expected: &picconvert.PicConverter{"../testdata", []string{"gif"}, "jpg"}, + }, + { + name: "pre: gif, post: png", + path: "../testdata", + pre: "gif", + post: "png", + expected: &picconvert.PicConverter{"../testdata", []string{"gif"}, "png"}, + }, + } + + for _, c := range cases { + c := c + t.Run(c.name, func(t *testing.T) { + actual := picconvert.NewPicConverter(c.path, c.pre, c.post) + // if actual != nil { fmt.Println(actual) } + if !reflect.DeepEqual(actual, c.expected) { + t.Errorf("want NewPicConverter(%s, %s, %s) = %v, but got %v", + c.path, c.pre, c.post, c.expected, actual) + } + }) + } +} diff --git a/kadai2/Mizushima/testdata/test01.gif b/kadai2/Mizushima/testdata/test01.gif new file mode 100644 index 0000000000000000000000000000000000000000..a517a9305a16a45582c06f9af6ac1641b2c18f92 GIT binary patch literal 1120 zcmXYwZ)n$56vm&}Z#6^Q%3PzYWebh2Tk7#&jbKqtr^GC~(LV?&4jR3P5_(f{r$b*v z(1oy+zbwwgS;CC`uZ^(|6TWJe|nPy|I$ z48>7cy$uSepn(on2M9!<0uy*ulK=@KK_!?3mq3Y-C=ykoNpy*n1W6)EC7C3bM9GjW zl2x)vcFB|iDI!Ism=u>nA%r4Sp$T1B>8QpQg(^(pl`IX=AR1JIX>bkH2#umqHJV1( zNKMcrnpBf%a!u3>&7xT~n`YNcEzlxbREueGEmT4&N>!TDl@)@DxER!62Cs-%fCaIj z7R-WMphZ{|i)zs=L3=i-i9@K+*a1ZndkK$21nn(9YPw*t3)RTE~PxK7W;#obLXZK7m@FHH+ zi+OP`biyf4b(+(ioC-qaMunu(RTZl!1TYALG8lt1kP(c+sEo$wjAR0nFe#HUITM+| zEX>Mm%+5>}un3E?7>lz^HLHSFxm_Wz=Ki<&|GBTWIe*oz0RG1OsjI0!fd=C8wAm{U z?jGLUF!PS5P9EGdvbCvg)vlF?-uvS97J2TwlZW0vv2CRJ8+85FHDgWd^@rE?-FN12 z-{h7J(=c!Ey|4D)eEj~Ey{9&>Us`{muI__#9ks5tKg?gXrgavYreC>w``NYwLl15k zeC37h&1VLFe{9zc%X{m4-fW(6Vc(*oZS8HfQ}5J&{>OO7{u{sEa`(xB+rHnlZl6B8 zV&|`q_D^1YX{=@chr^%Y^sSF<+&a;5?5dw`>OQ~onNMzMn8Ws=h9je+dzQWY#>n7U z@5taacRsQIp?7Bu*DX2oK+Dlfqo)t9Xgq##;eyAHywvmB*vDTjJF)M?jvb3X-S*bR zz~pu9PanJZ(ZlU$XZ~`}i|O08FKeBRf6kx1arc5*ix$Vd%bs72&P$DRpR7&%+}HZ{ ikMqV;t$F=;U&Gv?@lDg0o*&=b)P4K8{%h-N===wY{tcG^ literal 0 HcmV?d00001 diff --git a/kadai2/Mizushima/testdata/test01.jpg b/kadai2/Mizushima/testdata/test01.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8b03b2d51708b08fe3477aee73f652c1efdf0129 GIT binary patch literal 1641 zcmbV~do+}39LJw`-WhX~F>c8oNk(qnQOin8jY5)HnKDH;Qwd3D3^ro!nh?< z*&??Z*D*+Ki_InDP8LaK(u!PW->FWWvuFR<-S7MS&hvfF`+I)p_j#U2`cm2l6zyzm zZ2$y8-~jRgX+LnK2C|p{I5+@9000hPUt`254kLtU`^G z(l;;515i{t0Qqs!POt@_AV@a82*$uN$)V9O1}lff$^wTc;&C_v4vQr$A`po3h_HAC z(js{UnU@ujZI|srhCB|7lV$ujNvi-E2i5^o7+MZcWC$ihQX$YnI>jI-Av=sOf>1CT zBZss^AR-Axib(q~j6#~lpb=c?9P%Ha$rvT=byjl9P6x5eeO1}JV9--`VtM@6?*yxh9;&PHhynrzGW-bX4`gKyPuqQyU_RSbv=B$u-( z|1$xBL1!b*M@B_oxX8JZcr__G<>zbHGqbX9<=oEY-6?ojSX5k6`slHsMp#={|D@sR z?=MQ^+>(FSpqm zu%}Fb<4|JFn8{2uRerstdM@pjkx}&A=7!_FnT)i{)KFh3sJ03_?jGg-;C2n8k{sG1 z1zHxK1EeL5n&sV42_RM1PlDhNWH}Mo)gFA~|-eq?Vy|Uia*e?Zw zxW%l^xVbeFoj!VOnO83Bde3E69<|T?f~Z9_o|ErwVQ*eLpK)rkw`2cRpBVi|Y2Az; z`ck44%GbBJoICTTu6!-$=6wNGj+aH-#!2Fmosyk# zn%(#4Ohz!D*h21MZH#}?)YZ|oj6xCbxU%&%9*XZvU?R{z6y-z=rvxP7-S+}-! zfHZBW6~3&JYt%mxZQhoZEh*lZDrB-eq{pMs*4!CAi1ADRu|?;I^VOR&D0$? zM-`*jx&==3m)8p8NBwpzixY}Go*MSDm@e9fSRuK(7dwX-G21uBM?PQeK}g`ON}8Cw zlr?Q;5b(m`gwjsuAq>B$f<`NuFH2cH>iJ8G<#$pL@h}t$RHjP-+c|J-K((D+rt3F2 zzIvGYpg>~scoTn2?aXtYKtuiQ)1ZbWO14*B;Ng>IrXRR(G^JpLnfT~}aKEw?IEd*& zu6UPwhrO)Q+Y$5V@G{NW-Fh`^;JnnEmIl)dUWi4}Uy4$$rHbovOl(##^8k~ z^RYIk6ZKlcJi3FC3JzLPcg1%HNcGJ5oCWRqG|`Ky1ya+T&f(0l4aq*%#iPU;ua!5v zA`HDN%8qc>3?}Mb7xbiRG^NG*(KtP$)Ovfj@yZ~E|8RP~liz)tZ2w&mm(tq2ct$-U z$|LuqL%cU(1?BgrNaOU`ZKKIPL9X`GDR!*|lrFAUX>%T?ia2McD0roiLpz)3Ht*&% zs~v)G4J8D^E_{hYcU0D3ucoK*G`E)Sq8-UAzeY1Hi8!NP?d83wyzoSo@vQ-l13w4=LC6`8BEr^pmO_Q45V}TIckt+=yZ>+k~fXy!V&}wc^G} ip$|S*W%rYhbOx3D!TVXe&)LG>A2T`n;2#2(cKihsEUiEQ literal 0 HcmV?d00001 diff --git a/kadai2/Mizushima/testdata/test01.png b/kadai2/Mizushima/testdata/test01.png new file mode 100644 index 0000000000000000000000000000000000000000..17fbc75babc9ca1708fa680591b92d4ba81fe789 GIT binary patch literal 415 zcmeAS@N?(olHy`uVBq!ia0vp^ra)}S!2~3~ik2AyDaPU;cPEB*=VV?2IV|apzK#qG z8~eHcB(eheoCO|{#S9F5he4R}c>anMpav697srqY_qW#{@-{07uqHe&Qh1gqH7!x1 zdij#8T^}CYoM!U-)ref3AMNsQtJuzRnoxTd1oy+zbwwgS;CC`uZ^(|6TWJe|nPy|I$ z48>7cy$uSepn(on2M9!<0uy*ulK=@KK_!?3mq3Y-C=ykoNpy*n1W6)EC7C3bM9GjW zl2x)vcFB|iDI!Ism=u>nA%r4Sp$T1B>8QpQg(^(pl`IX=AR1JIX>bkH2#umqHJV1( zNKMcrnpBf%a!u3>&7xT~n`YNcEzlxbREueGEmT4&N>!TDl@)@DxER!62Cs-%fCaIj z7R-WMphZ{|i)zs=L3=i-i9@K+*a1ZndkK$21nn(9YPw*t3)RTE~PxK7W;#obLXZK7m@FHH+ zi+OP`biyf4b(+(ioC-qaMunu(RTZl!1TYALG8lt1kP(c+sEo$wjAR0nFe#HUITM+| zEX>Mm%+5>}un3E?7>lz^HLHSFxm_Wz=Ki<&|GBTWIe*oz0RG1OsjI0!fd=C8wAm{U z?jGLUF!PS5P9EGdvbCvg)vlF?-uvS97J2TwlZW0vv2CRJ8+85FHDgWd^@rE?-FN12 z-{h7J(=c!Ey|4D)eEj~Ey{9&>Us`{muI__#9ks5tKg?gXrgavYreC>w``NYwLl15k zeC37h&1VLFe{9zc%X{m4-fW(6Vc(*oZS8HfQ}5J&{>OO7{u{sEa`(xB+rHnlZl6B8 zV&|`q_D^1YX{=@chr^%Y^sSF<+&a;5?5dw`>OQ~onNMzMn8Ws=h9je+dzQWY#>n7U z@5taacRsQIp?7Bu*DX2oK+Dlfqo)t9Xgq##;eyAHywvmB*vDTjJF)M?jvb3X-S*bR zz~pu9PanJZ(ZlU$XZ~`}i|O08FKeBRf6kx1arc5*ix$Vd%bs72&P$DRpR7&%+}HZ{ ikMqV;t$F=;U&Gv?@lDg0o*&=b)P4K8{%h-N===wY{tcG^ literal 0 HcmV?d00001 diff --git a/kadai2/Mizushima/testdata/testdata/test01.jpg b/kadai2/Mizushima/testdata/testdata/test01.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8b03b2d51708b08fe3477aee73f652c1efdf0129 GIT binary patch literal 1641 zcmbV~do+}39LJw`-WhX~F>c8oNk(qnQOin8jY5)HnKDH;Qwd3D3^ro!nh?< z*&??Z*D*+Ki_InDP8LaK(u!PW->FWWvuFR<-S7MS&hvfF`+I)p_j#U2`cm2l6zyzm zZ2$y8-~jRgX+LnK2C|p{I5+@9000hPUt`254kLtU`^G z(l;;515i{t0Qqs!POt@_AV@a82*$uN$)V9O1}lff$^wTc;&C_v4vQr$A`po3h_HAC z(js{UnU@ujZI|srhCB|7lV$ujNvi-E2i5^o7+MZcWC$ihQX$YnI>jI-Av=sOf>1CT zBZss^AR-Axib(q~j6#~lpb=c?9P%Ha$rvT=byjl9P6x5eeO1}JV9--`VtM@6?*yxh9;&PHhynrzGW-bX4`gKyPuqQyU_RSbv=B$u-( z|1$xBL1!b*M@B_oxX8JZcr__G<>zbHGqbX9<=oEY-6?ojSX5k6`slHsMp#={|D@sR z?=MQ^+>(FSpqm zu%}Fb<4|JFn8{2uRerstdM@pjkx}&A=7!_FnT)i{)KFh3sJ03_?jGg-;C2n8k{sG1 z1zHxK1EeL5n&sV42_RM1PlDhNWH}Mo)gFA~|-eq?Vy|Uia*e?Zw zxW%l^xVbeFoj!VOnO83Bde3E69<|T?f~Z9_o|ErwVQ*eLpK)rkw`2cRpBVi|Y2Az; z`ck44%GbBJoICTTu6!-$=6wNGj+aH-#!2Fmosyk# zn%(#4Ohz!D*h21MZH#}?)YZ|oj6xCbxU%&%9*XZvU?R{z6y-z=rvxP7-S+}-! zfHZBW6~3&JYt%mxZQhoZEh*lZDrB-eq{pMs*4!CAi1ADRu|?;I^VOR&D0$? zM-`*jx&==3m)8p8NBwpzixY}Go*MSDm@e9fSRuK(7dwX-G21uBM?PQeK}g`ON}8Cw zlr?Q;5b(m`gwjsuAq>B$f<`NuFH2cH>iJ8G<#$pL@h}t$RHjP-+c|J-K((D+rt3F2 zzIvGYpg>~scoTn2?aXtYKtuiQ)1ZbWO14*B;Ng>IrXRR(G^JpLnfT~}aKEw?IEd*& zu6UPwhrO)Q+Y$5V@G{NW-Fh`^;JnnEmIl)dUWi4}Uy4$$rHbovOl(##^8k~ z^RYIk6ZKlcJi3FC3JzLPcg1%HNcGJ5oCWRqG|`Ky1ya+T&f(0l4aq*%#iPU;ua!5v zA`HDN%8qc>3?}Mb7xbiRG^NG*(KtP$)Ovfj@yZ~E|8RP~liz)tZ2w&mm(tq2ct$-U z$|LuqL%cU(1?BgrNaOU`ZKKIPL9X`GDR!*|lrFAUX>%T?ia2McD0roiLpz)3Ht*&% zs~v)G4J8D^E_{hYcU0D3ucoK*G`E)Sq8-UAzeY1Hi8!NP?d83wyzoSo@vQ-l13w4=LC6`8BEr^pmO_Q45V}TIckt+=yZ>+k~fXy!V&}wc^G} ip$|S*W%rYhbOx3D!TVXe&)LG>A2T`n;2#2(cKihsEUiEQ literal 0 HcmV?d00001 diff --git a/kadai2/Mizushima/testdata/testdata/test01.png b/kadai2/Mizushima/testdata/testdata/test01.png new file mode 100644 index 0000000000000000000000000000000000000000..17fbc75babc9ca1708fa680591b92d4ba81fe789 GIT binary patch literal 415 zcmeAS@N?(olHy`uVBq!ia0vp^ra)}S!2~3~ik2AyDaPU;cPEB*=VV?2IV|apzK#qG z8~eHcB(eheoCO|{#S9F5he4R}c>anMpav697srqY_qW#{@-{07uqHe&Qh1gqH7!x1 zdij#8T^}CYoM!U-)ref3AMNsQtJuzRnoxTd Date: Fri, 11 Jun 2021 18:36:40 +0900 Subject: [PATCH 02/51] =?UTF-8?q?main=E3=81=AE=E3=83=86=E3=82=B9=E3=83=88?= =?UTF-8?q?=E3=82=92=E8=BF=BD=E5=8A=A0=20=E3=81=9D=E3=81=AE=E4=BB=96=20mai?= =?UTF-8?q?n.go=20=E3=83=86=E3=82=B9=E3=83=88=E3=81=AE=E8=BF=BD=E5=8A=A0?= =?UTF-8?q?=E3=81=AB=E3=82=88=E3=82=8A=E4=BB=95=E6=A7=98=E5=A4=89=E6=9B=B4?= =?UTF-8?q?=20picconvert/picconvert.go=20godoc=E3=81=A7=E3=81=AE=E8=A6=8B?= =?UTF-8?q?=E6=A0=84=E3=81=88=E3=81=8C=E5=B0=91=E3=81=97=E3=81=A7=E3=82=82?= =?UTF-8?q?=E8=89=AF=E3=81=8F=E3=81=AA=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB?= =?UTF-8?q?=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88=E8=BF=BD=E5=8A=A0=20piccon?= =?UTF-8?q?vert=5Ftest.go=20TestGlob=E3=81=A7=E3=81=AE=E3=83=86=E3=82=B9?= =?UTF-8?q?=E3=83=88=E3=82=B1=E3=83=BC=E3=82=B9=E8=BF=BD=E5=8A=A0=E3=80=81?= =?UTF-8?q?testdata=E3=82=92=E3=82=B3=E3=83=94=E3=83=BC=E3=81=97=E3=81=AA?= =?UTF-8?q?=E3=81=84=E3=82=88=E3=81=86=E3=81=AB=E5=A4=89=E6=9B=B4=20Makefi?= =?UTF-8?q?le=20=E5=90=84=E3=82=B3=E3=83=9E=E3=83=B3=E3=83=89=E3=82=92go?= =?UTF-8?q?=E3=81=AE=E3=82=82=E3=81=AE=E3=81=AB=E5=A4=89=E6=9B=B4=20?= =?UTF-8?q?=E7=AD=89=E3=80=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai2/Mizushima/.gitignore | 3 +- kadai2/Mizushima/Makefile | 27 +++- kadai2/Mizushima/README.md | 140 +++++++++++++++--- kadai2/Mizushima/main.go | 42 ++++-- kadai2/Mizushima/main_test.go | 101 +++++++++++++ kadai2/Mizushima/picconvert/picconvert.go | 4 + .../Mizushima/picconvert/picconvert_test.go | 17 ++- 7 files changed, 288 insertions(+), 46 deletions(-) create mode 100644 kadai2/Mizushima/main_test.go diff --git a/kadai2/Mizushima/.gitignore b/kadai2/Mizushima/.gitignore index e4ec1c79..07784897 100644 --- a/kadai2/Mizushima/.gitignore +++ b/kadai2/Mizushima/.gitignore @@ -1,4 +1,5 @@ testdata/*_converted.* testdata/*/*_converted.* bin -*/profile \ No newline at end of file +*/profile +profile \ No newline at end of file diff --git a/kadai2/Mizushima/Makefile b/kadai2/Mizushima/Makefile index 1f3cd5e1..9f445a00 100644 --- a/kadai2/Mizushima/Makefile +++ b/kadai2/Mizushima/Makefile @@ -1,14 +1,25 @@ -NAME := bin/converter +# +# 原典：https://frasco.io/golang-dont-afraid-of-makefiles-785f3ec7eb32 +# -all: - go build -o $(NAME) +BINARY_NAME := bin/converter +GOCMD=go +GOBUILD=$(GOCMD) build +GOCLEAN=$(GOCMD) clean +GOTEST=$(GOCMD) test +GOGET=$(GOCMD) get -test: clean all - cd ./picconvert; go test -v -cover +all: build + +build: + $(GOBUILD) -o $(BINARY_NAME) -v + +test: clean all + $(GOTEST) -v -cover + cd ./picconvert; $(GOTEST) -v -cover clean: - rm -rf ./testdata/*_converted.* - rm -rf ./testdata/testdata/*_converted.* - rm -rf ./bin/* + $(GOCLEAN) + rm -rf $(BINARY_NAME) .PHONY: test clean diff --git a/kadai2/Mizushima/README.md b/kadai2/Mizushima/README.md index cc3b042f..52d952eb 100644 --- a/kadai2/Mizushima/README.md +++ b/kadai2/Mizushima/README.md @@ -1,26 +1,20 @@ -## 画像変換コマンド +## 課題2 テストを書いてみよう -### 下記の仕様を満たす -- ディレクトリを指定 -- 指定したディレクトリ以下のJPGファイルをPNGに変換（デフォルト） -- ディレクトリ以下は再帰的に処理する -- 変換前と変換後の画像形式を指定できる（オプション） +### 1回目の課題のテストを作る +- テストのしやすさを考えてリファクタリングしてみる +- テストのカバレッジを取ってみる +- テーブル駆動テストを行う +- テストヘルパーを作ってみる -### 下記を満たすように開発 -- mainパッケージと分離する -- 自作パッケージと標準パッケージと準標準パッケージのみ使う -- 標準標準パッケージ：golang.org/x以下のパッケージ -- ユーザ定義型を作ってみる -- GoDocを生成してみる -- Go Modulesを使ってみる ### コマンドラインオプション - | オプション | 説明 | デフォルト | - | --- | --- | --- | - | -pre | 変換前フォーマット | jpeg | - | -post | 変換後フォーマット | png | + | オプション | 説明 | デフォルト | 必須 | + | --- | --- | --- | --- | + | -path | 変換したい画像があるディレクトリ| カレントディレクトリ | + | -pre | 変換前フォーマット | jpeg | | + | -post | 変換後フォーマット | png | | ### 対応している画像フォーマット @@ -38,7 +32,7 @@ $ ./bin/converter [directory] [options...] ``` -### テスト +### テストの方法 - バイナリビルド & テスト ```bash $ make test @@ -47,3 +41,113 @@ $ make test ```bash $ make clean ``` + +### テスト結果 +```bash +=== RUN TestIsSupportedFormat +=== RUN TestIsSupportedFormat/jpg +=== RUN TestIsSupportedFormat/jpeg +=== RUN TestIsSupportedFormat/png +=== RUN TestIsSupportedFormat/gif +=== RUN TestIsSupportedFormat/xls +--- PASS: TestIsSupportedFormat (0.00s) + --- PASS: TestIsSupportedFormat/jpg (0.00s) + --- PASS: TestIsSupportedFormat/jpeg (0.00s) + --- PASS: TestIsSupportedFormat/png (0.00s) + --- PASS: TestIsSupportedFormat/gif (0.00s) + --- PASS: TestIsSupportedFormat/xls (0.00s) +=== RUN TestValidate +=== RUN TestValidate/jpg_and_png +=== RUN TestValidate/jpeg_and_png +=== RUN TestValidate/jpg_and_gif +=== RUN TestValidate/jpeg_and_gif +=== RUN TestValidate/png_and_jpg +=== RUN TestValidate/png_and_jpeg +=== RUN TestValidate/png_and_gif +=== RUN TestValidate/gif_and_jpeg +=== RUN TestValidate/gif_and_jpg +=== RUN TestValidate/gif_and_png +=== RUN TestValidate/xls_and_gif +=== RUN TestValidate/gif_and_xls +=== RUN TestValidate/jpg_and_jpeg +=== RUN TestValidate/jpeg_and_jpg +=== RUN TestValidate/png_and_png +=== RUN TestValidate/gif_and_gif +--- PASS: TestValidate (0.00s) + --- PASS: TestValidate/jpg_and_png (0.00s) + --- PASS: TestValidate/jpeg_and_png (0.00s) + --- PASS: TestValidate/jpg_and_gif (0.00s) + --- PASS: TestValidate/jpeg_and_gif (0.00s) + --- PASS: TestValidate/png_and_jpg (0.00s) + --- PASS: TestValidate/png_and_jpeg (0.00s) + --- PASS: TestValidate/png_and_gif (0.00s) + --- PASS: TestValidate/gif_and_jpeg (0.00s) + --- PASS: TestValidate/gif_and_jpg (0.00s) + --- PASS: TestValidate/gif_and_png (0.00s) + --- PASS: TestValidate/xls_and_gif (0.00s) + --- PASS: TestValidate/gif_and_xls (0.00s) + --- PASS: TestValidate/jpg_and_jpeg (0.00s) + --- PASS: TestValidate/jpeg_and_jpg (0.00s) + --- PASS: TestValidate/png_and_png (0.00s) + --- PASS: TestValidate/gif_and_gif (0.00s) +PASS +coverage: 73.1% of statements +ok github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai2/Mizushima 0.003s + +=== RUN TestGlob +=== RUN TestGlob/jpeg_files +=== RUN TestGlob/png_files +=== RUN TestGlob/gif_files +=== RUN TestGlob/xls_files +--- PASS: TestGlob (0.00s) + --- PASS: TestGlob/jpeg_files (0.00s) + --- PASS: TestGlob/png_files (0.00s) + --- PASS: TestGlob/gif_files (0.00s) + --- PASS: TestGlob/xls_files (0.00s) +=== RUN TestPicConverter_Conv +=== RUN TestPicConverter_Conv/pre:_jpeg,_after:_png +=== RUN TestPicConverter_Conv/pre:_jpeg,_after:_gif +=== RUN TestPicConverter_Conv/pre:_jpg,_after:_png +=== RUN TestPicConverter_Conv/pre:_jpg,_after:_gif +=== RUN TestPicConverter_Conv/pre:_png,_after:_jpg +=== RUN TestPicConverter_Conv/pre:_png,_after:_gif +=== RUN TestPicConverter_Conv/pre:_gif,_after:_jpg +=== RUN TestPicConverter_Conv/pre:_gif,_after:_png +=== RUN TestPicConverter_Conv/pre:_jpg,_after:_xls +=== RUN TestPicConverter_Conv/no_file +--- PASS: TestPicConverter_Conv (0.08s) + --- PASS: TestPicConverter_Conv/pre:_jpeg,_after:_png (0.00s) + --- PASS: TestPicConverter_Conv/pre:_jpeg,_after:_gif (0.01s) + --- PASS: TestPicConverter_Conv/pre:_jpg,_after:_png (0.00s) + --- PASS: TestPicConverter_Conv/pre:_jpg,_after:_gif (0.01s) + --- PASS: TestPicConverter_Conv/pre:_png,_after:_jpg (0.00s) + --- PASS: TestPicConverter_Conv/pre:_png,_after:_gif (0.01s) + --- PASS: TestPicConverter_Conv/pre:_gif,_after:_jpg (0.00s) + --- PASS: TestPicConverter_Conv/pre:_gif,_after:_png (0.05s) + --- PASS: TestPicConverter_Conv/pre:_jpg,_after:_xls (0.00s) + --- PASS: TestPicConverter_Conv/no_file (0.00s) +=== RUN TestNewPicConverter +=== RUN TestNewPicConverter/pre:_jpg,_post:_png +=== RUN TestNewPicConverter/pre:_jpeg,_post:_png +=== RUN TestNewPicConverter/pre:_jpg,_post:_gif +=== RUN TestNewPicConverter/pre:_png,_post:_jpeg +=== RUN TestNewPicConverter/pre:_png,_post:_jpg +=== RUN TestNewPicConverter/pre:_png,_post:_gif +=== RUN TestNewPicConverter/pre:_gif,_post:_jpeg +=== RUN TestNewPicConverter/pre:_gif,_post:_jpg +=== RUN TestNewPicConverter/pre:_gif,_post:_png +--- PASS: TestNewPicConverter (0.00s) + --- PASS: TestNewPicConverter/pre:_jpg,_post:_png (0.00s) + --- PASS: TestNewPicConverter/pre:_jpeg,_post:_png (0.00s) + --- PASS: TestNewPicConverter/pre:_jpg,_post:_gif (0.00s) + --- PASS: TestNewPicConverter/pre:_png,_post:_jpeg (0.00s) + --- PASS: TestNewPicConverter/pre:_png,_post:_jpg (0.00s) + --- PASS: TestNewPicConverter/pre:_png,_post:_gif (0.00s) + --- PASS: TestNewPicConverter/pre:_gif,_post:_jpeg (0.00s) + --- PASS: TestNewPicConverter/pre:_gif,_post:_jpg (0.00s) + --- PASS: TestNewPicConverter/pre:_gif,_post:_png (0.00s) +PASS +coverage: 87.8% of statements +ok github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai2/Mizushima/picconvert 0.087s + +``` diff --git a/kadai2/Mizushima/main.go b/kadai2/Mizushima/main.go index 8477e1e7..c80ad40d 100644 --- a/kadai2/Mizushima/main.go +++ b/kadai2/Mizushima/main.go @@ -2,22 +2,27 @@ package main import ( "flag" + "fmt" "log" + "testing" picconvert "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai2/Mizushima/picconvert" ) var preFormat string var afterFormat string +var srcPath string func init() { + testing.Init() flag.StringVar(&preFormat, "pre", "jpeg", "the file format before conversion") flag.StringVar(&afterFormat, "post", "png", "the file format after conversion") + flag.StringVar(&srcPath, "path", ".", "the source directory for conversion") flag.Parse() } // isSupportedFormat returns true if "format" is supported, othrewise returns false. -func isSupportedFormat(format string) bool { +func IsSupportedFormat(format string) bool { ext := []string{"jpg", "jpeg", "png", "gif"} for _, e := range ext { @@ -28,32 +33,37 @@ func isSupportedFormat(format string) bool { return false } -// validate occurs a error if there is something wrong with +// validate returns error if there is something wrong with // the entered parameters. -func validate() { - if (preFormat == afterFormat) || - (preFormat == "jpeg" && afterFormat == "jpg") || - (preFormat == "jpg" && afterFormat == "jpeg") { - log.Fatal("the parameter of -pre is same as that of -post.") +func Validate() error { + // fmt.Printf("%v, %v, %v\n", preFormat, afterFormat, srcPath) + if preFormat == afterFormat { + return fmt.Errorf("the parameter of -pre is same as that of -post") } - if len(flag.Args()) == 0 { - log.Fatal("please input the path.") + if (preFormat == "jpeg" && afterFormat == "jpg") || (preFormat == "jpg" && afterFormat == "jpeg") { + return fmt.Errorf("the parameter of -pre is same as that of -post. 'jpg' is considered same as 'jpeg'") } - if !isSupportedFormat(preFormat) { - log.Fatal(preFormat, " is not supported.") - } else if !isSupportedFormat(afterFormat) { - log.Fatal(afterFormat, " is not supported.") + if !IsSupportedFormat(preFormat) { + return fmt.Errorf("-pre %s is not supported", preFormat) + } else if !IsSupportedFormat(afterFormat) { + return fmt.Errorf("-post %s is not supported", afterFormat) } + + return nil } func main() { // fmt.Println("converting" ,preFormat, "to", afterFormat) - validate() - c := picconvert.NewPicConverter(flag.Args()[0], preFormat, afterFormat) - err := c.Conv() + err := Validate() + if err != nil { + log.Fatal(err) + } + + c := picconvert.NewPicConverter(srcPath, preFormat, afterFormat) + err = c.Conv() if err != nil { log.Fatal(err) } diff --git a/kadai2/Mizushima/main_test.go b/kadai2/Mizushima/main_test.go new file mode 100644 index 00000000..b310a64b --- /dev/null +++ b/kadai2/Mizushima/main_test.go @@ -0,0 +1,101 @@ +package main + +import ( + "flag" + "fmt" + "testing" +) + +func TestIsSupportedFormat(t *testing.T) { + cases := []struct { + name string + format string + expected bool + }{ + {name: "jpg", format: "jpg", expected: true}, + {name: "jpeg", format: "jpeg", expected: true}, + {name: "png", format: "png", expected: true}, + {name: "gif", format: "gif", expected: true}, + {name: "xls", format: "xls", expected: false}, + } + + for _, c := range cases { + // c := c + t.Run(c.name, func(t *testing.T) { + actual := IsSupportedFormat(c.format) + // if actual != nil { fmt.Println(actual) } + if actual != c.expected { + t.Errorf("want IsSupportedFormat(%s) = %v, got %v", c.format, c.expected, actual) + } + }) + } +} + +func TestValidate(t *testing.T) { + cases := []struct { + name string + pre string + post string + path string + expected error + }{ + {name: "jpg and png", pre: "jpg", post: "png", path: ".", expected: nil}, + {name: "jpeg and png", pre: "jpeg", post: "png", path: ".", expected: nil}, + {name: "jpg and gif", pre: "jpg", post: "gif", path: ".", expected: nil}, + {name: "jpeg and gif", pre: "jpeg", post: "gif", path: ".", expected: nil}, + {name: "png and jpg", pre: "png", post: "jpg", path: ".", expected: nil}, + {name: "png and jpeg", pre: "png", post: "jpeg", path: ".", expected: nil}, + {name: "png and gif", pre: "png", post: "gif", path: ".", expected: nil}, + {name: "gif and jpeg", pre: "gif", post: "jpeg", path: ".", expected: nil}, + {name: "gif and jpg", pre: "gif", post: "jpg", path: ".", expected: nil}, + {name: "gif and png", pre: "gif", post: "png", path: ".", expected: nil}, + {name: "xls and gif", pre: "xls", post: "gif", path: ".", expected: fmt.Errorf("-pre xls is not supported")}, + {name: "gif and xls", pre: "gif", post: "xls", path: ".", expected: fmt.Errorf("-post xls is not supported")}, + { + name: "jpg and jpeg", + pre: "jpg", + post: "jpeg", + path: ".", + expected: fmt.Errorf("the parameter of -pre is same as that of -post. 'jpg' is considered same as 'jpeg'"), + }, + { + name: "jpeg and jpg", + pre: "jpeg", + post: "jpg", + path: ".", + expected: fmt.Errorf("the parameter of -pre is same as that of -post. 'jpg' is considered same as 'jpeg'"), + }, + { + name: "png and png", + pre: "png", + post: "png", + path: ".", + expected: fmt.Errorf("the parameter of -pre is same as that of -post"), + }, + { + name: "gif and gif", + pre: "gif", + post: "gif", + path: ".", + expected: fmt.Errorf("the parameter of -pre is same as that of -post"), + }, + } + + for _, c := range cases { + // c := c + t.Run(c.name, func(t *testing.T) { + flag.CommandLine.Set("pre", c.pre) + flag.CommandLine.Set("post", c.post) + flag.CommandLine.Set("path", c.path) + actual := Validate() + // if actual != nil { fmt.Println(actual) } + if actual != nil && c.expected != nil && actual.Error() != c.expected.Error() { + t.Errorf("want Validate() = %v, but got %v", c.expected.Error(), actual.Error()) + } else if actual != nil && c.expected == nil { + t.Errorf("want Validate() = nil, but got %v", actual.Error()) + } else if actual == nil && c.expected != nil { + t.Errorf("want Validate() = %v, but got nil", c.expected.Error()) + } + }) + } +} diff --git a/kadai2/Mizushima/picconvert/picconvert.go b/kadai2/Mizushima/picconvert/picconvert.go index d9d77800..65d488aa 100644 --- a/kadai2/Mizushima/picconvert/picconvert.go +++ b/kadai2/Mizushima/picconvert/picconvert.go @@ -1,3 +1,7 @@ +// Package picconvert implements a picture format converter +// for command line tool in main package. +// + package picconvert import ( diff --git a/kadai2/Mizushima/picconvert/picconvert_test.go b/kadai2/Mizushima/picconvert/picconvert_test.go index c9233889..1095af3a 100644 --- a/kadai2/Mizushima/picconvert/picconvert_test.go +++ b/kadai2/Mizushima/picconvert/picconvert_test.go @@ -41,6 +41,12 @@ func TestGlob(t *testing.T) { expected1: []string{tmpTestDir + "/test01.gif", tmpTestDir + "/" + filepath.Base(tmpTestDir) + "/test01.gif"}, expected2: nil, }, + {name: "xls files", + path: tmpTestDir, + format: []string{"xls"}, + expected1: []string{}, + expected2: nil, + }, } for _, c := range cases { @@ -48,8 +54,13 @@ func TestGlob(t *testing.T) { t.Run(c.name, func(t *testing.T) { actual, err := picconvert.Glob(c.path, c.format) // if actual != nil { fmt.Println(actual) } - if !reflect.DeepEqual(actual, c.expected1) && (err == nil && c.expected2 == nil) { - t.Errorf("want Glob(%s, %s) = %v, got %v", c.path, c.format, c.expected1, actual) + if !reflect.DeepEqual(actual, c.expected1) && + !(len(actual) == 0 && len(c.expected1) == 0) && + (err == nil && c.expected2 == nil) { + t.Errorf("want Glob(%s, %s) = (%v, nil), got %v, nil", c.path, c.format, c.expected1, actual) + } else if !reflect.DeepEqual(actual, c.expected1) && (err != nil && c.expected2 != nil) { + t.Errorf("want Glob(%s, %s) = (%v, %v), got %v, %v", + c.path, c.format, c.expected1, c.expected2.Error(), actual, err.Error()) } }) } @@ -90,7 +101,7 @@ func TestPicConverter_Conv(t *testing.T) { }) } - testDeleteConveterd(t, tmpTestDir) + defer testDeleteConveterd(t, tmpTestDir) } // testDeleteConveterd returns a slice of the file paths that meets the "format". From 1da407c866c6c2199a811d378c71ddb52aea435f Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Fri, 11 Jun 2021 18:42:23 +0900 Subject: [PATCH 03/51] =?UTF-8?q?=E3=82=B3=E3=83=9E=E3=83=B3=E3=83=89?= =?UTF-8?q?=E3=83=A9=E3=82=A4=E3=83=B3=E3=82=AA=E3=83=97=E3=82=B7=E3=83=A7?= =?UTF-8?q?=E3=83=B3=E3=81=AE=E9=83=A8=E5=88=86=E3=82=92=E8=A8=82=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai2/Mizushima/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kadai2/Mizushima/README.md b/kadai2/Mizushima/README.md index 52d952eb..ebc39d27 100644 --- a/kadai2/Mizushima/README.md +++ b/kadai2/Mizushima/README.md @@ -10,11 +10,11 @@ ### コマンドラインオプション - | オプション | 説明 | デフォルト | 必須 | - | --- | --- | --- | --- | + | オプション | 説明 | デフォルト | + | --- | --- | --- | | -path | 変換したい画像があるディレクトリ| カレントディレクトリ | - | -pre | 変換前フォーマット | jpeg | | - | -post | 変換後フォーマット | png | | + | -pre | 変換前フォーマット | jpeg | + | -post | 変換後フォーマット | png | ### 対応している画像フォーマット From 766b979280e914d8fea56186645cf643478d49a1 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Fri, 11 Jun 2021 18:45:42 +0900 Subject: [PATCH 04/51] =?UTF-8?q?=E4=BD=BF=E3=81=84=E6=96=B9=E3=81=AE2?= =?UTF-8?q?=E3=82=92=E8=A8=82=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai2/Mizushima/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kadai2/Mizushima/README.md b/kadai2/Mizushima/README.md index ebc39d27..0de4d8cc 100644 --- a/kadai2/Mizushima/README.md +++ b/kadai2/Mizushima/README.md @@ -18,7 +18,9 @@ ### 対応している画像フォーマット -- png, jpeg(jpg), gif +- png +- jpeg(jpg) +- gif ### 使い方 @@ -28,7 +30,7 @@ $ make ``` 2. ディレクトリを指定して実行 ```bash -$ ./bin/converter [directory] [options...] +$ ./bin/converter [options...] ``` From a56f6941fed5e1d8f82c4ba65ac2983c1f346bc9 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Fri, 11 Jun 2021 19:34:20 +0900 Subject: [PATCH 05/51] =?UTF-8?q?go=20fmt=E9=81=A9=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai2/Mizushima/main_test.go | 32 +++++++++---------- .../Mizushima/picconvert/picconvert_test.go | 3 +- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/kadai2/Mizushima/main_test.go b/kadai2/Mizushima/main_test.go index b310a64b..1bad7760 100644 --- a/kadai2/Mizushima/main_test.go +++ b/kadai2/Mizushima/main_test.go @@ -52,31 +52,31 @@ func TestValidate(t *testing.T) { {name: "xls and gif", pre: "xls", post: "gif", path: ".", expected: fmt.Errorf("-pre xls is not supported")}, {name: "gif and xls", pre: "gif", post: "xls", path: ".", expected: fmt.Errorf("-post xls is not supported")}, { - name: "jpg and jpeg", - pre: "jpg", - post: "jpeg", - path: ".", + name: "jpg and jpeg", + pre: "jpg", + post: "jpeg", + path: ".", expected: fmt.Errorf("the parameter of -pre is same as that of -post. 'jpg' is considered same as 'jpeg'"), }, { - name: "jpeg and jpg", - pre: "jpeg", - post: "jpg", - path: ".", + name: "jpeg and jpg", + pre: "jpeg", + post: "jpg", + path: ".", expected: fmt.Errorf("the parameter of -pre is same as that of -post. 'jpg' is considered same as 'jpeg'"), }, { - name: "png and png", - pre: "png", - post: "png", - path: ".", + name: "png and png", + pre: "png", + post: "png", + path: ".", expected: fmt.Errorf("the parameter of -pre is same as that of -post"), }, { - name: "gif and gif", - pre: "gif", - post: "gif", - path: ".", + name: "gif and gif", + pre: "gif", + post: "gif", + path: ".", expected: fmt.Errorf("the parameter of -pre is same as that of -post"), }, } diff --git a/kadai2/Mizushima/picconvert/picconvert_test.go b/kadai2/Mizushima/picconvert/picconvert_test.go index 1095af3a..f178e40f 100644 --- a/kadai2/Mizushima/picconvert/picconvert_test.go +++ b/kadai2/Mizushima/picconvert/picconvert_test.go @@ -11,7 +11,6 @@ import ( picconvert "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai2/Mizushima/picconvert" ) - func TestGlob(t *testing.T) { tmpTestDir := "../testdata" @@ -54,7 +53,7 @@ func TestGlob(t *testing.T) { t.Run(c.name, func(t *testing.T) { actual, err := picconvert.Glob(c.path, c.format) // if actual != nil { fmt.Println(actual) } - if !reflect.DeepEqual(actual, c.expected1) && + if !reflect.DeepEqual(actual, c.expected1) && !(len(actual) == 0 && len(c.expected1) == 0) && (err == nil && c.expected2 == nil) { t.Errorf("want Glob(%s, %s) = (%v, nil), got %v, nil", c.path, c.format, c.expected1, actual) From 1393b38c813ace15f2233a3130d595f86beaf1b4 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Mon, 14 Jun 2021 10:18:39 +0900 Subject: [PATCH 06/51] =?UTF-8?q?README.md=20=E3=81=9D=E3=82=8C=E3=81=9E?= =?UTF-8?q?=E3=82=8C=E3=81=AE=E3=82=AB=E3=83=90=E3=83=AC=E3=83=83=E3=82=B8?= =?UTF-8?q?=E3=81=8C=E5=88=86=E3=81=8B=E3=82=8A=E3=82=84=E3=81=99=E3=81=8F?= =?UTF-8?q?=E3=81=AA=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai2/Mizushima/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kadai2/Mizushima/README.md b/kadai2/Mizushima/README.md index 0de4d8cc..906a83f6 100644 --- a/kadai2/Mizushima/README.md +++ b/kadai2/Mizushima/README.md @@ -45,6 +45,9 @@ $ make clean ``` ### テスト結果 +main_test.go : 73.1% +picconvert_test.go : 87.8% + ```bash === RUN TestIsSupportedFormat === RUN TestIsSupportedFormat/jpg From 054a7e22169457ec75b34d1216b04725c279cc39 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Mon, 14 Jun 2021 17:42:01 +0900 Subject: [PATCH 07/51] kadai3-1 first commit --- kadai3-1/Mizushima/main.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 kadai3-1/Mizushima/main.go diff --git a/kadai3-1/Mizushima/main.go b/kadai3-1/Mizushima/main.go new file mode 100644 index 00000000..7d011f82 --- /dev/null +++ b/kadai3-1/Mizushima/main.go @@ -0,0 +1,33 @@ +package main + +import ( + "bufio" + "fmt" + "io" + "os" +) + +func main() { + ch := input(os.Stdin) + for { + fmt.Print(">") + fmt.Println(<-ch) + } +} + +func input(r io.Reader) <-chan string { + // TODO: チャネルを作る + ch := make(chan string) + go func() { + s := bufio.NewScanner(r) + for s.Scan() { + // TODO: チャネルに読み込んだ文字列を送る + str := s.Text() + ch <- str + } + // TODO: チャネルを閉じる + close(ch) + }() + // TODO: チャネルを返す + return ch +} From 0738582807dad767645135a6a099575ab9e57252 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Mon, 14 Jun 2021 19:18:15 +0900 Subject: [PATCH 08/51] =?UTF-8?q?=E3=81=A8=E3=82=8A=E3=81=82=E3=81=88?= =?UTF-8?q?=E3=81=9A=E3=82=BF=E3=82=A4=E3=83=94=E3=83=B3=E3=82=B0=E3=81=A8?= =?UTF-8?q?=E6=AD=A3=E8=A7=A3=E5=88=A4=E5=AE=9A=E3=81=A7=E3=81=8D=E3=82=8B?= =?UTF-8?q?=E3=82=88=E3=81=86=E3=81=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-1/Mizushima/main.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/kadai3-1/Mizushima/main.go b/kadai3-1/Mizushima/main.go index 7d011f82..278d04f7 100644 --- a/kadai3-1/Mizushima/main.go +++ b/kadai3-1/Mizushima/main.go @@ -4,14 +4,24 @@ import ( "bufio" "fmt" "io" + "math/rand" "os" + "time" ) func main() { + + words := []string{ "ahoaho", "bakabaka", "unkounko" } + rand.Seed(time.Now().UnixNano()) ch := input(os.Stdin) for { - fmt.Print(">") - fmt.Println(<-ch) + idx := rand.Intn(3) + fmt.Printf("> %s\n", words[idx]) + if <-ch == words[idx] { + fmt.Println("> しぇいかい") + } else { + fmt.Println("> ぶっぶー") + } } } From 92b63c2ebe9bca49989c1141ebaf802dad851245 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Tue, 15 Jun 2021 14:08:28 +0900 Subject: [PATCH 09/51] =?UTF-8?q?=E6=99=82=E9=96=93=E5=88=B6=E9=99=90?= =?UTF-8?q?=E3=81=A8go.mod=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-1/Mizushima/go.mod | 3 +++ kadai3-1/Mizushima/main.go | 33 ++++++++++++++++++++++++++++----- 2 files changed, 31 insertions(+), 5 deletions(-) create mode 100644 kadai3-1/Mizushima/go.mod diff --git a/kadai3-1/Mizushima/go.mod b/kadai3-1/Mizushima/go.mod new file mode 100644 index 00000000..9cf24dd6 --- /dev/null +++ b/kadai3-1/Mizushima/go.mod @@ -0,0 +1,3 @@ +module github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-1/Mizushima + +go 1.16 diff --git a/kadai3-1/Mizushima/main.go b/kadai3-1/Mizushima/main.go index 278d04f7..2974d752 100644 --- a/kadai3-1/Mizushima/main.go +++ b/kadai3-1/Mizushima/main.go @@ -2,6 +2,7 @@ package main import ( "bufio" + "context" "fmt" "io" "math/rand" @@ -11,16 +12,38 @@ import ( func main() { - words := []string{ "ahoaho", "bakabaka", "unkounko" } + words := []string{"ahoaho", "bakabaka", "unkounko"} rand.Seed(time.Now().UnixNano()) + + bc := context.Background() + t := 20 * time.Second + ctx, cancel := context.WithTimeout(bc, t) + defer cancel() + + fmt.Println("> タイピングゲームを始めましゅ") + fmt.Println("> 英単語が出てきますので、同じ単語をタイプしてくだしゃい!") + fmt.Println("> 制限時間は20秒です") + ch := input(os.Stdin) + score := 0 + for { + idx := rand.Intn(3) fmt.Printf("> %s\n", words[idx]) - if <-ch == words[idx] { - fmt.Println("> しぇいかい") - } else { - fmt.Println("> ぶっぶー") + + select { + case <-time.After(1 * time.Second): + if <-ch == words[idx] { + fmt.Println("> しぇえか～い") + score++ + } else { + fmt.Println("> ぶっぶー") + } + case <-ctx.Done(): + fmt.Println("\n終了!") + fmt.Printf("%d問正解です!\n", score) + return } } } From bcd289cf34ce54817341b3423d5e0a9d7aa21a0d Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Tue, 15 Jun 2021 17:51:23 +0900 Subject: [PATCH 10/51] =?UTF-8?q?=E5=8D=98=E8=AA=9E=E3=82=92CSV=E3=81=8B?= =?UTF-8?q?=E3=82=89=E8=AA=AD=E3=81=BF=E8=BE=BC=E3=82=80=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-1/Mizushima/main.go | 45 +++++++++++++++++++++++++++++++----- kadai3-1/Mizushima/words.csv | 1 + 2 files changed, 40 insertions(+), 6 deletions(-) create mode 100644 kadai3-1/Mizushima/words.csv diff --git a/kadai3-1/Mizushima/main.go b/kadai3-1/Mizushima/main.go index 2974d752..ae85e111 100644 --- a/kadai3-1/Mizushima/main.go +++ b/kadai3-1/Mizushima/main.go @@ -3,8 +3,10 @@ package main import ( "bufio" "context" + "encoding/csv" "fmt" "io" + "log" "math/rand" "os" "time" @@ -12,7 +14,13 @@ import ( func main() { - words := []string{"ahoaho", "bakabaka", "unkounko"} + words, err := readCSV("./words.csv") + if err != nil { + log.Fatal(err) + } + + fmt.Println(words) + rand.Seed(time.Now().UnixNano()) bc := context.Background() @@ -29,10 +37,14 @@ func main() { for { - idx := rand.Intn(3) + idx := rand.Intn(len(words)) fmt.Printf("> %s\n", words[idx]) select { + case <-ctx.Done(): + fmt.Println("\n終了!") + fmt.Printf("%d問正解です!\n", score) + return case <-time.After(1 * time.Second): if <-ch == words[idx] { fmt.Println("> しぇえか～い") @@ -40,10 +52,6 @@ func main() { } else { fmt.Println("> ぶっぶー") } - case <-ctx.Done(): - fmt.Println("\n終了!") - fmt.Printf("%d問正解です!\n", score) - return } } } @@ -64,3 +72,28 @@ func input(r io.Reader) <-chan string { // TODO: チャネルを返す return ch } + +func readCSV(path string) ([]string, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + + csvFile := csv.NewReader(file) + csvFile.TrimLeadingSpace = true + + var ret []string + var row []string + + for { + row, err = csvFile.Read() + if err != nil { + break + } + + ret = append(ret, row...) + } + + return ret, nil +} \ No newline at end of file diff --git a/kadai3-1/Mizushima/words.csv b/kadai3-1/Mizushima/words.csv new file mode 100644 index 00000000..3e6c7b89 --- /dev/null +++ b/kadai3-1/Mizushima/words.csv @@ -0,0 +1 @@ +America,American,Angle,April,August,Bacon,Barber,Battery,Bible,Bill,Bush,Cage,Chinese,Christ,Christmas,Coward,Crane,Crow,December,Earnest,Echo,England,English,Eve,February,Forth,Fox,Frank,French,Friday,Grace,Grant,Gray,Ham,Hardy,Host,Hunt,January,Japan,Japanese,July,June,Lamb,Lily,London,March,Mass,Mat,Mill,Monday,Mrs,Ms,November,October,Pan,Pat,Poll,Pound,Punch,Ray,Rich,Rob,Rugby,Saturday,September,Singer,Spark,Sunday,Superior,Swift,Thursday,Tuesday,Turkey,Violet,Wednesday,absence,absolute,abstract,abuse,accent,access,accident,accord,achievement,acid,acre,active,addition,administration,admiration,admission,advance,advantage,adventure,advertisement,advice,affair,agency,agreement,aid,aim,alarm,alphabet,altogether,ambition,analysis,ancient,anxiety,appeal,appearance,application,appointment,approach,argument,army,arrangement,arrival,ash,aspect,assembly,assignment,association,assumption,atmosphere,atom,attraction,audience,authority,average,ax,background,baggage,balance,ballet,bank,banker,basic,bathe,battery,battle,bay,bean,bear,beast,beat,beauty,beef,beer,belief,bench,bend,benefit,billion,bind,birth,biscuit,bite,bitter,blade,blanket,bloom,blossom,blow,boil,bomb,bond,bone,border,bother,bowl,brain,brake,brass,breadth,breath,breeze,brick,brief,brilliant,broad,bronze,brow,brush,bubble,bucket,bug,bull,burden,burn,bush,butterfly,button,cabbage,cable,calf,calm,camel,camp,cancel,cancer,candy,cane,canvas,capacity,capital,capture,carbon,career,cargo,carpet,carriage,carrot,cast,casual,cattle,caution,celebration,cent,cereal,ceremony,certainty,chain,chalk,channel,chapel,character,charge,charity,charm,chart,chase,cheat,cheek,cheer,cheese,cherry,chew,chicken,chip,chocolate,chorus,cinema,circumstance,civilization,classic,clip,cloth,coach,coal,coast,cocktail,cocoa,coin,collapse,collection,college,colony,comb,combination,comedy,comfort,comic,command,comment,commission,communication,community,companion,comparison,competition,complaint,composition,concentration,conception,concern,concert,conclusion,concrete,conference,confidence,congress,connection,conscious,consciousness,consequence,consideration,constitution,construction,content,continent,contract,contrary,contrast,contribution,convenience,convention,conversation,conviction,copper,cord,core,corn,correction,costume,cough,countryside,county,courage,crack,craft,crane,crawl,creation,credit,crime,criticism,crop,crossing,crown,crush,crystal,cure,curiosity,curl,current,curse,curtain,custom,damage,damn,danger,dash,dawn,daylight,daytime,debate,debt,decrease,deer,defeat,defense,definition,degree,delay,delicious,delight,delivery,demand,democracy,demonstration,density,departure,deposit,depression,depth,description,desert,design,desire,destiny,detail,determination,development,devil,diamond,diet,difficulty,dig,dignity,dip,direction,disadvantage,disappointment,disaster,discipline,discovery,display,disposal,dispute,distinction,distribution,division,divorce,doctrine,domestic,double,doubt,downstairs,dozen,draft,drag,dragon,drama,drawing,drift,drill,duck,dust,duty,dynamic,eagle,earnest,economics,economy,education,eighteenth,eighth,elect,election,electric,element,eleventh,emotion,emphasis,empire,employment,engagement,enterprise,entertainment,enthusiasm,entrance,entry,environment,error,escape,essence,establishment,estate,eternal,eve,evidence,evil,examination,exception,excess,exchange,excitement,excuse,executive,exhaust,exhibition,existence,expansion,expectation,expedition,expenditure,expense,experiment,explosion,export,exposure,express,expression,extension,extent,external,extreme,facility,failure,faith,fancy,fare,farewell,fashion,fate,fault,favor,favorite,feather,feature,fee,feed,fellow,fever,fiber,fiction,fifteenth,fiftieth,finance,fit,flag,flame,flash,flavor,flesh,flight,flock,flood,flow,focus,fog,fold,folk,formation,formula,fortune,foundation,fourteenth,fox,fraction,frame,freedom,freeze,frontier,frost,fuel,fund,fundamental,funeral,funny,fur,gain,gallery,garment,gaze,gear,generation,generosity,genius,geography,gesture,ghost,gift,glass,globe,glory,glow,goat,goose,gossip,grace,grain,grammar,grant,grape,grasp,grave,gray,grief,grip,gross,growth,guard,gum,gym,habit,hall,ham,hang,harmony,harvest,hate,headquarters,heaven,height,hell,herd,heroic,hide,highway,hint,holy,honey,honor,hop,horn,horror,host,household,housewife,humanity,humor,hundredth,hunger,identity,illustration,imagination,imitation,impact,imperial,import,impression,improvement,impulse,inch,inevitable,influence,initiative,inspiration,instance,instant,institute,institution,instruction,insult,insurance,intellect,intelligence,intention,interior,internal,intersection,intimate,introduction,invention,investment,invisible,invitation,issue,jail,jar,jaw,jealousy,jelly,jerk,journal,joy,judge,juice,jungle,justice,kick,kindness,kingdom,labor,lace,lack,lamb,landing,landscape,lap,latest,launch,laundry,leadership,leaf,lean,leather,legend,lemon,lettuce,liberty,lid,lift,limit,lion,lip,liquid,liquor,listener,load,loaf,loan,location,lock,lord,loss,lower,loyalty,lump,luxury,mad,madam,majority,maker,management,manner,manufacture,maple,marble,march,marriage,mass,match,mathematics,meadow,means,meantime,measurement,melody,membership,mention,mercy,merit,mess,metal,meter,mile,military,mineral,ministry,minor,minority,mist,mix,mixture,monument,mood,moral,motion,mount,mouse,movement,murder,muscle,mushroom,mutter,mystery,myth,navy,necessity,neck,needle,negative,negotiation,neighborhood,nerve,net,network,neutral,nickel,nineteen,nineteenth,ninety,ninth,nonsense,normal,northern,northwest,notion,nut,nylon,oak,objection,obligation,observation,occasion,occupation,odd,offense,olive,onion,operation,opposition,option,orange,ordinary,organization,orient,origin,original,overall,overflow,pace,pack,painting,palace,pale,panic,paradise,parcel,pardon,parliament,passage,passion,passive,pasture,pat,patience,patrol,payment,pea,peach,peanut,pear,pearl,pence,penny,pepper,percentage,perception,performance,perfume,personality,perspective,philosophy,pie,pig,pigeon,pill,pine,pink,pitch,pity,plastic,plate,pleasure,plot,plow,poison,policy,polish,politics,poll,pond,pool,pop,port,portion,positive,possession,possibility,potato,pound,poverty,powder,praise,prayer,precious,preparation,prescription,presence,presentation,preserve,press,pressure,pride,prime,principal,principle,print,priority,prison,privilege,procedure,procession,production,profession,profit,progress,promotion,pronunciation,proof,property,proportion,proposal,prospect,protection,protest,province,provision,psychology,publication,pudding,pump,punch,punishment,purchase,purple,purse,pursuit,push,puzzle,quality,quantity,rabbit,rack,racket,radar,rage,rail,range,rank,ratio,rattle,raw,ray,reaction,reader,reality,realization,rear,recall,receipt,receiver,reception,recognition,recommendation,recovery,reduction,reference,reflection,refusal,regard,region,register,registration,regret,regulation,relation,release,relief,religion,remark,repair,repetition,representation,reputation,request,rescue,resemblance,reservation,reserve,residence,resistance,resolution,resolve,resort,resource,respect,response,responsibility,restriction,revenue,reverse,review,revolution,reward,ribbon,rifle,risk,roast,robe,rocket,rod,roll,root,rope,rough,row,rub,rubber,rugby,ruin,rush,sack,sacrifice,saddle,safety,sail,sake,salad,sale,salmon,sand,satellite,satisfaction,sauce,saving,scale,scandal,scare,scatter,scent,scholarship,score,scorn,scout,scramble,scratch,scream,screen,seal,seaside,secret,section,security,seed,selection,self,senate,sensation,sentence,separate,sequence,series,session,set,settlement,seventeen,seventeenth,seventh,sex,shade,shadow,shake,shallow,shame,shave,sheep,shell,shelter,shepherd,shock,shoulder,sickness,silence,silk,sin,sir,sixteenth,sixtieth,skate,skill,skirt,slip,slope,smart,smell,smoke,smooth,snap,soap,sock,soda,soil,solution,sorrow,soul,sour,southeast,southern,southwest,spade,spark,specific,spectacle,spelling,spice,spill,spin,spirit,split,spoil,sponge,spot,spray,squeeze,squirrel,staff,stage,stain,stair,stake,statement,status,steal,steep,stereo,stew,stick,stir,stock,stomach,stool,storage,strain,straw,strawberry,strength,stress,stretch,string,stroke,structure,substance,suburb,suck,suggestion,suicide,sum,summit,sunshine,supper,supply,surface,surgery,surrender,survey,survival,suspicion,swan,sweat,swell,swing,switch,sword,sympathy,tail,talent,tap,tape,taste,technique,technology,telegraph,temper,tension,tenth,territory,terror,text,theater,theory,thief,thirst,thirteenth,thirtieth,thorn,thread,thrust,thunder,tide,timber,tin,tissue,title,tobacco,toilet,toll,tomato,ton,tone,tongue,toss,trace,track,tragedy,training,transfer,translation,transport,transportation,trap,treasure,treat,treatment,treaty,tremble,trial,triumph,trunk,trust,truth,tube,tune,turkey,twelfth,twentieth,twin,twist,ultimate,unconscious,underground,underneath,union,universe,unknown,upright,upset,urge,usual,utility,vacation,van,variation,variety,vegetable,vein,verse,vice,video,violet,virgin,virtue,vision,visitor,vital,vitamin,vocabulary,volleyball,volume,vote,voyage,waste,wax,weakness,wealth,weed,weep,weight,whale,wheel,whip,whisper,width,wine,wire,wisdom,wise,wit,witness,wonder,worship,worth,wrap,wreck,youth,zero \ No newline at end of file From f52c186f39d9ab6e4d9f3b4122ed4fb000833652 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Tue, 15 Jun 2021 19:44:06 +0900 Subject: [PATCH 11/51] =?UTF-8?q?=E3=83=A2=E3=82=B8=E3=83=A5=E3=83=BC?= =?UTF-8?q?=E3=83=AB=E5=88=86=E5=89=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-1/Mizushima/{ => gamedata}/words.csv | 0 kadai3-1/Mizushima/main.go | 94 +--------------- kadai3-1/Mizushima/typing/typing.go | 115 ++++++++++++++++++++ 3 files changed, 118 insertions(+), 91 deletions(-) rename kadai3-1/Mizushima/{ => gamedata}/words.csv (100%) create mode 100644 kadai3-1/Mizushima/typing/typing.go diff --git a/kadai3-1/Mizushima/words.csv b/kadai3-1/Mizushima/gamedata/words.csv similarity index 100% rename from kadai3-1/Mizushima/words.csv rename to kadai3-1/Mizushima/gamedata/words.csv diff --git a/kadai3-1/Mizushima/main.go b/kadai3-1/Mizushima/main.go index ae85e111..56e8b70b 100644 --- a/kadai3-1/Mizushima/main.go +++ b/kadai3-1/Mizushima/main.go @@ -1,99 +1,11 @@ package main import ( - "bufio" - "context" - "encoding/csv" - "fmt" - "io" - "log" - "math/rand" "os" - "time" + + typing "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-1/Mizushima/typing" ) func main() { - - words, err := readCSV("./words.csv") - if err != nil { - log.Fatal(err) - } - - fmt.Println(words) - - rand.Seed(time.Now().UnixNano()) - - bc := context.Background() - t := 20 * time.Second - ctx, cancel := context.WithTimeout(bc, t) - defer cancel() - - fmt.Println("> タイピングゲームを始めましゅ") - fmt.Println("> 英単語が出てきますので、同じ単語をタイプしてくだしゃい!") - fmt.Println("> 制限時間は20秒です") - - ch := input(os.Stdin) - score := 0 - - for { - - idx := rand.Intn(len(words)) - fmt.Printf("> %s\n", words[idx]) - - select { - case <-ctx.Done(): - fmt.Println("\n終了!") - fmt.Printf("%d問正解です!\n", score) - return - case <-time.After(1 * time.Second): - if <-ch == words[idx] { - fmt.Println("> しぇえか～い") - score++ - } else { - fmt.Println("> ぶっぶー") - } - } - } -} - -func input(r io.Reader) <-chan string { - // TODO: チャネルを作る - ch := make(chan string) - go func() { - s := bufio.NewScanner(r) - for s.Scan() { - // TODO: チャネルに読み込んだ文字列を送る - str := s.Text() - ch <- str - } - // TODO: チャネルを閉じる - close(ch) - }() - // TODO: チャネルを返す - return ch + typing.Game(os.Stdin, os.Stdout) } - -func readCSV(path string) ([]string, error) { - file, err := os.Open(path) - if err != nil { - return nil, err - } - defer file.Close() - - csvFile := csv.NewReader(file) - csvFile.TrimLeadingSpace = true - - var ret []string - var row []string - - for { - row, err = csvFile.Read() - if err != nil { - break - } - - ret = append(ret, row...) - } - - return ret, nil -} \ No newline at end of file diff --git a/kadai3-1/Mizushima/typing/typing.go b/kadai3-1/Mizushima/typing/typing.go new file mode 100644 index 00000000..40d429d7 --- /dev/null +++ b/kadai3-1/Mizushima/typing/typing.go @@ -0,0 +1,115 @@ +package typing + +import ( + "bufio" + "bytes" + "context" + "encoding/csv" + "fmt" + "io" + "log" + "math/rand" + "os" + "strings" + "time" +) + + + +func Game(r io.Reader, w io.Writer) { + + words, err := readCSV("./gamedata/words.csv") + if err != nil { + log.Fatal(err) + } + + fmt.Println(words) + + rand.Seed(time.Now().UnixNano()) + + bc := context.Background() + t := 20 * time.Second + ctx, cancel := context.WithTimeout(bc, t) + defer cancel() + + fmt.Println("> タイピングゲームを始めましゅ") + fmt.Println("> 英単語が出てきますので、同じ単語をタイプしてくだしゃい!") + fmt.Println("> 制限時間は20秒です") + + ch := input(os.Stdin) + score := 0 + + for { + + idx := rand.Intn(len(words)) + fmt.Printf("> %s\n", words[idx]) + + select { + case <-ctx.Done(): + fmt.Println("\n終了!") + fmt.Printf("%d問正解です!\n", score) + return + case <-time.After(1 * time.Second): + if <-ch == words[idx] { + fmt.Println("> しぇえか～い") + score++ + } else { + fmt.Println("> ぶっぶー") + } + } + } +} + +func input(r io.Reader) <-chan string { + // TODO: チャネルを作る + ch := make(chan string) + go func() { + s := bufio.NewScanner(r) + for s.Scan() { + str := s.Text() + ch <- str + } + close(ch) + }() + return ch +} + +func readCSV(path string) ([]string, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + + csvFile := csv.NewReader(file) + csvFile.TrimLeadingSpace = true + + var ret []string + var row []string + + for { + row, err = csvFile.Read() + if err != nil { + break + } + + ret = append(ret, row...) + } + + return ret, nil +} + +func GetStdoutAsString(f func()) string { + tmpStdout := os.Stdout + defer func() { + os.Stdout = tmpStdout + }() + r, w, _ := os.Pipe() + os.Stdout = w + + f() + w.Close() + var buf bytes.Buffer + buf.ReadFrom(r) + return strings.TrimRight(buf.String(), "\n") +} \ No newline at end of file From 0c045955896dba971e8f0e1440028da2cdc088b9 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Thu, 17 Jun 2021 11:15:18 +0900 Subject: [PATCH 12/51] =?UTF-8?q?gigitignore,Makefile,typing=E3=81=AE?= =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E3=82=92=E8=BF=BD=E5=8A=A0,?= =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E8=BF=BD=E5=8A=A0=E3=81=AB=E4=BC=B4?= =?UTF-8?q?=E3=81=84typing/typing.go=E3=81=A8main.go=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-1/Mizushima/.gitignore | 1 + kadai3-1/Mizushima/Makefile | 21 +++++ kadai3-1/Mizushima/README.md | 37 ++++++++ kadai3-1/Mizushima/main.go | 14 ++- kadai3-1/Mizushima/typing/typing.go | 114 ++++++++++++++--------- kadai3-1/Mizushima/typing/typing_test.go | 68 ++++++++++++++ 6 files changed, 208 insertions(+), 47 deletions(-) create mode 100644 kadai3-1/Mizushima/.gitignore create mode 100644 kadai3-1/Mizushima/Makefile create mode 100644 kadai3-1/Mizushima/README.md create mode 100644 kadai3-1/Mizushima/typing/typing_test.go diff --git a/kadai3-1/Mizushima/.gitignore b/kadai3-1/Mizushima/.gitignore new file mode 100644 index 00000000..c5e82d74 --- /dev/null +++ b/kadai3-1/Mizushima/.gitignore @@ -0,0 +1 @@ +bin \ No newline at end of file diff --git a/kadai3-1/Mizushima/Makefile b/kadai3-1/Mizushima/Makefile new file mode 100644 index 00000000..42c3aca9 --- /dev/null +++ b/kadai3-1/Mizushima/Makefile @@ -0,0 +1,21 @@ +BINARY_NAME := bin/typing +GOCMD=go +GOBUILD=$(GOCMD) build +GOCLEAN=$(GOCMD) clean +GOTEST=$(GOCMD) test +GOGET=$(GOCMD) get + +all: build + +build: + $(GOBUILD) -o $(BINARY_NAME) -v + +test: clean all + $(GOTEST) -v -cover + cd ./typing; $(GOTEST) -v -cover + +clean: + $(GOCLEAN) + rm -rf $(BINARY_NAME) + +.PHONY: test clean diff --git a/kadai3-1/Mizushima/README.md b/kadai3-1/Mizushima/README.md new file mode 100644 index 00000000..20e55b38 --- /dev/null +++ b/kadai3-1/Mizushima/README.md @@ -0,0 +1,37 @@ +## 課題3-1 タイピングゲームを作ろう +- 標準出力に英単語を出す（出すものは自由） +- 標準入力から1行受け取る +- 制限時間内に何問解けたか表示する + + +### コマンドラインオプション + + | オプション | 説明 | デフォルト | + | --- | --- | --- | + | -limit | ゲーム全体の制限時間を秒数で設定する | 20 | + +### インストール方法 +```bash +go get github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-1/Mizushima +``` + +### 使い方 +1. バイナリビルド（実行ファイル作成） +```bash +$ make +``` +2. ディレクトリを指定して実行 +```bash +$ ./bin/typing [option] +``` + + +### テストの方法 +- バイナリビルド & テスト +```bash +$ make test +``` +- テスト後の処理(掃除) +```bash +$ make clean +``` diff --git a/kadai3-1/Mizushima/main.go b/kadai3-1/Mizushima/main.go index 56e8b70b..b65f218a 100644 --- a/kadai3-1/Mizushima/main.go +++ b/kadai3-1/Mizushima/main.go @@ -1,11 +1,23 @@ package main import ( + "flag" + "log" "os" + "time" typing "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-1/Mizushima/typing" ) func main() { - typing.Game(os.Stdin, os.Stdout) + wordsPath := "./gamedata/words.csv" + + tm := flag.Duration("limit", 20, "Time limit of the game") + flag.Parse() + + TimeLimit := time.Second * (*tm) + _, err := typing.Game(os.Stdin, os.Stdout, wordsPath, TimeLimit, false) + if err != nil { + log.Fatal(err) + } } diff --git a/kadai3-1/Mizushima/typing/typing.go b/kadai3-1/Mizushima/typing/typing.go index 40d429d7..b04cfaa1 100644 --- a/kadai3-1/Mizushima/typing/typing.go +++ b/kadai3-1/Mizushima/typing/typing.go @@ -2,74 +2,98 @@ package typing import ( "bufio" - "bytes" - "context" "encoding/csv" "fmt" "io" "log" "math/rand" "os" - "strings" "time" ) +func Game(r io.Reader, w io.Writer, wordsPath string, t time.Duration, isTest bool) (int, error) { - -func Game(r io.Reader, w io.Writer) { - - words, err := readCSV("./gamedata/words.csv") + words, err := readCSV(wordsPath) if err != nil { - log.Fatal(err) + return -1, err } - fmt.Println(words) + limit := time.After(t) rand.Seed(time.Now().UnixNano()) + var indices []int + if !isTest { + indices = rand.Perm(len(words)) + } else { + indices = incSlice(len(words)) + } - bc := context.Background() - t := 20 * time.Second - ctx, cancel := context.WithTimeout(bc, t) - defer cancel() + if !isTest { + _, err = fmt.Fprintln(w, "> Typing game start\nPlease type the displayed word") + if err != nil { + return -1, err + } - fmt.Println("> タイピングゲームを始めましゅ") - fmt.Println("> 英単語が出てきますので、同じ単語をタイプしてくだしゃい!") - fmt.Println("> 制限時間は20秒です") + _, err = fmt.Fprintf(w, "> Time limit is %d seconds\n", int(t.Seconds())) + if err != nil { + return -1, err + } + } - ch := input(os.Stdin) + ch := input(r) score := 0 + var idx int = 0 + var word string for { - idx := rand.Intn(len(words)) - fmt.Printf("> %s\n", words[idx]) + word = words[indices[idx]] + + _, err = fmt.Fprintf(w, "> %s\n", word) + if err != nil { + return -1, err + } + + idx++ select { - case <-ctx.Done(): - fmt.Println("\n終了!") - fmt.Printf("%d問正解です!\n", score) - return - case <-time.After(1 * time.Second): - if <-ch == words[idx] { - fmt.Println("> しぇえか～い") + case <-limit: + _, err = fmt.Fprintf(w, "\nGame ends!\nThe number of correct answers is %d\n", score) + if err != nil { + return -1, err + } + return score, nil + case ans := <-ch: + // fmt.Printf("ans: %s\n", ans) + // fmt.Printf("word: %s\n", word) + if ans == word { + if !isTest { + _, err = fmt.Fprintln(w, "> しぇえか～い") + if err != nil { + return -1, err + } + } score++ } else { - fmt.Println("> ぶっぶー") + if !isTest { + _, err = fmt.Fprintln(w, "> ぶっぶー") + if err != nil { + return -1, err + } + } } } } } func input(r io.Reader) <-chan string { - // TODO: チャネルを作る ch := make(chan string) go func() { s := bufio.NewScanner(r) for s.Scan() { - str := s.Text() - ch <- str + ch <- s.Text() } - close(ch) + // close(ch) }() return ch } @@ -79,7 +103,12 @@ func readCSV(path string) ([]string, error) { if err != nil { return nil, err } - defer file.Close() + + defer func() { + if err = file.Close(); err != nil { + log.Fatal(err) + } + }() csvFile := csv.NewReader(file) csvFile.TrimLeadingSpace = true @@ -92,24 +121,17 @@ func readCSV(path string) ([]string, error) { if err != nil { break } - + ret = append(ret, row...) } return ret, nil } -func GetStdoutAsString(f func()) string { - tmpStdout := os.Stdout - defer func() { - os.Stdout = tmpStdout - }() - r, w, _ := os.Pipe() - os.Stdout = w - - f() - w.Close() - var buf bytes.Buffer - buf.ReadFrom(r) - return strings.TrimRight(buf.String(), "\n") +func incSlice(n int) []int { + var res []int + for i := 0; i < n; i++ { + res = append(res, i) + } + return res } \ No newline at end of file diff --git a/kadai3-1/Mizushima/typing/typing_test.go b/kadai3-1/Mizushima/typing/typing_test.go new file mode 100644 index 00000000..c647a7a5 --- /dev/null +++ b/kadai3-1/Mizushima/typing/typing_test.go @@ -0,0 +1,68 @@ +package typing_test + +import ( + "bytes" + "strings" + "testing" + "time" + + typing "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-1/Mizushima/typing" +) + +func TestGame(t *testing.T) { + t.Helper() + + cases := []struct { + name string + tm time.Duration + ans []string + expected int + }{ + { + name: "No typo", + tm: 3 * time.Second, + ans: []string{ + "America", + "American", + "Angle", + "April", + "August", + "Bacon", + "Barber", + "Battery", + "Bible", + "Bill", + }, + expected: 10, + }, + { + name: "One typo", + tm: 3 * time.Second, + ans: []string{ + "America", + "American", + "typo", + "April", + "August", + "Bacon", + "Barber", + "Battery", + "Bible", + "Bill", + }, + expected: 9, + }, + } + + // []string{"America","American","Angle","April","August","Bacon","Barber","Battery","Bible","Bill"} + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + output := new(bytes.Buffer) + input := bytes.NewBufferString(strings.Join(c.ans, "\n")) + actual, _ := typing.Game(input, output, "../gamedata/words.csv", c.tm, true) + if actual != c.expected { + t.Errorf("wanted %d, but got %d", c.expected, actual) + } + }) + } +} From f772cf5a2c555f532f337616711e95ed52f687d2 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Thu, 17 Jun 2021 11:23:49 +0900 Subject: [PATCH 13/51] =?UTF-8?q?README=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-1/Mizushima/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/kadai3-1/Mizushima/README.md b/kadai3-1/Mizushima/README.md index 20e55b38..455e99e2 100644 --- a/kadai3-1/Mizushima/README.md +++ b/kadai3-1/Mizushima/README.md @@ -10,6 +10,7 @@ | --- | --- | --- | | -limit | ゲーム全体の制限時間を秒数で設定する | 20 | + ### インストール方法 ```bash go get github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-1/Mizushima From 5117bb9acfce5e4f8d2d1fb349eb6945fe563024 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Thu, 17 Jun 2021 11:26:37 +0900 Subject: [PATCH 14/51] =?UTF-8?q?go=20fmt=E9=81=A9=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-1/Mizushima/typing/typing.go | 6 +-- kadai3-1/Mizushima/typing/typing_test.go | 50 ++++++++++++------------ 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/kadai3-1/Mizushima/typing/typing.go b/kadai3-1/Mizushima/typing/typing.go index b04cfaa1..a5b0c363 100644 --- a/kadai3-1/Mizushima/typing/typing.go +++ b/kadai3-1/Mizushima/typing/typing.go @@ -48,7 +48,7 @@ func Game(r io.Reader, w io.Writer, wordsPath string, t time.Duration, isTest bo for { word = words[indices[idx]] - + _, err = fmt.Fprintf(w, "> %s\n", word) if err != nil { return -1, err @@ -63,7 +63,7 @@ func Game(r io.Reader, w io.Writer, wordsPath string, t time.Duration, isTest bo return -1, err } return score, nil - case ans := <-ch: + case ans := <-ch: // fmt.Printf("ans: %s\n", ans) // fmt.Printf("word: %s\n", word) if ans == word { @@ -134,4 +134,4 @@ func incSlice(n int) []int { res = append(res, i) } return res -} \ No newline at end of file +} diff --git a/kadai3-1/Mizushima/typing/typing_test.go b/kadai3-1/Mizushima/typing/typing_test.go index c647a7a5..99d53eb5 100644 --- a/kadai3-1/Mizushima/typing/typing_test.go +++ b/kadai3-1/Mizushima/typing/typing_test.go @@ -11,7 +11,7 @@ import ( func TestGame(t *testing.T) { t.Helper() - + cases := []struct { name string tm time.Duration @@ -19,35 +19,35 @@ func TestGame(t *testing.T) { expected int }{ { - name: "No typo", - tm: 3 * time.Second, - ans: []string{ - "America", - "American", - "Angle", - "April", - "August", - "Bacon", - "Barber", - "Battery", - "Bible", + name: "No typo", + tm: 3 * time.Second, + ans: []string{ + "America", + "American", + "Angle", + "April", + "August", + "Bacon", + "Barber", + "Battery", + "Bible", "Bill", }, expected: 10, }, { - name: "One typo", - tm: 3 * time.Second, - ans: []string{ - "America", - "American", - "typo", - "April", - "August", - "Bacon", - "Barber", - "Battery", - "Bible", + name: "One typo", + tm: 3 * time.Second, + ans: []string{ + "America", + "American", + "typo", + "April", + "August", + "Bacon", + "Barber", + "Battery", + "Bible", "Bill", }, expected: 9, From a3f369455cea8ee0d2182fcc412536271dea62bd Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Thu, 17 Jun 2021 11:39:10 +0900 Subject: [PATCH 15/51] =?UTF-8?q?README=E3=81=AB=E3=83=87=E3=82=A3?= =?UTF-8?q?=E3=83=AC=E3=82=AF=E3=83=88=E3=83=AA=E6=A7=8B=E6=88=90=E3=82=92?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-1/Mizushima/README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/kadai3-1/Mizushima/README.md b/kadai3-1/Mizushima/README.md index 455e99e2..e1c2b049 100644 --- a/kadai3-1/Mizushima/README.md +++ b/kadai3-1/Mizushima/README.md @@ -36,3 +36,18 @@ $ make test ```bash $ make clean ``` + +### ディレクトリ構成 +``` +. +├─ gamedata +│ └─ words.csv +├─ typing +│ └─ typing.go +│ └─ typing_test.go +├─ .gitignore +├─ go.mod +├─ main.go +├─ Makefile +└─ README.md +``` \ No newline at end of file From bbe053dba742a254485437fd90f4b08cfba28bc8 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Thu, 17 Jun 2021 12:50:59 +0900 Subject: [PATCH 16/51] =?UTF-8?q?=E8=AA=B2=E9=A1=8C2=E3=81=AE=E3=82=B3?= =?UTF-8?q?=E3=83=9F=E3=83=83=E3=83=88=E3=81=8C=E6=AE=8B=E3=81=A3=E3=81=A6?= =?UTF-8?q?=E3=81=84=E3=81=9F=E3=81=AE=E3=81=A7=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai2/Mizushima/.gitignore | 5 - kadai2/Mizushima/Makefile | 25 --- kadai2/Mizushima/README.md | 158 -------------- kadai2/Mizushima/go.mod | 7 - kadai2/Mizushima/go.sum | 0 kadai2/Mizushima/main.go | 70 ------ kadai2/Mizushima/main_test.go | 101 --------- kadai2/Mizushima/picconvert/go.mod | 5 - kadai2/Mizushima/picconvert/picconvert.go | 122 ----------- .../Mizushima/picconvert/picconvert_test.go | 206 ------------------ kadai2/Mizushima/testdata/test01.gif | Bin 1120 -> 0 bytes kadai2/Mizushima/testdata/test01.jpg | Bin 1641 -> 0 bytes kadai2/Mizushima/testdata/test01.png | Bin 415 -> 0 bytes kadai2/Mizushima/testdata/testdata/test01.gif | Bin 1120 -> 0 bytes kadai2/Mizushima/testdata/testdata/test01.jpg | Bin 1641 -> 0 bytes kadai2/Mizushima/testdata/testdata/test01.png | Bin 415 -> 0 bytes 16 files changed, 699 deletions(-) delete mode 100644 kadai2/Mizushima/.gitignore delete mode 100644 kadai2/Mizushima/Makefile delete mode 100644 kadai2/Mizushima/README.md delete mode 100644 kadai2/Mizushima/go.mod delete mode 100644 kadai2/Mizushima/go.sum delete mode 100644 kadai2/Mizushima/main.go delete mode 100644 kadai2/Mizushima/main_test.go delete mode 100644 kadai2/Mizushima/picconvert/go.mod delete mode 100644 kadai2/Mizushima/picconvert/picconvert.go delete mode 100644 kadai2/Mizushima/picconvert/picconvert_test.go delete mode 100644 kadai2/Mizushima/testdata/test01.gif delete mode 100644 kadai2/Mizushima/testdata/test01.jpg delete mode 100644 kadai2/Mizushima/testdata/test01.png delete mode 100644 kadai2/Mizushima/testdata/testdata/test01.gif delete mode 100644 kadai2/Mizushima/testdata/testdata/test01.jpg delete mode 100644 kadai2/Mizushima/testdata/testdata/test01.png diff --git a/kadai2/Mizushima/.gitignore b/kadai2/Mizushima/.gitignore deleted file mode 100644 index 07784897..00000000 --- a/kadai2/Mizushima/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -testdata/*_converted.* -testdata/*/*_converted.* -bin -*/profile -profile \ No newline at end of file diff --git a/kadai2/Mizushima/Makefile b/kadai2/Mizushima/Makefile deleted file mode 100644 index 9f445a00..00000000 --- a/kadai2/Mizushima/Makefile +++ /dev/null @@ -1,25 +0,0 @@ -# -# 原典：https://frasco.io/golang-dont-afraid-of-makefiles-785f3ec7eb32 -# - -BINARY_NAME := bin/converter -GOCMD=go -GOBUILD=$(GOCMD) build -GOCLEAN=$(GOCMD) clean -GOTEST=$(GOCMD) test -GOGET=$(GOCMD) get - -all: build - -build: - $(GOBUILD) -o $(BINARY_NAME) -v - -test: clean all - $(GOTEST) -v -cover - cd ./picconvert; $(GOTEST) -v -cover - -clean: - $(GOCLEAN) - rm -rf $(BINARY_NAME) - -.PHONY: test clean diff --git a/kadai2/Mizushima/README.md b/kadai2/Mizushima/README.md deleted file mode 100644 index 906a83f6..00000000 --- a/kadai2/Mizushima/README.md +++ /dev/null @@ -1,158 +0,0 @@ -## 課題2 テストを書いてみよう - - -### 1回目の課題のテストを作る -- テストのしやすさを考えてリファクタリングしてみる -- テストのカバレッジを取ってみる -- テーブル駆動テストを行う -- テストヘルパーを作ってみる - - -### コマンドラインオプション - - | オプション | 説明 | デフォルト | - | --- | --- | --- | - | -path | 変換したい画像があるディレクトリ| カレントディレクトリ | - | -pre | 変換前フォーマット | jpeg | - | -post | 変換後フォーマット | png | - - -### 対応している画像フォーマット -- png -- jpeg(jpg) -- gif - - -### 使い方 -1. バイナリビルド（実行ファイル作成） -```bash -$ make -``` -2. ディレクトリを指定して実行 -```bash -$ ./bin/converter [options...] -``` - - -### テストの方法 -- バイナリビルド & テスト -```bash -$ make test -``` -- テスト後の処理(掃除) -```bash -$ make clean -``` - -### テスト結果 -main_test.go : 73.1% -picconvert_test.go : 87.8% - -```bash -=== RUN TestIsSupportedFormat -=== RUN TestIsSupportedFormat/jpg -=== RUN TestIsSupportedFormat/jpeg -=== RUN TestIsSupportedFormat/png -=== RUN TestIsSupportedFormat/gif -=== RUN TestIsSupportedFormat/xls ---- PASS: TestIsSupportedFormat (0.00s) - --- PASS: TestIsSupportedFormat/jpg (0.00s) - --- PASS: TestIsSupportedFormat/jpeg (0.00s) - --- PASS: TestIsSupportedFormat/png (0.00s) - --- PASS: TestIsSupportedFormat/gif (0.00s) - --- PASS: TestIsSupportedFormat/xls (0.00s) -=== RUN TestValidate -=== RUN TestValidate/jpg_and_png -=== RUN TestValidate/jpeg_and_png -=== RUN TestValidate/jpg_and_gif -=== RUN TestValidate/jpeg_and_gif -=== RUN TestValidate/png_and_jpg -=== RUN TestValidate/png_and_jpeg -=== RUN TestValidate/png_and_gif -=== RUN TestValidate/gif_and_jpeg -=== RUN TestValidate/gif_and_jpg -=== RUN TestValidate/gif_and_png -=== RUN TestValidate/xls_and_gif -=== RUN TestValidate/gif_and_xls -=== RUN TestValidate/jpg_and_jpeg -=== RUN TestValidate/jpeg_and_jpg -=== RUN TestValidate/png_and_png -=== RUN TestValidate/gif_and_gif ---- PASS: TestValidate (0.00s) - --- PASS: TestValidate/jpg_and_png (0.00s) - --- PASS: TestValidate/jpeg_and_png (0.00s) - --- PASS: TestValidate/jpg_and_gif (0.00s) - --- PASS: TestValidate/jpeg_and_gif (0.00s) - --- PASS: TestValidate/png_and_jpg (0.00s) - --- PASS: TestValidate/png_and_jpeg (0.00s) - --- PASS: TestValidate/png_and_gif (0.00s) - --- PASS: TestValidate/gif_and_jpeg (0.00s) - --- PASS: TestValidate/gif_and_jpg (0.00s) - --- PASS: TestValidate/gif_and_png (0.00s) - --- PASS: TestValidate/xls_and_gif (0.00s) - --- PASS: TestValidate/gif_and_xls (0.00s) - --- PASS: TestValidate/jpg_and_jpeg (0.00s) - --- PASS: TestValidate/jpeg_and_jpg (0.00s) - --- PASS: TestValidate/png_and_png (0.00s) - --- PASS: TestValidate/gif_and_gif (0.00s) -PASS -coverage: 73.1% of statements -ok github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai2/Mizushima 0.003s - -=== RUN TestGlob -=== RUN TestGlob/jpeg_files -=== RUN TestGlob/png_files -=== RUN TestGlob/gif_files -=== RUN TestGlob/xls_files ---- PASS: TestGlob (0.00s) - --- PASS: TestGlob/jpeg_files (0.00s) - --- PASS: TestGlob/png_files (0.00s) - --- PASS: TestGlob/gif_files (0.00s) - --- PASS: TestGlob/xls_files (0.00s) -=== RUN TestPicConverter_Conv -=== RUN TestPicConverter_Conv/pre:_jpeg,_after:_png -=== RUN TestPicConverter_Conv/pre:_jpeg,_after:_gif -=== RUN TestPicConverter_Conv/pre:_jpg,_after:_png -=== RUN TestPicConverter_Conv/pre:_jpg,_after:_gif -=== RUN TestPicConverter_Conv/pre:_png,_after:_jpg -=== RUN TestPicConverter_Conv/pre:_png,_after:_gif -=== RUN TestPicConverter_Conv/pre:_gif,_after:_jpg -=== RUN TestPicConverter_Conv/pre:_gif,_after:_png -=== RUN TestPicConverter_Conv/pre:_jpg,_after:_xls -=== RUN TestPicConverter_Conv/no_file ---- PASS: TestPicConverter_Conv (0.08s) - --- PASS: TestPicConverter_Conv/pre:_jpeg,_after:_png (0.00s) - --- PASS: TestPicConverter_Conv/pre:_jpeg,_after:_gif (0.01s) - --- PASS: TestPicConverter_Conv/pre:_jpg,_after:_png (0.00s) - --- PASS: TestPicConverter_Conv/pre:_jpg,_after:_gif (0.01s) - --- PASS: TestPicConverter_Conv/pre:_png,_after:_jpg (0.00s) - --- PASS: TestPicConverter_Conv/pre:_png,_after:_gif (0.01s) - --- PASS: TestPicConverter_Conv/pre:_gif,_after:_jpg (0.00s) - --- PASS: TestPicConverter_Conv/pre:_gif,_after:_png (0.05s) - --- PASS: TestPicConverter_Conv/pre:_jpg,_after:_xls (0.00s) - --- PASS: TestPicConverter_Conv/no_file (0.00s) -=== RUN TestNewPicConverter -=== RUN TestNewPicConverter/pre:_jpg,_post:_png -=== RUN TestNewPicConverter/pre:_jpeg,_post:_png -=== RUN TestNewPicConverter/pre:_jpg,_post:_gif -=== RUN TestNewPicConverter/pre:_png,_post:_jpeg -=== RUN TestNewPicConverter/pre:_png,_post:_jpg -=== RUN TestNewPicConverter/pre:_png,_post:_gif -=== RUN TestNewPicConverter/pre:_gif,_post:_jpeg -=== RUN TestNewPicConverter/pre:_gif,_post:_jpg -=== RUN TestNewPicConverter/pre:_gif,_post:_png ---- PASS: TestNewPicConverter (0.00s) - --- PASS: TestNewPicConverter/pre:_jpg,_post:_png (0.00s) - --- PASS: TestNewPicConverter/pre:_jpeg,_post:_png (0.00s) - --- PASS: TestNewPicConverter/pre:_jpg,_post:_gif (0.00s) - --- PASS: TestNewPicConverter/pre:_png,_post:_jpeg (0.00s) - --- PASS: TestNewPicConverter/pre:_png,_post:_jpg (0.00s) - --- PASS: TestNewPicConverter/pre:_png,_post:_gif (0.00s) - --- PASS: TestNewPicConverter/pre:_gif,_post:_jpeg (0.00s) - --- PASS: TestNewPicConverter/pre:_gif,_post:_jpg (0.00s) - --- PASS: TestNewPicConverter/pre:_gif,_post:_png (0.00s) -PASS -coverage: 87.8% of statements -ok github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai2/Mizushima/picconvert 0.087s - -``` diff --git a/kadai2/Mizushima/go.mod b/kadai2/Mizushima/go.mod deleted file mode 100644 index 876315e1..00000000 --- a/kadai2/Mizushima/go.mod +++ /dev/null @@ -1,7 +0,0 @@ -module github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai2/Mizushima - -go 1.16 - -replace github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai2/Mizushima/picconvert => ./picconvert - -require github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai2/Mizushima/picconvert v0.0.0-00010101000000-000000000000 diff --git a/kadai2/Mizushima/go.sum b/kadai2/Mizushima/go.sum deleted file mode 100644 index e69de29b..00000000 diff --git a/kadai2/Mizushima/main.go b/kadai2/Mizushima/main.go deleted file mode 100644 index c80ad40d..00000000 --- a/kadai2/Mizushima/main.go +++ /dev/null @@ -1,70 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "log" - "testing" - - picconvert "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai2/Mizushima/picconvert" -) - -var preFormat string -var afterFormat string -var srcPath string - -func init() { - testing.Init() - flag.StringVar(&preFormat, "pre", "jpeg", "the file format before conversion") - flag.StringVar(&afterFormat, "post", "png", "the file format after conversion") - flag.StringVar(&srcPath, "path", ".", "the source directory for conversion") - flag.Parse() -} - -// isSupportedFormat returns true if "format" is supported, othrewise returns false. -func IsSupportedFormat(format string) bool { - ext := []string{"jpg", "jpeg", "png", "gif"} - - for _, e := range ext { - if e == format { - return true - } - } - return false -} - -// validate returns error if there is something wrong with -// the entered parameters. -func Validate() error { - // fmt.Printf("%v, %v, %v\n", preFormat, afterFormat, srcPath) - if preFormat == afterFormat { - return fmt.Errorf("the parameter of -pre is same as that of -post") - } - - if (preFormat == "jpeg" && afterFormat == "jpg") || (preFormat == "jpg" && afterFormat == "jpeg") { - return fmt.Errorf("the parameter of -pre is same as that of -post. 'jpg' is considered same as 'jpeg'") - } - - if !IsSupportedFormat(preFormat) { - return fmt.Errorf("-pre %s is not supported", preFormat) - } else if !IsSupportedFormat(afterFormat) { - return fmt.Errorf("-post %s is not supported", afterFormat) - } - - return nil -} - -func main() { - // fmt.Println("converting" ,preFormat, "to", afterFormat) - err := Validate() - if err != nil { - log.Fatal(err) - } - - c := picconvert.NewPicConverter(srcPath, preFormat, afterFormat) - - err = c.Conv() - if err != nil { - log.Fatal(err) - } -} diff --git a/kadai2/Mizushima/main_test.go b/kadai2/Mizushima/main_test.go deleted file mode 100644 index 1bad7760..00000000 --- a/kadai2/Mizushima/main_test.go +++ /dev/null @@ -1,101 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "testing" -) - -func TestIsSupportedFormat(t *testing.T) { - cases := []struct { - name string - format string - expected bool - }{ - {name: "jpg", format: "jpg", expected: true}, - {name: "jpeg", format: "jpeg", expected: true}, - {name: "png", format: "png", expected: true}, - {name: "gif", format: "gif", expected: true}, - {name: "xls", format: "xls", expected: false}, - } - - for _, c := range cases { - // c := c - t.Run(c.name, func(t *testing.T) { - actual := IsSupportedFormat(c.format) - // if actual != nil { fmt.Println(actual) } - if actual != c.expected { - t.Errorf("want IsSupportedFormat(%s) = %v, got %v", c.format, c.expected, actual) - } - }) - } -} - -func TestValidate(t *testing.T) { - cases := []struct { - name string - pre string - post string - path string - expected error - }{ - {name: "jpg and png", pre: "jpg", post: "png", path: ".", expected: nil}, - {name: "jpeg and png", pre: "jpeg", post: "png", path: ".", expected: nil}, - {name: "jpg and gif", pre: "jpg", post: "gif", path: ".", expected: nil}, - {name: "jpeg and gif", pre: "jpeg", post: "gif", path: ".", expected: nil}, - {name: "png and jpg", pre: "png", post: "jpg", path: ".", expected: nil}, - {name: "png and jpeg", pre: "png", post: "jpeg", path: ".", expected: nil}, - {name: "png and gif", pre: "png", post: "gif", path: ".", expected: nil}, - {name: "gif and jpeg", pre: "gif", post: "jpeg", path: ".", expected: nil}, - {name: "gif and jpg", pre: "gif", post: "jpg", path: ".", expected: nil}, - {name: "gif and png", pre: "gif", post: "png", path: ".", expected: nil}, - {name: "xls and gif", pre: "xls", post: "gif", path: ".", expected: fmt.Errorf("-pre xls is not supported")}, - {name: "gif and xls", pre: "gif", post: "xls", path: ".", expected: fmt.Errorf("-post xls is not supported")}, - { - name: "jpg and jpeg", - pre: "jpg", - post: "jpeg", - path: ".", - expected: fmt.Errorf("the parameter of -pre is same as that of -post. 'jpg' is considered same as 'jpeg'"), - }, - { - name: "jpeg and jpg", - pre: "jpeg", - post: "jpg", - path: ".", - expected: fmt.Errorf("the parameter of -pre is same as that of -post. 'jpg' is considered same as 'jpeg'"), - }, - { - name: "png and png", - pre: "png", - post: "png", - path: ".", - expected: fmt.Errorf("the parameter of -pre is same as that of -post"), - }, - { - name: "gif and gif", - pre: "gif", - post: "gif", - path: ".", - expected: fmt.Errorf("the parameter of -pre is same as that of -post"), - }, - } - - for _, c := range cases { - // c := c - t.Run(c.name, func(t *testing.T) { - flag.CommandLine.Set("pre", c.pre) - flag.CommandLine.Set("post", c.post) - flag.CommandLine.Set("path", c.path) - actual := Validate() - // if actual != nil { fmt.Println(actual) } - if actual != nil && c.expected != nil && actual.Error() != c.expected.Error() { - t.Errorf("want Validate() = %v, but got %v", c.expected.Error(), actual.Error()) - } else if actual != nil && c.expected == nil { - t.Errorf("want Validate() = nil, but got %v", actual.Error()) - } else if actual == nil && c.expected != nil { - t.Errorf("want Validate() = %v, but got nil", c.expected.Error()) - } - }) - } -} diff --git a/kadai2/Mizushima/picconvert/go.mod b/kadai2/Mizushima/picconvert/go.mod deleted file mode 100644 index c16f9342..00000000 --- a/kadai2/Mizushima/picconvert/go.mod +++ /dev/null @@ -1,5 +0,0 @@ -module github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai2/Mizushima/picconvert - -go 1.16 - -replace "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai2/Mizushima/picconvert" => ./picconvert \ No newline at end of file diff --git a/kadai2/Mizushima/picconvert/picconvert.go b/kadai2/Mizushima/picconvert/picconvert.go deleted file mode 100644 index 65d488aa..00000000 --- a/kadai2/Mizushima/picconvert/picconvert.go +++ /dev/null @@ -1,122 +0,0 @@ -// Package picconvert implements a picture format converter -// for command line tool in main package. -// - -package picconvert - -import ( - "fmt" - "image" - "image/gif" - "image/jpeg" - "image/png" - "os" - "path/filepath" -) - -// PicConverter is user-defined type for converting -// a picture file format has root path, pre-conversion format, -// and after-conversion format. -type PicConverter struct { - Path string - PreFormat []string - AfterFormat string -} - -// Conv converts the picture file format. -func (p *PicConverter) Conv() error { - files, err := Glob(p.Path, p.PreFormat) - - if err != nil { - return err - } - - if files == nil { - return fmt.Errorf("there's no %s file", p.PreFormat) - } - - // prosessing for each file. - for _, file := range files { - // fmt.Println("from:", file) - f, err := os.Open(file) - if err != nil { - return err - } - defer f.Close() - - // reading the image file. - img, _, err := image.Decode(f) - if err != nil { - return err - } - - // creating filepath for output. - output, err := os.Create(fmt.Sprintf( - "%s_converted.%s", - baseName(file), p.AfterFormat)) - - if err != nil { - return err - } - - // converting the file. - switch p.AfterFormat { - case "png": - err = png.Encode(output, img) - case "jpg": - case "jpeg": - err = jpeg.Encode(output, img, nil) - case "gif": - err = gif.Encode(output, img, nil) - default: - err = fmt.Errorf("%s is not supported", p.AfterFormat) - } - - if err != nil { - return err - } - - // fmt.Printf("converted %s\n", output.Name()) - } - - return nil -} - -// NewPicConverter is the constructor for a PicConverter. -func NewPicConverter(Path string, PreFormat string, AfterFormat string) *PicConverter { - res := new(PicConverter) - res.Path = Path - - if PreFormat == "jpeg" || PreFormat == "jpg" { - res.PreFormat = []string{"jpeg", "jpg"} - } else { - res.PreFormat = []string{PreFormat} - } - - res.AfterFormat = AfterFormat - return res -} - -// Glob returns a slice of the file paths that meets the "format". -func Glob(path string, format []string) ([]string, error) { - var res []string - - var err error - for _, f := range format { - err = filepath.Walk(path, - func(path string, info os.FileInfo, err error) error { - if filepath.Ext(path) == "."+f && !info.IsDir() { - res = append(res, path) - } - return nil - }) - } - - return res, err -} - -// baseName returns the filepath without a extension. -func baseName(filePath string) string { - ext := filepath.Ext(filePath) - return filePath[:len(filePath)-len(ext)] -} diff --git a/kadai2/Mizushima/picconvert/picconvert_test.go b/kadai2/Mizushima/picconvert/picconvert_test.go deleted file mode 100644 index f178e40f..00000000 --- a/kadai2/Mizushima/picconvert/picconvert_test.go +++ /dev/null @@ -1,206 +0,0 @@ -package picconvert_test - -import ( - "fmt" - "os" - "path/filepath" - "reflect" - "strings" - "testing" - - picconvert "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai2/Mizushima/picconvert" -) - -func TestGlob(t *testing.T) { - tmpTestDir := "../testdata" - - // make test cases - cases := []struct { - name string - path string - format []string - expected1 []string - expected2 error - }{ - {name: "jpeg files", - path: tmpTestDir, - format: []string{"jpg", "jpeg"}, - expected1: []string{tmpTestDir + "/test01.jpg", tmpTestDir + "/" + filepath.Base(tmpTestDir) + "/test01.jpg"}, - expected2: nil, - }, - {name: "png files", - path: tmpTestDir, - format: []string{"png"}, - expected1: []string{tmpTestDir + "/test01.png", tmpTestDir + "/" + filepath.Base(tmpTestDir) + "/test01.png"}, - expected2: nil, - }, - {name: "gif files", - path: tmpTestDir, - format: []string{"gif"}, - expected1: []string{tmpTestDir + "/test01.gif", tmpTestDir + "/" + filepath.Base(tmpTestDir) + "/test01.gif"}, - expected2: nil, - }, - {name: "xls files", - path: tmpTestDir, - format: []string{"xls"}, - expected1: []string{}, - expected2: nil, - }, - } - - for _, c := range cases { - c := c - t.Run(c.name, func(t *testing.T) { - actual, err := picconvert.Glob(c.path, c.format) - // if actual != nil { fmt.Println(actual) } - if !reflect.DeepEqual(actual, c.expected1) && - !(len(actual) == 0 && len(c.expected1) == 0) && - (err == nil && c.expected2 == nil) { - t.Errorf("want Glob(%s, %s) = (%v, nil), got %v, nil", c.path, c.format, c.expected1, actual) - } else if !reflect.DeepEqual(actual, c.expected1) && (err != nil && c.expected2 != nil) { - t.Errorf("want Glob(%s, %s) = (%v, %v), got %v, %v", - c.path, c.format, c.expected1, c.expected2.Error(), actual, err.Error()) - } - }) - } -} - -// TestPicConverter_Conv is the test for (p PicConverter)Conv() -func TestPicConverter_Conv(t *testing.T) { - tmpTestDir := "../testdata" - - // make test cases - cases := []struct { - name string - pc *picconvert.PicConverter - expected error - }{ - {name: "pre: jpeg, after: png", pc: picconvert.NewPicConverter(tmpTestDir, "jpeg", "png"), expected: nil}, - {name: "pre: jpeg, after: gif", pc: picconvert.NewPicConverter(tmpTestDir, "jpeg", "gif"), expected: nil}, - {name: "pre: jpg, after: png", pc: picconvert.NewPicConverter(tmpTestDir, "jpg", "png"), expected: nil}, - {name: "pre: jpg, after: gif", pc: picconvert.NewPicConverter(tmpTestDir, "jpg", "gif"), expected: nil}, - {name: "pre: png, after: jpg", pc: picconvert.NewPicConverter(tmpTestDir, "png", "jpg"), expected: nil}, - {name: "pre: png, after: gif", pc: picconvert.NewPicConverter(tmpTestDir, "png", "gif"), expected: nil}, - {name: "pre: gif, after: jpg", pc: picconvert.NewPicConverter(tmpTestDir, "gif", "jpg"), expected: nil}, - {name: "pre: gif, after: png", pc: picconvert.NewPicConverter(tmpTestDir, "gif", "png"), expected: nil}, - {name: "pre: jpg, after: xls", pc: picconvert.NewPicConverter(tmpTestDir, "jpg", "xls"), expected: fmt.Errorf("xls is not supported")}, - {name: "no file", pc: picconvert.NewPicConverter(tmpTestDir+"/test02.png", "gif", "png"), expected: fmt.Errorf("there's no [gif] file")}, - } - - for _, c := range cases { - c := c - t.Run(c.name, func(t *testing.T) { - actual := c.pc.Conv() - // if actual != nil { fmt.Println(actual) } - if actual != nil && actual.Error() != c.expected.Error() { - t.Errorf("want p.Conv() = %v, got %v", c.expected, actual) - } else if _, err := os.Stat(fmt.Sprintf("%s/test01_converted.%s", tmpTestDir, c.pc.AfterFormat)); err != nil && c.pc.AfterFormat != "xls" { - t.Errorf("%s file wasn't made", c.pc.AfterFormat) - } - }) - } - - defer testDeleteConveterd(t, tmpTestDir) -} - -// testDeleteConveterd returns a slice of the file paths that meets the "format". -func testDeleteConveterd(t *testing.T, path string) { - err := filepath.Walk(path, - func(path string, info os.FileInfo, err error) error { - if strings.Contains(path, "_converted") && !info.IsDir() { - os.Remove(path) - } - return nil - }) - - if err != nil { - t.Error(err) - } -} - -func TestNewPicConverter(t *testing.T) { - - cases := []struct { - name string - path string - pre string - post string - expected *picconvert.PicConverter - }{ - { - name: "pre: jpg, post: png", - path: "../testdata", - pre: "jpg", - post: "png", - expected: &picconvert.PicConverter{"../testdata", []string{"jpeg", "jpg"}, "png"}, - }, - { - name: "pre: jpeg, post: png", - path: "../testdata", - pre: "jpeg", - post: "png", - expected: &picconvert.PicConverter{"../testdata", []string{"jpeg", "jpg"}, "png"}, - }, - { - name: "pre: jpg, post: gif", - path: "../testdata", - pre: "jpg", - post: "gif", - expected: &picconvert.PicConverter{"../testdata", []string{"jpeg", "jpg"}, "gif"}, - }, - { - name: "pre: png, post: jpeg", - path: "../testdata", - pre: "png", - post: "jpeg", - expected: &picconvert.PicConverter{"../testdata", []string{"png"}, "jpeg"}, - }, - { - name: "pre: png, post: jpg", - path: "../testdata", - pre: "png", - post: "jpg", - expected: &picconvert.PicConverter{"../testdata", []string{"png"}, "jpg"}, - }, - { - name: "pre: png, post: gif", - path: "../testdata", - pre: "png", - post: "gif", - expected: &picconvert.PicConverter{"../testdata", []string{"png"}, "gif"}, - }, - { - name: "pre: gif, post: jpeg", - path: "../testdata", - pre: "gif", - post: "jpeg", - expected: &picconvert.PicConverter{"../testdata", []string{"gif"}, "jpeg"}, - }, - { - name: "pre: gif, post: jpg", - path: "../testdata", - pre: "gif", - post: "jpg", - expected: &picconvert.PicConverter{"../testdata", []string{"gif"}, "jpg"}, - }, - { - name: "pre: gif, post: png", - path: "../testdata", - pre: "gif", - post: "png", - expected: &picconvert.PicConverter{"../testdata", []string{"gif"}, "png"}, - }, - } - - for _, c := range cases { - c := c - t.Run(c.name, func(t *testing.T) { - actual := picconvert.NewPicConverter(c.path, c.pre, c.post) - // if actual != nil { fmt.Println(actual) } - if !reflect.DeepEqual(actual, c.expected) { - t.Errorf("want NewPicConverter(%s, %s, %s) = %v, but got %v", - c.path, c.pre, c.post, c.expected, actual) - } - }) - } -} diff --git a/kadai2/Mizushima/testdata/test01.gif b/kadai2/Mizushima/testdata/test01.gif deleted file mode 100644 index a517a9305a16a45582c06f9af6ac1641b2c18f92..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1120 zcmXYwZ)n$56vm&}Z#6^Q%3PzYWebh2Tk7#&jbKqtr^GC~(LV?&4jR3P5_(f{r$b*v z(1oy+zbwwgS;CC`uZ^(|6TWJe|nPy|I$ z48>7cy$uSepn(on2M9!<0uy*ulK=@KK_!?3mq3Y-C=ykoNpy*n1W6)EC7C3bM9GjW zl2x)vcFB|iDI!Ism=u>nA%r4Sp$T1B>8QpQg(^(pl`IX=AR1JIX>bkH2#umqHJV1( zNKMcrnpBf%a!u3>&7xT~n`YNcEzlxbREueGEmT4&N>!TDl@)@DxER!62Cs-%fCaIj z7R-WMphZ{|i)zs=L3=i-i9@K+*a1ZndkK$21nn(9YPw*t3)RTE~PxK7W;#obLXZK7m@FHH+ zi+OP`biyf4b(+(ioC-qaMunu(RTZl!1TYALG8lt1kP(c+sEo$wjAR0nFe#HUITM+| zEX>Mm%+5>}un3E?7>lz^HLHSFxm_Wz=Ki<&|GBTWIe*oz0RG1OsjI0!fd=C8wAm{U z?jGLUF!PS5P9EGdvbCvg)vlF?-uvS97J2TwlZW0vv2CRJ8+85FHDgWd^@rE?-FN12 z-{h7J(=c!Ey|4D)eEj~Ey{9&>Us`{muI__#9ks5tKg?gXrgavYreC>w``NYwLl15k zeC37h&1VLFe{9zc%X{m4-fW(6Vc(*oZS8HfQ}5J&{>OO7{u{sEa`(xB+rHnlZl6B8 zV&|`q_D^1YX{=@chr^%Y^sSF<+&a;5?5dw`>OQ~onNMzMn8Ws=h9je+dzQWY#>n7U z@5taacRsQIp?7Bu*DX2oK+Dlfqo)t9Xgq##;eyAHywvmB*vDTjJF)M?jvb3X-S*bR zz~pu9PanJZ(ZlU$XZ~`}i|O08FKeBRf6kx1arc5*ix$Vd%bs72&P$DRpR7&%+}HZ{ ikMqV;t$F=;U&Gv?@lDg0o*&=b)P4K8{%h-N===wY{tcG^ diff --git a/kadai2/Mizushima/testdata/test01.jpg b/kadai2/Mizushima/testdata/test01.jpg deleted file mode 100644 index 8b03b2d51708b08fe3477aee73f652c1efdf0129..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1641 zcmbV~do+}39LJw`-WhX~F>c8oNk(qnQOin8jY5)HnKDH;Qwd3D3^ro!nh?< z*&??Z*D*+Ki_InDP8LaK(u!PW->FWWvuFR<-S7MS&hvfF`+I)p_j#U2`cm2l6zyzm zZ2$y8-~jRgX+LnK2C|p{I5+@9000hPUt`254kLtU`^G z(l;;515i{t0Qqs!POt@_AV@a82*$uN$)V9O1}lff$^wTc;&C_v4vQr$A`po3h_HAC z(js{UnU@ujZI|srhCB|7lV$ujNvi-E2i5^o7+MZcWC$ihQX$YnI>jI-Av=sOf>1CT zBZss^AR-Axib(q~j6#~lpb=c?9P%Ha$rvT=byjl9P6x5eeO1}JV9--`VtM@6?*yxh9;&PHhynrzGW-bX4`gKyPuqQyU_RSbv=B$u-( z|1$xBL1!b*M@B_oxX8JZcr__G<>zbHGqbX9<=oEY-6?ojSX5k6`slHsMp#={|D@sR z?=MQ^+>(FSpqm zu%}Fb<4|JFn8{2uRerstdM@pjkx}&A=7!_FnT)i{)KFh3sJ03_?jGg-;C2n8k{sG1 z1zHxK1EeL5n&sV42_RM1PlDhNWH}Mo)gFA~|-eq?Vy|Uia*e?Zw zxW%l^xVbeFoj!VOnO83Bde3E69<|T?f~Z9_o|ErwVQ*eLpK)rkw`2cRpBVi|Y2Az; z`ck44%GbBJoICTTu6!-$=6wNGj+aH-#!2Fmosyk# zn%(#4Ohz!D*h21MZH#}?)YZ|oj6xCbxU%&%9*XZvU?R{z6y-z=rvxP7-S+}-! zfHZBW6~3&JYt%mxZQhoZEh*lZDrB-eq{pMs*4!CAi1ADRu|?;I^VOR&D0$? zM-`*jx&==3m)8p8NBwpzixY}Go*MSDm@e9fSRuK(7dwX-G21uBM?PQeK}g`ON}8Cw zlr?Q;5b(m`gwjsuAq>B$f<`NuFH2cH>iJ8G<#$pL@h}t$RHjP-+c|J-K((D+rt3F2 zzIvGYpg>~scoTn2?aXtYKtuiQ)1ZbWO14*B;Ng>IrXRR(G^JpLnfT~}aKEw?IEd*& zu6UPwhrO)Q+Y$5V@G{NW-Fh`^;JnnEmIl)dUWi4}Uy4$$rHbovOl(##^8k~ z^RYIk6ZKlcJi3FC3JzLPcg1%HNcGJ5oCWRqG|`Ky1ya+T&f(0l4aq*%#iPU;ua!5v zA`HDN%8qc>3?}Mb7xbiRG^NG*(KtP$)Ovfj@yZ~E|8RP~liz)tZ2w&mm(tq2ct$-U z$|LuqL%cU(1?BgrNaOU`ZKKIPL9X`GDR!*|lrFAUX>%T?ia2McD0roiLpz)3Ht*&% zs~v)G4J8D^E_{hYcU0D3ucoK*G`E)Sq8-UAzeY1Hi8!NP?d83wyzoSo@vQ-l13w4=LC6`8BEr^pmO_Q45V}TIckt+=yZ>+k~fXy!V&}wc^G} ip$|S*W%rYhbOx3D!TVXe&)LG>A2T`n;2#2(cKihsEUiEQ diff --git a/kadai2/Mizushima/testdata/test01.png b/kadai2/Mizushima/testdata/test01.png deleted file mode 100644 index 17fbc75babc9ca1708fa680591b92d4ba81fe789..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 415 zcmeAS@N?(olHy`uVBq!ia0vp^ra)}S!2~3~ik2AyDaPU;cPEB*=VV?2IV|apzK#qG z8~eHcB(eheoCO|{#S9F5he4R}c>anMpav697srqY_qW#{@-{07uqHe&Qh1gqH7!x1 zdij#8T^}CYoM!U-)ref3AMNsQtJuzRnoxTd1oy+zbwwgS;CC`uZ^(|6TWJe|nPy|I$ z48>7cy$uSepn(on2M9!<0uy*ulK=@KK_!?3mq3Y-C=ykoNpy*n1W6)EC7C3bM9GjW zl2x)vcFB|iDI!Ism=u>nA%r4Sp$T1B>8QpQg(^(pl`IX=AR1JIX>bkH2#umqHJV1( zNKMcrnpBf%a!u3>&7xT~n`YNcEzlxbREueGEmT4&N>!TDl@)@DxER!62Cs-%fCaIj z7R-WMphZ{|i)zs=L3=i-i9@K+*a1ZndkK$21nn(9YPw*t3)RTE~PxK7W;#obLXZK7m@FHH+ zi+OP`biyf4b(+(ioC-qaMunu(RTZl!1TYALG8lt1kP(c+sEo$wjAR0nFe#HUITM+| zEX>Mm%+5>}un3E?7>lz^HLHSFxm_Wz=Ki<&|GBTWIe*oz0RG1OsjI0!fd=C8wAm{U z?jGLUF!PS5P9EGdvbCvg)vlF?-uvS97J2TwlZW0vv2CRJ8+85FHDgWd^@rE?-FN12 z-{h7J(=c!Ey|4D)eEj~Ey{9&>Us`{muI__#9ks5tKg?gXrgavYreC>w``NYwLl15k zeC37h&1VLFe{9zc%X{m4-fW(6Vc(*oZS8HfQ}5J&{>OO7{u{sEa`(xB+rHnlZl6B8 zV&|`q_D^1YX{=@chr^%Y^sSF<+&a;5?5dw`>OQ~onNMzMn8Ws=h9je+dzQWY#>n7U z@5taacRsQIp?7Bu*DX2oK+Dlfqo)t9Xgq##;eyAHywvmB*vDTjJF)M?jvb3X-S*bR zz~pu9PanJZ(ZlU$XZ~`}i|O08FKeBRf6kx1arc5*ix$Vd%bs72&P$DRpR7&%+}HZ{ ikMqV;t$F=;U&Gv?@lDg0o*&=b)P4K8{%h-N===wY{tcG^ diff --git a/kadai2/Mizushima/testdata/testdata/test01.jpg b/kadai2/Mizushima/testdata/testdata/test01.jpg deleted file mode 100644 index 8b03b2d51708b08fe3477aee73f652c1efdf0129..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1641 zcmbV~do+}39LJw`-WhX~F>c8oNk(qnQOin8jY5)HnKDH;Qwd3D3^ro!nh?< z*&??Z*D*+Ki_InDP8LaK(u!PW->FWWvuFR<-S7MS&hvfF`+I)p_j#U2`cm2l6zyzm zZ2$y8-~jRgX+LnK2C|p{I5+@9000hPUt`254kLtU`^G z(l;;515i{t0Qqs!POt@_AV@a82*$uN$)V9O1}lff$^wTc;&C_v4vQr$A`po3h_HAC z(js{UnU@ujZI|srhCB|7lV$ujNvi-E2i5^o7+MZcWC$ihQX$YnI>jI-Av=sOf>1CT zBZss^AR-Axib(q~j6#~lpb=c?9P%Ha$rvT=byjl9P6x5eeO1}JV9--`VtM@6?*yxh9;&PHhynrzGW-bX4`gKyPuqQyU_RSbv=B$u-( z|1$xBL1!b*M@B_oxX8JZcr__G<>zbHGqbX9<=oEY-6?ojSX5k6`slHsMp#={|D@sR z?=MQ^+>(FSpqm zu%}Fb<4|JFn8{2uRerstdM@pjkx}&A=7!_FnT)i{)KFh3sJ03_?jGg-;C2n8k{sG1 z1zHxK1EeL5n&sV42_RM1PlDhNWH}Mo)gFA~|-eq?Vy|Uia*e?Zw zxW%l^xVbeFoj!VOnO83Bde3E69<|T?f~Z9_o|ErwVQ*eLpK)rkw`2cRpBVi|Y2Az; z`ck44%GbBJoICTTu6!-$=6wNGj+aH-#!2Fmosyk# zn%(#4Ohz!D*h21MZH#}?)YZ|oj6xCbxU%&%9*XZvU?R{z6y-z=rvxP7-S+}-! zfHZBW6~3&JYt%mxZQhoZEh*lZDrB-eq{pMs*4!CAi1ADRu|?;I^VOR&D0$? zM-`*jx&==3m)8p8NBwpzixY}Go*MSDm@e9fSRuK(7dwX-G21uBM?PQeK}g`ON}8Cw zlr?Q;5b(m`gwjsuAq>B$f<`NuFH2cH>iJ8G<#$pL@h}t$RHjP-+c|J-K((D+rt3F2 zzIvGYpg>~scoTn2?aXtYKtuiQ)1ZbWO14*B;Ng>IrXRR(G^JpLnfT~}aKEw?IEd*& zu6UPwhrO)Q+Y$5V@G{NW-Fh`^;JnnEmIl)dUWi4}Uy4$$rHbovOl(##^8k~ z^RYIk6ZKlcJi3FC3JzLPcg1%HNcGJ5oCWRqG|`Ky1ya+T&f(0l4aq*%#iPU;ua!5v zA`HDN%8qc>3?}Mb7xbiRG^NG*(KtP$)Ovfj@yZ~E|8RP~liz)tZ2w&mm(tq2ct$-U z$|LuqL%cU(1?BgrNaOU`ZKKIPL9X`GDR!*|lrFAUX>%T?ia2McD0roiLpz)3Ht*&% zs~v)G4J8D^E_{hYcU0D3ucoK*G`E)Sq8-UAzeY1Hi8!NP?d83wyzoSo@vQ-l13w4=LC6`8BEr^pmO_Q45V}TIckt+=yZ>+k~fXy!V&}wc^G} ip$|S*W%rYhbOx3D!TVXe&)LG>A2T`n;2#2(cKihsEUiEQ diff --git a/kadai2/Mizushima/testdata/testdata/test01.png b/kadai2/Mizushima/testdata/testdata/test01.png deleted file mode 100644 index 17fbc75babc9ca1708fa680591b92d4ba81fe789..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 415 zcmeAS@N?(olHy`uVBq!ia0vp^ra)}S!2~3~ik2AyDaPU;cPEB*=VV?2IV|apzK#qG z8~eHcB(eheoCO|{#S9F5he4R}c>anMpav697srqY_qW#{@-{07uqHe&Qh1gqH7!x1 zdij#8T^}CYoM!U-)ref3AMNsQtJuzRnoxTd Date: Fri, 18 Jun 2021 16:42:27 +0900 Subject: [PATCH 17/51] kadai3-2 first commit --- kadai3-2/Mizushima/.gitignore | 2 + kadai3-2/Mizushima/go.mod | 8 ++++ kadai3-2/Mizushima/go.sum | 6 +++ kadai3-2/Mizushima/main.go | 55 +++++++++++++++++++++++ kadai3-2/Mizushima/options/options.go | 63 +++++++++++++++++++++++++++ 5 files changed, 134 insertions(+) create mode 100644 kadai3-2/Mizushima/.gitignore create mode 100644 kadai3-2/Mizushima/go.mod create mode 100644 kadai3-2/Mizushima/go.sum create mode 100644 kadai3-2/Mizushima/main.go create mode 100644 kadai3-2/Mizushima/options/options.go diff --git a/kadai3-2/Mizushima/.gitignore b/kadai3-2/Mizushima/.gitignore new file mode 100644 index 00000000..9705278b --- /dev/null +++ b/kadai3-2/Mizushima/.gitignore @@ -0,0 +1,2 @@ +http.request* +*.jpg diff --git a/kadai3-2/Mizushima/go.mod b/kadai3-2/Mizushima/go.mod new file mode 100644 index 00000000..bceb76f1 --- /dev/null +++ b/kadai3-2/Mizushima/go.mod @@ -0,0 +1,8 @@ +module github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima + +go 1.16 + +require ( + github.com/jessevdk/go-flags v1.5.0 + github.com/pkg/errors v0.9.1 +) diff --git a/kadai3-2/Mizushima/go.sum b/kadai3-2/Mizushima/go.sum new file mode 100644 index 00000000..3c8e9a7b --- /dev/null +++ b/kadai3-2/Mizushima/go.sum @@ -0,0 +1,6 @@ +github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 h1:EZ2mChiOa8udjfp6rRmswTbtZN/QzUQp4ptM4rnjHvc= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/kadai3-2/Mizushima/main.go b/kadai3-2/Mizushima/main.go new file mode 100644 index 00000000..df22a99c --- /dev/null +++ b/kadai3-2/Mizushima/main.go @@ -0,0 +1,55 @@ +package main + +import ( + "fmt" + "log" + "os" + + "github/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/options" +) + +// go run main.go -url https://4.bp.blogspot.com/-2-Ny23XgrF0/Ws69gszw2jI/AAAAAAABLdU/unbzWD_U8foWBwPKWQdGP1vEDoQoYjgZwCLcBGAs/s1600/top_banner.jpg + + +func main() { + opts, err := options.ParseArgs(os.Args[1:]) + if err != nil { + log.Fatal(err) + } + + fmt.Println(opts) + + // fmt.Println("url:", *url) + // *filePath += filepath.Base(*url) + // // fmt.Println("file_path:", *file_path) + + // if err := DownloadFile(*filePath, *url); err != nil { + // log.Fatal(err) + // } +} + +// func DownloadFile(filepath string, url string) error { +// req, err := http.NewRequest("GET", url, nil) +// if err != nil { +// return errors.Wrap(err, fmt.Sprintf("failed to split NewRequest for get: %d", r.worker)) +// } + +// fmt.Printf("%#v\n", req) + +// req.Header.Set("Range", fmt.Sprintf()) + +// resp, err := http.DefaultClient.Do(req) +// if err != nil { +// return err +// } +// defer resp.Body.Close() + +// out, err := os.Create(filepath) +// if err != nil { +// return err +// } +// defer out.Close() + +// _, err = io.Copy(out, resp.Body) +// return err +// } \ No newline at end of file diff --git a/kadai3-2/Mizushima/options/options.go b/kadai3-2/Mizushima/options/options.go new file mode 100644 index 00000000..985ffcdd --- /dev/null +++ b/kadai3-2/Mizushima/options/options.go @@ -0,0 +1,63 @@ +package main + +import ( + "bytes" + "fmt" + "os" + + "github.com/jessevdk/go-flags" + "github.com/pkg/errors" +) + +type Options struct { + Help bool `short:"h" long:"help" description:"print usage and exit"` + Procs int `short:"p" long:"procs" description:"the number of split to download"` + Output string `short:"o" long:"output" description:"path and file name of the file downloaded"` +} + +func (opts *Options) parse(argv []string) ([]string, error) { + p := flags.NewParser(opts, flags.PrintErrors) + args, err := p.ParseArgs(argv) + + if err != nil { + os.Stderr.Write(opts.usage()) + return nil, errors.Wrap(err, "invalid command line options") + } + + return args, nil +} + +func (opts Options) usage() []byte { + buf := bytes.Buffer{} + + fmt.Fprintln(&buf, + `Usage: pd [options] URL + Options: + -h, --help print usage and exit + -p, --procs the number of split to download + -o, --output path and file name of the file downloaded + `, + ) + + return buf.Bytes() +} + +func ParseOptions(argv []string) (*Options, error) { + var opts Options + if len(argv) == 0 { + os.Stdout.Write(opts.usage()) + return nil, errors.New("no options") + } + + o, err := opts.parse(argv) + if err != nil { + return nil, err + } + + if opts.Help { + os.Stdout.Write(opts.usage()) + return nil, errors.New("print usage") + } + + return &opts, nil +} \ No newline at end of file From 9d475ed258ae1e59af2dbbe2716b78035bb5808b Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Fri, 18 Jun 2021 17:25:47 +0900 Subject: [PATCH 18/51] =?UTF-8?q?=E3=82=AA=E3=83=97=E3=82=B7=E3=83=A7?= =?UTF-8?q?=E3=83=B3=E5=91=A8=E3=82=8A=E3=81=AE=E4=BF=AE=E6=AD=A3=E5=8F=8A?= =?UTF-8?q?=E3=81=B3=E3=83=80=E3=82=A6=E3=83=B3=E3=83=AD=E3=83=BC=E3=83=89?= =?UTF-8?q?=E3=81=AF=E3=81=A7=E3=81=8D=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/go.mod | 7 ++- kadai3-2/Mizushima/main.go | 72 +++++++++++++++------------ kadai3-2/Mizushima/options/go.mod | 8 +++ kadai3-2/Mizushima/options/go.sum | 6 +++ kadai3-2/Mizushima/options/options.go | 27 +++++----- 5 files changed, 72 insertions(+), 48 deletions(-) create mode 100644 kadai3-2/Mizushima/options/go.mod create mode 100644 kadai3-2/Mizushima/options/go.sum diff --git a/kadai3-2/Mizushima/go.mod b/kadai3-2/Mizushima/go.mod index bceb76f1..056eb223 100644 --- a/kadai3-2/Mizushima/go.mod +++ b/kadai3-2/Mizushima/go.mod @@ -2,7 +2,6 @@ module github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima go 1.16 -require ( - github.com/jessevdk/go-flags v1.5.0 - github.com/pkg/errors v0.9.1 -) +replace github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/options => ./options + +require github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/options v0.0.0-00010101000000-000000000000 diff --git a/kadai3-2/Mizushima/main.go b/kadai3-2/Mizushima/main.go index df22a99c..f5c90479 100644 --- a/kadai3-2/Mizushima/main.go +++ b/kadai3-2/Mizushima/main.go @@ -2,54 +2,64 @@ package main import ( "fmt" + "io" "log" + "net/http" "os" + "path/filepath" - "github/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/options" + options "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/options" ) -// go run main.go -url https://4.bp.blogspot.com/-2-Ny23XgrF0/Ws69gszw2jI/AAAAAAABLdU/unbzWD_U8foWBwPKWQdGP1vEDoQoYjgZwCLcBGAs/s1600/top_banner.jpg +// go run main.go https://4.bp.blogspot.com/-2-Ny23XgrF0/Ws69gszw2jI/AAAAAAABLdU/unbzWD_U8foWBwPKWQdGP1vEDoQoYjgZwCLcBGAs/s1600/top_banner.jpg func main() { - opts, err := options.ParseArgs(os.Args[1:]) + opts, url, err := options.ParseOptions(os.Args[1:]) if err != nil { log.Fatal(err) } - fmt.Println(opts) + fmt.Printf("opts:%#v\n", opts) + fmt.Println(url) - // fmt.Println("url:", *url) - // *filePath += filepath.Base(*url) - // // fmt.Println("file_path:", *file_path) - - // if err := DownloadFile(*filePath, *url); err != nil { - // log.Fatal(err) - // } + if err := DownloadFile(opts.Output, url); err != nil { + log.Fatal(err) + } } -// func DownloadFile(filepath string, url string) error { -// req, err := http.NewRequest("GET", url, nil) -// if err != nil { -// return errors.Wrap(err, fmt.Sprintf("failed to split NewRequest for get: %d", r.worker)) -// } +func DownloadFile(path string, urls []string) error { + + for _, url := range urls { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return err + } -// fmt.Printf("%#v\n", req) + fmt.Printf("%#v\n", req) -// req.Header.Set("Range", fmt.Sprintf()) + req.Header.Set("Range", "byte=0-499") -// resp, err := http.DefaultClient.Do(req) -// if err != nil { -// return err -// } -// defer resp.Body.Close() + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() -// out, err := os.Create(filepath) -// if err != nil { -// return err -// } -// defer out.Close() + if path[len(path)-1] != '/' { + path += "/" + } + + out, err := os.Create(path+filepath.Base(url)) + if err != nil { + return err + } + defer out.Close() -// _, err = io.Copy(out, resp.Body) -// return err -// } \ No newline at end of file + _, err = io.Copy(out, resp.Body) + if err != nil { + return err + } + } + return nil +} \ No newline at end of file diff --git a/kadai3-2/Mizushima/options/go.mod b/kadai3-2/Mizushima/options/go.mod new file mode 100644 index 00000000..3614e828 --- /dev/null +++ b/kadai3-2/Mizushima/options/go.mod @@ -0,0 +1,8 @@ +module github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/options + +go 1.16 + +require ( + github.com/jessevdk/go-flags v1.5.0 + github.com/pkg/errors v0.9.1 +) diff --git a/kadai3-2/Mizushima/options/go.sum b/kadai3-2/Mizushima/options/go.sum new file mode 100644 index 00000000..3c8e9a7b --- /dev/null +++ b/kadai3-2/Mizushima/options/go.sum @@ -0,0 +1,6 @@ +github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 h1:EZ2mChiOa8udjfp6rRmswTbtZN/QzUQp4ptM4rnjHvc= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/kadai3-2/Mizushima/options/options.go b/kadai3-2/Mizushima/options/options.go index 985ffcdd..46401038 100644 --- a/kadai3-2/Mizushima/options/options.go +++ b/kadai3-2/Mizushima/options/options.go @@ -1,4 +1,4 @@ -package main +package options import ( "bytes" @@ -12,7 +12,7 @@ import ( type Options struct { Help bool `short:"h" long:"help" description:"print usage and exit"` Procs int `short:"p" long:"procs" description:"the number of split to download"` - Output string `short:"o" long:"output" description:"path and file name of the file downloaded"` + Output string `short:"o" long:"output" description:"directory name of the file downloaded" required:"./"` } func (opts *Options) parse(argv []string) ([]string, error) { @@ -31,33 +31,34 @@ func (opts Options) usage() []byte { buf := bytes.Buffer{} fmt.Fprintln(&buf, - `Usage: pd [options] URL - Options: - -h, --help print usage and exit - -p, --procs the number of split to download - -o, --output path and file name of the file downloaded - `, + `Usage: pd [options] URL + + Options: + -h, --help print usage and exit + -p, --procs the number of split to download + -o, --output path and file name of the file downloaded + `, ) return buf.Bytes() } -func ParseOptions(argv []string) (*Options, error) { +func ParseOptions(argv []string) (*Options, []string, error) { var opts Options if len(argv) == 0 { os.Stdout.Write(opts.usage()) - return nil, errors.New("no options") + return nil, nil, errors.New("no options") } o, err := opts.parse(argv) if err != nil { - return nil, err + return nil, nil, err } if opts.Help { os.Stdout.Write(opts.usage()) - return nil, errors.New("print usage") + return nil, nil, errors.New("print usage") } - return &opts, nil + return &opts, o, nil } \ No newline at end of file From c3b5dea3dfc1a106b58407e5471d7c90c99caab6 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Fri, 18 Jun 2021 19:48:16 +0900 Subject: [PATCH 19/51] =?UTF-8?q?options/options.go=E3=81=AEimport?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E3=80=81TODO=E3=83=AA=E3=82=B9=E3=83=88?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/TODO.md | 10 ++++++++++ kadai3-2/Mizushima/options/options.go | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 kadai3-2/Mizushima/TODO.md diff --git a/kadai3-2/Mizushima/TODO.md b/kadai3-2/Mizushima/TODO.md new file mode 100644 index 00000000..3ef46bb3 --- /dev/null +++ b/kadai3-2/Mizushima/TODO.md @@ -0,0 +1,10 @@ +### TODOリスト +順不同 + +- go routineによる分割ダウンロード処理 + - fileの大きさの事前取得 + - range access の range取得 +- Makefile作成 + - テスト作成 + - ダミーサーバー +- -oオプションの初期値設定がうまくいかない \ No newline at end of file diff --git a/kadai3-2/Mizushima/options/options.go b/kadai3-2/Mizushima/options/options.go index 46401038..f00515a0 100644 --- a/kadai3-2/Mizushima/options/options.go +++ b/kadai3-2/Mizushima/options/options.go @@ -5,7 +5,7 @@ import ( "fmt" "os" - "github.com/jessevdk/go-flags" + flags "github.com/jessevdk/go-flags" "github.com/pkg/errors" ) From 274bae4553aee504eb41541c9c7797da82a2a34a Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Mon, 21 Jun 2021 15:03:38 +0900 Subject: [PATCH 20/51] =?UTF-8?q?range=E3=83=AA=E3=82=AF=E3=82=A8=E3=82=B9?= =?UTF-8?q?=E3=83=88=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/download/download.go | 44 ++++++++++++++++ kadai3-2/Mizushima/getheader/getHeader.go | 34 +++++++++++++ kadai3-2/Mizushima/getheader/go.mod | 3 ++ kadai3-2/Mizushima/main.go | 61 ++++++++--------------- kadai3-2/Mizushima/request/go.mod | 3 ++ kadai3-2/Mizushima/request/request.go | 28 +++++++++++ 6 files changed, 132 insertions(+), 41 deletions(-) create mode 100644 kadai3-2/Mizushima/download/download.go create mode 100644 kadai3-2/Mizushima/getheader/getHeader.go create mode 100644 kadai3-2/Mizushima/getheader/go.mod create mode 100644 kadai3-2/Mizushima/request/go.mod create mode 100644 kadai3-2/Mizushima/request/request.go diff --git a/kadai3-2/Mizushima/download/download.go b/kadai3-2/Mizushima/download/download.go new file mode 100644 index 00000000..9ddc40ca --- /dev/null +++ b/kadai3-2/Mizushima/download/download.go @@ -0,0 +1,44 @@ +package download + +import ( + "io" + "net/http" + "os" + "path/filepath" +) + +func DownloadFile(path string, urls []string) error { + + for _, url := range urls { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return err + } + + // fmt.Printf("%#v\n", req) + + req.Header.Set("Range", "byte=0-499") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if path[len(path)-1] != '/' { + path += "/" + } + + out, err := os.Create(path + filepath.Base(url)) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, resp.Body) + if err != nil { + return err + } + } + return nil +} \ No newline at end of file diff --git a/kadai3-2/Mizushima/getheader/getHeader.go b/kadai3-2/Mizushima/getheader/getHeader.go new file mode 100644 index 00000000..59d907c0 --- /dev/null +++ b/kadai3-2/Mizushima/getheader/getHeader.go @@ -0,0 +1,34 @@ +package getheader + +import ( + "fmt" + "io" + "net/http" +) + +// get all headers +func Headers(w io.Writer, r *http.Response) { + h := r.Header + fmt.Fprintln(w, h) +} + +// get the specified header +func Header(w io.Writer, r *http.Response, header string) ([]string, error) { + h, is := r.Header[header] + fmt.Println(h) + if !is { + return nil, fmt.Errorf("cannot find %s header", header) + } + fmt.Fprintf(w, "Header[%s] = %s\n", header, h) + return h, nil +} + +// get the specified header by commas +func HeaderComma(w io.Writer, r *http.Response, header string) (string, error) { + h := r.Header.Get(header) + // if !is { + // return "error", fmt.Errorf("cannot find %s header", header) + // } + fmt.Fprintf(w, "Header[%s] = %s\n", header, h) + return h, nil +} diff --git a/kadai3-2/Mizushima/getheader/go.mod b/kadai3-2/Mizushima/getheader/go.mod new file mode 100644 index 00000000..52086171 --- /dev/null +++ b/kadai3-2/Mizushima/getheader/go.mod @@ -0,0 +1,3 @@ +module github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/getheader + +go 1.16 diff --git a/kadai3-2/Mizushima/main.go b/kadai3-2/Mizushima/main.go index f5c90479..67e05787 100644 --- a/kadai3-2/Mizushima/main.go +++ b/kadai3-2/Mizushima/main.go @@ -2,17 +2,19 @@ package main import ( "fmt" - "io" "log" - "net/http" + "net/http/httputil" "os" - "path/filepath" + download "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/download" + getheader "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/getheader" options "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/options" + request "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/request" ) -// go run main.go https://4.bp.blogspot.com/-2-Ny23XgrF0/Ws69gszw2jI/AAAAAAABLdU/unbzWD_U8foWBwPKWQdGP1vEDoQoYjgZwCLcBGAs/s1600/top_banner.jpg - +// go run main.go https://4.bp.blogspot.com/-2-Ny23XgrF0/Ws69gszw2jI/AAAAAAABLdU/unbzWD_U8foWBwPKWQdGP1vEDoQoYjgZwCLcBGAs/s1600/top_banner.jpg -o kadai3-2 +// go run main.go http://i.imgur.com/z4d4kWk.jpg -o . +// go run main.go https://misc.laboradian.com/test/003/ -o . func main() { opts, url, err := options.ParseOptions(os.Args[1:]) @@ -23,43 +25,20 @@ func main() { fmt.Printf("opts:%#v\n", opts) fmt.Println(url) - if err := DownloadFile(opts.Output, url); err != nil { - log.Fatal(err) + resp, err := request.Request("GET", url[0], "Range", "bytes=281-294") + if err != nil { + log.Fatalf("err: %s\n", err) } -} - -func DownloadFile(path string, urls []string) error { - - for _, url := range urls { - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return err - } + defer resp.Body.Close() - fmt.Printf("%#v\n", req) + // getheader.Headers(os.Stdout, resp) + dump, _ := httputil.DumpResponse(resp, false) + fmt.Printf("response:\n%s\n", dump) - req.Header.Set("Range", "byte=0-499") - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - - if path[len(path)-1] != '/' { - path += "/" - } - - out, err := os.Create(path+filepath.Base(url)) - if err != nil { - return err - } - defer out.Close() - - _, err = io.Copy(out, resp.Body) - if err != nil { - return err - } + fmt.Printf("response status code: %s\n", resp.Status) + outs, _ := getheader.HeaderComma(os.Stdout, resp, "Accept-Ranges") + fmt.Println(outs == "bytes") + if err := download.DownloadFile(opts.Output, url); err != nil { + log.Fatal(err) } - return nil -} \ No newline at end of file +} diff --git a/kadai3-2/Mizushima/request/go.mod b/kadai3-2/Mizushima/request/go.mod new file mode 100644 index 00000000..3a9f90c5 --- /dev/null +++ b/kadai3-2/Mizushima/request/go.mod @@ -0,0 +1,3 @@ +module github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/request + +go 1.16 diff --git a/kadai3-2/Mizushima/request/request.go b/kadai3-2/Mizushima/request/request.go new file mode 100644 index 00000000..d7a5c3a2 --- /dev/null +++ b/kadai3-2/Mizushima/request/request.go @@ -0,0 +1,28 @@ +package request + +import ( + "fmt" + "net/http" + "net/http/httputil" +) + +func Request(method string, url string, setH string, setV string) (*http.Response, error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + req.Header.Set(setH, setV) + + dump, _ := httputil.DumpRequestOut(req, false) + fmt.Printf("request:\n%s\n", dump) + + client := new(http.Client) + resp, err := client.Do(req) + if err != nil { + return nil, err + } + return resp, nil +} + + From 3dfddf907ebabbd9da0e5b7db51c8d61af3b1109 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Mon, 21 Jun 2021 15:27:00 +0900 Subject: [PATCH 21/51] =?UTF-8?q?request=E3=81=AE=E5=87=A6=E7=90=86?= =?UTF-8?q?=E3=82=92=E5=A4=89=E6=9B=B4=E3=80=81root=E3=83=95=E3=82=A9?= =?UTF-8?q?=E3=83=AB=E3=83=80=E3=81=AEgo.mod=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/download/download.go | 15 ++++----------- kadai3-2/Mizushima/go.mod | 13 +++++++++++-- kadai3-2/Mizushima/main.go | 5 +++++ 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/kadai3-2/Mizushima/download/download.go b/kadai3-2/Mizushima/download/download.go index 9ddc40ca..f8987e47 100644 --- a/kadai3-2/Mizushima/download/download.go +++ b/kadai3-2/Mizushima/download/download.go @@ -2,24 +2,17 @@ package download import ( "io" - "net/http" "os" "path/filepath" + + "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/request" ) func DownloadFile(path string, urls []string) error { for _, url := range urls { - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return err - } - - // fmt.Printf("%#v\n", req) - - req.Header.Set("Range", "byte=0-499") - - resp, err := http.DefaultClient.Do(req) + + resp, err := request.Request("GET", url, "Range", "bytes=281-294") if err != nil { return err } diff --git a/kadai3-2/Mizushima/go.mod b/kadai3-2/Mizushima/go.mod index 056eb223..704c83e1 100644 --- a/kadai3-2/Mizushima/go.mod +++ b/kadai3-2/Mizushima/go.mod @@ -2,6 +2,15 @@ module github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima go 1.16 -replace github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/options => ./options +replace ( + github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/download => ./download + github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/getheader => ./getheader + github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/options => ./options + github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/request => ./request +) -require github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/options v0.0.0-00010101000000-000000000000 +require ( + github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/getheader v0.0.0-00010101000000-000000000000 + github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/options v0.0.0-00010101000000-000000000000 + github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/request v0.0.0-00010101000000-000000000000 +) diff --git a/kadai3-2/Mizushima/main.go b/kadai3-2/Mizushima/main.go index 67e05787..3aeada79 100644 --- a/kadai3-2/Mizushima/main.go +++ b/kadai3-2/Mizushima/main.go @@ -5,6 +5,7 @@ import ( "log" "net/http/httputil" "os" + "runtime" download "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/download" getheader "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/getheader" @@ -22,6 +23,10 @@ func main() { log.Fatal(err) } + if opts.Procs == 0 { + opts.Procs = runtime.NumCPU() + } + fmt.Printf("opts:%#v\n", opts) fmt.Println(url) From a44ebffc47367c7bdc5954bec8a994e9a1d7ff62 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Mon, 21 Jun 2021 19:31:51 +0900 Subject: [PATCH 22/51] =?UTF-8?q?=E3=83=80=E3=82=A6=E3=83=B3=E3=83=AD?= =?UTF-8?q?=E3=83=BC=E3=83=89=E3=81=AE=E5=87=A6=E7=90=86=E3=82=92=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0=E3=80=81=E9=80=94=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/download/download.go | 55 +++++++++++---- kadai3-2/Mizushima/getheader/getHeader.go | 16 ++++- kadai3-2/Mizushima/main.go | 84 ++++++++++++++++++----- kadai3-2/Mizushima/options/options.go | 22 +++--- kadai3-2/Mizushima/request/request.go | 9 ++- 5 files changed, 142 insertions(+), 44 deletions(-) diff --git a/kadai3-2/Mizushima/download/download.go b/kadai3-2/Mizushima/download/download.go index f8987e47..17555e37 100644 --- a/kadai3-2/Mizushima/download/download.go +++ b/kadai3-2/Mizushima/download/download.go @@ -1,16 +1,18 @@ package download import ( + "fmt" "io" + "io/ioutil" "os" - "path/filepath" + "sync" "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/request" ) -func DownloadFile(path string, urls []string) error { +var mu sync.Mutex - for _, url := range urls { +func DownloadFile(url string, out *os.File) error { resp, err := request.Request("GET", url, "Range", "bytes=281-294") if err != nil { @@ -18,20 +20,47 @@ func DownloadFile(path string, urls []string) error { } defer resp.Body.Close() - if path[len(path)-1] != '/' { - path += "/" - } - - out, err := os.Create(path + filepath.Base(url)) - if err != nil { - return err - } - defer out.Close() - + _, err = io.Copy(out, resp.Body) if err != nil { return err } + + return nil +} + +func PDownload(url string, out *os.File, fileSize int, idx int, part int, procs int) error { + mu.Lock() + fmt.Printf("%d/%d downloading...\n", idx, procs) + var start, end int + if idx == 0 { + start = 0 + } else { + start = idx*part + 1 + } + + // 最後だったら + if idx == procs+1 { + end = fileSize + } else { + end = (idx+1) * part + } + + bytes := fmt.Sprintf("bytes=%d-%d", start, end) + resp, err := request.Request("GET", url, "Range", bytes) + if err != nil { + return err + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + length, err := out.Write(body) + if err != nil { + return err } + fmt.Printf("%d/%d was written len=%d\n", idx, procs, length) + mu.Unlock() return nil } \ No newline at end of file diff --git a/kadai3-2/Mizushima/getheader/getHeader.go b/kadai3-2/Mizushima/getheader/getHeader.go index 59d907c0..0c5839d7 100644 --- a/kadai3-2/Mizushima/getheader/getHeader.go +++ b/kadai3-2/Mizushima/getheader/getHeader.go @@ -4,6 +4,8 @@ import ( "fmt" "io" "net/http" + "os" + "strconv" ) // get all headers @@ -13,7 +15,7 @@ func Headers(w io.Writer, r *http.Response) { } // get the specified header -func Header(w io.Writer, r *http.Response, header string) ([]string, error) { +func ResHeader(w io.Writer, r *http.Response, header string) ([]string, error) { h, is := r.Header[header] fmt.Println(h) if !is { @@ -24,7 +26,7 @@ func Header(w io.Writer, r *http.Response, header string) ([]string, error) { } // get the specified header by commas -func HeaderComma(w io.Writer, r *http.Response, header string) (string, error) { +func ResHeaderComma(w io.Writer, r *http.Response, header string) (string, error) { h := r.Header.Get(header) // if !is { // return "error", fmt.Errorf("cannot find %s header", header) @@ -32,3 +34,13 @@ func HeaderComma(w io.Writer, r *http.Response, header string) (string, error) { fmt.Fprintf(w, "Header[%s] = %s\n", header, h) return h, nil } + + +func GetSize(resp *http.Response) (int, error) { + contLen, err := ResHeader(os.Stdout, resp, "Content-Length") + if err != nil { + return -1, err + } + return strconv.Atoi(contLen[0]) +} + diff --git a/kadai3-2/Mizushima/main.go b/kadai3-2/Mizushima/main.go index 3aeada79..9d27a77c 100644 --- a/kadai3-2/Mizushima/main.go +++ b/kadai3-2/Mizushima/main.go @@ -1,16 +1,19 @@ package main import ( + "context" + "errors" "fmt" "log" - "net/http/httputil" + "net/http" "os" + "path/filepath" "runtime" + "time" download "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/download" getheader "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/getheader" options "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/options" - request "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/request" ) // go run main.go https://4.bp.blogspot.com/-2-Ny23XgrF0/Ws69gszw2jI/AAAAAAABLdU/unbzWD_U8foWBwPKWQdGP1vEDoQoYjgZwCLcBGAs/s1600/top_banner.jpg -o kadai3-2 @@ -18,7 +21,7 @@ import ( // go run main.go https://misc.laboradian.com/test/003/ -o . func main() { - opts, url, err := options.ParseOptions(os.Args[1:]) + opts, urls, err := options.ParseOptions(os.Args[1:]) if err != nil { log.Fatal(err) } @@ -28,22 +31,71 @@ func main() { } fmt.Printf("opts:%#v\n", opts) - fmt.Println(url) + fmt.Println(urls) + + if len(opts.Output) > 0 && opts.Output[len(opts.Output)-1] != '/' { + opts.Output += "/" + } - resp, err := request.Request("GET", url[0], "Range", "bytes=281-294") - if err != nil { - log.Fatalf("err: %s\n", err) + for _, url := range urls { + + resp, err := http.Get(url) + if err != nil { + log.Fatalf("err: %s\n", err) + } + defer resp.Body.Close() + + fileSize, err := getheader.GetSize(resp) + if err != nil { + log.Fatalf("err: %s\n", err) + } + partial := fileSize / opts.Procs + + ctx, cancel := context.WithTimeout(context.Background(),time.Duration(opts.Tm)*time.Minute) + out, err := os.Create(opts.Output + filepath.Base(url)) + if err != nil { + log.Fatalf("err: %s\n", err) + } + defer out.Close() + + accept, err := getheader.ResHeader(os.Stdout, resp, "Accept-Ranges") + if err != nil { + log.Fatalf("err: %s\n", err) + } else if accept[0] != "bytes" { + download.DownloadFile(url, out) + continue; + } + + err = pararel(url, out, fileSize, partial, opts.Procs, ctx) + if err != nil { + log.Fatalf("err: %s\n", err) + } + cancel() } - defer resp.Body.Close() - // getheader.Headers(os.Stdout, resp) - dump, _ := httputil.DumpResponse(resp, false) - fmt.Printf("response:\n%s\n", dump) + + // dump, _ := httputil.DumpResponse(resp, false) + // fmt.Printf("response:\n%s\n", dump) - fmt.Printf("response status code: %s\n", resp.Status) - outs, _ := getheader.HeaderComma(os.Stdout, resp, "Accept-Ranges") - fmt.Println(outs == "bytes") - if err := download.DownloadFile(opts.Output, url); err != nil { - log.Fatal(err) + // fmt.Printf("response status code: %s\n", resp.Status) + // outs, _ := getheader.ResHeaderComma(os.Stdout, resp, "Accept-Ranges") + // fmt.Println(outs == "bytes") + // if err := download.DownloadFile(opts.Output, url); err != nil { + // log.Fatal(err) + // } +} + +func pararel(url string, file *os.File, fileSize int, part int, procs int, ctx context.Context) error { + // fmt.Println(url) + ch := make(chan int) + for i := 0; i < procs; i++ { + ch <- i + select { + case <-ctx.Done(): + return errors.New("time limit exceeded") + case <-ch: + download.PDownload(url, file, fileSize, i, part, procs) + } } + return nil } diff --git a/kadai3-2/Mizushima/options/options.go b/kadai3-2/Mizushima/options/options.go index f00515a0..c1c8892f 100644 --- a/kadai3-2/Mizushima/options/options.go +++ b/kadai3-2/Mizushima/options/options.go @@ -6,13 +6,14 @@ import ( "os" flags "github.com/jessevdk/go-flags" - "github.com/pkg/errors" + errors "github.com/pkg/errors" ) type Options struct { - Help bool `short:"h" long:"help" description:"print usage and exit"` - Procs int `short:"p" long:"procs" description:"the number of split to download"` - Output string `short:"o" long:"output" description:"directory name of the file downloaded" required:"./"` + Help bool `short:"h" long:"help"` + Procs int `short:"p" long:"procs"` + Output string `short:"o" long:"output" default:"./"` + Tm int `short:"t" long:"time" default:"3"` } func (opts *Options) parse(argv []string) ([]string, error) { @@ -30,13 +31,14 @@ func (opts *Options) parse(argv []string) ([]string, error) { func (opts Options) usage() []byte { buf := bytes.Buffer{} - fmt.Fprintln(&buf, - `Usage: pd [options] URL + fmt.Fprintln(&buf, + `Usage: pd [options] URL Options: - -h, --help print usage and exit - -p, --procs the number of split to download - -o, --output path and file name of the file downloaded + -h, -help print usage and exit + -p, -procs the number of split to download + -o, -output path and file name of the file downloaded + -t, -time Time limit minuts until the download is completed `, ) @@ -61,4 +63,4 @@ func ParseOptions(argv []string) (*Options, []string, error) { } return &opts, o, nil -} \ No newline at end of file +} diff --git a/kadai3-2/Mizushima/request/request.go b/kadai3-2/Mizushima/request/request.go index d7a5c3a2..1c78b11c 100644 --- a/kadai3-2/Mizushima/request/request.go +++ b/kadai3-2/Mizushima/request/request.go @@ -6,14 +6,17 @@ import ( "net/http/httputil" ) +// Request returns a response from url and a error. func Request(method string, url string, setH string, setV string) (*http.Response, error) { - req, err := http.NewRequest("GET", url, nil) + req, err := http.NewRequest(method, url, nil) if err != nil { return nil, err } - req.Header.Set(setH, setV) - + if len(setH) == 0 { + req.Header.Set(setH, setV) + } + dump, _ := httputil.DumpRequestOut(req, false) fmt.Printf("request:\n%s\n", dump) From d89f81d66a1445b5ea21db55292636d770ba14d5 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Tue, 22 Jun 2021 17:53:26 +0900 Subject: [PATCH 23/51] =?UTF-8?q?=E4=B8=A6=E8=A1=8C=E3=83=80=E3=82=A6?= =?UTF-8?q?=E3=83=B3=E3=83=AD=E3=83=BC=E3=83=89=E5=87=A6=E7=90=86=E4=BB=96?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/.gitignore | 1 + kadai3-2/Mizushima/Makefile | 20 +++ kadai3-2/Mizushima/TODO.md | 15 ++- kadai3-2/Mizushima/download/download.go | 127 ++++++++++++++---- kadai3-2/Mizushima/download/go.mod | 10 ++ kadai3-2/Mizushima/download/go.sum | 2 + .../Mizushima/dummy_server/dummy_server.go | 9 ++ kadai3-2/Mizushima/getheader/getHeader.go | 10 +- kadai3-2/Mizushima/go.mod | 1 + kadai3-2/Mizushima/go.sum | 2 + kadai3-2/Mizushima/listen/listen.go | 26 ++++ kadai3-2/Mizushima/main.go | 98 +++++++++----- kadai3-2/Mizushima/options/options.go | 12 +- kadai3-2/Mizushima/request/request.go | 4 +- 14 files changed, 262 insertions(+), 75 deletions(-) create mode 100644 kadai3-2/Mizushima/Makefile create mode 100644 kadai3-2/Mizushima/download/go.mod create mode 100644 kadai3-2/Mizushima/download/go.sum create mode 100644 kadai3-2/Mizushima/dummy_server/dummy_server.go create mode 100644 kadai3-2/Mizushima/listen/listen.go diff --git a/kadai3-2/Mizushima/.gitignore b/kadai3-2/Mizushima/.gitignore index 9705278b..be2027cc 100644 --- a/kadai3-2/Mizushima/.gitignore +++ b/kadai3-2/Mizushima/.gitignore @@ -1,2 +1,3 @@ http.request* *.jpg +documents \ No newline at end of file diff --git a/kadai3-2/Mizushima/Makefile b/kadai3-2/Mizushima/Makefile new file mode 100644 index 00000000..2bc4d956 --- /dev/null +++ b/kadai3-2/Mizushima/Makefile @@ -0,0 +1,20 @@ +BINARY_NAME := bin/paraDW +GOCMD=go +GOBUILD=$(GOCMD) build +GOCLEAN=$(GOCMD) clean +GOTEST=$(GOCMD) test +GOGET=$(GOCMD) get + +all: build + +build: + $(GOBUILD) -o $(BINARY_NAME) -v + +test: clean all + $(BINARY_NAME) http://i.imgur.com/z4d4kWk.jpg -o ./test_download + +clean: + $(GOCLEAN) + rm -rf $(BINARY_NAME) test_download/* + +.PHONY: test clean diff --git a/kadai3-2/Mizushima/TODO.md b/kadai3-2/Mizushima/TODO.md index 3ef46bb3..a0e56164 100644 --- a/kadai3-2/Mizushima/TODO.md +++ b/kadai3-2/Mizushima/TODO.md @@ -7,4 +7,17 @@ - Makefile作成 - テスト作成 - ダミーサーバー -- -oオプションの初期値設定がうまくいかない \ No newline at end of file + +### 処理の流れ +- オプションからURL取得（複数可）一つ一つのURLにつき、 + - ~~Access-Range=bytesかどうか~~ + - ~~否なら、通常のダウンロード~~ + - ~~完成版ファイルを作成~~ + - ~~一時フォルダを作る~~ + - ~~並行してダウンロードし連番の一時ファイルに保存~~ + - ~~完成版フォルダに一時ファイルを順番にコピー~~ + - ~~一時フォルダ毎削除~~ + +### 参考にしたもの +[pget](https://qiita.com/codehex/items/d0a500ac387d39a34401) (goroutineを使ったダウンロード処理、コマンドラインオプションの処理等々) +https://github.com/gopherdojo/dojo3/pull/50 (ctrl+cを押したときのキャンセル処理など) \ No newline at end of file diff --git a/kadai3-2/Mizushima/download/download.go b/kadai3-2/Mizushima/download/download.go index 17555e37..283c8316 100644 --- a/kadai3-2/Mizushima/download/download.go +++ b/kadai3-2/Mizushima/download/download.go @@ -1,66 +1,135 @@ package download import ( + "context" "fmt" "io" "io/ioutil" "os" - "sync" + "strconv" "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/request" + "golang.org/x/sync/errgroup" ) -var mu sync.Mutex +type PDownloader struct { + url string + output *os.File + fileSize uint + part uint + procs uint +} -func DownloadFile(url string, out *os.File) error { - - resp, err := request.Request("GET", url, "Range", "bytes=281-294") - if err != nil { - return err +func newPDownloader(url string, output *os.File, fileSize uint, part uint, procs uint) *PDownloader { + return &PDownloader{ + url: url, + output: output, + fileSize: fileSize, + part: part, + procs: procs, } - defer resp.Body.Close() +} - - _, err = io.Copy(out, resp.Body) +// +func Downloader(url string, + output *os.File, fileSize uint, part uint, procs uint, isPara bool, + tmpDirName string, ctx context.Context) error { + pd := newPDownloader(url, output, fileSize, part, procs) + if !isPara { + fmt.Printf("%s do not accept range access. so downloading in single process\n", url) + err := pd.DownloadFile() if err != nil { return err } + } else { + grp, ctx := errgroup.WithContext(context.Background()) + pd.PDownload(grp, tmpDirName, procs, ctx) + + if err := grp.Wait(); err != nil { + return err + } + } + return nil +} + +// non-pararel download +func (pd *PDownloader)DownloadFile() error { + + resp, err := request.Request("GET", pd.url, "Range", "bytes=281-294") + if err != nil { + return err + } + defer resp.Body.Close() + + _, err = io.Copy(pd.output, resp.Body) + if err != nil { + return err + } return nil } -func PDownload(url string, out *os.File, fileSize int, idx int, part int, procs int) error { - mu.Lock() - fmt.Printf("%d/%d downloading...\n", idx, procs) - var start, end int - if idx == 0 { - start = 0 - } else { - start = idx*part + 1 +func (pd *PDownloader)PDownload(grp *errgroup.Group, + tmpDirName string, procs uint, ctx context.Context) error { + // fmt.Printf("%d/%d downloading...\n", idx, pd.procs) + var start, end, idx uint + + for idx = uint(0); idx < procs; idx++ { + if idx == 0 { + start = 0 + } else { + start = idx*pd.part + 1 + } + + // 最後だったら + if idx == pd.procs-1 { + end = pd.fileSize + } else { + end = (idx+1) * pd.part + } + + // idxを代入し直す + // https://qiita.com/harhogefoo/items/7ccb4e353a4a01cfa773 + idx := idx + fmt.Printf("start: %d, end: %d, pd.part: %d\n", start, end, pd.part) + bytes := fmt.Sprintf("bytes=%d-%d", start, end) + + grp.Go(func() error { + fmt.Printf("grp.Go: tmpDirName: %s, bytes %s, idx: %d\n", tmpDirName, bytes, idx) + return pd.ReqToMakeCopy(tmpDirName, bytes, idx) + }) } + return nil +} - // 最後だったら - if idx == procs+1 { - end = fileSize - } else { - end = (idx+1) * part +func (pd *PDownloader)ReqToMakeCopy(tmpDirName, bytes string, idx uint) error { + fmt.Printf("ReqToMakeCopy: tmpDirName: %s, bytes %s, idx: %d\n", tmpDirName, bytes, idx) + resp, err := request.Request("GET", pd.url, "Range", bytes) + if err != nil { + return err } - bytes := fmt.Sprintf("bytes=%d-%d", start, end) - resp, err := request.Request("GET", url, "Range", bytes) + tmpOut, err := os.Create(tmpDirName+"/"+strconv.Itoa(int(idx))) if err != nil { return err } + // fmt.Printf("tmpOut.Name(): %s\n", tmpOut.Name()) + defer func(){ + err = tmpOut.Close() + if err != nil { + fmt.Fprintf(os.Stderr, "err: tmpOut.Close(): %s", err.Error()) + } + }() body, err := ioutil.ReadAll(resp.Body) if err != nil { return err } - length, err := out.Write(body) + length, err := tmpOut.Write(body) if err != nil { return err } - fmt.Printf("%d/%d was written len=%d\n", idx, procs, length) - mu.Unlock() + fmt.Printf("%d/%d was downloaded len=%d\n", idx, pd.procs, length) return nil -} \ No newline at end of file +} + diff --git a/kadai3-2/Mizushima/download/go.mod b/kadai3-2/Mizushima/download/go.mod new file mode 100644 index 00000000..240ce7d4 --- /dev/null +++ b/kadai3-2/Mizushima/download/go.mod @@ -0,0 +1,10 @@ +module github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/download + +go 1.16 + +replace github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/request => ../request + +require ( + github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/request v0.0.0-00010101000000-000000000000 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c +) diff --git a/kadai3-2/Mizushima/download/go.sum b/kadai3-2/Mizushima/download/go.sum new file mode 100644 index 00000000..5c00efd3 --- /dev/null +++ b/kadai3-2/Mizushima/download/go.sum @@ -0,0 +1,2 @@ +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/kadai3-2/Mizushima/dummy_server/dummy_server.go b/kadai3-2/Mizushima/dummy_server/dummy_server.go new file mode 100644 index 00000000..9c33c1c4 --- /dev/null +++ b/kadai3-2/Mizushima/dummy_server/dummy_server.go @@ -0,0 +1,9 @@ +package main + +import ( + "net/http" +) + +func main() { + http.ListenAndServe(":12345", nil) +} \ No newline at end of file diff --git a/kadai3-2/Mizushima/getheader/getHeader.go b/kadai3-2/Mizushima/getheader/getHeader.go index 0c5839d7..2f538cee 100644 --- a/kadai3-2/Mizushima/getheader/getHeader.go +++ b/kadai3-2/Mizushima/getheader/getHeader.go @@ -36,11 +36,15 @@ func ResHeaderComma(w io.Writer, r *http.Response, header string) (string, error } -func GetSize(resp *http.Response) (int, error) { +func GetSize(resp *http.Response) (uint, error) { contLen, err := ResHeader(os.Stdout, resp, "Content-Length") if err != nil { - return -1, err + return 0, err } - return strconv.Atoi(contLen[0]) + ret, err := strconv.ParseUint(contLen[0], 10, 32) + if err != nil { + return 0, err + } + return uint(ret), nil } diff --git a/kadai3-2/Mizushima/go.mod b/kadai3-2/Mizushima/go.mod index 704c83e1..d2a4f562 100644 --- a/kadai3-2/Mizushima/go.mod +++ b/kadai3-2/Mizushima/go.mod @@ -10,6 +10,7 @@ replace ( ) require ( + github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/download v0.0.0-00010101000000-000000000000 github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/getheader v0.0.0-00010101000000-000000000000 github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/options v0.0.0-00010101000000-000000000000 github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/request v0.0.0-00010101000000-000000000000 diff --git a/kadai3-2/Mizushima/go.sum b/kadai3-2/Mizushima/go.sum index 3c8e9a7b..282a40a4 100644 --- a/kadai3-2/Mizushima/go.sum +++ b/kadai3-2/Mizushima/go.sum @@ -2,5 +2,7 @@ github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LF github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 h1:EZ2mChiOa8udjfp6rRmswTbtZN/QzUQp4ptM4rnjHvc= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/kadai3-2/Mizushima/listen/listen.go b/kadai3-2/Mizushima/listen/listen.go new file mode 100644 index 00000000..46b0e6ce --- /dev/null +++ b/kadai3-2/Mizushima/listen/listen.go @@ -0,0 +1,26 @@ +package listen + +import ( + "context" + "fmt" + "io" + "os" + "os/signal" + "syscall" +) + +func Listen(ctx context.Context, w io.Writer, f func()) (context.Context, func()) { + ctx, cancel := context.WithCancel(ctx) + + ch := make(chan os.Signal, 2) + signal.Notify(ch, os.Interrupt, syscall.SIGTERM) + go func() { + <-ch + fmt.Fprintln(w, "\n^Csignal pressed. interrupt.") + cancel() + f() + os.Exit(0) + }() + + return ctx, cancel +} \ No newline at end of file diff --git a/kadai3-2/Mizushima/main.go b/kadai3-2/Mizushima/main.go index 9d27a77c..f855a0fe 100644 --- a/kadai3-2/Mizushima/main.go +++ b/kadai3-2/Mizushima/main.go @@ -2,20 +2,22 @@ package main import ( "context" - "errors" "fmt" "log" - "net/http" + "net/http/httputil" "os" "path/filepath" "runtime" - "time" + "strconv" download "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/download" getheader "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/getheader" + listen "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/listen" options "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/options" + "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/request" ) +// for test // go run main.go https://4.bp.blogspot.com/-2-Ny23XgrF0/Ws69gszw2jI/AAAAAAABLdU/unbzWD_U8foWBwPKWQdGP1vEDoQoYjgZwCLcBGAs/s1600/top_banner.jpg -o kadai3-2 // go run main.go http://i.imgur.com/z4d4kWk.jpg -o . // go run main.go https://misc.laboradian.com/test/003/ -o . @@ -27,7 +29,7 @@ func main() { } if opts.Procs == 0 { - opts.Procs = runtime.NumCPU() + opts.Procs = uint(runtime.NumCPU()) } fmt.Printf("opts:%#v\n", opts) @@ -37,65 +39,89 @@ func main() { opts.Output += "/" } - for _, url := range urls { + for i, url := range urls { - resp, err := http.Get(url) + resp, err := request.Request("HEAD", url, "", "") if err != nil { log.Fatalf("err: %s\n", err) } - defer resp.Body.Close() + fmt.Println("response:") + bytes, err := httputil.DumpResponse(resp, true) + if err != nil { + log.Fatalf("err: httputil.DumpResponse: %s\n", err) + } + fmt.Println(string(bytes)) + fileSize, err := getheader.GetSize(resp) if err != nil { log.Fatalf("err: %s\n", err) } + resp.Body.Close() + partial := fileSize / opts.Procs - ctx, cancel := context.WithTimeout(context.Background(),time.Duration(opts.Tm)*time.Minute) - out, err := os.Create(opts.Output + filepath.Base(url)) + out, err := os.OpenFile(opts.Output + filepath.Base(url), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0755) if err != nil { - log.Fatalf("err: %s\n", err) + log.Fatalf("err: os.Create: %s\n", err) } - defer out.Close() - accept, err := getheader.ResHeader(os.Stdout, resp, "Accept-Ranges") + // make a temporary directory + tmpDirName := opts.Output + strconv.Itoa(i) + err = os.Mkdir(tmpDirName, 0777) if err != nil { - log.Fatalf("err: %s\n", err) + log.Fatalf("err: os.Mkdir: %s\n", err) + } + + // ctx, cancel := context.WithTimeout(context.Background(),time.Duration(opts.Tm)*time.Minute) + ctx := context.Background() + clean := func() { os.RemoveAll(tmpDirName)} + ctx, cancel := listen.Listen(ctx, os.Stdout, clean) + + var isPara bool = true + accept, err := getheader.ResHeader(os.Stdout, resp, "Accept-Ranges") + if err != nil && err.Error() == "cannot find Accept-Ranges header" { + isPara = false + } else if err != nil { + log.Fatalf("err: getheader.ResHeader: %s\n", err) } else if accept[0] != "bytes" { - download.DownloadFile(url, out) + isPara = false continue; } - err = pararel(url, out, fileSize, partial, opts.Procs, ctx) + err = download.Downloader(url, out, fileSize, partial, opts.Procs, isPara, tmpDirName, ctx) + // err = pararel(url, out, fileSize, partial, opts.Procs, tmpDirName, ctx) if err != nil { log.Fatalf("err: %s\n", err) } + + fmt.Println("download complete") + + err = MergeFiles(tmpDirName, opts.Procs, fileSize, out) + if err != nil { + log.Fatalf("err: MergeFiles: %s\n", err) + } + + // delete the tmporary directory + if err := os.RemoveAll(tmpDirName); err != nil { + log.Fatalf("err: RemoveAll: %s\n", err) + } + cancel() + out.Close() } - - - // dump, _ := httputil.DumpResponse(resp, false) - // fmt.Printf("response:\n%s\n", dump) - - // fmt.Printf("response status code: %s\n", resp.Status) - // outs, _ := getheader.ResHeaderComma(os.Stdout, resp, "Accept-Ranges") - // fmt.Println(outs == "bytes") - // if err := download.DownloadFile(opts.Output, url); err != nil { - // log.Fatal(err) - // } } -func pararel(url string, file *os.File, fileSize int, part int, procs int, ctx context.Context) error { - // fmt.Println(url) - ch := make(chan int) - for i := 0; i < procs; i++ { - ch <- i - select { - case <-ctx.Done(): - return errors.New("time limit exceeded") - case <-ch: - download.PDownload(url, file, fileSize, i, part, procs) +func MergeFiles(tmpDirName string, procs, fileSize uint, output *os.File) error { + for i := uint(0); i < procs; i++ { + + body, err := os.ReadFile(tmpDirName+"/"+strconv.Itoa(int(i))) + if err != nil { + return err } + + fmt.Fprint(output, string(body)) + fmt.Printf("target file: %s, len=%d written\n", output.Name(), len(string(body))) } return nil } diff --git a/kadai3-2/Mizushima/options/options.go b/kadai3-2/Mizushima/options/options.go index c1c8892f..be8fde01 100644 --- a/kadai3-2/Mizushima/options/options.go +++ b/kadai3-2/Mizushima/options/options.go @@ -1,3 +1,4 @@ +// This package implements command-line options for this application package options import ( @@ -9,13 +10,15 @@ import ( errors "github.com/pkg/errors" ) +// struct for options type Options struct { Help bool `short:"h" long:"help"` - Procs int `short:"p" long:"procs"` + Procs uint `short:"p" long:"procs"` Output string `short:"o" long:"output" default:"./"` - Tm int `short:"t" long:"time" default:"3"` + Tm int `short:"t" long:"time" default:"1"` } +// parse options func (opts *Options) parse(argv []string) ([]string, error) { p := flags.NewParser(opts, flags.PrintErrors) args, err := p.ParseArgs(argv) @@ -28,6 +31,7 @@ func (opts *Options) parse(argv []string) ([]string, error) { return args, nil } +// usage prints a description of avilable options func (opts Options) usage() []byte { buf := bytes.Buffer{} @@ -37,8 +41,8 @@ func (opts Options) usage() []byte { Options: -h, -help print usage and exit -p, -procs the number of split to download - -o, -output path and file name of the file downloaded - -t, -time Time limit minuts until the download is completed + -o, -output path of the file downloaded + -t, -time Time limit minuts until the each download is completed `, ) diff --git a/kadai3-2/Mizushima/request/request.go b/kadai3-2/Mizushima/request/request.go index 1c78b11c..5bb39d28 100644 --- a/kadai3-2/Mizushima/request/request.go +++ b/kadai3-2/Mizushima/request/request.go @@ -6,14 +6,14 @@ import ( "net/http/httputil" ) -// Request returns a response from url and a error. +// Request throws a request and returns a response from url and a error. func Request(method string, url string, setH string, setV string) (*http.Response, error) { req, err := http.NewRequest(method, url, nil) if err != nil { return nil, err } - if len(setH) == 0 { + if len(setH) != 0 { req.Header.Set(setH, setV) } From 21f4b854d2a0de58f13081f9a1b3b09566c6e5f3 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Tue, 22 Jun 2021 17:59:46 +0900 Subject: [PATCH 24/51] =?UTF-8?q?gitignore=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/.gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kadai3-2/Mizushima/.gitignore b/kadai3-2/Mizushima/.gitignore index be2027cc..5276ed00 100644 --- a/kadai3-2/Mizushima/.gitignore +++ b/kadai3-2/Mizushima/.gitignore @@ -1,3 +1,4 @@ http.request* *.jpg -documents \ No newline at end of file +documents +bin/* \ No newline at end of file From 0bb34f441269d4c911eef07c075fce7732739d42 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Tue, 22 Jun 2021 18:22:25 +0900 Subject: [PATCH 25/51] =?UTF-8?q?kadai3-1=E3=81=8C=E6=AE=8B=E3=81=A3?= =?UTF-8?q?=E3=81=A6=E3=81=84=E3=81=9F=E3=81=AE=E3=81=A7=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-1/Mizushima/Makefile | 21 ---- kadai3-1/Mizushima/README.md | 53 --------- kadai3-1/Mizushima/gamedata/words.csv | 1 - kadai3-1/Mizushima/go.mod | 3 - kadai3-1/Mizushima/main.go | 23 ---- kadai3-1/Mizushima/typing/typing.go | 137 ----------------------- kadai3-1/Mizushima/typing/typing_test.go | 68 ----------- 7 files changed, 306 deletions(-) delete mode 100644 kadai3-1/Mizushima/Makefile delete mode 100644 kadai3-1/Mizushima/README.md delete mode 100644 kadai3-1/Mizushima/gamedata/words.csv delete mode 100644 kadai3-1/Mizushima/go.mod delete mode 100644 kadai3-1/Mizushima/main.go delete mode 100644 kadai3-1/Mizushima/typing/typing.go delete mode 100644 kadai3-1/Mizushima/typing/typing_test.go diff --git a/kadai3-1/Mizushima/Makefile b/kadai3-1/Mizushima/Makefile deleted file mode 100644 index 42c3aca9..00000000 --- a/kadai3-1/Mizushima/Makefile +++ /dev/null @@ -1,21 +0,0 @@ -BINARY_NAME := bin/typing -GOCMD=go -GOBUILD=$(GOCMD) build -GOCLEAN=$(GOCMD) clean -GOTEST=$(GOCMD) test -GOGET=$(GOCMD) get - -all: build - -build: - $(GOBUILD) -o $(BINARY_NAME) -v - -test: clean all - $(GOTEST) -v -cover - cd ./typing; $(GOTEST) -v -cover - -clean: - $(GOCLEAN) - rm -rf $(BINARY_NAME) - -.PHONY: test clean diff --git a/kadai3-1/Mizushima/README.md b/kadai3-1/Mizushima/README.md deleted file mode 100644 index e1c2b049..00000000 --- a/kadai3-1/Mizushima/README.md +++ /dev/null @@ -1,53 +0,0 @@ -## 課題3-1 タイピングゲームを作ろう -- 標準出力に英単語を出す（出すものは自由） -- 標準入力から1行受け取る -- 制限時間内に何問解けたか表示する - - -### コマンドラインオプション - - | オプション | 説明 | デフォルト | - | --- | --- | --- | - | -limit | ゲーム全体の制限時間を秒数で設定する | 20 | - - -### インストール方法 -```bash -go get github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-1/Mizushima -``` - -### 使い方 -1. バイナリビルド（実行ファイル作成） -```bash -$ make -``` -2. ディレクトリを指定して実行 -```bash -$ ./bin/typing [option] -``` - - -### テストの方法 -- バイナリビルド & テスト -```bash -$ make test -``` -- テスト後の処理(掃除) -```bash -$ make clean -``` - -### ディレクトリ構成 -``` -. -├─ gamedata -│ └─ words.csv -├─ typing -│ └─ typing.go -│ └─ typing_test.go -├─ .gitignore -├─ go.mod -├─ main.go -├─ Makefile -└─ README.md -``` \ No newline at end of file diff --git a/kadai3-1/Mizushima/gamedata/words.csv b/kadai3-1/Mizushima/gamedata/words.csv deleted file mode 100644 index 3e6c7b89..00000000 --- a/kadai3-1/Mizushima/gamedata/words.csv +++ /dev/null @@ -1 +0,0 @@ -America,American,Angle,April,August,Bacon,Barber,Battery,Bible,Bill,Bush,Cage,Chinese,Christ,Christmas,Coward,Crane,Crow,December,Earnest,Echo,England,English,Eve,February,Forth,Fox,Frank,French,Friday,Grace,Grant,Gray,Ham,Hardy,Host,Hunt,January,Japan,Japanese,July,June,Lamb,Lily,London,March,Mass,Mat,Mill,Monday,Mrs,Ms,November,October,Pan,Pat,Poll,Pound,Punch,Ray,Rich,Rob,Rugby,Saturday,September,Singer,Spark,Sunday,Superior,Swift,Thursday,Tuesday,Turkey,Violet,Wednesday,absence,absolute,abstract,abuse,accent,access,accident,accord,achievement,acid,acre,active,addition,administration,admiration,admission,advance,advantage,adventure,advertisement,advice,affair,agency,agreement,aid,aim,alarm,alphabet,altogether,ambition,analysis,ancient,anxiety,appeal,appearance,application,appointment,approach,argument,army,arrangement,arrival,ash,aspect,assembly,assignment,association,assumption,atmosphere,atom,attraction,audience,authority,average,ax,background,baggage,balance,ballet,bank,banker,basic,bathe,battery,battle,bay,bean,bear,beast,beat,beauty,beef,beer,belief,bench,bend,benefit,billion,bind,birth,biscuit,bite,bitter,blade,blanket,bloom,blossom,blow,boil,bomb,bond,bone,border,bother,bowl,brain,brake,brass,breadth,breath,breeze,brick,brief,brilliant,broad,bronze,brow,brush,bubble,bucket,bug,bull,burden,burn,bush,butterfly,button,cabbage,cable,calf,calm,camel,camp,cancel,cancer,candy,cane,canvas,capacity,capital,capture,carbon,career,cargo,carpet,carriage,carrot,cast,casual,cattle,caution,celebration,cent,cereal,ceremony,certainty,chain,chalk,channel,chapel,character,charge,charity,charm,chart,chase,cheat,cheek,cheer,cheese,cherry,chew,chicken,chip,chocolate,chorus,cinema,circumstance,civilization,classic,clip,cloth,coach,coal,coast,cocktail,cocoa,coin,collapse,collection,college,colony,comb,combination,comedy,comfort,comic,command,comment,commission,communication,community,companion,comparison,competition,complaint,composition,concentration,conception,concern,concert,conclusion,concrete,conference,confidence,congress,connection,conscious,consciousness,consequence,consideration,constitution,construction,content,continent,contract,contrary,contrast,contribution,convenience,convention,conversation,conviction,copper,cord,core,corn,correction,costume,cough,countryside,county,courage,crack,craft,crane,crawl,creation,credit,crime,criticism,crop,crossing,crown,crush,crystal,cure,curiosity,curl,current,curse,curtain,custom,damage,damn,danger,dash,dawn,daylight,daytime,debate,debt,decrease,deer,defeat,defense,definition,degree,delay,delicious,delight,delivery,demand,democracy,demonstration,density,departure,deposit,depression,depth,description,desert,design,desire,destiny,detail,determination,development,devil,diamond,diet,difficulty,dig,dignity,dip,direction,disadvantage,disappointment,disaster,discipline,discovery,display,disposal,dispute,distinction,distribution,division,divorce,doctrine,domestic,double,doubt,downstairs,dozen,draft,drag,dragon,drama,drawing,drift,drill,duck,dust,duty,dynamic,eagle,earnest,economics,economy,education,eighteenth,eighth,elect,election,electric,element,eleventh,emotion,emphasis,empire,employment,engagement,enterprise,entertainment,enthusiasm,entrance,entry,environment,error,escape,essence,establishment,estate,eternal,eve,evidence,evil,examination,exception,excess,exchange,excitement,excuse,executive,exhaust,exhibition,existence,expansion,expectation,expedition,expenditure,expense,experiment,explosion,export,exposure,express,expression,extension,extent,external,extreme,facility,failure,faith,fancy,fare,farewell,fashion,fate,fault,favor,favorite,feather,feature,fee,feed,fellow,fever,fiber,fiction,fifteenth,fiftieth,finance,fit,flag,flame,flash,flavor,flesh,flight,flock,flood,flow,focus,fog,fold,folk,formation,formula,fortune,foundation,fourteenth,fox,fraction,frame,freedom,freeze,frontier,frost,fuel,fund,fundamental,funeral,funny,fur,gain,gallery,garment,gaze,gear,generation,generosity,genius,geography,gesture,ghost,gift,glass,globe,glory,glow,goat,goose,gossip,grace,grain,grammar,grant,grape,grasp,grave,gray,grief,grip,gross,growth,guard,gum,gym,habit,hall,ham,hang,harmony,harvest,hate,headquarters,heaven,height,hell,herd,heroic,hide,highway,hint,holy,honey,honor,hop,horn,horror,host,household,housewife,humanity,humor,hundredth,hunger,identity,illustration,imagination,imitation,impact,imperial,import,impression,improvement,impulse,inch,inevitable,influence,initiative,inspiration,instance,instant,institute,institution,instruction,insult,insurance,intellect,intelligence,intention,interior,internal,intersection,intimate,introduction,invention,investment,invisible,invitation,issue,jail,jar,jaw,jealousy,jelly,jerk,journal,joy,judge,juice,jungle,justice,kick,kindness,kingdom,labor,lace,lack,lamb,landing,landscape,lap,latest,launch,laundry,leadership,leaf,lean,leather,legend,lemon,lettuce,liberty,lid,lift,limit,lion,lip,liquid,liquor,listener,load,loaf,loan,location,lock,lord,loss,lower,loyalty,lump,luxury,mad,madam,majority,maker,management,manner,manufacture,maple,marble,march,marriage,mass,match,mathematics,meadow,means,meantime,measurement,melody,membership,mention,mercy,merit,mess,metal,meter,mile,military,mineral,ministry,minor,minority,mist,mix,mixture,monument,mood,moral,motion,mount,mouse,movement,murder,muscle,mushroom,mutter,mystery,myth,navy,necessity,neck,needle,negative,negotiation,neighborhood,nerve,net,network,neutral,nickel,nineteen,nineteenth,ninety,ninth,nonsense,normal,northern,northwest,notion,nut,nylon,oak,objection,obligation,observation,occasion,occupation,odd,offense,olive,onion,operation,opposition,option,orange,ordinary,organization,orient,origin,original,overall,overflow,pace,pack,painting,palace,pale,panic,paradise,parcel,pardon,parliament,passage,passion,passive,pasture,pat,patience,patrol,payment,pea,peach,peanut,pear,pearl,pence,penny,pepper,percentage,perception,performance,perfume,personality,perspective,philosophy,pie,pig,pigeon,pill,pine,pink,pitch,pity,plastic,plate,pleasure,plot,plow,poison,policy,polish,politics,poll,pond,pool,pop,port,portion,positive,possession,possibility,potato,pound,poverty,powder,praise,prayer,precious,preparation,prescription,presence,presentation,preserve,press,pressure,pride,prime,principal,principle,print,priority,prison,privilege,procedure,procession,production,profession,profit,progress,promotion,pronunciation,proof,property,proportion,proposal,prospect,protection,protest,province,provision,psychology,publication,pudding,pump,punch,punishment,purchase,purple,purse,pursuit,push,puzzle,quality,quantity,rabbit,rack,racket,radar,rage,rail,range,rank,ratio,rattle,raw,ray,reaction,reader,reality,realization,rear,recall,receipt,receiver,reception,recognition,recommendation,recovery,reduction,reference,reflection,refusal,regard,region,register,registration,regret,regulation,relation,release,relief,religion,remark,repair,repetition,representation,reputation,request,rescue,resemblance,reservation,reserve,residence,resistance,resolution,resolve,resort,resource,respect,response,responsibility,restriction,revenue,reverse,review,revolution,reward,ribbon,rifle,risk,roast,robe,rocket,rod,roll,root,rope,rough,row,rub,rubber,rugby,ruin,rush,sack,sacrifice,saddle,safety,sail,sake,salad,sale,salmon,sand,satellite,satisfaction,sauce,saving,scale,scandal,scare,scatter,scent,scholarship,score,scorn,scout,scramble,scratch,scream,screen,seal,seaside,secret,section,security,seed,selection,self,senate,sensation,sentence,separate,sequence,series,session,set,settlement,seventeen,seventeenth,seventh,sex,shade,shadow,shake,shallow,shame,shave,sheep,shell,shelter,shepherd,shock,shoulder,sickness,silence,silk,sin,sir,sixteenth,sixtieth,skate,skill,skirt,slip,slope,smart,smell,smoke,smooth,snap,soap,sock,soda,soil,solution,sorrow,soul,sour,southeast,southern,southwest,spade,spark,specific,spectacle,spelling,spice,spill,spin,spirit,split,spoil,sponge,spot,spray,squeeze,squirrel,staff,stage,stain,stair,stake,statement,status,steal,steep,stereo,stew,stick,stir,stock,stomach,stool,storage,strain,straw,strawberry,strength,stress,stretch,string,stroke,structure,substance,suburb,suck,suggestion,suicide,sum,summit,sunshine,supper,supply,surface,surgery,surrender,survey,survival,suspicion,swan,sweat,swell,swing,switch,sword,sympathy,tail,talent,tap,tape,taste,technique,technology,telegraph,temper,tension,tenth,territory,terror,text,theater,theory,thief,thirst,thirteenth,thirtieth,thorn,thread,thrust,thunder,tide,timber,tin,tissue,title,tobacco,toilet,toll,tomato,ton,tone,tongue,toss,trace,track,tragedy,training,transfer,translation,transport,transportation,trap,treasure,treat,treatment,treaty,tremble,trial,triumph,trunk,trust,truth,tube,tune,turkey,twelfth,twentieth,twin,twist,ultimate,unconscious,underground,underneath,union,universe,unknown,upright,upset,urge,usual,utility,vacation,van,variation,variety,vegetable,vein,verse,vice,video,violet,virgin,virtue,vision,visitor,vital,vitamin,vocabulary,volleyball,volume,vote,voyage,waste,wax,weakness,wealth,weed,weep,weight,whale,wheel,whip,whisper,width,wine,wire,wisdom,wise,wit,witness,wonder,worship,worth,wrap,wreck,youth,zero \ No newline at end of file diff --git a/kadai3-1/Mizushima/go.mod b/kadai3-1/Mizushima/go.mod deleted file mode 100644 index 9cf24dd6..00000000 --- a/kadai3-1/Mizushima/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-1/Mizushima - -go 1.16 diff --git a/kadai3-1/Mizushima/main.go b/kadai3-1/Mizushima/main.go deleted file mode 100644 index b65f218a..00000000 --- a/kadai3-1/Mizushima/main.go +++ /dev/null @@ -1,23 +0,0 @@ -package main - -import ( - "flag" - "log" - "os" - "time" - - typing "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-1/Mizushima/typing" -) - -func main() { - wordsPath := "./gamedata/words.csv" - - tm := flag.Duration("limit", 20, "Time limit of the game") - flag.Parse() - - TimeLimit := time.Second * (*tm) - _, err := typing.Game(os.Stdin, os.Stdout, wordsPath, TimeLimit, false) - if err != nil { - log.Fatal(err) - } -} diff --git a/kadai3-1/Mizushima/typing/typing.go b/kadai3-1/Mizushima/typing/typing.go deleted file mode 100644 index a5b0c363..00000000 --- a/kadai3-1/Mizushima/typing/typing.go +++ /dev/null @@ -1,137 +0,0 @@ -package typing - -import ( - "bufio" - "encoding/csv" - "fmt" - "io" - "log" - "math/rand" - "os" - "time" -) - -func Game(r io.Reader, w io.Writer, wordsPath string, t time.Duration, isTest bool) (int, error) { - - words, err := readCSV(wordsPath) - if err != nil { - return -1, err - } - - limit := time.After(t) - - rand.Seed(time.Now().UnixNano()) - var indices []int - if !isTest { - indices = rand.Perm(len(words)) - } else { - indices = incSlice(len(words)) - } - - if !isTest { - _, err = fmt.Fprintln(w, "> Typing game start\nPlease type the displayed word") - if err != nil { - return -1, err - } - - _, err = fmt.Fprintf(w, "> Time limit is %d seconds\n", int(t.Seconds())) - if err != nil { - return -1, err - } - } - - ch := input(r) - score := 0 - var idx int = 0 - var word string - - for { - - word = words[indices[idx]] - - _, err = fmt.Fprintf(w, "> %s\n", word) - if err != nil { - return -1, err - } - - idx++ - - select { - case <-limit: - _, err = fmt.Fprintf(w, "\nGame ends!\nThe number of correct answers is %d\n", score) - if err != nil { - return -1, err - } - return score, nil - case ans := <-ch: - // fmt.Printf("ans: %s\n", ans) - // fmt.Printf("word: %s\n", word) - if ans == word { - if !isTest { - _, err = fmt.Fprintln(w, "> しぇえか～い") - if err != nil { - return -1, err - } - } - score++ - } else { - if !isTest { - _, err = fmt.Fprintln(w, "> ぶっぶー") - if err != nil { - return -1, err - } - } - } - } - } -} - -func input(r io.Reader) <-chan string { - ch := make(chan string) - go func() { - s := bufio.NewScanner(r) - for s.Scan() { - ch <- s.Text() - } - // close(ch) - }() - return ch -} - -func readCSV(path string) ([]string, error) { - file, err := os.Open(path) - if err != nil { - return nil, err - } - - defer func() { - if err = file.Close(); err != nil { - log.Fatal(err) - } - }() - - csvFile := csv.NewReader(file) - csvFile.TrimLeadingSpace = true - - var ret []string - var row []string - - for { - row, err = csvFile.Read() - if err != nil { - break - } - - ret = append(ret, row...) - } - - return ret, nil -} - -func incSlice(n int) []int { - var res []int - for i := 0; i < n; i++ { - res = append(res, i) - } - return res -} diff --git a/kadai3-1/Mizushima/typing/typing_test.go b/kadai3-1/Mizushima/typing/typing_test.go deleted file mode 100644 index 99d53eb5..00000000 --- a/kadai3-1/Mizushima/typing/typing_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package typing_test - -import ( - "bytes" - "strings" - "testing" - "time" - - typing "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-1/Mizushima/typing" -) - -func TestGame(t *testing.T) { - t.Helper() - - cases := []struct { - name string - tm time.Duration - ans []string - expected int - }{ - { - name: "No typo", - tm: 3 * time.Second, - ans: []string{ - "America", - "American", - "Angle", - "April", - "August", - "Bacon", - "Barber", - "Battery", - "Bible", - "Bill", - }, - expected: 10, - }, - { - name: "One typo", - tm: 3 * time.Second, - ans: []string{ - "America", - "American", - "typo", - "April", - "August", - "Bacon", - "Barber", - "Battery", - "Bible", - "Bill", - }, - expected: 9, - }, - } - - // []string{"America","American","Angle","April","August","Bacon","Barber","Battery","Bible","Bill"} - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - output := new(bytes.Buffer) - input := bytes.NewBufferString(strings.Join(c.ans, "\n")) - actual, _ := typing.Game(input, output, "../gamedata/words.csv", c.tm, true) - if actual != c.expected { - t.Errorf("wanted %d, but got %d", c.expected, actual) - } - }) - } -} From 1d024fc2e6a70d3d31f12a2c75c9c70264eb1716 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Tue, 22 Jun 2021 19:28:39 +0900 Subject: [PATCH 26/51] =?UTF-8?q?=E3=82=A8=E3=83=A9=E3=83=BC=E5=8F=8A?= =?UTF-8?q?=E3=81=B3=E3=82=AD=E3=83=A3=E3=83=B3=E3=82=BB=E3=83=AB=E5=BE=8C?= =?UTF-8?q?=E3=81=AE=E5=87=A6=E7=90=86=E3=81=AE=E5=BE=8C=E7=89=87=E3=81=A5?= =?UTF-8?q?=E3=81=91=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/.gitignore | 1 - kadai3-2/Mizushima/download/download.go | 25 ++++++++++++++++------- kadai3-2/Mizushima/getheader/getHeader.go | 9 ++++---- kadai3-2/Mizushima/listen/listen.go | 1 + kadai3-2/Mizushima/main.go | 11 ++++++++-- kadai3-2/Mizushima/options/options.go | 2 -- 6 files changed, 33 insertions(+), 16 deletions(-) diff --git a/kadai3-2/Mizushima/.gitignore b/kadai3-2/Mizushima/.gitignore index 5276ed00..11ad24f3 100644 --- a/kadai3-2/Mizushima/.gitignore +++ b/kadai3-2/Mizushima/.gitignore @@ -1,4 +1,3 @@ -http.request* *.jpg documents bin/* \ No newline at end of file diff --git a/kadai3-2/Mizushima/download/download.go b/kadai3-2/Mizushima/download/download.go index 283c8316..9a3e1f7e 100644 --- a/kadai3-2/Mizushima/download/download.go +++ b/kadai3-2/Mizushima/download/download.go @@ -1,3 +1,5 @@ +// download package implements parallel download and non-parallel +// download. package download import ( @@ -12,14 +14,16 @@ import ( "golang.org/x/sync/errgroup" ) +//PDownloader is user-defined struct type PDownloader struct { - url string - output *os.File - fileSize uint - part uint - procs uint + url string // URL for the download + output *os.File // Where to save the downloaded file + fileSize uint // size of the downloaded file + part uint // Number of divided bytes + procs uint // Number of parallel download process } +// newPDownloader is constructor for PDownloader. func newPDownloader(url string, output *os.File, fileSize uint, part uint, procs uint) *PDownloader { return &PDownloader{ url: url, @@ -30,6 +34,9 @@ func newPDownloader(url string, output *os.File, fileSize uint, part uint, procs } } +// Downloader gets elements of PDownloader, the download is parallel or not, temprary +// directory name and context.Context, and drives DownloadFile method if isPara is false +// or PDownload if isPrara is true. // func Downloader(url string, output *os.File, fileSize uint, part uint, procs uint, isPara bool, @@ -52,7 +59,7 @@ func Downloader(url string, return nil } -// non-pararel download +// DownloadFile drives a non-parallel download func (pd *PDownloader)DownloadFile() error { resp, err := request.Request("GET", pd.url, "Range", "bytes=281-294") @@ -69,6 +76,7 @@ func (pd *PDownloader)DownloadFile() error { return nil } +// PDownload drives parallel download. func (pd *PDownloader)PDownload(grp *errgroup.Group, tmpDirName string, procs uint, ctx context.Context) error { // fmt.Printf("%d/%d downloading...\n", idx, pd.procs) @@ -81,7 +89,7 @@ func (pd *PDownloader)PDownload(grp *errgroup.Group, start = idx*pd.part + 1 } - // 最後だったら + // if idx is the end if idx == pd.procs-1 { end = pd.fileSize } else { @@ -102,6 +110,9 @@ func (pd *PDownloader)PDownload(grp *errgroup.Group, return nil } +// ReqToMakeCopy sends a get request with "Range" field with "bytes" range. +// And gets response and make a copy to a temprary file in temprary directory from response body. +// func (pd *PDownloader)ReqToMakeCopy(tmpDirName, bytes string, idx uint) error { fmt.Printf("ReqToMakeCopy: tmpDirName: %s, bytes %s, idx: %d\n", tmpDirName, bytes, idx) resp, err := request.Request("GET", pd.url, "Range", bytes) diff --git a/kadai3-2/Mizushima/getheader/getHeader.go b/kadai3-2/Mizushima/getheader/getHeader.go index 2f538cee..ad04e3e8 100644 --- a/kadai3-2/Mizushima/getheader/getHeader.go +++ b/kadai3-2/Mizushima/getheader/getHeader.go @@ -1,3 +1,4 @@ +// getheader package implements to read a response header of http package getheader import ( @@ -8,13 +9,13 @@ import ( "strconv" ) -// get all headers +// Headers returns all headers func Headers(w io.Writer, r *http.Response) { h := r.Header fmt.Fprintln(w, h) } -// get the specified header +// ResHeader returns the value of the specified header func ResHeader(w io.Writer, r *http.Response, header string) ([]string, error) { h, is := r.Header[header] fmt.Println(h) @@ -25,7 +26,7 @@ func ResHeader(w io.Writer, r *http.Response, header string) ([]string, error) { return h, nil } -// get the specified header by commas +// ResHeaderComma returns the value of the specified response header by commas. func ResHeaderComma(w io.Writer, r *http.Response, header string) (string, error) { h := r.Header.Get(header) // if !is { @@ -35,7 +36,7 @@ func ResHeaderComma(w io.Writer, r *http.Response, header string) (string, error return h, nil } - +// GetSize returns size from response header. func GetSize(resp *http.Response) (uint, error) { contLen, err := ResHeader(os.Stdout, resp, "Content-Length") if err != nil { diff --git a/kadai3-2/Mizushima/listen/listen.go b/kadai3-2/Mizushima/listen/listen.go index 46b0e6ce..cc0ea30e 100644 --- a/kadai3-2/Mizushima/listen/listen.go +++ b/kadai3-2/Mizushima/listen/listen.go @@ -9,6 +9,7 @@ import ( "syscall" ) +// Listen returns a context for keyboad(ctrl + c) interrupt. func Listen(ctx context.Context, w io.Writer, f func()) (context.Context, func()) { ctx, cancel := context.WithCancel(ctx) diff --git a/kadai3-2/Mizushima/main.go b/kadai3-2/Mizushima/main.go index f855a0fe..9a3d5428 100644 --- a/kadai3-2/Mizushima/main.go +++ b/kadai3-2/Mizushima/main.go @@ -70,12 +70,19 @@ func main() { tmpDirName := opts.Output + strconv.Itoa(i) err = os.Mkdir(tmpDirName, 0777) if err != nil { + out.Close() + if err2 := os.Remove(opts.Output + filepath.Base(url)); err2 != nil { + log.Fatalf("err: os.Mkdir: %s\nerr: os.Remove: %s\n", err, err2) + } log.Fatalf("err: os.Mkdir: %s\n", err) } // ctx, cancel := context.WithTimeout(context.Background(),time.Duration(opts.Tm)*time.Minute) ctx := context.Background() - clean := func() { os.RemoveAll(tmpDirName)} + clean := func() { + os.RemoveAll(tmpDirName) + os.Remove(opts.Output + filepath.Base(url)) + } ctx, cancel := listen.Listen(ctx, os.Stdout, clean) var isPara bool = true @@ -83,6 +90,7 @@ func main() { if err != nil && err.Error() == "cannot find Accept-Ranges header" { isPara = false } else if err != nil { + clean() log.Fatalf("err: getheader.ResHeader: %s\n", err) } else if accept[0] != "bytes" { isPara = false @@ -90,7 +98,6 @@ func main() { } err = download.Downloader(url, out, fileSize, partial, opts.Procs, isPara, tmpDirName, ctx) - // err = pararel(url, out, fileSize, partial, opts.Procs, tmpDirName, ctx) if err != nil { log.Fatalf("err: %s\n", err) } diff --git a/kadai3-2/Mizushima/options/options.go b/kadai3-2/Mizushima/options/options.go index be8fde01..e51aaf75 100644 --- a/kadai3-2/Mizushima/options/options.go +++ b/kadai3-2/Mizushima/options/options.go @@ -15,7 +15,6 @@ type Options struct { Help bool `short:"h" long:"help"` Procs uint `short:"p" long:"procs"` Output string `short:"o" long:"output" default:"./"` - Tm int `short:"t" long:"time" default:"1"` } // parse options @@ -42,7 +41,6 @@ func (opts Options) usage() []byte { -h, -help print usage and exit -p, -procs the number of split to download -o, -output path of the file downloaded - -t, -time Time limit minuts until the each download is completed `, ) From 0b0eb0584491e459a58f40a7260a565c60439c5b Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Tue, 22 Jun 2021 19:33:24 +0900 Subject: [PATCH 27/51] =?UTF-8?q?gitignore=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/.gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kadai3-2/Mizushima/.gitignore b/kadai3-2/Mizushima/.gitignore index 11ad24f3..eb5867b3 100644 --- a/kadai3-2/Mizushima/.gitignore +++ b/kadai3-2/Mizushima/.gitignore @@ -1,3 +1,4 @@ *.jpg documents -bin/* \ No newline at end of file +bin/* +test_download/* \ No newline at end of file From b9d9b41ea33dc843cef7f571d5d45c56243b2f2a Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Tue, 22 Jun 2021 19:49:30 +0900 Subject: [PATCH 28/51] =?UTF-8?q?=E3=82=A8=E3=83=A9=E3=83=BC=E3=82=84?= =?UTF-8?q?=E3=82=AD=E3=83=A3=E3=83=B3=E3=82=BB=E3=83=AB=E3=81=A7=E9=80=94?= =?UTF-8?q?=E4=B8=AD=E7=B5=82=E4=BA=86=E3=81=97=E3=81=9F=E6=99=82=E3=81=AE?= =?UTF-8?q?=E5=87=A6=E7=90=86=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/Makefile | 1 + kadai3-2/Mizushima/main.go | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/kadai3-2/Mizushima/Makefile b/kadai3-2/Mizushima/Makefile index 2bc4d956..4dad8bb7 100644 --- a/kadai3-2/Mizushima/Makefile +++ b/kadai3-2/Mizushima/Makefile @@ -12,6 +12,7 @@ build: test: clean all $(BINARY_NAME) http://i.imgur.com/z4d4kWk.jpg -o ./test_download + $(BINARY_NAME) http://socia-enterprise.co.jp -o ./test_download clean: $(GOCLEAN) diff --git a/kadai3-2/Mizushima/main.go b/kadai3-2/Mizushima/main.go index 9a3d5428..f8c826d4 100644 --- a/kadai3-2/Mizushima/main.go +++ b/kadai3-2/Mizushima/main.go @@ -80,8 +80,14 @@ func main() { // ctx, cancel := context.WithTimeout(context.Background(),time.Duration(opts.Tm)*time.Minute) ctx := context.Background() clean := func() { - os.RemoveAll(tmpDirName) - os.Remove(opts.Output + filepath.Base(url)) + out.Close() + // delete the tmporary directory + if err := os.RemoveAll(tmpDirName); err != nil { + log.Fatalf("err: RemoveAll: %s\n", err) + } + if err := os.Remove(opts.Output + filepath.Base(url)); err != nil { + log.Fatalf("err: os.Remove: %s\n", err) + } } ctx, cancel := listen.Listen(ctx, os.Stdout, clean) From 0ce8d211fdc319c4d6505bf56697c80c3dfb5299 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Wed, 23 Jun 2021 11:45:00 +0900 Subject: [PATCH 29/51] =?UTF-8?q?go=20fmt=E9=81=A9=E7=94=A8=E3=80=81?= =?UTF-8?q?=E3=82=AA=E3=83=97=E3=82=B7=E3=83=A7=E3=83=B3=E3=81=AE=E5=87=A6?= =?UTF-8?q?=E7=90=86=E3=82=92main=E9=96=A2=E6=95=B0=E3=81=AB=E7=A7=BB?= =?UTF-8?q?=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/Makefile | 5 +- kadai3-2/Mizushima/TODO.md | 1 + kadai3-2/Mizushima/download/download.go | 70 +++++++------- kadai3-2/Mizushima/getheader/getHeader.go | 1 - kadai3-2/Mizushima/go.mod | 5 +- kadai3-2/Mizushima/listen/listen.go | 4 +- kadai3-2/Mizushima/main.go | 110 ++++++++++++++++------ kadai3-2/Mizushima/options/go.mod | 8 -- kadai3-2/Mizushima/options/go.sum | 6 -- kadai3-2/Mizushima/options/options.go | 68 ------------- kadai3-2/Mizushima/request/request.go | 13 +-- 11 files changed, 129 insertions(+), 162 deletions(-) delete mode 100644 kadai3-2/Mizushima/options/go.mod delete mode 100644 kadai3-2/Mizushima/options/go.sum delete mode 100644 kadai3-2/Mizushima/options/options.go diff --git a/kadai3-2/Mizushima/Makefile b/kadai3-2/Mizushima/Makefile index 4dad8bb7..3220a6ef 100644 --- a/kadai3-2/Mizushima/Makefile +++ b/kadai3-2/Mizushima/Makefile @@ -5,14 +5,15 @@ GOCLEAN=$(GOCMD) clean GOTEST=$(GOCMD) test GOGET=$(GOCMD) get -all: build +all: build test build: $(GOBUILD) -o $(BINARY_NAME) -v -test: clean all +test: $(BINARY_NAME) http://i.imgur.com/z4d4kWk.jpg -o ./test_download $(BINARY_NAME) http://socia-enterprise.co.jp -o ./test_download + $(BINARY_NAME) http://i.imgur.com/z4d4kWk.jpg http://socia-enterprise.co.jp -o ./test_download clean: $(GOCLEAN) diff --git a/kadai3-2/Mizushima/TODO.md b/kadai3-2/Mizushima/TODO.md index a0e56164..02d36e8a 100644 --- a/kadai3-2/Mizushima/TODO.md +++ b/kadai3-2/Mizushima/TODO.md @@ -17,6 +17,7 @@ - ~~並行してダウンロードし連番の一時ファイルに保存~~ - ~~完成版フォルダに一時ファイルを順番にコピー~~ - ~~一時フォルダ毎削除~~ + - 時間制限を設ける ### 参考にしたもの [pget](https://qiita.com/codehex/items/d0a500ac387d39a34401) (goroutineを使ったダウンロード処理、コマンドラインオプションの処理等々) diff --git a/kadai3-2/Mizushima/download/download.go b/kadai3-2/Mizushima/download/download.go index 9a3e1f7e..46ac5322 100644 --- a/kadai3-2/Mizushima/download/download.go +++ b/kadai3-2/Mizushima/download/download.go @@ -16,42 +16,42 @@ import ( //PDownloader is user-defined struct type PDownloader struct { - url string // URL for the download - output *os.File // Where to save the downloaded file - fileSize uint // size of the downloaded file - part uint // Number of divided bytes - procs uint // Number of parallel download process + url string // URL for the download + output *os.File // Where to save the downloaded file + fileSize uint // size of the downloaded file + part uint // Number of divided bytes + procs uint // Number of parallel download process } // newPDownloader is constructor for PDownloader. func newPDownloader(url string, output *os.File, fileSize uint, part uint, procs uint) *PDownloader { return &PDownloader{ - url: url, - output: output, - fileSize: fileSize, - part: part, - procs: procs, - } + url: url, + output: output, + fileSize: fileSize, + part: part, + procs: procs, + } } -// Downloader gets elements of PDownloader, the download is parallel or not, temprary +// Downloader gets elements of PDownloader, the download is parallel or not, temprary // directory name and context.Context, and drives DownloadFile method if isPara is false // or PDownload if isPrara is true. -// -func Downloader(url string, - output *os.File, fileSize uint, part uint, procs uint, isPara bool, +// +func Downloader(url string, + output *os.File, fileSize uint, part uint, procs uint, isPara bool, tmpDirName string, ctx context.Context) error { pd := newPDownloader(url, output, fileSize, part, procs) if !isPara { - fmt.Printf("%s do not accept range access. so downloading in single process\n", url) - err := pd.DownloadFile() + fmt.Printf("%s do not accept range access: downloading by single process\n", url) + err := pd.DownloadFile(ctx) if err != nil { return err } } else { grp, ctx := errgroup.WithContext(context.Background()) pd.PDownload(grp, tmpDirName, procs, ctx) - + if err := grp.Wait(); err != nil { return err } @@ -60,9 +60,9 @@ func Downloader(url string, } // DownloadFile drives a non-parallel download -func (pd *PDownloader)DownloadFile() error { - - resp, err := request.Request("GET", pd.url, "Range", "bytes=281-294") +func (pd *PDownloader) DownloadFile(ctx context.Context) error { + + resp, err := request.Request(ctx, "GET", pd.url, "Range", "bytes=281-294") if err != nil { return err } @@ -72,14 +72,13 @@ func (pd *PDownloader)DownloadFile() error { if err != nil { return err } - + return nil } // PDownload drives parallel download. -func (pd *PDownloader)PDownload(grp *errgroup.Group, +func (pd *PDownloader) PDownload(grp *errgroup.Group, tmpDirName string, procs uint, ctx context.Context) error { - // fmt.Printf("%d/%d downloading...\n", idx, pd.procs) var start, end, idx uint for idx = uint(0); idx < procs; idx++ { @@ -88,44 +87,44 @@ func (pd *PDownloader)PDownload(grp *errgroup.Group, } else { start = idx*pd.part + 1 } - + // if idx is the end if idx == pd.procs-1 { end = pd.fileSize } else { - end = (idx+1) * pd.part + end = (idx + 1) * pd.part } - + // idxを代入し直す // https://qiita.com/harhogefoo/items/7ccb4e353a4a01cfa773 - idx := idx - fmt.Printf("start: %d, end: %d, pd.part: %d\n", start, end, pd.part) + idx := idx + // fmt.Printf("start: %d, end: %d, pd.part: %d\n", start, end, pd.part) bytes := fmt.Sprintf("bytes=%d-%d", start, end) grp.Go(func() error { fmt.Printf("grp.Go: tmpDirName: %s, bytes %s, idx: %d\n", tmpDirName, bytes, idx) - return pd.ReqToMakeCopy(tmpDirName, bytes, idx) + return pd.ReqToMakeCopy(tmpDirName, bytes, idx, ctx) }) } return nil } -// ReqToMakeCopy sends a get request with "Range" field with "bytes" range. +// ReqToMakeCopy sends a "GET" request with "Range" field with "bytes" range. // And gets response and make a copy to a temprary file in temprary directory from response body. // -func (pd *PDownloader)ReqToMakeCopy(tmpDirName, bytes string, idx uint) error { +func (pd *PDownloader) ReqToMakeCopy(tmpDirName, bytes string, idx uint, ctx context.Context) error { fmt.Printf("ReqToMakeCopy: tmpDirName: %s, bytes %s, idx: %d\n", tmpDirName, bytes, idx) - resp, err := request.Request("GET", pd.url, "Range", bytes) + resp, err := request.Request(ctx, "GET", pd.url, "Range", bytes) if err != nil { return err } - tmpOut, err := os.Create(tmpDirName+"/"+strconv.Itoa(int(idx))) + tmpOut, err := os.Create(tmpDirName + "/" + strconv.Itoa(int(idx))) if err != nil { return err } // fmt.Printf("tmpOut.Name(): %s\n", tmpOut.Name()) - defer func(){ + defer func() { err = tmpOut.Close() if err != nil { fmt.Fprintf(os.Stderr, "err: tmpOut.Close(): %s", err.Error()) @@ -143,4 +142,3 @@ func (pd *PDownloader)ReqToMakeCopy(tmpDirName, bytes string, idx uint) error { fmt.Printf("%d/%d was downloaded len=%d\n", idx, pd.procs, length) return nil } - diff --git a/kadai3-2/Mizushima/getheader/getHeader.go b/kadai3-2/Mizushima/getheader/getHeader.go index ad04e3e8..f99413a2 100644 --- a/kadai3-2/Mizushima/getheader/getHeader.go +++ b/kadai3-2/Mizushima/getheader/getHeader.go @@ -48,4 +48,3 @@ func GetSize(resp *http.Response) (uint, error) { } return uint(ret), nil } - diff --git a/kadai3-2/Mizushima/go.mod b/kadai3-2/Mizushima/go.mod index d2a4f562..580c93bd 100644 --- a/kadai3-2/Mizushima/go.mod +++ b/kadai3-2/Mizushima/go.mod @@ -5,13 +5,14 @@ go 1.16 replace ( github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/download => ./download github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/getheader => ./getheader - github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/options => ./options + github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/listen => ./listen github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/request => ./request ) require ( github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/download v0.0.0-00010101000000-000000000000 github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/getheader v0.0.0-00010101000000-000000000000 - github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/options v0.0.0-00010101000000-000000000000 github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/request v0.0.0-00010101000000-000000000000 + github.com/jessevdk/go-flags v1.5.0 + github.com/pkg/errors v0.9.1 ) diff --git a/kadai3-2/Mizushima/listen/listen.go b/kadai3-2/Mizushima/listen/listen.go index cc0ea30e..10a931d6 100644 --- a/kadai3-2/Mizushima/listen/listen.go +++ b/kadai3-2/Mizushima/listen/listen.go @@ -17,11 +17,11 @@ func Listen(ctx context.Context, w io.Writer, f func()) (context.Context, func() signal.Notify(ch, os.Interrupt, syscall.SIGTERM) go func() { <-ch - fmt.Fprintln(w, "\n^Csignal pressed. interrupt.") + fmt.Fprintln(w, "\n^Csignal : interrupt.") cancel() f() os.Exit(0) }() return ctx, cancel -} \ No newline at end of file +} diff --git a/kadai3-2/Mizushima/main.go b/kadai3-2/Mizushima/main.go index f8c826d4..8ab0c49a 100644 --- a/kadai3-2/Mizushima/main.go +++ b/kadai3-2/Mizushima/main.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "context" "fmt" "log" @@ -9,11 +10,14 @@ import ( "path/filepath" "runtime" "strconv" + "time" - download "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/download" - getheader "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/getheader" - listen "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/listen" - options "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/options" + flags "github.com/jessevdk/go-flags" + errors "github.com/pkg/errors" + + "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/download" + "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/getheader" + "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/listen" "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/request" ) @@ -22,46 +26,97 @@ import ( // go run main.go http://i.imgur.com/z4d4kWk.jpg -o . // go run main.go https://misc.laboradian.com/test/003/ -o . +// struct for options +type Options struct { + Help bool `short:"h" long:"help"` + Procs uint `short:"p" long:"procs"` + Output string `short:"o" long:"output" default:"./"` +} + +// parse options +func (opts *Options) parse(argv []string) ([]string, error) { + p := flags.NewParser(opts, flags.PrintErrors) + args, err := p.ParseArgs(argv) + + if err != nil { + os.Stderr.Write(opts.usage()) + return nil, errors.Wrap(err, "invalid command line options") + } + + return args, nil +} + +// usage prints a description of avilable options +func (opts Options) usage() []byte { + buf := bytes.Buffer{} + + fmt.Fprintln(&buf, + `Usage: pd [options] URL + + Options: + -h, --help print usage and exit + -p, --procs the number of split to download (default: the number of CPU cores) + -o, --output path of the file downloaded (default: current directory) + `, + ) + + return buf.Bytes() +} + func main() { - opts, urls, err := options.ParseOptions(os.Args[1:]) + + // parse options + var opts Options + argv := os.Args[1:] + if len(argv) == 0 { + os.Stdout.Write(opts.usage()) + log.Fatalf("err: %s\n", errors.New("no options")) + } + + urls, err := opts.parse(argv) if err != nil { - log.Fatal(err) + log.Fatalf("err: %s\n", err) } + if opts.Help { + os.Stdout.Write(opts.usage()) + log.Fatalf("err: %s\n", errors.New("print usage")) + } + + // if opts.Procs == 0 { opts.Procs = uint(runtime.NumCPU()) } - fmt.Printf("opts:%#v\n", opts) - fmt.Println(urls) - if len(opts.Output) > 0 && opts.Output[len(opts.Output)-1] != '/' { opts.Output += "/" } + // download from each url in urls for i, url := range urls { - resp, err := request.Request("HEAD", url, "", "") + // make a empty context + ctx := context.Background() + ctxTimeout, cancelTimeout := context.WithTimeout(ctx, 10*time.Second) + defer cancelTimeout() + + resp, err := request.Request(ctxTimeout, "HEAD", url, "", "") if err != nil { log.Fatalf("err: %s\n", err) } - - fmt.Println("response:") - bytes, err := httputil.DumpResponse(resp, true) - if err != nil { - log.Fatalf("err: httputil.DumpResponse: %s\n", err) - } - fmt.Println(string(bytes)) + + h, _ := httputil.DumpResponse(resp, false) + fmt.Printf("response:\n%s", h) fileSize, err := getheader.GetSize(resp) if err != nil { - log.Fatalf("err: %s\n", err) + log.Fatalf("err: getheader.GetSize: %s\n", err) } resp.Body.Close() partial := fileSize / opts.Procs - out, err := os.OpenFile(opts.Output + filepath.Base(url), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0755) + out, err := os.OpenFile(opts.Output+filepath.Base(url), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0755) if err != nil { log.Fatalf("err: os.Create: %s\n", err) } @@ -78,8 +133,7 @@ func main() { } // ctx, cancel := context.WithTimeout(context.Background(),time.Duration(opts.Tm)*time.Minute) - ctx := context.Background() - clean := func() { + clean := func() { out.Close() // delete the tmporary directory if err := os.RemoveAll(tmpDirName); err != nil { @@ -89,33 +143,33 @@ func main() { log.Fatalf("err: os.Remove: %s\n", err) } } - ctx, cancel := listen.Listen(ctx, os.Stdout, clean) + ctx, cancel := listen.Listen(ctxTimeout, os.Stdout, clean) var isPara bool = true accept, err := getheader.ResHeader(os.Stdout, resp, "Accept-Ranges") if err != nil && err.Error() == "cannot find Accept-Ranges header" { - isPara = false + isPara = false } else if err != nil { clean() log.Fatalf("err: getheader.ResHeader: %s\n", err) } else if accept[0] != "bytes" { isPara = false - continue; + continue } - + err = download.Downloader(url, out, fileSize, partial, opts.Procs, isPara, tmpDirName, ctx) if err != nil { log.Fatalf("err: %s\n", err) } - fmt.Println("download complete") + fmt.Printf("download complete: %s\n", url) err = MergeFiles(tmpDirName, opts.Procs, fileSize, out) if err != nil { log.Fatalf("err: MergeFiles: %s\n", err) } - // delete the tmporary directory + // delete the tmporary directory only if err := os.RemoveAll(tmpDirName); err != nil { log.Fatalf("err: RemoveAll: %s\n", err) } @@ -128,7 +182,7 @@ func main() { func MergeFiles(tmpDirName string, procs, fileSize uint, output *os.File) error { for i := uint(0); i < procs; i++ { - body, err := os.ReadFile(tmpDirName+"/"+strconv.Itoa(int(i))) + body, err := os.ReadFile(tmpDirName + "/" + strconv.Itoa(int(i))) if err != nil { return err } diff --git a/kadai3-2/Mizushima/options/go.mod b/kadai3-2/Mizushima/options/go.mod deleted file mode 100644 index 3614e828..00000000 --- a/kadai3-2/Mizushima/options/go.mod +++ /dev/null @@ -1,8 +0,0 @@ -module github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/options - -go 1.16 - -require ( - github.com/jessevdk/go-flags v1.5.0 - github.com/pkg/errors v0.9.1 -) diff --git a/kadai3-2/Mizushima/options/go.sum b/kadai3-2/Mizushima/options/go.sum deleted file mode 100644 index 3c8e9a7b..00000000 --- a/kadai3-2/Mizushima/options/go.sum +++ /dev/null @@ -1,6 +0,0 @@ -github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= -github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 h1:EZ2mChiOa8udjfp6rRmswTbtZN/QzUQp4ptM4rnjHvc= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/kadai3-2/Mizushima/options/options.go b/kadai3-2/Mizushima/options/options.go deleted file mode 100644 index e51aaf75..00000000 --- a/kadai3-2/Mizushima/options/options.go +++ /dev/null @@ -1,68 +0,0 @@ -// This package implements command-line options for this application -package options - -import ( - "bytes" - "fmt" - "os" - - flags "github.com/jessevdk/go-flags" - errors "github.com/pkg/errors" -) - -// struct for options -type Options struct { - Help bool `short:"h" long:"help"` - Procs uint `short:"p" long:"procs"` - Output string `short:"o" long:"output" default:"./"` -} - -// parse options -func (opts *Options) parse(argv []string) ([]string, error) { - p := flags.NewParser(opts, flags.PrintErrors) - args, err := p.ParseArgs(argv) - - if err != nil { - os.Stderr.Write(opts.usage()) - return nil, errors.Wrap(err, "invalid command line options") - } - - return args, nil -} - -// usage prints a description of avilable options -func (opts Options) usage() []byte { - buf := bytes.Buffer{} - - fmt.Fprintln(&buf, - `Usage: pd [options] URL - - Options: - -h, -help print usage and exit - -p, -procs the number of split to download - -o, -output path of the file downloaded - `, - ) - - return buf.Bytes() -} - -func ParseOptions(argv []string) (*Options, []string, error) { - var opts Options - if len(argv) == 0 { - os.Stdout.Write(opts.usage()) - return nil, nil, errors.New("no options") - } - - o, err := opts.parse(argv) - if err != nil { - return nil, nil, err - } - - if opts.Help { - os.Stdout.Write(opts.usage()) - return nil, nil, errors.New("print usage") - } - - return &opts, o, nil -} diff --git a/kadai3-2/Mizushima/request/request.go b/kadai3-2/Mizushima/request/request.go index 5bb39d28..fb48772a 100644 --- a/kadai3-2/Mizushima/request/request.go +++ b/kadai3-2/Mizushima/request/request.go @@ -1,14 +1,14 @@ package request import ( + "context" "fmt" "net/http" - "net/http/httputil" ) // Request throws a request and returns a response from url and a error. -func Request(method string, url string, setH string, setV string) (*http.Response, error) { - req, err := http.NewRequest(method, url, nil) +func Request(ctx context.Context, method string, url string, setH string, setV string) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, method, url, nil) if err != nil { return nil, err } @@ -16,16 +16,11 @@ func Request(method string, url string, setH string, setV string) (*http.Respons if len(setH) != 0 { req.Header.Set(setH, setV) } - - dump, _ := httputil.DumpRequestOut(req, false) - fmt.Printf("request:\n%s\n", dump) client := new(http.Client) resp, err := client.Do(req) if err != nil { - return nil, err + return nil, fmt.Errorf("request.Request err: %s", err) } return resp, nil } - - From 886a1a9b78ace4cc88c02ed40e682d993133028a Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Thu, 24 Jun 2021 09:38:57 +0900 Subject: [PATCH 30/51] =?UTF-8?q?=E3=82=A8=E3=83=A9=E3=83=BC=E5=87=A6?= =?UTF-8?q?=E7=90=86=E5=BF=98=E3=82=8C=E4=BF=AE=E6=AD=A3=E3=80=81errors.Wr?= =?UTF-8?q?ap=E3=82=92fmt.Errorf=E3=81=AB=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/TODO.md | 5 +- kadai3-2/Mizushima/download/download.go | 30 ++-- kadai3-2/Mizushima/dummy_server/dummy.go | 25 ++++ .../Mizushima/dummy_server/dummy_server.go | 9 -- kadai3-2/Mizushima/getheader/getHeader.go | 19 ++- kadai3-2/Mizushima/listen/listen.go | 7 +- kadai3-2/Mizushima/main.go | 137 +++++++++++++----- kadai3-2/Mizushima/request/request.go | 4 +- 8 files changed, 164 insertions(+), 72 deletions(-) create mode 100644 kadai3-2/Mizushima/dummy_server/dummy.go delete mode 100644 kadai3-2/Mizushima/dummy_server/dummy_server.go diff --git a/kadai3-2/Mizushima/TODO.md b/kadai3-2/Mizushima/TODO.md index 02d36e8a..09d72102 100644 --- a/kadai3-2/Mizushima/TODO.md +++ b/kadai3-2/Mizushima/TODO.md @@ -17,7 +17,10 @@ - ~~並行してダウンロードし連番の一時ファイルに保存~~ - ~~完成版フォルダに一時ファイルを順番にコピー~~ - ~~一時フォルダ毎削除~~ - - 時間制限を設ける + - ~~時間制限を設ける~~ + - 空のctx -> ctx.WithTimeout(child) in main function※ここで時間制限設定 -> ctx.WithCancel(child) in listen.Listen -> request.Requestに渡してhttp.NewRequestWithContextに渡すことでリクエスト時のタイムアウト設定 + - ~~ファイルが追記モードで開かざるを得ないので、続けて同じファイルをダウンロードすると後ろに追記されてしまう~~ + - 先に同じ名前のファイルがあったら消す ### 参考にしたもの [pget](https://qiita.com/codehex/items/d0a500ac387d39a34401) (goroutineを使ったダウンロード処理、コマンドラインオプションの処理等々) diff --git a/kadai3-2/Mizushima/download/download.go b/kadai3-2/Mizushima/download/download.go index 46ac5322..0cdd7f4f 100644 --- a/kadai3-2/Mizushima/download/download.go +++ b/kadai3-2/Mizushima/download/download.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "io/ioutil" + "net/url" "os" "strconv" @@ -16,7 +17,7 @@ import ( //PDownloader is user-defined struct type PDownloader struct { - url string // URL for the download + url *url.URL // URL for the download output *os.File // Where to save the downloaded file fileSize uint // size of the downloaded file part uint // Number of divided bytes @@ -24,7 +25,7 @@ type PDownloader struct { } // newPDownloader is constructor for PDownloader. -func newPDownloader(url string, output *os.File, fileSize uint, part uint, procs uint) *PDownloader { +func newPDownloader(url *url.URL, output *os.File, fileSize uint, part uint, procs uint) *PDownloader { return &PDownloader{ url: url, output: output, @@ -38,7 +39,7 @@ func newPDownloader(url string, output *os.File, fileSize uint, part uint, procs // directory name and context.Context, and drives DownloadFile method if isPara is false // or PDownload if isPrara is true. // -func Downloader(url string, +func Downloader(url *url.URL, output *os.File, fileSize uint, part uint, procs uint, isPara bool, tmpDirName string, ctx context.Context) error { pd := newPDownloader(url, output, fileSize, part, procs) @@ -49,8 +50,10 @@ func Downloader(url string, return err } } else { - grp, ctx := errgroup.WithContext(context.Background()) - pd.PDownload(grp, tmpDirName, procs, ctx) + grp, ctx := errgroup.WithContext(ctx) + if err := pd.PDownload(grp, tmpDirName, procs, ctx); err != nil { + return err + } if err := grp.Wait(); err != nil { return err @@ -60,13 +63,15 @@ func Downloader(url string, } // DownloadFile drives a non-parallel download -func (pd *PDownloader) DownloadFile(ctx context.Context) error { +func (pd *PDownloader) DownloadFile(ctx context.Context) (err error) { - resp, err := request.Request(ctx, "GET", pd.url, "Range", "bytes=281-294") + resp, err := request.Request(ctx, "GET", pd.url.String(), "", "") if err != nil { return err } - defer resp.Body.Close() + defer func() { + err = resp.Body.Close() + }() _, err = io.Copy(pd.output, resp.Body) if err != nil { @@ -113,8 +118,8 @@ func (pd *PDownloader) PDownload(grp *errgroup.Group, // And gets response and make a copy to a temprary file in temprary directory from response body. // func (pd *PDownloader) ReqToMakeCopy(tmpDirName, bytes string, idx uint, ctx context.Context) error { - fmt.Printf("ReqToMakeCopy: tmpDirName: %s, bytes %s, idx: %d\n", tmpDirName, bytes, idx) - resp, err := request.Request(ctx, "GET", pd.url, "Range", bytes) + // fmt.Printf("ReqToMakeCopy: tmpDirName: %s, bytes %s, idx: %d\n", tmpDirName, bytes, idx) + resp, err := request.Request(ctx, "GET", pd.url.String(), "Range", bytes) if err != nil { return err } @@ -125,9 +130,8 @@ func (pd *PDownloader) ReqToMakeCopy(tmpDirName, bytes string, idx uint, ctx con } // fmt.Printf("tmpOut.Name(): %s\n", tmpOut.Name()) defer func() { - err = tmpOut.Close() - if err != nil { - fmt.Fprintf(os.Stderr, "err: tmpOut.Close(): %s", err.Error()) + if err = tmpOut.Close(); err != nil { + fmt.Fprintf(os.Stderr, "err: tmpOut.Close(): %w", err.Error()) } }() diff --git a/kadai3-2/Mizushima/dummy_server/dummy.go b/kadai3-2/Mizushima/dummy_server/dummy.go new file mode 100644 index 00000000..4e108d1a --- /dev/null +++ b/kadai3-2/Mizushima/dummy_server/dummy.go @@ -0,0 +1,25 @@ +package main + +import ( + "fmt" + "net/http" + "time" +) + +func handleFnLate(w http.ResponseWriter, r *http.Request) { + // w.Header().Set("Accept-Range", "bytes") + time.Sleep(30*time.Second) + fmt.Fprint(w, "Hello\n") +} + +func handleFn(w http.ResponseWriter, r *http.Request) { + // w.Header().Set("Accept-Range", "bytes") + // time.Sleep(30*time.Second) + fmt.Fprint(w, "Hello\n") +} + +func main() { + http.HandleFunc("/late", handleFnLate) + http.HandleFunc("/", handleFn) + http.ListenAndServe(":12345", nil) +} \ No newline at end of file diff --git a/kadai3-2/Mizushima/dummy_server/dummy_server.go b/kadai3-2/Mizushima/dummy_server/dummy_server.go deleted file mode 100644 index 9c33c1c4..00000000 --- a/kadai3-2/Mizushima/dummy_server/dummy_server.go +++ /dev/null @@ -1,9 +0,0 @@ -package main - -import ( - "net/http" -) - -func main() { - http.ListenAndServe(":12345", nil) -} \ No newline at end of file diff --git a/kadai3-2/Mizushima/getheader/getHeader.go b/kadai3-2/Mizushima/getheader/getHeader.go index f99413a2..b70c627c 100644 --- a/kadai3-2/Mizushima/getheader/getHeader.go +++ b/kadai3-2/Mizushima/getheader/getHeader.go @@ -10,19 +10,24 @@ import ( ) // Headers returns all headers -func Headers(w io.Writer, r *http.Response) { +func Headers(w io.Writer, r *http.Response) error { h := r.Header - fmt.Fprintln(w, h) + if _, err := fmt.Fprintln(w, h); err != nil { + return err + } + return nil } -// ResHeader returns the value of the specified header +// ResHeader returns the value of the specified header, and writes http response header on io.Writer. func ResHeader(w io.Writer, r *http.Response, header string) ([]string, error) { h, is := r.Header[header] - fmt.Println(h) + // fmt.Println(h) if !is { return nil, fmt.Errorf("cannot find %s header", header) } - fmt.Fprintf(w, "Header[%s] = %s\n", header, h) + if _, err := fmt.Fprintf(w, "Header[%s] = %s\n", header, h); err != nil { + return nil, err + } return h, nil } @@ -32,7 +37,9 @@ func ResHeaderComma(w io.Writer, r *http.Response, header string) (string, error // if !is { // return "error", fmt.Errorf("cannot find %s header", header) // } - fmt.Fprintf(w, "Header[%s] = %s\n", header, h) + if _, err := fmt.Fprintf(w, "Header[%s] = %s\n", header, h); err != nil { + return "", err + } return h, nil } diff --git a/kadai3-2/Mizushima/listen/listen.go b/kadai3-2/Mizushima/listen/listen.go index 10a931d6..819179f3 100644 --- a/kadai3-2/Mizushima/listen/listen.go +++ b/kadai3-2/Mizushima/listen/listen.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "log" "os" "os/signal" "syscall" @@ -17,7 +18,11 @@ func Listen(ctx context.Context, w io.Writer, f func()) (context.Context, func() signal.Notify(ch, os.Interrupt, syscall.SIGTERM) go func() { <-ch - fmt.Fprintln(w, "\n^Csignal : interrupt.") + _, err := fmt.Fprintln(w, "\n^Csignal : interrupt.") + if err != nil { + cancel() + log.Fatalf("err: listen.Listen: %s\n", err) + } cancel() f() os.Exit(0) diff --git a/kadai3-2/Mizushima/main.go b/kadai3-2/Mizushima/main.go index 8ab0c49a..81e40f85 100644 --- a/kadai3-2/Mizushima/main.go +++ b/kadai3-2/Mizushima/main.go @@ -3,9 +3,11 @@ package main import ( "bytes" "context" + "errors" "fmt" "log" "net/http/httputil" + "net/url" "os" "path/filepath" "runtime" @@ -13,7 +15,6 @@ import ( "time" flags "github.com/jessevdk/go-flags" - errors "github.com/pkg/errors" "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/download" "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/getheader" @@ -31,16 +32,19 @@ type Options struct { Help bool `short:"h" long:"help"` Procs uint `short:"p" long:"procs"` Output string `short:"o" long:"output" default:"./"` + Tm int `short:"t" long:"timeout" default:"120"` } // parse options func (opts *Options) parse(argv []string) ([]string, error) { p := flags.NewParser(opts, flags.PrintErrors) args, err := p.ParseArgs(argv) - if err != nil { - os.Stderr.Write(opts.usage()) - return nil, errors.Wrap(err, "invalid command line options") + _, err2 := os.Stderr.Write(opts.usage()) + if err2 != nil { + return nil, fmt.Errorf("%w: invalid command line options: cannot print usage: %w", err, err2) + } + return nil, fmt.Errorf("%w: invalid command line options", err) } return args, nil @@ -51,12 +55,13 @@ func (opts Options) usage() []byte { buf := bytes.Buffer{} fmt.Fprintln(&buf, - `Usage: pd [options] URL + `Usage: paraDW [options] URL (URL2, URL3, ...) Options: -h, --help print usage and exit -p, --procs the number of split to download (default: the number of CPU cores) -o, --output path of the file downloaded (default: current directory) + -t, --timeout Time limit of return of http response in seconds (default: 120) `, ) @@ -69,78 +74,115 @@ func main() { var opts Options argv := os.Args[1:] if len(argv) == 0 { - os.Stdout.Write(opts.usage()) - log.Fatalf("err: %s\n", errors.New("no options")) + if _, err := os.Stdout.Write(opts.usage()); err != nil { + log.Fatalf("err: %w: %w\n", errors.New("no options"), err) + } + log.Fatalf("err: %w\n", errors.New("no options")) } - urls, err := opts.parse(argv) + urlsStr, err := opts.parse(argv) if err != nil { - log.Fatalf("err: %s\n", err) + log.Fatalf("err: %w\n", err) } + var urls []*url.URL + for _, u := range urlsStr { + url, err := url.ParseRequestURI(u) + if err != nil { + log.Fatalf("err: url.ParseRequestURI: %w\n", err) + } + urls = append(urls, url) + } + + fmt.Printf("timeout: %d\n", opts.Tm) + if opts.Help { - os.Stdout.Write(opts.usage()) - log.Fatalf("err: %s\n", errors.New("print usage")) + if _, err := os.Stdout.Write(opts.usage()); err != nil { + log.Fatalf("err: cannot print usage: %w", err) + } + log.Fatal(errors.New("print usage")) } - // + // if procs was inputted, set the number of runtime.NumCPU() to opts.Procs. if opts.Procs == 0 { opts.Procs = uint(runtime.NumCPU()) } + // if opts.Output inputted and the end of opts.Output is not '/', + // add '/'. if len(opts.Output) > 0 && opts.Output[len(opts.Output)-1] != '/' { opts.Output += "/" } // download from each url in urls - for i, url := range urls { + for i, urlObj := range urls { // make a empty context ctx := context.Background() - ctxTimeout, cancelTimeout := context.WithTimeout(ctx, 10*time.Second) + ctxTimeout, cancelTimeout := context.WithTimeout(ctx, time.Duration(opts.Tm)*time.Second) defer cancelTimeout() - resp, err := request.Request(ctxTimeout, "HEAD", url, "", "") + // send "HEAD" request, and gets response. + resp, err := request.Request(ctxTimeout, "HEAD", urlObj.String(), "", "") if err != nil { - log.Fatalf("err: %s\n", err) + log.Fatalf("err: %w\n", err) } + // show response header h, _ := httputil.DumpResponse(resp, false) fmt.Printf("response:\n%s", h) + // get the size from the response header. fileSize, err := getheader.GetSize(resp) if err != nil { - log.Fatalf("err: getheader.GetSize: %s\n", err) + log.Fatalf("err: getheader.GetSize: %w\n", err) + } + if err = resp.Body.Close(); err != nil { + log.Fatalf("err: %w", err) } - resp.Body.Close() + // How many bytes to download at a time partial := fileSize / opts.Procs - out, err := os.OpenFile(opts.Output+filepath.Base(url), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0755) + outputPath := opts.Output + filepath.Base(urlObj.String()) + // if there is the same file in opts.Output, delete that file in advance. + if isExists(outputPath) { + err := os.Remove(outputPath) + if err != nil { + log.Fatalf("err: isExists: os.Remove: %w\n", err) + } + } + + // make a file for download + out, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0755) if err != nil { - log.Fatalf("err: os.Create: %s\n", err) + log.Fatalf("err: os.Create: %w\n", err) } - // make a temporary directory + // make a temporary directory for parallel download tmpDirName := opts.Output + strconv.Itoa(i) err = os.Mkdir(tmpDirName, 0777) if err != nil { - out.Close() - if err2 := os.Remove(opts.Output + filepath.Base(url)); err2 != nil { - log.Fatalf("err: os.Mkdir: %s\nerr: os.Remove: %s\n", err, err2) + if err3 := out.Close(); err3 != nil { + log.Fatalf("err: %w", err3) + } + if err2 := os.Remove(opts.Output + filepath.Base(urlObj.String())); err2 != nil { + log.Fatalf("err: os.Mkdir: %w\nerr: os.Remove: %w\n", err, err2) } - log.Fatalf("err: os.Mkdir: %s\n", err) + log.Fatalf("err: os.Mkdir: %w\n", err) } // ctx, cancel := context.WithTimeout(context.Background(),time.Duration(opts.Tm)*time.Minute) clean := func() { - out.Close() + if err := out.Close(); err != nil { + log.Fatalf("err: out.Close: %w\n", err) + } // delete the tmporary directory if err := os.RemoveAll(tmpDirName); err != nil { - log.Fatalf("err: RemoveAll: %s\n", err) + log.Fatalf("err: RemoveAll: %w\n", err) } - if err := os.Remove(opts.Output + filepath.Base(url)); err != nil { - log.Fatalf("err: os.Remove: %s\n", err) + if err := os.Remove(opts.Output + filepath.Base(urlObj.String())); err != nil { + log.Fatalf("err: os.Remove: %w\n", err) } } ctx, cancel := listen.Listen(ctxTimeout, os.Stdout, clean) @@ -151,34 +193,41 @@ func main() { isPara = false } else if err != nil { clean() - log.Fatalf("err: getheader.ResHeader: %s\n", err) - } else if accept[0] != "bytes" { + log.Fatalf("err: getheader.ResHeader: %w\n", err) + } else if accept[0] != "bytes" || opts.Procs == 1 { isPara = false continue } - err = download.Downloader(url, out, fileSize, partial, opts.Procs, isPara, tmpDirName, ctx) + // drive a download process + err = download.Downloader(urlObj, out, fileSize, partial, opts.Procs, isPara, tmpDirName, ctx) if err != nil { - log.Fatalf("err: %s\n", err) + log.Fatalf("err: %w\n", err) } - fmt.Printf("download complete: %s\n", url) + fmt.Printf("download complete: %s\n", urlObj.String()) - err = MergeFiles(tmpDirName, opts.Procs, fileSize, out) - if err != nil { - log.Fatalf("err: MergeFiles: %s\n", err) + // Merge the temporary files into "out", when parallel download executed. + if isPara { + err = MergeFiles(tmpDirName, opts.Procs, fileSize, out) + if err != nil { + log.Fatalf("err: MergeFiles: %w\n", err) + } } // delete the tmporary directory only if err := os.RemoveAll(tmpDirName); err != nil { - log.Fatalf("err: RemoveAll: %s\n", err) + log.Fatalf("err: RemoveAll: %w\n", err) } cancel() - out.Close() + if err = out.Close(); err != nil { + log.Fatalf("err: %w", err) + } } } +// MergeFiles merges temporary files made for parallel download into "output". func MergeFiles(tmpDirName string, procs, fileSize uint, output *os.File) error { for i := uint(0); i < procs; i++ { @@ -187,8 +236,16 @@ func MergeFiles(tmpDirName string, procs, fileSize uint, output *os.File) error return err } - fmt.Fprint(output, string(body)) + if _, err = fmt.Fprint(output, string(body)); err != nil { + return err + } fmt.Printf("target file: %s, len=%d written\n", output.Name(), len(string(body))) } return nil } + +// +func isExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} diff --git a/kadai3-2/Mizushima/request/request.go b/kadai3-2/Mizushima/request/request.go index fb48772a..d899ce6c 100644 --- a/kadai3-2/Mizushima/request/request.go +++ b/kadai3-2/Mizushima/request/request.go @@ -7,8 +7,8 @@ import ( ) // Request throws a request and returns a response from url and a error. -func Request(ctx context.Context, method string, url string, setH string, setV string) (*http.Response, error) { - req, err := http.NewRequestWithContext(ctx, method, url, nil) +func Request(ctx context.Context, method string, urlStr string, setH string, setV string) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, method, urlStr, nil) if err != nil { return nil, err } From bcbfcb62de68fa80f51d2317efe0b2be85da0ab7 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Thu, 24 Jun 2021 19:48:28 +0900 Subject: [PATCH 31/51] =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/download/download.go | 10 +- kadai3-2/Mizushima/download/download_test.go | 40 +++++ kadai3-2/Mizushima/getheader/geHeader_test.go | 137 ++++++++++++++++++ kadai3-2/Mizushima/getheader/getHeader.go | 21 --- kadai3-2/Mizushima/listen/listen.go | 4 +- kadai3-2/Mizushima/listen/listen_test.go | 39 +++++ kadai3-2/Mizushima/main.go | 9 +- 7 files changed, 229 insertions(+), 31 deletions(-) create mode 100644 kadai3-2/Mizushima/download/download_test.go create mode 100644 kadai3-2/Mizushima/getheader/geHeader_test.go create mode 100644 kadai3-2/Mizushima/listen/listen_test.go diff --git a/kadai3-2/Mizushima/download/download.go b/kadai3-2/Mizushima/download/download.go index 0cdd7f4f..9c5c1684 100644 --- a/kadai3-2/Mizushima/download/download.go +++ b/kadai3-2/Mizushima/download/download.go @@ -67,7 +67,7 @@ func (pd *PDownloader) DownloadFile(ctx context.Context) (err error) { resp, err := request.Request(ctx, "GET", pd.url.String(), "", "") if err != nil { - return err + return } defer func() { err = resp.Body.Close() @@ -75,7 +75,7 @@ func (pd *PDownloader) DownloadFile(ctx context.Context) (err error) { _, err = io.Copy(pd.output, resp.Body) if err != nil { - return err + return } return nil @@ -117,7 +117,7 @@ func (pd *PDownloader) PDownload(grp *errgroup.Group, // ReqToMakeCopy sends a "GET" request with "Range" field with "bytes" range. // And gets response and make a copy to a temprary file in temprary directory from response body. // -func (pd *PDownloader) ReqToMakeCopy(tmpDirName, bytes string, idx uint, ctx context.Context) error { +func (pd *PDownloader) ReqToMakeCopy(tmpDirName, bytes string, idx uint, ctx context.Context) (err error) { // fmt.Printf("ReqToMakeCopy: tmpDirName: %s, bytes %s, idx: %d\n", tmpDirName, bytes, idx) resp, err := request.Request(ctx, "GET", pd.url.String(), "Range", bytes) if err != nil { @@ -130,9 +130,7 @@ func (pd *PDownloader) ReqToMakeCopy(tmpDirName, bytes string, idx uint, ctx con } // fmt.Printf("tmpOut.Name(): %s\n", tmpOut.Name()) defer func() { - if err = tmpOut.Close(); err != nil { - fmt.Fprintf(os.Stderr, "err: tmpOut.Close(): %w", err.Error()) - } + err = tmpOut.Close(); }() body, err := ioutil.ReadAll(resp.Body) diff --git a/kadai3-2/Mizushima/download/download_test.go b/kadai3-2/Mizushima/download/download_test.go new file mode 100644 index 00000000..7d57ff1c --- /dev/null +++ b/kadai3-2/Mizushima/download/download_test.go @@ -0,0 +1,40 @@ +package download_test + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "testing" +) + +func TestDownloader_SingleProcess(t *testing.T) { + +} + +func newTestServer(t *testing.T, + handler func(t *testing.T, w http.ResponseWriter, r *http.Request)) (*httptest.Server, func()) { + + t.Helper() + + ts := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + handler(t, w, r) + }, + )) + + return ts, func() { ts.Close()} +} + +func nonRangeAccessHandler(t *testing.T, w http.ResponseWriter, r *http.Request) { + t.Helper() + + body, err := os.ReadFile("../003") + if err != nil { + t.Fatal(err) + } + w.Header().Set("Content-Length", "311") + w.WriteHeader(http.StatusFound) + fmt.Fprint(w, body) + +} diff --git a/kadai3-2/Mizushima/getheader/geHeader_test.go b/kadai3-2/Mizushima/getheader/geHeader_test.go new file mode 100644 index 00000000..3467c423 --- /dev/null +++ b/kadai3-2/Mizushima/getheader/geHeader_test.go @@ -0,0 +1,137 @@ +package getheader_test + +import ( + "errors" + "net/http" + "os" + "reflect" + "testing" + + "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/getheader" +) + + +var heads1 = []string { + "Content-Length", + "Accept-Ranges", + "Content-Type", + "Access-Control-Allow-Methods", + } + +var heads2 = []string { + "Accept-Ranges", + "Content-Type", + "Access-Control-Allow-Methods", + } + +var vals1 = [][]string{{"146515"}, {"bytes"}, {"image/jpeg"}, {"GET", "OPTIONS"}} + +var vals2 = [][]string{{"bytes"}, {"image/jpeg"}, {"GET", "OPTIONS"}} + +func Test_ResHeader(t *testing.T) { + + cases := []struct { + name string + input string + heads []string + vals [][]string + expected []string + }{ + { + name: "case 1", + input: "Content-Length", + heads: heads1, + vals: vals1, + expected: []string{"146515"}, + }, + { + name: "case 2", + input: "Accept-Ranges", + heads: heads2, + vals: vals2, + expected: []string{"bytes"}, + }, + } + + for _, c := range cases { + c := c + t.Run(c.name, func(t *testing.T) { + resp, err := makeResponse(t, c.heads, c.vals) + if err != nil { + t.Error(err) + } + actual, err := getheader.ResHeader(os.Stdout, resp, c.input) + if err != nil { + t.Error(err) + } + if !reflect.DeepEqual(actual, c.expected) { + t.Errorf("expected %s, but got %s", c.expected, actual) + } + }) + } +} + +func Test_GetSize(t *testing.T) { + + cases := []struct { + name string + heads []string + vals [][]string + expected uint + }{ + { + name: "case 1", + heads: heads1, + vals: vals1, + expected: uint(146515), + }, + { + name: "case 2", + heads: heads2, + vals: vals2, + expected: 0, + }, + } + + for _, c := range cases { + c := c + t.Run(c.name, func(t *testing.T) { + resp, err := makeResponse(t, c.heads, c.vals) + + if err != nil { + t.Error(err) + } + actual, err := getheader.GetSize(resp) + if err != nil { + if actual != 0 || c.expected != 0 { + t.Error(err) + } else { + if err.Error() != "cannot find Content-Length header" { + t.Errorf("expected error: cannot find Content-Length header, but %w", err) + } + } + } + if actual != c.expected { + t.Errorf("expected %d, but got %d", c.expected, actual) + } + }) + } +} + +func makeResponse(t *testing.T, heads []string, vals [][]string) (*http.Response, error) { + var resp = make(map[string][]string) + + if len(heads) != len(vals) { + return nil, errors.New("expected the length of heads and vals sre same") + } + + for i := 0; i < len(heads); i++ { + if _, ok := resp[heads[i]]; ok { + return nil, errors.New("Duplicate elements in heads") + } + + resp[heads[i]] = vals[i] + } + + return &http.Response{Header: http.Header(resp)}, nil +} \ No newline at end of file diff --git a/kadai3-2/Mizushima/getheader/getHeader.go b/kadai3-2/Mizushima/getheader/getHeader.go index b70c627c..1165d5f5 100644 --- a/kadai3-2/Mizushima/getheader/getHeader.go +++ b/kadai3-2/Mizushima/getheader/getHeader.go @@ -9,15 +9,6 @@ import ( "strconv" ) -// Headers returns all headers -func Headers(w io.Writer, r *http.Response) error { - h := r.Header - if _, err := fmt.Fprintln(w, h); err != nil { - return err - } - return nil -} - // ResHeader returns the value of the specified header, and writes http response header on io.Writer. func ResHeader(w io.Writer, r *http.Response, header string) ([]string, error) { h, is := r.Header[header] @@ -31,18 +22,6 @@ func ResHeader(w io.Writer, r *http.Response, header string) ([]string, error) { return h, nil } -// ResHeaderComma returns the value of the specified response header by commas. -func ResHeaderComma(w io.Writer, r *http.Response, header string) (string, error) { - h := r.Header.Get(header) - // if !is { - // return "error", fmt.Errorf("cannot find %s header", header) - // } - if _, err := fmt.Fprintf(w, "Header[%s] = %s\n", header, h); err != nil { - return "", err - } - return h, nil -} - // GetSize returns size from response header. func GetSize(resp *http.Response) (uint, error) { contLen, err := ResHeader(os.Stdout, resp, "Content-Length") diff --git a/kadai3-2/Mizushima/listen/listen.go b/kadai3-2/Mizushima/listen/listen.go index 819179f3..200f6588 100644 --- a/kadai3-2/Mizushima/listen/listen.go +++ b/kadai3-2/Mizushima/listen/listen.go @@ -10,6 +10,8 @@ import ( "syscall" ) +var osExit = os.Exit + // Listen returns a context for keyboad(ctrl + c) interrupt. func Listen(ctx context.Context, w io.Writer, f func()) (context.Context, func()) { ctx, cancel := context.WithCancel(ctx) @@ -25,7 +27,7 @@ func Listen(ctx context.Context, w io.Writer, f func()) (context.Context, func() } cancel() f() - os.Exit(0) + osExit(0) }() return ctx, cancel diff --git a/kadai3-2/Mizushima/listen/listen_test.go b/kadai3-2/Mizushima/listen/listen_test.go new file mode 100644 index 00000000..c8877722 --- /dev/null +++ b/kadai3-2/Mizushima/listen/listen_test.go @@ -0,0 +1,39 @@ +package listen + +import ( + "bytes" + "context" + "os" + "testing" + "time" +) + + +func Test_Listen(t *testing.T) { + cleanFn := func(){} + + doneCh := make(chan struct{}) + osExit = func(code int) { doneCh <- struct{}{} } + + output := new(bytes.Buffer) + _, cancel := Listen(context.Background(), output, cleanFn) + defer cancel() + + proc, err := os.FindProcess(os.Getpid()) + if err != nil { + t.Fatalf("err: %s", err) + } + + err = proc.Signal(os.Interrupt) + if err != nil { + t.Fatalf("err: %s", err) + } + + select { + case <-doneCh: + return + case <-time.After(100 * time.Millisecond): + t.Fatal("timeout") + } + +} \ No newline at end of file diff --git a/kadai3-2/Mizushima/main.go b/kadai3-2/Mizushima/main.go index 81e40f85..b36cfc6c 100644 --- a/kadai3-2/Mizushima/main.go +++ b/kadai3-2/Mizushima/main.go @@ -129,8 +129,12 @@ func main() { } // show response header - h, _ := httputil.DumpResponse(resp, false) - fmt.Printf("response:\n%s", h) + fmt.Printf("response:\n") + if b, err := httputil.DumpResponse(resp, false); err != nil { + log. Fatalf("err: %w", err) + } else { + fmt.Printf("%s\n", b) + } // get the size from the response header. fileSize, err := getheader.GetSize(resp) @@ -172,7 +176,6 @@ func main() { log.Fatalf("err: os.Mkdir: %w\n", err) } - // ctx, cancel := context.WithTimeout(context.Background(),time.Duration(opts.Tm)*time.Minute) clean := func() { if err := out.Close(); err != nil { log.Fatalf("err: out.Close: %w\n", err) From 91a0b2be3091e7b1e6817116b316b761c4b111fa Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Fri, 25 Jun 2021 11:31:55 +0900 Subject: [PATCH 32/51] =?UTF-8?q?download=E3=81=AE=E3=83=86=E3=82=B9?= =?UTF-8?q?=E3=83=88=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/download/download_test.go | 120 ++++++++++++++++++- 1 file changed, 116 insertions(+), 4 deletions(-) diff --git a/kadai3-2/Mizushima/download/download_test.go b/kadai3-2/Mizushima/download/download_test.go index 7d57ff1c..1cf10497 100644 --- a/kadai3-2/Mizushima/download/download_test.go +++ b/kadai3-2/Mizushima/download/download_test.go @@ -1,15 +1,74 @@ package download_test import ( + "bytes" + "context" "fmt" + "io/ioutil" "net/http" "net/http/httptest" + "net/url" "os" + "reflect" + "strconv" "testing" + + "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/download" ) +var testdataPathMap = map[int]string { + 0 : "../documents/003", + 1 : "../documents/z4d4kWk.jpg", + 2 : "../documents/http.request.txt", +} + func TestDownloader_SingleProcess(t *testing.T) { - + ts, clean := newTestServer(t, nonRangeAccessHandler) + defer clean() + + // get a url.URL object + urlObj := getURLObject(t, ts.URL) + + // get a file for output + output, clean := makeTempFile(t) + defer clean() + + resp, err := http.Get(ts.URL) + if err != nil { + t.Error(err) + } + defer resp.Body.Close() + + // get the file size to be downloaded. + size := GetSize(t, resp) + + // this test is non-parallel download. + part := size + procs := uint(1) + isPara := false + tmpDirName := "" + ctx := context.Background() + + t.Run("case 1", func(t *testing.T) { + err := download.Downloader(urlObj, output, size, part, procs, isPara, tmpDirName, ctx) + if err != nil { + t.Error(err) + } + actual := new(bytes.Buffer).Bytes() + _, err = output.Read(actual) + if err != nil { + t.Error(err) + } + expected, err := os.ReadFile(testdataPathMap[0]) + if err != nil { + t.Error(err) + } + + if reflect.DeepEqual(actual, expected) { + t.Errorf("expected %s, but got %s", expected, actual) + } + }) + } func newTestServer(t *testing.T, @@ -23,18 +82,71 @@ func newTestServer(t *testing.T, }, )) - return ts, func() { ts.Close()} + return ts, func() { ts.Close() } } func nonRangeAccessHandler(t *testing.T, w http.ResponseWriter, r *http.Request) { t.Helper() - body, err := os.ReadFile("../003") + body, err := os.ReadFile(testdataPathMap[0]) if err != nil { t.Fatal(err) } w.Header().Set("Content-Length", "311") - w.WriteHeader(http.StatusFound) + w.WriteHeader(http.StatusOK) fmt.Fprint(w, body) +} + +func getURLObject(t *testing.T, urlStr string) *url.URL { + t.Helper() + + urlObj, err := url.ParseRequestURI(urlStr) + if err != nil { + t.Error(err) + } + + return urlObj +} + +func makeTempFile(t *testing.T) (*os.File, func()) { + t.Helper() + dir, err := ioutil.TempDir("", "test_download") + if err != nil { + t.Error(err) + } + + out, err := os.Create(dir+"/test") + if err != nil { + t.Error(err) + } + + return out, + func() { + err = out.Close() + if err != nil { + t.Error(err) + } + err = os.RemoveAll(dir) + if err != nil { + t.Error(err) + } + } +} + +// GetSize returns size from response header. +func GetSize(t *testing.T, r *http.Response) uint { + t.Helper() + + contLen, is := r.Header["Content-Length"] + // fmt.Println(h) + if !is { + t.Errorf("cannot find Content-Length header") + } + + ret, err := strconv.ParseUint(contLen[0], 10, 32) + if err != nil { + t.Error(err) + } + return uint(ret) } From 7d03be47a023d1d6e7c46eaf7e60aaa966590c0f Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Fri, 25 Jun 2021 18:35:52 +0900 Subject: [PATCH 33/51] =?UTF-8?q?download.go=E3=81=AE=E3=83=86=E3=82=B9?= =?UTF-8?q?=E3=83=88=E4=BF=AE=E6=AD=A3=E9=80=94=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/download/download.go | 16 +- kadai3-2/Mizushima/download/download_test.go | 201 +++++++++++++++++- kadai3-2/Mizushima/getheader/geHeader_test.go | 3 +- kadai3-2/Mizushima/listen/listen_test.go | 2 + kadai3-2/Mizushima/main.go | 51 ++--- kadai3-2/Mizushima/request/request.go | 2 +- 6 files changed, 235 insertions(+), 40 deletions(-) diff --git a/kadai3-2/Mizushima/download/download.go b/kadai3-2/Mizushima/download/download.go index 9c5c1684..2b639ffd 100644 --- a/kadai3-2/Mizushima/download/download.go +++ b/kadai3-2/Mizushima/download/download.go @@ -81,7 +81,7 @@ func (pd *PDownloader) DownloadFile(ctx context.Context) (err error) { return nil } -// PDownload drives parallel download. +// PDownload drives parallel download. downloaded file is in temporary directory named tmpDirName. func (pd *PDownloader) PDownload(grp *errgroup.Group, tmpDirName string, procs uint, ctx context.Context) error { var start, end, idx uint @@ -128,15 +128,25 @@ func (pd *PDownloader) ReqToMakeCopy(tmpDirName, bytes string, idx uint, ctx con if err != nil { return err } - // fmt.Printf("tmpOut.Name(): %s\n", tmpOut.Name()) + fmt.Printf("tmpOut.Name(): %s\n", tmpOut.Name()) defer func() { err = tmpOut.Close(); }() + // b := make([]byte, 1000) + // resp.Body.Read(b) + // fmt.Printf("resp.body: %s\n", string(b)) + body, err := ioutil.ReadAll(resp.Body) if err != nil { - return err + // fmt.Printf("err: %s\n", err) + if err != io.EOF && err != io.ErrUnexpectedEOF { + return err + } } + + // fmt.Printf("response body: length: %d\n", len(body)) + length, err := tmpOut.Write(body) if err != nil { return err diff --git a/kadai3-2/Mizushima/download/download_test.go b/kadai3-2/Mizushima/download/download_test.go index 1cf10497..7e18d70a 100644 --- a/kadai3-2/Mizushima/download/download_test.go +++ b/kadai3-2/Mizushima/download/download_test.go @@ -3,6 +3,7 @@ package download_test import ( "bytes" "context" + "errors" "fmt" "io/ioutil" "net/http" @@ -10,19 +11,24 @@ import ( "net/url" "os" "reflect" + "runtime" "strconv" + "strings" "testing" + "time" "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/download" ) -var testdataPathMap = map[int]string { - 0 : "../documents/003", - 1 : "../documents/z4d4kWk.jpg", - 2 : "../documents/http.request.txt", +var testdataPathMap = map[int][]string { + 0 : {"../documents/003","311"}, + 1 : {"../documents/z4d4kWk.jpg", "146515"}, + // 2 : "../documents/http.request.txt", } func TestDownloader_SingleProcess(t *testing.T) { + t.Helper() + ts, clean := newTestServer(t, nonRangeAccessHandler) defer clean() @@ -54,12 +60,14 @@ func TestDownloader_SingleProcess(t *testing.T) { if err != nil { t.Error(err) } + actual := new(bytes.Buffer).Bytes() _, err = output.Read(actual) if err != nil { t.Error(err) } - expected, err := os.ReadFile(testdataPathMap[0]) + + expected, err := os.ReadFile(testdataPathMap[0][0]) if err != nil { t.Error(err) } @@ -68,7 +76,128 @@ func TestDownloader_SingleProcess(t *testing.T) { t.Errorf("expected %s, but got %s", expected, actual) } }) +} + +func TestDownloader_SingleProcessTimeout(t *testing.T) { + t.Helper() + + ts, clean := newTestServer(t, nonRangeAccessTooLateHandler) + defer clean() + + // get a url.URL object + urlObj := getURLObject(t, ts.URL) + + // get a file for output + output, clean := makeTempFile(t) + defer clean() + + resp, err := http.Get(ts.URL) + if err != nil { + t.Error(err) + } + defer resp.Body.Close() + + // get the file size to be downloaded. + size := GetSize(t, resp) + + // this test is non-parallel download. + part := size + procs := uint(1) + isPara := false + tmpDirName := "" + ctx, cancel := context.WithTimeout(context.Background(), 1 * time.Second) + defer cancel() + t.Run("case 1", func(t *testing.T) { + actual := download.Downloader(urlObj, output, size, part, procs, isPara, tmpDirName, ctx) + if err != nil { + t.Error(err) + } + expected := fmt.Errorf("request.Request err: Get \"%s\": %w", urlObj, context.DeadlineExceeded) + if actual.Error() != expected.Error() { + t.Errorf("expected %s, \nbut got %s", expected, actual) + } + }) +} + +func TestDownloader_ParallelProcess(t *testing.T) { + t.Helper() + + ts, clean := newTestServer(t, rangeAccessHandler) + defer clean() + + // get a url.URL object + urlObj := getURLObject(t, ts.URL) + + // get a file for output + output, clean := makeTempFile(t) + defer clean() + + resp, err := http.Get(ts.URL) + if err != nil { + t.Error(err) + } + defer resp.Body.Close() + + // get the file size to be downloaded. + size := GetSize(t, resp) + + // this test is non-parallel download. + procs := uint(runtime.NumCPU()) + part := size / procs + isPara := true + tmpDirName := "test" + ctx := context.Background() + + t.Run("case 1", func(t *testing.T) { + if err := os.Mkdir(tmpDirName, 0775); err != nil { + t.Error(err) + } + err := download.Downloader(urlObj, output, size, part, procs, isPara, tmpDirName, ctx) + if err != nil { + t.Errorf("err: %w", err) + } + defer func() { + err := os.RemoveAll(tmpDirName) + if err != nil { + t.Error(err) + } + }() + + // make expected data + byteData, err := os.ReadFile(testdataPathMap[1][0]) + if err != nil { + t.Error(err) + } + + for i := uint(0); i < procs; i++ { + var start, end uint + if i == 0 { + start = 0 + } else { + start = i * part + uint(1) + } + + if end == procs -1 { + end = size + } else { + end = (i + 1) * part + } + + expected := byteData[start:end+1] + fmt.Printf("length of expected: %d\n", len(expected)) + + actual, err := os.ReadFile(tmpDirName+"/"+strconv.Itoa(int(i))) + if err != nil { + t.Error(err) + } + fmt.Printf("length of actual: %d\n", len(actual)) + + if !reflect.DeepEqual(actual, expected) { + t.Error("expected is not equal to actual") + } + } + }) } func newTestServer(t *testing.T, @@ -88,15 +217,73 @@ func newTestServer(t *testing.T, func nonRangeAccessHandler(t *testing.T, w http.ResponseWriter, r *http.Request) { t.Helper() - body, err := os.ReadFile(testdataPathMap[0]) + body, err := os.ReadFile(testdataPathMap[0][0]) if err != nil { t.Fatal(err) } - w.Header().Set("Content-Length", "311") + w.Header().Set("Content-Length", testdataPathMap[0][1]) w.WriteHeader(http.StatusOK) fmt.Fprint(w, body) } +func nonRangeAccessTooLateHandler(t *testing.T, w http.ResponseWriter, r *http.Request) { + t.Helper() + + body, err := os.ReadFile(testdataPathMap[0][0]) + if err != nil { + t.Fatal(err) + } + + time.Sleep(3 * time.Second) + w.Header().Set("Content-Length", testdataPathMap[0][1]) + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, body) +} + +func rangeAccessHandler(t *testing.T, w http.ResponseWriter, r *http.Request) { + t.Helper() + + w.Header().Set("Content-Length", testdataPathMap[1][1]) + w.Header().Set("Access-Range", "bytes") + + rangeHeader := r.Header.Get("Range") + + body := retBody(t, rangeHeader, testdataPathMap[1][0]) + w.WriteHeader(http.StatusPartialContent) + fmt.Fprint(w, body) +} + +func retBody(t *testing.T, rangeHeader string, testdataPath string) []byte { + b, err := os.ReadFile(testdataPath) + if err != nil { + t.Fatal(err) + } + + if rangeHeader == "" { + return b + } + + rangeVals := strings.Split(rangeHeader, "=") + if rangeVals[0] != "bytes" { + t.Fatal(errors.New("err : Range header expected \"bytes\"")) + } + + rangeBytes := strings.Split(rangeVals[1], "-") + start, err := strconv.Atoi(rangeBytes[0]) + if err != nil { + t.Fatal(err) + } + + end, err := strconv.Atoi(rangeBytes[1]) + if err != nil { + t.Fatal(err) + } + + fmt.Printf("length of b: %d\n", len(b)) + fmt.Printf("length of b[start:end+1]: %d\n", len(b[start:end+1])) + return b[start:end+1] +} + func getURLObject(t *testing.T, urlStr string) *url.URL { t.Helper() diff --git a/kadai3-2/Mizushima/getheader/geHeader_test.go b/kadai3-2/Mizushima/getheader/geHeader_test.go index 3467c423..e1f401bb 100644 --- a/kadai3-2/Mizushima/getheader/geHeader_test.go +++ b/kadai3-2/Mizushima/getheader/geHeader_test.go @@ -29,7 +29,8 @@ var vals1 = [][]string{{"146515"}, {"bytes"}, {"image/jpeg"}, {"GET", "OPTIONS"} var vals2 = [][]string{{"bytes"}, {"image/jpeg"}, {"GET", "OPTIONS"}} func Test_ResHeader(t *testing.T) { - + t.Helper() + cases := []struct { name string input string diff --git a/kadai3-2/Mizushima/listen/listen_test.go b/kadai3-2/Mizushima/listen/listen_test.go index c8877722..78fca902 100644 --- a/kadai3-2/Mizushima/listen/listen_test.go +++ b/kadai3-2/Mizushima/listen/listen_test.go @@ -10,6 +10,8 @@ import ( func Test_Listen(t *testing.T) { + t.Helper() + cleanFn := func(){} doneCh := make(chan struct{}) diff --git a/kadai3-2/Mizushima/main.go b/kadai3-2/Mizushima/main.go index b36cfc6c..81766b54 100644 --- a/kadai3-2/Mizushima/main.go +++ b/kadai3-2/Mizushima/main.go @@ -22,11 +22,6 @@ import ( "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/request" ) -// for test -// go run main.go https://4.bp.blogspot.com/-2-Ny23XgrF0/Ws69gszw2jI/AAAAAAABLdU/unbzWD_U8foWBwPKWQdGP1vEDoQoYjgZwCLcBGAs/s1600/top_banner.jpg -o kadai3-2 -// go run main.go http://i.imgur.com/z4d4kWk.jpg -o . -// go run main.go https://misc.laboradian.com/test/003/ -o . - // struct for options type Options struct { Help bool `short:"h" long:"help"` @@ -42,7 +37,7 @@ func (opts *Options) parse(argv []string) ([]string, error) { if err != nil { _, err2 := os.Stderr.Write(opts.usage()) if err2 != nil { - return nil, fmt.Errorf("%w: invalid command line options: cannot print usage: %w", err, err2) + return nil, fmt.Errorf("%s: invalid command line options: cannot print usage: %s", err, err2) } return nil, fmt.Errorf("%w: invalid command line options", err) } @@ -75,21 +70,21 @@ func main() { argv := os.Args[1:] if len(argv) == 0 { if _, err := os.Stdout.Write(opts.usage()); err != nil { - log.Fatalf("err: %w: %w\n", errors.New("no options"), err) + log.Fatalf("err: %s: %s\n", errors.New("no options"), err) } - log.Fatalf("err: %w\n", errors.New("no options")) + log.Fatalf("err: %s\n", errors.New("no options")) } urlsStr, err := opts.parse(argv) if err != nil { - log.Fatalf("err: %w\n", err) + log.Fatalf("err: %s\n", err) } var urls []*url.URL for _, u := range urlsStr { url, err := url.ParseRequestURI(u) if err != nil { - log.Fatalf("err: url.ParseRequestURI: %w\n", err) + log.Fatalf("err: url.ParseRequestURI: %s\n", err) } urls = append(urls, url) } @@ -98,7 +93,7 @@ func main() { if opts.Help { if _, err := os.Stdout.Write(opts.usage()); err != nil { - log.Fatalf("err: cannot print usage: %w", err) + log.Fatalf("err: cannot print usage: %s", err) } log.Fatal(errors.New("print usage")) } @@ -125,13 +120,13 @@ func main() { // send "HEAD" request, and gets response. resp, err := request.Request(ctxTimeout, "HEAD", urlObj.String(), "", "") if err != nil { - log.Fatalf("err: %w\n", err) + log.Fatalf("err: %s\n", err) } // show response header fmt.Printf("response:\n") if b, err := httputil.DumpResponse(resp, false); err != nil { - log. Fatalf("err: %w", err) + log. Fatalf("err: %s", err) } else { fmt.Printf("%s\n", b) } @@ -139,10 +134,10 @@ func main() { // get the size from the response header. fileSize, err := getheader.GetSize(resp) if err != nil { - log.Fatalf("err: getheader.GetSize: %w\n", err) + log.Fatalf("err: getheader.GetSize: %s\n", err) } if err = resp.Body.Close(); err != nil { - log.Fatalf("err: %w", err) + log.Fatalf("err: %s", err) } // How many bytes to download at a time @@ -153,14 +148,14 @@ func main() { if isExists(outputPath) { err := os.Remove(outputPath) if err != nil { - log.Fatalf("err: isExists: os.Remove: %w\n", err) + log.Fatalf("err: isExists: os.Remove: %s\n", err) } } // make a file for download out, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0755) if err != nil { - log.Fatalf("err: os.Create: %w\n", err) + log.Fatalf("err: os.Create: %s\n", err) } // make a temporary directory for parallel download @@ -168,24 +163,24 @@ func main() { err = os.Mkdir(tmpDirName, 0777) if err != nil { if err3 := out.Close(); err3 != nil { - log.Fatalf("err: %w", err3) + log.Fatalf("err: %s", err3) } if err2 := os.Remove(opts.Output + filepath.Base(urlObj.String())); err2 != nil { - log.Fatalf("err: os.Mkdir: %w\nerr: os.Remove: %w\n", err, err2) + log.Fatalf("err: os.Mkdir: %s\nerr: os.Remove: %s\n", err, err2) } - log.Fatalf("err: os.Mkdir: %w\n", err) + log.Fatalf("err: os.Mkdir: %s\n", err) } clean := func() { if err := out.Close(); err != nil { - log.Fatalf("err: out.Close: %w\n", err) + log.Fatalf("err: out.Close: %s\n", err) } // delete the tmporary directory if err := os.RemoveAll(tmpDirName); err != nil { - log.Fatalf("err: RemoveAll: %w\n", err) + log.Fatalf("err: RemoveAll: %s\n", err) } if err := os.Remove(opts.Output + filepath.Base(urlObj.String())); err != nil { - log.Fatalf("err: os.Remove: %w\n", err) + log.Fatalf("err: os.Remove: %s\n", err) } } ctx, cancel := listen.Listen(ctxTimeout, os.Stdout, clean) @@ -196,7 +191,7 @@ func main() { isPara = false } else if err != nil { clean() - log.Fatalf("err: getheader.ResHeader: %w\n", err) + log.Fatalf("err: getheader.ResHeader: %s\n", err) } else if accept[0] != "bytes" || opts.Procs == 1 { isPara = false continue @@ -205,7 +200,7 @@ func main() { // drive a download process err = download.Downloader(urlObj, out, fileSize, partial, opts.Procs, isPara, tmpDirName, ctx) if err != nil { - log.Fatalf("err: %w\n", err) + log.Fatalf("err: %s\n", err) } fmt.Printf("download complete: %s\n", urlObj.String()) @@ -214,18 +209,18 @@ func main() { if isPara { err = MergeFiles(tmpDirName, opts.Procs, fileSize, out) if err != nil { - log.Fatalf("err: MergeFiles: %w\n", err) + log.Fatalf("err: MergeFiles: %s\n", err) } } // delete the tmporary directory only if err := os.RemoveAll(tmpDirName); err != nil { - log.Fatalf("err: RemoveAll: %w\n", err) + log.Fatalf("err: RemoveAll: %s\n", err) } cancel() if err = out.Close(); err != nil { - log.Fatalf("err: %w", err) + log.Fatalf("err: %s", err) } } } diff --git a/kadai3-2/Mizushima/request/request.go b/kadai3-2/Mizushima/request/request.go index d899ce6c..865c0bcc 100644 --- a/kadai3-2/Mizushima/request/request.go +++ b/kadai3-2/Mizushima/request/request.go @@ -6,7 +6,7 @@ import ( "net/http" ) -// Request throws a request and returns a response from url and a error. +// Request throws a request and returns a response object from url and a error. func Request(ctx context.Context, method string, urlStr string, setH string, setV string) (*http.Response, error) { req, err := http.NewRequestWithContext(ctx, method, urlStr, nil) if err != nil { From a3c94f98a13e263bda30cd771a9df03cda17a79e Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Fri, 25 Jun 2021 18:38:18 +0900 Subject: [PATCH 34/51] =?UTF-8?q?go=20fmt=E9=81=A9=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kadai3-2/Mizushima/main.go b/kadai3-2/Mizushima/main.go index 81766b54..29a5031f 100644 --- a/kadai3-2/Mizushima/main.go +++ b/kadai3-2/Mizushima/main.go @@ -126,7 +126,7 @@ func main() { // show response header fmt.Printf("response:\n") if b, err := httputil.DumpResponse(resp, false); err != nil { - log. Fatalf("err: %s", err) + log.Fatalf("err: %s", err) } else { fmt.Printf("%s\n", b) } From 5a04cf266d2481a7ac84ce01eeb7f2a9db7990e8 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Mon, 28 Jun 2021 19:19:15 +0900 Subject: [PATCH 35/51] =?UTF-8?q?=E3=83=80=E3=82=A6=E3=83=B3=E3=83=AD?= =?UTF-8?q?=E3=83=BC=E3=83=89=E3=81=AE=E3=83=86=E3=82=B9=E3=83=88=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0=E3=80=81request.go=E3=81=AE=E3=83=86=E3=82=B9?= =?UTF-8?q?=E3=83=88=E8=BF=BD=E5=8A=A0=EF=BC=88=E9=80=94=E4=B8=AD=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/download/download.go | 9 +- kadai3-2/Mizushima/download/download_test.go | 117 ++++------- kadai3-2/Mizushima/request/request_test.go | 201 +++++++++++++++++++ 3 files changed, 250 insertions(+), 77 deletions(-) create mode 100644 kadai3-2/Mizushima/request/request_test.go diff --git a/kadai3-2/Mizushima/download/download.go b/kadai3-2/Mizushima/download/download.go index 2b639ffd..e36685ac 100644 --- a/kadai3-2/Mizushima/download/download.go +++ b/kadai3-2/Mizushima/download/download.go @@ -17,7 +17,7 @@ import ( //PDownloader is user-defined struct type PDownloader struct { - url *url.URL // URL for the download + url *url.URL // URL for the download output *os.File // Where to save the downloaded file fileSize uint // size of the downloaded file part uint // Number of divided bytes @@ -81,7 +81,8 @@ func (pd *PDownloader) DownloadFile(ctx context.Context) (err error) { return nil } -// PDownload drives parallel download. downloaded file is in temporary directory named tmpDirName. +// PDownload drives parallel download. downloaded file is in temporary +// directory named tmpDirName. func (pd *PDownloader) PDownload(grp *errgroup.Group, tmpDirName string, procs uint, ctx context.Context) error { var start, end, idx uint @@ -128,9 +129,9 @@ func (pd *PDownloader) ReqToMakeCopy(tmpDirName, bytes string, idx uint, ctx con if err != nil { return err } - fmt.Printf("tmpOut.Name(): %s\n", tmpOut.Name()) + // fmt.Printf("tmpOut.Name(): %s\n", tmpOut.Name()) defer func() { - err = tmpOut.Close(); + err = tmpOut.Close() }() // b := make([]byte, 1000) diff --git a/kadai3-2/Mizushima/download/download_test.go b/kadai3-2/Mizushima/download/download_test.go index 7e18d70a..f2131281 100644 --- a/kadai3-2/Mizushima/download/download_test.go +++ b/kadai3-2/Mizushima/download/download_test.go @@ -20,16 +20,16 @@ import ( "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/download" ) -var testdataPathMap = map[int][]string { - 0 : {"../documents/003","311"}, - 1 : {"../documents/z4d4kWk.jpg", "146515"}, +var testdataPathMap = map[int][]string{ + 0: {"../documents/003", "311"}, + 1: {"../documents/z4d4kWk.jpg", "146515"}, // 2 : "../documents/http.request.txt", } func TestDownloader_SingleProcess(t *testing.T) { t.Helper() - ts, clean := newTestServer(t, nonRangeAccessHandler) + ts, clean := newTestServer(t, nonRangeAccessHandler, 0) defer clean() // get a url.URL object @@ -46,7 +46,7 @@ func TestDownloader_SingleProcess(t *testing.T) { defer resp.Body.Close() // get the file size to be downloaded. - size := GetSize(t, resp) + size := getSizeForTest(t, resp) // this test is non-parallel download. part := size @@ -81,7 +81,7 @@ func TestDownloader_SingleProcess(t *testing.T) { func TestDownloader_SingleProcessTimeout(t *testing.T) { t.Helper() - ts, clean := newTestServer(t, nonRangeAccessTooLateHandler) + ts, clean := newTestServer(t, nonRangeAccessTooLateHandler, 0) defer clean() // get a url.URL object @@ -98,14 +98,14 @@ func TestDownloader_SingleProcessTimeout(t *testing.T) { defer resp.Body.Close() // get the file size to be downloaded. - size := GetSize(t, resp) + size := getSizeForTest(t, resp) // this test is non-parallel download. part := size procs := uint(1) isPara := false tmpDirName := "" - ctx, cancel := context.WithTimeout(context.Background(), 1 * time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() t.Run("case 1", func(t *testing.T) { @@ -123,7 +123,8 @@ func TestDownloader_SingleProcessTimeout(t *testing.T) { func TestDownloader_ParallelProcess(t *testing.T) { t.Helper() - ts, clean := newTestServer(t, rangeAccessHandler) + // make a server for test. + ts, clean := newTestServer(t, rangeAccessHandler, 1) defer clean() // get a url.URL object @@ -133,6 +134,7 @@ func TestDownloader_ParallelProcess(t *testing.T) { output, clean := makeTempFile(t) defer clean() + // get a response from test server. resp, err := http.Get(ts.URL) if err != nil { t.Error(err) @@ -140,9 +142,9 @@ func TestDownloader_ParallelProcess(t *testing.T) { defer resp.Body.Close() // get the file size to be downloaded. - size := GetSize(t, resp) + size := getSizeForTest(t, resp) - // this test is non-parallel download. + // this test is parallel download. procs := uint(runtime.NumCPU()) part := size / procs isPara := true @@ -163,98 +165,67 @@ func TestDownloader_ParallelProcess(t *testing.T) { t.Error(err) } }() - - // make expected data - byteData, err := os.ReadFile(testdataPathMap[1][0]) - if err != nil { - t.Error(err) - } - - for i := uint(0); i < procs; i++ { - var start, end uint - if i == 0 { - start = 0 - } else { - start = i * part + uint(1) - } - - if end == procs -1 { - end = size - } else { - end = (i + 1) * part - } - - expected := byteData[start:end+1] - fmt.Printf("length of expected: %d\n", len(expected)) - - actual, err := os.ReadFile(tmpDirName+"/"+strconv.Itoa(int(i))) - if err != nil { - t.Error(err) - } - fmt.Printf("length of actual: %d\n", len(actual)) - - if !reflect.DeepEqual(actual, expected) { - t.Error("expected is not equal to actual") - } - } }) } -func newTestServer(t *testing.T, - handler func(t *testing.T, w http.ResponseWriter, r *http.Request)) (*httptest.Server, func()) { - +func newTestServer(t *testing.T, + handler func(t *testing.T, w http.ResponseWriter, r *http.Request, testDataKey int), + testDataKey int) (*httptest.Server, func()) { + t.Helper() ts := httptest.NewServer(http.HandlerFunc( func(w http.ResponseWriter, r *http.Request) { - handler(t, w, r) + handler(t, w, r, testDataKey) }, )) - + return ts, func() { ts.Close() } } -func nonRangeAccessHandler(t *testing.T, w http.ResponseWriter, r *http.Request) { +func nonRangeAccessHandler(t *testing.T, w http.ResponseWriter, r *http.Request, testDataKey int) { t.Helper() - body, err := os.ReadFile(testdataPathMap[0][0]) + body, err := os.ReadFile(testdataPathMap[testDataKey][0]) if err != nil { t.Fatal(err) } - w.Header().Set("Content-Length", testdataPathMap[0][1]) + w.Header().Set("Content-Length", testdataPathMap[testDataKey][1]) w.WriteHeader(http.StatusOK) fmt.Fprint(w, body) } -func nonRangeAccessTooLateHandler(t *testing.T, w http.ResponseWriter, r *http.Request) { +func nonRangeAccessTooLateHandler(t *testing.T, w http.ResponseWriter, r *http.Request, testDataKey int) { t.Helper() - body, err := os.ReadFile(testdataPathMap[0][0]) + body, err := os.ReadFile(testdataPathMap[testDataKey][0]) if err != nil { t.Fatal(err) } time.Sleep(3 * time.Second) - w.Header().Set("Content-Length", testdataPathMap[0][1]) + w.Header().Set("Content-Length", testdataPathMap[testDataKey][1]) w.WriteHeader(http.StatusOK) fmt.Fprint(w, body) } -func rangeAccessHandler(t *testing.T, w http.ResponseWriter, r *http.Request) { +func rangeAccessHandler(t *testing.T, w http.ResponseWriter, r *http.Request, testDataKey int) { t.Helper() - w.Header().Set("Content-Length", testdataPathMap[1][1]) + w.Header().Set("Content-Length", testdataPathMap[testDataKey][1]) w.Header().Set("Access-Range", "bytes") - + rangeHeader := r.Header.Get("Range") - body := retBody(t, rangeHeader, testdataPathMap[1][0]) + body := retBody(t, rangeHeader, testdataPathMap[testDataKey][0]) w.WriteHeader(http.StatusPartialContent) fmt.Fprint(w, body) } -func retBody(t *testing.T, rangeHeader string, testdataPath string) []byte { - b, err := os.ReadFile(testdataPath) +func retBody(t *testing.T, rangeHeader string, testDataPath string) []byte { + t.Helper() + + b, err := os.ReadFile(testDataPath) if err != nil { t.Fatal(err) } @@ -279,9 +250,9 @@ func retBody(t *testing.T, rangeHeader string, testdataPath string) []byte { t.Fatal(err) } - fmt.Printf("length of b: %d\n", len(b)) - fmt.Printf("length of b[start:end+1]: %d\n", len(b[start:end+1])) - return b[start:end+1] + // fmt.Printf("length of b: %d\n", len(b)) + // fmt.Printf("length of b[start:end+1]: %d\n", len(b[start:end+1])) + return b[start : end+1] } func getURLObject(t *testing.T, urlStr string) *url.URL { @@ -300,29 +271,29 @@ func makeTempFile(t *testing.T) (*os.File, func()) { dir, err := ioutil.TempDir("", "test_download") if err != nil { - t.Error(err) + t.Fatal(err) } - out, err := os.Create(dir+"/test") + out, err := os.Create(dir + "/test") if err != nil { - t.Error(err) + t.Fatal(err) } - return out, + return out, func() { err = out.Close() if err != nil { - t.Error(err) + t.Fatal(err) } err = os.RemoveAll(dir) if err != nil { - t.Error(err) + t.Fatal(err) } } } // GetSize returns size from response header. -func GetSize(t *testing.T, r *http.Response) uint { +func getSizeForTest(t *testing.T, r *http.Response) uint { t.Helper() contLen, is := r.Header["Content-Length"] @@ -330,7 +301,7 @@ func GetSize(t *testing.T, r *http.Response) uint { if !is { t.Errorf("cannot find Content-Length header") } - + ret, err := strconv.ParseUint(contLen[0], 10, 32) if err != nil { t.Error(err) diff --git a/kadai3-2/Mizushima/request/request_test.go b/kadai3-2/Mizushima/request/request_test.go new file mode 100644 index 00000000..2617e9eb --- /dev/null +++ b/kadai3-2/Mizushima/request/request_test.go @@ -0,0 +1,201 @@ +package request_test + +import ( + "errors" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + "os" + "reflect" + "strconv" + "strings" + "testing" + "time" + + "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/request" +) + +var testdataPathMap = map[int][]string{ + 0: {"../documents/003", "311"}, + 1: {"../documents/z4d4kWk.jpg", "146515"}, + // 2 : "../documents/http.request.txt", +} + +func Test_Request(t *testing.T) { + t.Helper() + + cases := []struct { + name string + key int // key for testdataPathMap + handler http.HandlerFunc + expected *http.Response + }{ + { + name: "case 1", + key: 0, + handler: nonRangeAccessHandler, + expected: &http.Response{}, + }, + { + name: "case 2", + key: 1, + handler: rangeAccessHandler, + expected: &http.Response{}, + }, + } + + for _, c := range cases { + c := c + t.Run(c.name, func(t *testing.T) { + ts, clean := newTestServer(t, c.handler, c.key) + actual, err := request.Request(context.BackGround(), "GET", ts.URL, "", "") + if reflect.DeepEqual(actual, c.expected) { + t.Errorf("expected %v, but got %v", c.expected, actual) + } + }) + } +} + + +func newTestServer(t *testing.T, + handler func(t *testing.T, w http.ResponseWriter, r *http.Request, testDataKey int), + testDataKey int) (*httptest.Server, func()) { + + t.Helper() + + ts := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + handler(t, w, r, testDataKey) + }, + )) + + return ts, func() { ts.Close() } +} + +func nonRangeAccessHandler(t *testing.T, w http.ResponseWriter, r *http.Request, testDataKey int) { + t.Helper() + + body, err := os.ReadFile(testdataPathMap[testDataKey][0]) + if err != nil { + t.Fatal(err) + } + w.Header().Set("Content-Length", testdataPathMap[testDataKey][1]) + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, body) +} + +func nonRangeAccessTooLateHandler(t *testing.T, w http.ResponseWriter, r *http.Request, testDataKey int) { + t.Helper() + + body, err := os.ReadFile(testdataPathMap[testDataKey][0]) + if err != nil { + t.Fatal(err) + } + + time.Sleep(3 * time.Second) + w.Header().Set("Content-Length", testdataPathMap[testDataKey][1]) + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, body) +} + +func rangeAccessHandler(t *testing.T, w http.ResponseWriter, r *http.Request, testDataKey int) { + t.Helper() + + w.Header().Set("Content-Length", testdataPathMap[testDataKey][1]) + w.Header().Set("Access-Range", "bytes") + + rangeHeader := r.Header.Get("Range") + + body := retBody(t, rangeHeader, testdataPathMap[testDataKey][0]) + w.WriteHeader(http.StatusPartialContent) + fmt.Fprint(w, body) +} + +func retBody(t *testing.T, rangeHeader string, testDataPath string) []byte { + t.Helper() + + b, err := os.ReadFile(testDataPath) + if err != nil { + t.Fatal(err) + } + + if rangeHeader == "" { + return b + } + + rangeVals := strings.Split(rangeHeader, "=") + if rangeVals[0] != "bytes" { + t.Fatal(errors.New("err : Range header expected \"bytes\"")) + } + + rangeBytes := strings.Split(rangeVals[1], "-") + start, err := strconv.Atoi(rangeBytes[0]) + if err != nil { + t.Fatal(err) + } + + end, err := strconv.Atoi(rangeBytes[1]) + if err != nil { + t.Fatal(err) + } + + // fmt.Printf("length of b: %d\n", len(b)) + // fmt.Printf("length of b[start:end+1]: %d\n", len(b[start:end+1])) + return b[start : end+1] +} + +func getURLObject(t *testing.T, urlStr string) *url.URL { + t.Helper() + + urlObj, err := url.ParseRequestURI(urlStr) + if err != nil { + t.Error(err) + } + + return urlObj +} + +func makeTempFile(t *testing.T) (*os.File, func()) { + t.Helper() + + dir, err := ioutil.TempDir("", "test_download") + if err != nil { + t.Fatal(err) + } + + out, err := os.Create(dir + "/test") + if err != nil { + t.Fatal(err) + } + + return out, + func() { + err = out.Close() + if err != nil { + t.Fatal(err) + } + err = os.RemoveAll(dir) + if err != nil { + t.Fatal(err) + } + } +} + +// GetSize returns size from response header. +func getSizeForTest(t *testing.T, r *http.Response) uint { + t.Helper() + + contLen, is := r.Header["Content-Length"] + // fmt.Println(h) + if !is { + t.Errorf("cannot find Content-Length header") + } + + ret, err := strconv.ParseUint(contLen[0], 10, 32) + if err != nil { + t.Error(err) + } + return uint(ret) +} \ No newline at end of file From 14102f589aa0b66faa138411695e7a9cf5db10d7 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Tue, 6 Jul 2021 14:40:08 +0900 Subject: [PATCH 36/51] =?UTF-8?q?test=E3=81=AE=E8=A8=98=E8=BF=B0=E3=82=92?= =?UTF-8?q?=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/Makefile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/kadai3-2/Mizushima/Makefile b/kadai3-2/Mizushima/Makefile index 3220a6ef..bca1571b 100644 --- a/kadai3-2/Mizushima/Makefile +++ b/kadai3-2/Mizushima/Makefile @@ -11,9 +11,7 @@ build: $(GOBUILD) -o $(BINARY_NAME) -v test: - $(BINARY_NAME) http://i.imgur.com/z4d4kWk.jpg -o ./test_download - $(BINARY_NAME) http://socia-enterprise.co.jp -o ./test_download - $(BINARY_NAME) http://i.imgur.com/z4d4kWk.jpg http://socia-enterprise.co.jp -o ./test_download + $(BINARY_NAME) https://golang.org/dl/go1.16.5.linux-amd64.tar.gz -o ./test_download clean: $(GOCLEAN) From 12869499933601cab721d158cd1faf010f731011 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Mon, 12 Jul 2021 18:32:10 +0900 Subject: [PATCH 37/51] =?UTF-8?q?getheader=5Ftest.go=E3=81=ABt.Helper()?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0=E3=80=81request=5Ftest.go=E3=81=AETest=5FReq?= =?UTF-8?q?uest=E9=96=A2=E6=95=B0=E3=81=AE=E4=BD=9C=E6=88=90=EF=BC=88?= =?UTF-8?q?=E9=80=94=E4=B8=AD=EF=BC=89=E3=80=81gitignore=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/.gitignore | 2 +- kadai3-2/Mizushima/getheader/geHeader_test.go | 2 + kadai3-2/Mizushima/request/request_test.go | 65 ++++++++++++++----- 3 files changed, 50 insertions(+), 19 deletions(-) diff --git a/kadai3-2/Mizushima/.gitignore b/kadai3-2/Mizushima/.gitignore index eb5867b3..6eb01878 100644 --- a/kadai3-2/Mizushima/.gitignore +++ b/kadai3-2/Mizushima/.gitignore @@ -1,4 +1,4 @@ *.jpg documents bin/* -test_download/* \ No newline at end of file +test_download/* diff --git a/kadai3-2/Mizushima/getheader/geHeader_test.go b/kadai3-2/Mizushima/getheader/geHeader_test.go index e1f401bb..00abc360 100644 --- a/kadai3-2/Mizushima/getheader/geHeader_test.go +++ b/kadai3-2/Mizushima/getheader/geHeader_test.go @@ -120,6 +120,8 @@ func Test_GetSize(t *testing.T) { } func makeResponse(t *testing.T, heads []string, vals [][]string) (*http.Response, error) { + t.Helper() + var resp = make(map[string][]string) if len(heads) != len(vals) { diff --git a/kadai3-2/Mizushima/request/request_test.go b/kadai3-2/Mizushima/request/request_test.go index 2617e9eb..63b29134 100644 --- a/kadai3-2/Mizushima/request/request_test.go +++ b/kadai3-2/Mizushima/request/request_test.go @@ -1,11 +1,14 @@ package request_test import ( + "context" "errors" "fmt" "io/ioutil" + "log" "net/http" "net/http/httptest" + "net/http/httputil" "net/url" "os" "reflect" @@ -18,31 +21,35 @@ import ( ) var testdataPathMap = map[int][]string{ - 0: {"../documents/003", "311"}, - 1: {"../documents/z4d4kWk.jpg", "146515"}, - // 2 : "../documents/http.request.txt", + 0: {"../documents/003", mustGetSize("../documents/003")}, + 1: {"../documents/z4d4kWk.jpg", mustGetSize("../documents/z4d4kWk.jpg")}, } func Test_Request(t *testing.T) { t.Helper() cases := []struct { - name string - key int // key for testdataPathMap - handler http.HandlerFunc + name string + key int // key for testdataPathMap + handler func(t *testing.T, w http.ResponseWriter, r *http.Request, testDataKey int) expected *http.Response }{ { - name: "case 1", - key: 0, - handler: nonRangeAccessHandler, + name: "case 1", + key: 0, + handler: nonRangeAccessHandler, expected: &http.Response{}, }, { - name: "case 2", - key: 1, - handler: rangeAccessHandler, - expected: &http.Response{}, + name: "case 2", + key: 1, + handler: rangeAccessHandler, + // http.Responseを作る方法を調べる + expected: &http.Response{ + Status: "206 Partial Content", + StatusCode: 206, + Proto: "HTTP/1.1", + }, }, } @@ -50,15 +57,27 @@ func Test_Request(t *testing.T) { c := c t.Run(c.name, func(t *testing.T) { ts, clean := newTestServer(t, c.handler, c.key) - actual, err := request.Request(context.BackGround(), "GET", ts.URL, "", "") - if reflect.DeepEqual(actual, c.expected) { - t.Errorf("expected %v, but got %v", c.expected, actual) + defer clean() + actual, err := request.Request(context.Background(), "GET", ts.URL, "", "") + if err != nil { + t.Fatal(err) + } + fmt.Println("actual:", actual.Header) + if !reflect.DeepEqual(actual.Header, c.expected.Header) { + dumped_expected, err := httputil.DumpResponse(c.expected, false) + if err != nil { + t.Fatal(err) + } + dumped_actual, err := httputil.DumpResponse(actual, false) + if err != nil { + t.Fatal(err) + } + t.Errorf("expected,\n%vbut got,\n%v", string(dumped_expected), string(dumped_actual)) } }) } } - func newTestServer(t *testing.T, handler func(t *testing.T, w http.ResponseWriter, r *http.Request, testDataKey int), testDataKey int) (*httptest.Server, func()) { @@ -183,6 +202,16 @@ func makeTempFile(t *testing.T) (*os.File, func()) { } } +func mustGetSize(path string) string { + + fileinfo, err := os.Stat(path) + if err != nil { + log.Fatal(err) + } + + return strconv.Itoa(int(fileinfo.Size())) +} + // GetSize returns size from response header. func getSizeForTest(t *testing.T, r *http.Response) uint { t.Helper() @@ -198,4 +227,4 @@ func getSizeForTest(t *testing.T, r *http.Response) uint { t.Error(err) } return uint(ret) -} \ No newline at end of file +} From 2d3a754beb6a67f0785f8663597dd206ab534875 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Mon, 12 Jul 2021 18:34:24 +0900 Subject: [PATCH 38/51] =?UTF-8?q?Test=5FRequest=E9=96=A2=E6=95=B0=E3=81=AE?= =?UTF-8?q?=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/request/request_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/kadai3-2/Mizushima/request/request_test.go b/kadai3-2/Mizushima/request/request_test.go index 63b29134..68664060 100644 --- a/kadai3-2/Mizushima/request/request_test.go +++ b/kadai3-2/Mizushima/request/request_test.go @@ -53,6 +53,19 @@ func Test_Request(t *testing.T) { }, } + /* Responseの内容 + actual: map[Access-Range:[bytes] Content-Length:[2066] Date:[Mon, 12 Jul 2021 09:22:22 GMT]] + request_test.go:74: expected, + HTTP/0.0 206 Partial Content + Content-Length: 0 + + but got, + HTTP/1.1 206 Partial Content + Content-Length: 2066 + Access-Range: bytes + Date: Mon, 12 Jul 2021 09:22:22 GMT + */ + for _, c := range cases { c := c t.Run(c.name, func(t *testing.T) { From a31b2bf6bd1153d606392a35046fd55f2da7961d Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Tue, 13 Jul 2021 17:01:19 +0900 Subject: [PATCH 39/51] =?UTF-8?q?README=E8=BF=BD=E5=8A=A0=E3=80=81?= =?UTF-8?q?=E4=B8=8D=E8=A6=81=E3=81=AApackage=E7=AD=89=E5=89=8A=E9=99=A4?= =?UTF-8?q?=E3=80=81testdate=E3=81=AB=E3=83=95=E3=82=A9=E3=83=AB=E3=83=80?= =?UTF-8?q?=E5=90=8D=E5=A4=89=E6=9B=B4=E3=80=81=E3=81=9D=E3=82=8C=E3=81=AB?= =?UTF-8?q?=E4=BC=B4=E3=81=84=E5=90=84=E3=83=86=E3=82=B9=E3=83=88=E9=96=A2?= =?UTF-8?q?=E6=95=B0=E3=81=AE=E4=BF=AE=E6=AD=A3=E3=80=81Makefile=E3=81=AEt?= =?UTF-8?q?est=E5=A4=89=E6=9B=B4=E4=BB=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/.gitignore | 3 - kadai3-2/Mizushima/Makefile | 14 +- kadai3-2/Mizushima/README.md | 81 ++++++++++ kadai3-2/Mizushima/TODO.md | 27 ---- kadai3-2/Mizushima/download/download_test.go | 15 +- kadai3-2/Mizushima/dummy_server/dummy.go | 25 --- kadai3-2/Mizushima/getheader/geHeader_test.go | 71 ++++----- kadai3-2/Mizushima/listen/listen_test.go | 7 +- kadai3-2/Mizushima/main.go | 4 +- kadai3-2/Mizushima/request/request_test.go | 150 ++++++++---------- kadai3-2/Mizushima/testdata/003 | 1 + kadai3-2/Mizushima/testdata/z4d4kWk.jpg | Bin 0 -> 2066 bytes 12 files changed, 211 insertions(+), 187 deletions(-) create mode 100644 kadai3-2/Mizushima/README.md delete mode 100644 kadai3-2/Mizushima/TODO.md delete mode 100644 kadai3-2/Mizushima/dummy_server/dummy.go create mode 100644 kadai3-2/Mizushima/testdata/003 create mode 100644 kadai3-2/Mizushima/testdata/z4d4kWk.jpg diff --git a/kadai3-2/Mizushima/.gitignore b/kadai3-2/Mizushima/.gitignore index 6eb01878..36f971e3 100644 --- a/kadai3-2/Mizushima/.gitignore +++ b/kadai3-2/Mizushima/.gitignore @@ -1,4 +1 @@ -*.jpg -documents bin/* -test_download/* diff --git a/kadai3-2/Mizushima/Makefile b/kadai3-2/Mizushima/Makefile index bca1571b..a22e4431 100644 --- a/kadai3-2/Mizushima/Makefile +++ b/kadai3-2/Mizushima/Makefile @@ -4,17 +4,23 @@ GOBUILD=$(GOCMD) build GOCLEAN=$(GOCMD) clean GOTEST=$(GOCMD) test GOGET=$(GOCMD) get +GOFMT=$(GOCMD) fmt all: build test build: $(GOBUILD) -o $(BINARY_NAME) -v -test: - $(BINARY_NAME) https://golang.org/dl/go1.16.5.linux-amd64.tar.gz -o ./test_download +test: build + cd download; $(GOTEST) -v -cover + cd getheader; $(GOTEST) -v -cover + cd listen; $(GOTEST) -v -cover + cd request; $(GOTEST) -v -cover clean: $(GOCLEAN) - rm -rf $(BINARY_NAME) test_download/* -.PHONY: test clean +format: + $(GOFMT) ./... + +.PHONY: test clean format diff --git a/kadai3-2/Mizushima/README.md b/kadai3-2/Mizushima/README.md new file mode 100644 index 00000000..351b79ea --- /dev/null +++ b/kadai3-2/Mizushima/README.md @@ -0,0 +1,81 @@ +## 課題3-1 分割ダウンローダを作ろう +- 分割ダウンロードを行う +- Rangeアクセスを用いる +- いくつかのゴルーチンでダウンロードしてマージする +- エラー処理を工夫する + - golang.org/x/sync/errgourpパッケージなどを使ってみる +- キャンセルが発生した場合の実装を行う + + + +### コマンドラインオプション + + | ショートオプション | ロングオプション | 説明 | デフォルト | + | --------- | --------- | --------- | --------- | + | -h | --help | 使い方を表示して終了 | - | + | -p \ | --procs \ | プロセス数を指定 | お使いのPCのコア数 | + | -o \ | --output \ | ダウンロードしたファイルをどこに保存するか指定する | カレントディレクトリ | + | -t \ | --timeout \ | サーバーへのリクエストを止める時間を秒数で指定 | 20 | + + +### インストール方法 +```bash +go get github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima +``` + +### 使い方 +1. 実行ファイル作成 +```bash +$ make build +``` +2. ディレクトリを指定して実行 +```bash +$ ./bin/paraDW [option] +``` + + +### テストの方法 +- バイナリビルド & テスト +```bash +$ make +``` +- テスト後の処理(掃除) +```bash +$ make clean +``` + +### ディレクトリ構成 +```bash +. +├── bin +│ └── paraDW # "make build" command required. +├── download +│ ├── download.go +│ ├── download_test.go +│ ├── go.mod +│ └── go.sum +├── getheader +│ ├── geHeader_test.go +│ ├── getHeader.go +│ └── go.mod +├── .gitignore +├── go.mod +├── go.sum +├── listen +│ ├── listen.go +│ └── listen_test.go +├── main.go +├── Makefile +├── README.md +├── request +│ ├── go.mod +│ ├── request.go +│ └── request_test.go +└── testdata + ├── 003 + └── z4d4kWk.jpg +``` + +### 参考にしたもの +[pget](https://qiita.com/codehex/items/d0a500ac387d39a34401) (goroutineを使ったダウンロード処理、コマンドラインオプションの処理等々) +https://github.com/gopherdojo/dojo3/pull/50 (ctrl+cを押したときのキャンセル処理など) \ No newline at end of file diff --git a/kadai3-2/Mizushima/TODO.md b/kadai3-2/Mizushima/TODO.md deleted file mode 100644 index 09d72102..00000000 --- a/kadai3-2/Mizushima/TODO.md +++ /dev/null @@ -1,27 +0,0 @@ -### TODOリスト -順不同 - -- go routineによる分割ダウンロード処理 - - fileの大きさの事前取得 - - range access の range取得 -- Makefile作成 - - テスト作成 - - ダミーサーバー - -### 処理の流れ -- オプションからURL取得（複数可）一つ一つのURLにつき、 - - ~~Access-Range=bytesかどうか~~ - - ~~否なら、通常のダウンロード~~ - - ~~完成版ファイルを作成~~ - - ~~一時フォルダを作る~~ - - ~~並行してダウンロードし連番の一時ファイルに保存~~ - - ~~完成版フォルダに一時ファイルを順番にコピー~~ - - ~~一時フォルダ毎削除~~ - - ~~時間制限を設ける~~ - - 空のctx -> ctx.WithTimeout(child) in main function※ここで時間制限設定 -> ctx.WithCancel(child) in listen.Listen -> request.Requestに渡してhttp.NewRequestWithContextに渡すことでリクエスト時のタイムアウト設定 - - ~~ファイルが追記モードで開かざるを得ないので、続けて同じファイルをダウンロードすると後ろに追記されてしまう~~ - - 先に同じ名前のファイルがあったら消す - -### 参考にしたもの -[pget](https://qiita.com/codehex/items/d0a500ac387d39a34401) (goroutineを使ったダウンロード処理、コマンドラインオプションの処理等々) -https://github.com/gopherdojo/dojo3/pull/50 (ctrl+cを押したときのキャンセル処理など) \ No newline at end of file diff --git a/kadai3-2/Mizushima/download/download_test.go b/kadai3-2/Mizushima/download/download_test.go index f2131281..9a99526e 100644 --- a/kadai3-2/Mizushima/download/download_test.go +++ b/kadai3-2/Mizushima/download/download_test.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io/ioutil" + "log" "net/http" "net/http/httptest" "net/url" @@ -21,8 +22,8 @@ import ( ) var testdataPathMap = map[int][]string{ - 0: {"../documents/003", "311"}, - 1: {"../documents/z4d4kWk.jpg", "146515"}, + 0: {"../testdata/003", mustGetSize("../testdata/003")}, + 1: {"../testdata/z4d4kWk.jpg", mustGetSize("../testdata/z4d4kWk.jpg")}, // 2 : "../documents/http.request.txt", } @@ -308,3 +309,13 @@ func getSizeForTest(t *testing.T, r *http.Response) uint { } return uint(ret) } + +func mustGetSize(path string) string { + + fileinfo, err := os.Stat(path) + if err != nil { + log.Fatal(err) + } + + return strconv.Itoa(int(fileinfo.Size())) +} diff --git a/kadai3-2/Mizushima/dummy_server/dummy.go b/kadai3-2/Mizushima/dummy_server/dummy.go deleted file mode 100644 index 4e108d1a..00000000 --- a/kadai3-2/Mizushima/dummy_server/dummy.go +++ /dev/null @@ -1,25 +0,0 @@ -package main - -import ( - "fmt" - "net/http" - "time" -) - -func handleFnLate(w http.ResponseWriter, r *http.Request) { - // w.Header().Set("Accept-Range", "bytes") - time.Sleep(30*time.Second) - fmt.Fprint(w, "Hello\n") -} - -func handleFn(w http.ResponseWriter, r *http.Request) { - // w.Header().Set("Accept-Range", "bytes") - // time.Sleep(30*time.Second) - fmt.Fprint(w, "Hello\n") -} - -func main() { - http.HandleFunc("/late", handleFnLate) - http.HandleFunc("/", handleFn) - http.ListenAndServe(":12345", nil) -} \ No newline at end of file diff --git a/kadai3-2/Mizushima/getheader/geHeader_test.go b/kadai3-2/Mizushima/getheader/geHeader_test.go index 00abc360..4ba88d76 100644 --- a/kadai3-2/Mizushima/getheader/geHeader_test.go +++ b/kadai3-2/Mizushima/getheader/geHeader_test.go @@ -10,19 +10,18 @@ import ( "github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima/getheader" ) - -var heads1 = []string { - "Content-Length", - "Accept-Ranges", - "Content-Type", +var heads1 = []string{ + "Content-Length", + "Accept-Ranges", + "Content-Type", "Access-Control-Allow-Methods", - } +} -var heads2 = []string { - "Accept-Ranges", - "Content-Type", +var heads2 = []string{ + "Accept-Ranges", + "Content-Type", "Access-Control-Allow-Methods", - } +} var vals1 = [][]string{{"146515"}, {"bytes"}, {"image/jpeg"}, {"GET", "OPTIONS"}} @@ -30,26 +29,26 @@ var vals2 = [][]string{{"bytes"}, {"image/jpeg"}, {"GET", "OPTIONS"}} func Test_ResHeader(t *testing.T) { t.Helper() - + cases := []struct { - name string - input string - heads []string - vals [][]string + name string + input string + heads []string + vals [][]string expected []string }{ { - name: "case 1", - input: "Content-Length", - heads: heads1, - vals: vals1, + name: "case 1", + input: "Content-Length", + heads: heads1, + vals: vals1, expected: []string{"146515"}, }, { - name: "case 2", - input: "Accept-Ranges", - heads: heads2, - vals: vals2, + name: "case 2", + input: "Accept-Ranges", + heads: heads2, + vals: vals2, expected: []string{"bytes"}, }, } @@ -75,21 +74,21 @@ func Test_ResHeader(t *testing.T) { func Test_GetSize(t *testing.T) { cases := []struct { - name string - heads []string - vals [][]string + name string + heads []string + vals [][]string expected uint }{ { - name: "case 1", - heads: heads1, - vals: vals1, + name: "case 1", + heads: heads1, + vals: vals1, expected: uint(146515), }, { - name: "case 2", - heads: heads2, - vals: vals2, + name: "case 2", + heads: heads2, + vals: vals2, expected: 0, }, } @@ -98,7 +97,7 @@ func Test_GetSize(t *testing.T) { c := c t.Run(c.name, func(t *testing.T) { resp, err := makeResponse(t, c.heads, c.vals) - + if err != nil { t.Error(err) } @@ -121,7 +120,7 @@ func Test_GetSize(t *testing.T) { func makeResponse(t *testing.T, heads []string, vals [][]string) (*http.Response, error) { t.Helper() - + var resp = make(map[string][]string) if len(heads) != len(vals) { @@ -132,9 +131,9 @@ func makeResponse(t *testing.T, heads []string, vals [][]string) (*http.Response if _, ok := resp[heads[i]]; ok { return nil, errors.New("Duplicate elements in heads") } - + resp[heads[i]] = vals[i] } return &http.Response{Header: http.Header(resp)}, nil -} \ No newline at end of file +} diff --git a/kadai3-2/Mizushima/listen/listen_test.go b/kadai3-2/Mizushima/listen/listen_test.go index 78fca902..24991b8c 100644 --- a/kadai3-2/Mizushima/listen/listen_test.go +++ b/kadai3-2/Mizushima/listen/listen_test.go @@ -8,11 +8,10 @@ import ( "time" ) - func Test_Listen(t *testing.T) { t.Helper() - - cleanFn := func(){} + + cleanFn := func() {} doneCh := make(chan struct{}) osExit = func(code int) { doneCh <- struct{}{} } @@ -38,4 +37,4 @@ func Test_Listen(t *testing.T) { t.Fatal("timeout") } -} \ No newline at end of file +} diff --git a/kadai3-2/Mizushima/main.go b/kadai3-2/Mizushima/main.go index 29a5031f..e3cb7259 100644 --- a/kadai3-2/Mizushima/main.go +++ b/kadai3-2/Mizushima/main.go @@ -113,8 +113,7 @@ func main() { for i, urlObj := range urls { // make a empty context - ctx := context.Background() - ctxTimeout, cancelTimeout := context.WithTimeout(ctx, time.Duration(opts.Tm)*time.Second) + ctxTimeout, cancelTimeout := context.WithTimeout(context.Background(), time.Duration(opts.Tm)*time.Second) defer cancelTimeout() // send "HEAD" request, and gets response. @@ -218,6 +217,7 @@ func main() { log.Fatalf("err: RemoveAll: %s\n", err) } + // Clean up cancel() if err = out.Close(); err != nil { log.Fatalf("err: %s", err) diff --git a/kadai3-2/Mizushima/request/request_test.go b/kadai3-2/Mizushima/request/request_test.go index 68664060..8ed2bfd2 100644 --- a/kadai3-2/Mizushima/request/request_test.go +++ b/kadai3-2/Mizushima/request/request_test.go @@ -4,12 +4,10 @@ import ( "context" "errors" "fmt" - "io/ioutil" "log" "net/http" "net/http/httptest" "net/http/httputil" - "net/url" "os" "reflect" "strconv" @@ -21,61 +19,61 @@ import ( ) var testdataPathMap = map[int][]string{ - 0: {"../documents/003", mustGetSize("../documents/003")}, - 1: {"../documents/z4d4kWk.jpg", mustGetSize("../documents/z4d4kWk.jpg")}, + 0: {"../testdata/003", mustGetSize("../testdata/003")}, + 1: {"../testdata/z4d4kWk.jpg", mustGetSize("../testdata/z4d4kWk.jpg")}, } -func Test_Request(t *testing.T) { +func Test_RequestStandard(t *testing.T) { t.Helper() - cases := []struct { - name string + cases := map[string]struct { key int // key for testdataPathMap handler func(t *testing.T, w http.ResponseWriter, r *http.Request, testDataKey int) expected *http.Response }{ - { - name: "case 1", - key: 0, - handler: nonRangeAccessHandler, - expected: &http.Response{}, + "case 1": { + key: 0, + handler: nonRangeAccessHandler, + expected: &http.Response{ + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: map[string][]string{ + "Content-Length": {testdataPathMap[0][1]}, + "Date": {mustTimeLayout(t, time.Now())}, + }, + }, }, - { - name: "case 2", - key: 1, - handler: rangeAccessHandler, - // http.Responseを作る方法を調べる + "case 2": { + key: 1, + handler: rangeAccessHandler, expected: &http.Response{ - Status: "206 Partial Content", + Status: "206 Partial Content", StatusCode: 206, - Proto: "HTTP/1.1", + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: map[string][]string{ + "Access-Range": {"bytes"}, + "Content-Length": {testdataPathMap[1][1]}, + "Date": {mustTimeLayout(t, time.Now())}, + }, }, }, } - /* Responseの内容 - actual: map[Access-Range:[bytes] Content-Length:[2066] Date:[Mon, 12 Jul 2021 09:22:22 GMT]] - request_test.go:74: expected, - HTTP/0.0 206 Partial Content - Content-Length: 0 - - but got, - HTTP/1.1 206 Partial Content - Content-Length: 2066 - Access-Range: bytes - Date: Mon, 12 Jul 2021 09:22:22 GMT - */ - - for _, c := range cases { + for name, c := range cases { c := c - t.Run(c.name, func(t *testing.T) { + t.Run(name, func(t *testing.T) { ts, clean := newTestServer(t, c.handler, c.key) defer clean() actual, err := request.Request(context.Background(), "GET", ts.URL, "", "") if err != nil { t.Fatal(err) } - fmt.Println("actual:", actual.Header) + // fmt.Println("actual:", actual.Header) if !reflect.DeepEqual(actual.Header, c.expected.Header) { dumped_expected, err := httputil.DumpResponse(c.expected, false) if err != nil { @@ -91,6 +89,28 @@ func Test_Request(t *testing.T) { } } +func Test_RequestTimeout(t *testing.T) { + t.Helper() + + name := "case timeout" + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + t.Run(name, func(t *testing.T) { + ts, clean := newTestServer(t, nonRangeAccessTooLateHandler, 1) + defer clean() + + expected := fmt.Errorf("request.Request err: Get \"%s\": %w", ts.URL, context.DeadlineExceeded) + _, err := request.Request(ctx, "GET", ts.URL, "", "") + + // fmt.Println("actual:\n", actual) + if err.Error() != expected.Error() { + t.Errorf("expected %s, but got %s", expected.Error(), err.Error()) + } + }) +} + func newTestServer(t *testing.T, handler func(t *testing.T, w http.ResponseWriter, r *http.Request, testDataKey int), testDataKey int) (*httptest.Server, func()) { @@ -178,45 +198,9 @@ func retBody(t *testing.T, rangeHeader string, testDataPath string) []byte { return b[start : end+1] } -func getURLObject(t *testing.T, urlStr string) *url.URL { - t.Helper() - - urlObj, err := url.ParseRequestURI(urlStr) - if err != nil { - t.Error(err) - } - - return urlObj -} - -func makeTempFile(t *testing.T) (*os.File, func()) { - t.Helper() - - dir, err := ioutil.TempDir("", "test_download") - if err != nil { - t.Fatal(err) - } - - out, err := os.Create(dir + "/test") - if err != nil { - t.Fatal(err) - } - - return out, - func() { - err = out.Close() - if err != nil { - t.Fatal(err) - } - err = os.RemoveAll(dir) - if err != nil { - t.Fatal(err) - } - } -} - +// mustGetSize returns the size of the file in "path" as a string for "Content-Length" in http header. func mustGetSize(path string) string { - + fileinfo, err := os.Stat(path) if err != nil { log.Fatal(err) @@ -225,19 +209,17 @@ func mustGetSize(path string) string { return strconv.Itoa(int(fileinfo.Size())) } -// GetSize returns size from response header. -func getSizeForTest(t *testing.T, r *http.Response) uint { +// mustTimeLayout returns the time now in format like this : "Mon, 12 Jul 2021 09:22:22 GMT" +func mustTimeLayout(t *testing.T, tm time.Time) string { t.Helper() - contLen, is := r.Header["Content-Length"] - // fmt.Println(h) - if !is { - t.Errorf("cannot find Content-Length header") - } - - ret, err := strconv.ParseUint(contLen[0], 10, 32) + // get the gmt time + location, err := time.LoadLocation("GMT") if err != nil { - t.Error(err) + t.Fatal(err) } - return uint(ret) + tm = tm.In(location) + + // the layout is like this : "Mon, 12 Jul 2021 09:22:22 GMT" + return tm.Format(time.RFC1123) } diff --git a/kadai3-2/Mizushima/testdata/003 b/kadai3-2/Mizushima/testdata/003 new file mode 100644 index 00000000..d9b00569 --- /dev/null +++ b/kadai3-2/Mizushima/testdata/003 @@ -0,0 +1 @@ +ahoahoaho \ No newline at end of file diff --git a/kadai3-2/Mizushima/testdata/z4d4kWk.jpg b/kadai3-2/Mizushima/testdata/z4d4kWk.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a2cee5f24bfdae96df7b6da867b3fb2455c639ae GIT binary patch literal 2066 zcmbV}3pCqV9>@O?kw`*NucD(wTcuXLR&{Br>d_jM(soelT{W6|8`WgIpNF=Ttxg#& z^=3RXr4>YI=~TUmN(4nM(|?NeuOiuGw`ca8-E-#5&iCH$Irp6V``!Ef+$)$8%mFfI zZO+&LFc=KDA{>BV2CxFeU@+1AA~*srl7zT89D$TTB1J|?VkA*0Gzy7COQF#itWZcv zX`B>RTC^4&B5D_P2`3hXM2RZ?KMA@3Srp(1_`_kEfS4={E(;U%0_p$&LkQD~0)G)q z3@(n45Jo~{gaxfK!uW8wm@q6tTwGXvOSlh+%Od173@jz~IQt+q!{iMU?v$eNry9Ez zT!uj{Bj4~uNwlJp@?Mn#2M=i<)-g8utLZVb<5s7wZEWq%oIUUAcH!csZ`}R-0|JAt z249PajEatl{VpykIVCkMJtOmOUjDs;!oS}ydi1!g{7FS+RdrKyOKThD=cnyGv|f5& z|BIIcBco&E6O&WZ4EFrO;=85gmG`Thj~km?+)vQ*e(*98065>>0`khcPJqm&jS#jqvudWIqn~af6c*!f z>$@PH?k(Lqr1ZW-g)eu_OAhS4O{)!w?mRlWG}=;lzpaO&gfYm$<8PEsDVk#^P5<}2 zoHoVPmCs_C%OXka+Wr{>xQq4|8bxXA4lBpC{S4nQM zNfONq0gZ!&eZ}~8+k$<%mK;(^CG%B!`NIO`z#t7*IXg92-sxK((wqlpp%#nlnc6>! zeKFQ4oh4;%#>wh7<*RUF(l(vsp$Iq2ecXFBQ}kN_u?b0tyL;X4bOoGGMc&3;;~5Bm zL@J3+MV{CzOc^Htd~=ad1lyD*gtDbyJXcS76x^xQ*fQGseUwE?L0N2Q0y#OAEr(g(-kt}#qrM3{;)dz(aj zn5H&1>r6@xGssB*AmvDdC0GFvNPsjbeg7b)vZSA3 zKrZXT)8PZ1S+wJb`n!eoMdg(?`5t`<1F2UqL8>{&CkF%o=%_xk%$M{7H@LFU`BI+y zn@;rGc>58aSqO3dj0qu$qEf!VtmcD*;qCs3K|>d%r?*T#BZW6=#A|W9`}H9_i>{A_ z4$;2S;G|`kNJc&mDO-t)WTb4paBv)F`5&$MTc~y5F*Sr1sJl#s4&itTd7itu0>DQ- zg@EDN@2HYyO&J0p;h9-TnfKe^5=sAp*TKG?MyE$S@%fuaGH+GisAEu##s|J?JJ*)m zH1q3P$mPGdo7a@>nuN{9ke6a{Jhcrfo4PK!a~SHs4B9lz3xKHwnqu@vuh<}~j9TMH z%~#a+D2FQ95F5%R7fWTGyM5VDZ*@N3*~foF2JKmpwgBiSr#A8|z?d`=F5U}db_CZ! zfdlGTE4IU_`BjGNy(__bZngE#e(AvK<2xjXM6=M()Fkl?zU(?Xo+Hw?5_a_SkbZ zA$_o!8wy!0lwjjEARqRS%Fx_Hop0hT%b^Q>L#}t0N9Q;0*ZZZP{rbU!@ANjVo4pR{ zA*`a@ zmYrb3S{V4?>#?=^&2@W6FfcbHCO8UXWl{Um5B)404-Bk!C6 Date: Tue, 13 Jul 2021 17:03:15 +0900 Subject: [PATCH 40/51] =?UTF-8?q?README.md=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kadai3-2/Mizushima/README.md b/kadai3-2/Mizushima/README.md index 351b79ea..cc26b568 100644 --- a/kadai3-2/Mizushima/README.md +++ b/kadai3-2/Mizushima/README.md @@ -1,4 +1,4 @@ -## 課題3-1 分割ダウンローダを作ろう +## 課題3-2 分割ダウンローダを作ろう - 分割ダウンロードを行う - Rangeアクセスを用いる - いくつかのゴルーチンでダウンロードしてマージする From 2922f3e2dd7698839776bc0cb260b5bf59d52a83 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Tue, 13 Jul 2021 17:15:54 +0900 Subject: [PATCH 41/51] =?UTF-8?q?README.md=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kadai3-2/Mizushima/README.md b/kadai3-2/Mizushima/README.md index cc26b568..94f56aa3 100644 --- a/kadai3-2/Mizushima/README.md +++ b/kadai3-2/Mizushima/README.md @@ -32,7 +32,8 @@ $ make build ```bash $ ./bin/paraDW [option] ``` - +※ ダウンロード先がRangeアクセスに対応していれば、go routineを使った並行ダウンロードを +　行い、そうでなければ１プロセスのダウンロードを行います ### テストの方法 - バイナリビルド & テスト From b36d85e1681d87332cda64942d1fa7c7529f0a51 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Tue, 13 Jul 2021 17:30:51 +0900 Subject: [PATCH 42/51] =?UTF-8?q?README.md=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kadai3-2/Mizushima/README.md b/kadai3-2/Mizushima/README.md index 94f56aa3..ef6245f9 100644 --- a/kadai3-2/Mizushima/README.md +++ b/kadai3-2/Mizushima/README.md @@ -32,8 +32,7 @@ $ make build ```bash $ ./bin/paraDW [option] ``` -※ ダウンロード先がRangeアクセスに対応していれば、go routineを使った並行ダウンロードを -　行い、そうでなければ１プロセスのダウンロードを行います +※ ダウンロード先がRangeアクセスに対応していれば、go routineを使った並行ダウンロードを行い、そうでなければ１プロセスのダウンロードを行います ### テストの方法 - バイナリビルド & テスト From 648256b28f106460dd6b9fc214997ca189faa1a4 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Tue, 13 Jul 2021 18:41:22 +0900 Subject: [PATCH 43/51] =?UTF-8?q?README.md=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/kadai3-2/Mizushima/README.md b/kadai3-2/Mizushima/README.md index ef6245f9..a453cc64 100644 --- a/kadai3-2/Mizushima/README.md +++ b/kadai3-2/Mizushima/README.md @@ -14,7 +14,7 @@ | --------- | --------- | --------- | --------- | | -h | --help | 使い方を表示して終了 | - | | -p \ | --procs \ | プロセス数を指定 | お使いのPCのコア数 | - | -o \ | --output \ | ダウンロードしたファイルをどこに保存するか指定する | カレントディレクトリ | + | -o \ | --output \ | ダウンロードしたファイルをどこのディレクトリに保存するか指定する | カレントディレクトリ | | -t \ | --timeout \ | サーバーへのリクエストを止める時間を秒数で指定 | 20 | @@ -30,9 +30,11 @@ $ make build ``` 2. ディレクトリを指定して実行 ```bash -$ ./bin/paraDW [option] +$ ./bin/paraDW [option] URL(, URL, URL, ...) ``` -※ ダウンロード先がRangeアクセスに対応していれば、go routineを使った並行ダウンロードを行い、そうでなければ１プロセスのダウンロードを行います + +※ URLは複数指定できます +※ ダウンロード先がRangeアクセスに対応していれば、go routineを使った並行ダウンロードを行い、そうでなければ１プロセスのダウンロードを行います ### テストの方法 - バイナリビルド & テスト From 9a40d02791532601c9dc0b19c2559cb9e92c66fe Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Tue, 13 Jul 2021 19:18:36 +0900 Subject: [PATCH 44/51] =?UTF-8?q?README.md=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kadai3-2/Mizushima/README.md b/kadai3-2/Mizushima/README.md index a453cc64..1e48175e 100644 --- a/kadai3-2/Mizushima/README.md +++ b/kadai3-2/Mizushima/README.md @@ -15,7 +15,7 @@ | -h | --help | 使い方を表示して終了 | - | | -p \ | --procs \ | プロセス数を指定 | お使いのPCのコア数 | | -o \ | --output \ | ダウンロードしたファイルをどこのディレクトリに保存するか指定する | カレントディレクトリ | - | -t \ | --timeout \ | サーバーへのリクエストを止める時間を秒数で指定 | 20 | + | -t \ | --timeout \ | サーバーへのリクエストを止める時間を秒数で指定 | 120 | ### インストール方法 From b2cfb3a2b821632831b7ecc7e2f43a1e329e7ae2 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Wed, 14 Jul 2021 11:57:57 +0900 Subject: [PATCH 45/51] =?UTF-8?q?READEME.md=E4=BF=AE=E6=AD=A3=E3=80=81?= =?UTF-8?q?=E5=8F=8A=E3=81=B3download.go=E3=81=AE=E3=82=B3=E3=83=A1?= =?UTF-8?q?=E3=83=B3=E3=83=88=E3=81=AE=E3=82=B9=E3=83=9A=E3=83=AB=E3=83=9F?= =?UTF-8?q?=E3=82=B9=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/README.md | 4 ++-- kadai3-2/Mizushima/download/download.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kadai3-2/Mizushima/README.md b/kadai3-2/Mizushima/README.md index 1e48175e..00c0e8dd 100644 --- a/kadai3-2/Mizushima/README.md +++ b/kadai3-2/Mizushima/README.md @@ -28,9 +28,9 @@ go get github.com/MizushimaToshihiko/gopherdojo-studyroom/kadai3-2/Mizushima ```bash $ make build ``` -2. ディレクトリを指定して実行 +2. URLを指定してダウンロード実行 ```bash -$ ./bin/paraDW [option] URL(, URL, URL, ...) +$ ./bin/paraDW [option] URL( URL URL ...) ``` ※ URLは複数指定できます diff --git a/kadai3-2/Mizushima/download/download.go b/kadai3-2/Mizushima/download/download.go index e36685ac..ece75d7d 100644 --- a/kadai3-2/Mizushima/download/download.go +++ b/kadai3-2/Mizushima/download/download.go @@ -37,7 +37,7 @@ func newPDownloader(url *url.URL, output *os.File, fileSize uint, part uint, pro // Downloader gets elements of PDownloader, the download is parallel or not, temprary // directory name and context.Context, and drives DownloadFile method if isPara is false -// or PDownload if isPrara is true. +// or PDownload if isPara is true. // func Downloader(url *url.URL, output *os.File, fileSize uint, part uint, procs uint, isPara bool, From 880f69839796797ab5bd32cf46ca1bcc2d058ca9 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Fri, 16 Jul 2021 19:46:51 +0900 Subject: [PATCH 46/51] =?UTF-8?q?main=E3=81=AE=E3=83=86=E3=82=B9=E3=83=88?= =?UTF-8?q?=E4=BD=9C=E6=88=90=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/main_test.go | 65 +++++++++++++++++++++++++++ kadai3-2/Mizushima/testdata/empty.txt | 0 2 files changed, 65 insertions(+) create mode 100644 kadai3-2/Mizushima/main_test.go create mode 100644 kadai3-2/Mizushima/testdata/empty.txt diff --git a/kadai3-2/Mizushima/main_test.go b/kadai3-2/Mizushima/main_test.go new file mode 100644 index 00000000..a25ed411 --- /dev/null +++ b/kadai3-2/Mizushima/main_test.go @@ -0,0 +1,65 @@ +package main + +import ( + "log" + "os" + "runtime" + "strconv" + "testing" +) + + +var testdataPathMap = map[int][]string{ + 0: {"../testdata/003", mustGetSize("../testdata/003")}, + 1: {"../testdata/z4d4kWk.jpg", mustGetSize("../testdata/z4d4kWk.jpg")}, +} + +func Test_main(t *testing.T) { + cases := map[string]struct { + key int // key for testdataPathMap + }{ + "case 1" : { + key : 0, + }, + "case 2" : { + key : 1, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + testdataPath := testdataPathMap[c.key][0] + expected, err := os.Open(testdataPath) + if err != nil { + t.Fatal(err) + } + + defer func() { + if err := expected.Close(); err != nil { + t.Fatal(err) + } + }() + + opts := &Options{ + Help : false, + Procs: uint(runtime.NumCPU()), + Output: "./", + Tm: 3, + } + + + + }) + } +} + +// mustGetSize returns the size of the file in "path" as a string for "Content-Length" in http header. +func mustGetSize(path string) string { + + fileinfo, err := os.Stat(path) + if err != nil { + log.Fatal(err) + } + + return strconv.Itoa(int(fileinfo.Size())) +} \ No newline at end of file diff --git a/kadai3-2/Mizushima/testdata/empty.txt b/kadai3-2/Mizushima/testdata/empty.txt new file mode 100644 index 00000000..e69de29b From 1d54062f457bf6280b90cfcb3b9d9b9101f4ff9e Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Thu, 22 Jul 2021 02:09:17 +0000 Subject: [PATCH 47/51] =?UTF-8?q?main=5Ftest.--------=E6=AD=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kadai3-2/Mizushima/main_test.go | 65 --------------------------------- 1 file changed, 65 deletions(-) delete mode 100644 kadai3-2/Mizushima/main_test.go diff --git a/kadai3-2/Mizushima/main_test.go b/kadai3-2/Mizushima/main_test.go deleted file mode 100644 index a25ed411..00000000 --- a/kadai3-2/Mizushima/main_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package main - -import ( - "log" - "os" - "runtime" - "strconv" - "testing" -) - - -var testdataPathMap = map[int][]string{ - 0: {"../testdata/003", mustGetSize("../testdata/003")}, - 1: {"../testdata/z4d4kWk.jpg", mustGetSize("../testdata/z4d4kWk.jpg")}, -} - -func Test_main(t *testing.T) { - cases := map[string]struct { - key int // key for testdataPathMap - }{ - "case 1" : { - key : 0, - }, - "case 2" : { - key : 1, - }, - } - - for name, c := range cases { - t.Run(name, func(t *testing.T) { - testdataPath := testdataPathMap[c.key][0] - expected, err := os.Open(testdataPath) - if err != nil { - t.Fatal(err) - } - - defer func() { - if err := expected.Close(); err != nil { - t.Fatal(err) - } - }() - - opts := &Options{ - Help : false, - Procs: uint(runtime.NumCPU()), - Output: "./", - Tm: 3, - } - - - - }) - } -} - -// mustGetSize returns the size of the file in "path" as a string for "Content-Length" in http header. -func mustGetSize(path string) string { - - fileinfo, err := os.Stat(path) - if err != nil { - log.Fatal(err) - } - - return strconv.Itoa(int(fileinfo.Size())) -} \ No newline at end of file From 83cb3ef42820df5470326e3f9b216114800d6459 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Fri, 30 Jul 2021 12:02:41 +0000 Subject: [PATCH 48/51] modified .gitignore --- kadai3-2/Mizushima/.gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kadai3-2/Mizushima/.gitignore b/kadai3-2/Mizushima/.gitignore index 36f971e3..4083730c 100644 --- a/kadai3-2/Mizushima/.gitignore +++ b/kadai3-2/Mizushima/.gitignore @@ -1 +1,5 @@ bin/* +../../.devcontainer/ +../../Dockerfile +../../app/ +../../docker-compose.yml From 3852a4a1c9deb830fcdfd188d4b7b78b0333adf6 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Fri, 30 Jul 2021 12:17:07 +0000 Subject: [PATCH 49/51] modified main function in main.go --- kadai3-2/Mizushima/main.go | 182 +++++++++++++++++++------------------ 1 file changed, 92 insertions(+), 90 deletions(-) diff --git a/kadai3-2/Mizushima/main.go b/kadai3-2/Mizushima/main.go index e3cb7259..c2b0d1e7 100644 --- a/kadai3-2/Mizushima/main.go +++ b/kadai3-2/Mizushima/main.go @@ -111,118 +111,120 @@ func main() { // download from each url in urls for i, urlObj := range urls { + downloadFromUrl(i, opts, urlObj) + } +} - // make a empty context - ctxTimeout, cancelTimeout := context.WithTimeout(context.Background(), time.Duration(opts.Tm)*time.Second) - defer cancelTimeout() +// +func downloadFromUrl(i int, opts Options, urlObj *url.URL) { - // send "HEAD" request, and gets response. - resp, err := request.Request(ctxTimeout, "HEAD", urlObj.String(), "", "") - if err != nil { - log.Fatalf("err: %s\n", err) - } + // make a timeout context from a empty context + ctxTimeout, cancelTimeout := context.WithTimeout(context.Background(), time.Duration(opts.Tm)*time.Second) + defer cancelTimeout() - // show response header - fmt.Printf("response:\n") - if b, err := httputil.DumpResponse(resp, false); err != nil { - log.Fatalf("err: %s", err) - } else { - fmt.Printf("%s\n", b) - } + // send "HEAD" request, and gets response. + resp, err := request.Request(ctxTimeout, "HEAD", urlObj.String(), "", "") + if err != nil { + log.Fatalf("err: %s\n", err) + } - // get the size from the response header. - fileSize, err := getheader.GetSize(resp) - if err != nil { - log.Fatalf("err: getheader.GetSize: %s\n", err) - } - if err = resp.Body.Close(); err != nil { - log.Fatalf("err: %s", err) - } + // show response header + fmt.Printf("response:\n") + if b, err := httputil.DumpResponse(resp, false); err != nil { + log.Fatalf("err: %s", err) + } else { + fmt.Printf("%s\n", b) + } - // How many bytes to download at a time - partial := fileSize / opts.Procs + // get the size from the response header. + fileSize, err := getheader.GetSize(resp) + if err != nil { + log.Fatalf("err: getheader.GetSize: %s\n", err) + } + if err = resp.Body.Close(); err != nil { + log.Fatalf("err: %s", err) + } - outputPath := opts.Output + filepath.Base(urlObj.String()) - // if there is the same file in opts.Output, delete that file in advance. - if isExists(outputPath) { - err := os.Remove(outputPath) - if err != nil { - log.Fatalf("err: isExists: os.Remove: %s\n", err) - } - } + // How many bytes to download at a time + partial := fileSize / opts.Procs - // make a file for download - out, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0755) + outputPath := opts.Output + filepath.Base(urlObj.String()) + // if there is the same file in opts.Output, delete that file in advance. + if isExists(outputPath) { + err := os.Remove(outputPath) if err != nil { - log.Fatalf("err: os.Create: %s\n", err) + log.Fatalf("err: isExists: os.Remove: %s\n", err) } + } - // make a temporary directory for parallel download - tmpDirName := opts.Output + strconv.Itoa(i) - err = os.Mkdir(tmpDirName, 0777) - if err != nil { - if err3 := out.Close(); err3 != nil { - log.Fatalf("err: %s", err3) - } - if err2 := os.Remove(opts.Output + filepath.Base(urlObj.String())); err2 != nil { - log.Fatalf("err: os.Mkdir: %s\nerr: os.Remove: %s\n", err, err2) - } - log.Fatalf("err: os.Mkdir: %s\n", err) + // make a file for download + out, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0755) + if err != nil { + log.Fatalf("err: os.Create: %s\n", err) + } + defer func() { + if err := out.Close(); err != nil { + log.Fatalf("err: %s", err) } + }() - clean := func() { - if err := out.Close(); err != nil { - log.Fatalf("err: out.Close: %s\n", err) - } - // delete the tmporary directory - if err := os.RemoveAll(tmpDirName); err != nil { - log.Fatalf("err: RemoveAll: %s\n", err) - } - if err := os.Remove(opts.Output + filepath.Base(urlObj.String())); err != nil { - log.Fatalf("err: os.Remove: %s\n", err) - } - } - ctx, cancel := listen.Listen(ctxTimeout, os.Stdout, clean) - - var isPara bool = true - accept, err := getheader.ResHeader(os.Stdout, resp, "Accept-Ranges") - if err != nil && err.Error() == "cannot find Accept-Ranges header" { - isPara = false - } else if err != nil { - clean() - log.Fatalf("err: getheader.ResHeader: %s\n", err) - } else if accept[0] != "bytes" || opts.Procs == 1 { - isPara = false - continue + // make a temporary directory for parallel download + tmpDirName := opts.Output + strconv.Itoa(i) + err = os.Mkdir(tmpDirName, 0777) + if err != nil { + if err3 := out.Close(); err3 != nil { + log.Fatalf("err: %s", err3) } - - // drive a download process - err = download.Downloader(urlObj, out, fileSize, partial, opts.Procs, isPara, tmpDirName, ctx) - if err != nil { - log.Fatalf("err: %s\n", err) + if err2 := os.Remove(opts.Output + filepath.Base(urlObj.String())); err2 != nil { + log.Fatalf("err: os.Mkdir: %s\nerr: os.Remove: %s\n", err, err2) } + log.Fatalf("err: os.Mkdir: %s\n", err) + } - fmt.Printf("download complete: %s\n", urlObj.String()) - - // Merge the temporary files into "out", when parallel download executed. - if isPara { - err = MergeFiles(tmpDirName, opts.Procs, fileSize, out) - if err != nil { - log.Fatalf("err: MergeFiles: %s\n", err) - } + clean := func() { + if err := out.Close(); err != nil { + log.Fatalf("err: out.Close: %s\n", err) } - - // delete the tmporary directory only + // delete the tmporary directory if err := os.RemoveAll(tmpDirName); err != nil { log.Fatalf("err: RemoveAll: %s\n", err) } + if err := os.Remove(opts.Output + filepath.Base(urlObj.String())); err != nil { + log.Fatalf("err: os.Remove: %s\n", err) + } + } + ctx, cancel := listen.Listen(ctxTimeout, os.Stdout, clean) + defer cancel() + + var isPara bool = true + _, err = getheader.ResHeader(os.Stdout, resp, "Accept-Ranges") + if err != nil && err.Error() == "cannot find Accept-Ranges header" { + isPara = false + } else if err != nil { + clean() + log.Fatalf("err: getheader.ResHeader: %s\n", err) + } - // Clean up - cancel() - if err = out.Close(); err != nil { - log.Fatalf("err: %s", err) + // drive a download process + err = download.Downloader(urlObj, out, fileSize, partial, opts.Procs, isPara, tmpDirName, ctx) + if err != nil { + log.Fatalf("err: %s\n", err) + } + + fmt.Printf("download complete: %s\n", urlObj.String()) + + // Merge the temporary files into "out", when parallel download executed. + if isPara { + err = MergeFiles(tmpDirName, opts.Procs, fileSize, out) + if err != nil { + log.Fatalf("err: MergeFiles: %s\n", err) } } + + // delete the tmporary directory only + if err := os.RemoveAll(tmpDirName); err != nil { + log.Fatalf("err: RemoveAll: %s\n", err) + } } // MergeFiles merges temporary files made for parallel download into "output". From 8c1515381495ca54030aa560972ce4d65420cdd6 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Fri, 30 Jul 2021 12:25:48 +0000 Subject: [PATCH 50/51] modified main function in main.go --- kadai3-2/Mizushima/main.go | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/kadai3-2/Mizushima/main.go b/kadai3-2/Mizushima/main.go index c2b0d1e7..e16f681d 100644 --- a/kadai3-2/Mizushima/main.go +++ b/kadai3-2/Mizushima/main.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "log" - "net/http/httputil" "net/url" "os" "path/filepath" @@ -115,7 +114,7 @@ func main() { } } -// +// downloadFromUrl does the download processing from url object. func downloadFromUrl(i int, opts Options, urlObj *url.URL) { // make a timeout context from a empty context @@ -128,14 +127,6 @@ func downloadFromUrl(i int, opts Options, urlObj *url.URL) { log.Fatalf("err: %s\n", err) } - // show response header - fmt.Printf("response:\n") - if b, err := httputil.DumpResponse(resp, false); err != nil { - log.Fatalf("err: %s", err) - } else { - fmt.Printf("%s\n", b) - } - // get the size from the response header. fileSize, err := getheader.GetSize(resp) if err != nil { @@ -145,7 +136,7 @@ func downloadFromUrl(i int, opts Options, urlObj *url.URL) { log.Fatalf("err: %s", err) } - // How many bytes to download at a time + // get how many bytes to download at a time partial := fileSize / opts.Procs outputPath := opts.Output + filepath.Base(urlObj.String()) From 548c9aaeb9a6ba79b80a84b95f4244d0a0a1bcd2 Mon Sep 17 00:00:00 2001 From: MizushimaToshihiko Date: Sat, 31 Jul 2021 01:29:03 +0000 Subject: [PATCH 51/51] modified main.go and download/download.go corrected comment --- kadai3-2/Mizushima/download/download.go | 3 ++- kadai3-2/Mizushima/main.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/kadai3-2/Mizushima/download/download.go b/kadai3-2/Mizushima/download/download.go index ece75d7d..772c9d1d 100644 --- a/kadai3-2/Mizushima/download/download.go +++ b/kadai3-2/Mizushima/download/download.go @@ -15,7 +15,8 @@ import ( "golang.org/x/sync/errgroup" ) -//PDownloader is user-defined struct +// PDownloader is user-defined struct for the download process. +// It's not limited to parallel downloads. type PDownloader struct { url *url.URL // URL for the download output *os.File // Where to save the downloaded file diff --git a/kadai3-2/Mizushima/main.go b/kadai3-2/Mizushima/main.go index e16f681d..1bf14fe3 100644 --- a/kadai3-2/Mizushima/main.go +++ b/kadai3-2/Mizushima/main.go @@ -235,7 +235,7 @@ func MergeFiles(tmpDirName string, procs, fileSize uint, output *os.File) error return nil } -// +// isExists returns the file exists or not, in 'path'. func isExists(path string) bool { _, err := os.Stat(path) return err == nil