diff --git a/kadai3-2/haijima/.gitignore b/kadai3-2/haijima/.gitignore new file mode 100644 index 0000000..8427f66 --- /dev/null +++ b/kadai3-2/haijima/.gitignore @@ -0,0 +1,130 @@ + +# Created by https://www.gitignore.io/api/go,macos,jetbrains+all + +### Go ### +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +### Go Patch ### +/vendor/ +/Godeps/ + +### JetBrains+all ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/modules.xml +# .idea/*.iml +# .idea/modules + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +### JetBrains+all Patch ### +# Ignores the whole .idea folder and all .iml files +# See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 + +.idea/ + +# Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 + +*.iml +modules.xml +.idea/misc.xml +*.ipr + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + + +# End of https://www.gitignore.io/api/go,macos,jetbrains+all + diff --git a/kadai3-2/haijima/Makefile b/kadai3-2/haijima/Makefile new file mode 100644 index 0000000..da6db7d --- /dev/null +++ b/kadai3-2/haijima/Makefile @@ -0,0 +1,3 @@ +build: + go build -o gget main.go + diff --git a/kadai3-2/haijima/README.md b/kadai3-2/haijima/README.md new file mode 100644 index 0000000..70146b9 --- /dev/null +++ b/kadai3-2/haijima/README.md @@ -0,0 +1,23 @@ +# 課題3-2 +## 要求 +分割ダウンロードを行う +* [x] Rangeアクセスを用いる +* [x] いくつかのゴルーチンでダウンロードしてマージする +* [x] エラー処理を工夫する + * golang.org/x/sync/errgourpパッケージなどを使ってみる +* [x] キャンセルが発生した場合の実装を行う + +### できてないこと +* [ ] テスト + + +## How to build +``` +$ make +``` + + +## How to run +``` +$ ./gget [-n num] [-o outputdir] url +``` diff --git a/kadai3-2/haijima/main.go b/kadai3-2/haijima/main.go new file mode 100644 index 0000000..6f26096 --- /dev/null +++ b/kadai3-2/haijima/main.go @@ -0,0 +1,128 @@ +package main + +import ( + "context" + "flag" + "fmt" + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" + "io" + "io/ioutil" + "net/http" + "os" + "path" + "strconv" +) + +func main() { + var ( + dir = flag.String("o", ".", "output directory") + num = flag.Int("n", 3, "num of worker") + ) + flag.Parse() + + url := flag.Arg(0) + filename := path.Join(*dir, path.Base(url)) + file, err := os.Create(filename) + if err != nil { + fmt.Fprintf(os.Stderr, "error %+v\n", err) + } + defer file.Close() + + err = Exec(url, file, *num) + if err != nil { + fmt.Fprintf(os.Stderr, "error %+v\n", err) + os.Remove(filename) + } +} + +func Exec(url string, w io.Writer, num int) error { + length, ok := acceptsRangeRequest(url) + if num > 1 && ok { + b := length/num + 1 + eg, ctx := errgroup.WithContext(context.Background()) + + // Range access + tmpFiles := make([]io.Reader, num) + defer func() { + for _, tmpFile := range tmpFiles { + if f, ok := tmpFile.(io.ReadCloser); ok { + f.Close() + } + } + }() + + for i := 0; i < num; i++ { + i := i + eg.Go(func() error { + tmpFile, err := ioutil.TempFile("", path.Base(url)) + if err != nil { + return errors.WithStack(err) + } + tmpFiles[i] = tmpFile + header := map[string]string{"Range": fmt.Sprintf("bytes=%v-%v", i*b, (i+1)*b-1)} + return download(ctx, url, header, tmpFile) + }) + } + if err := eg.Wait(); err != nil { + return errors.WithStack(err) + } + + // Concatenate partial files + return concat(w, tmpFiles) + } else { + ctx, _ := context.WithCancel(context.Background()) + return download(ctx, url, nil, w) + } +} + +func acceptsRangeRequest(url string) (int, bool) { + resp, err := http.Head(url) + if err != nil { + return 0, false + } + defer resp.Body.Close() + unit := resp.Header.Get("Accept-Ranges") + length, err := strconv.Atoi(resp.Header.Get("Content-Length")) + if err != nil { + return 0, false + } + return length, unit == "bytes" +} + +func download(ctx context.Context, url string, h map[string]string, w io.Writer) error { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return errors.WithStack(err) + } + for k, v := range h { + req.Header.Set(k, v) + } + resp, err := http.DefaultClient.Do(req.WithContext(ctx)) + if err != nil { + return errors.WithStack(err) + } + defer resp.Body.Close() + + _, err = io.Copy(w, resp.Body) + if err != nil { + return errors.WithStack(err) + } + return nil +} + +func concat(dst io.Writer, srcs []io.Reader) error { + for _, src := range srcs { + err := func() error { + _, err := io.Copy(dst, src) + if err != nil { + return errors.WithStack(err) + } + return nil + }() + if err != nil { + return errors.WithStack(err) + } + } + return nil +}