kadai3-2 haijima

kadai3-2/haijima/.gitignore

@@ -0,0 +1,130 @@
+
+# Created by https://www.gitignore.io/api/go,macos,jetbrains+all
+
+### Go ###
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, build with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+### Go Patch ###
+/vendor/
+/Godeps/
+
+### JetBrains+all ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+### JetBrains+all Patch ###
+# Ignores the whole .idea folder and all .iml files
+# See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360
+
+.idea/
+
+# Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023
+
+*.iml
+modules.xml
+.idea/misc.xml
+*.ipr
+
+### macOS ###
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+
+# End of https://www.gitignore.io/api/go,macos,jetbrains+all
+

kadai3-2/haijima/Makefile

@@ -0,0 +1,3 @@
+build:
+	go build -o gget main.go
+

kadai3-2/haijima/README.md

@@ -0,0 +1,23 @@
+# 課題3-2
+## 要求
+分割ダウンロードを行う
+* [x] Rangeアクセスを用いる
+* [x] いくつかのゴルーチンでダウンロードしてマージする
+* [x] エラー処理を工夫する
+    * golang.org/x/sync/errgourpパッケージなどを使ってみる
+* [x] キャンセルが発生した場合の実装を行う
+
+### できてないこと
+* [ ] テスト
+
+
+## How to build
+```
+$ make
+```
+
+
+## How to run
+```
+$ ./gget [-n num] [-o outputdir] url
+```

kadai3-2/haijima/main.go

@@ -0,0 +1,128 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"github.com/pkg/errors"
+	"golang.org/x/sync/errgroup"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path"
+	"strconv"
+)
+
+func main() {
+	var (
+		dir = flag.String("o", ".", "output directory")
+		num = flag.Int("n", 3, "num of worker")
+	)
+	flag.Parse()
+
+	url := flag.Arg(0)
+	filename := path.Join(*dir, path.Base(url))
+	file, err := os.Create(filename)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error %+v\n", err)
+	}
+	defer file.Close()
+
+	err = Exec(url, file, *num)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error %+v\n", err)
+		os.Remove(filename)
+	}
+}
+
+func Exec(url string, w io.Writer, num int) error {
+	length, ok := acceptsRangeRequest(url)
+	if num > 1 && ok {
+		b := length/num + 1
+		eg, ctx := errgroup.WithContext(context.Background())
+
+		// Range access
+		tmpFiles := make([]io.Reader, num)
+		defer func() {
+			for _, tmpFile := range tmpFiles {
+				if f, ok := tmpFile.(io.ReadCloser); ok {
+					f.Close()
+				}
+			}
+		}()
+
+		for i := 0; i < num; i++ {
+			i := i
+			eg.Go(func() error {
+				tmpFile, err := ioutil.TempFile("", path.Base(url))
+				if err != nil {
+					return errors.WithStack(err)
+				}
+				tmpFiles[i] = tmpFile
+				header := map[string]string{"Range": fmt.Sprintf("bytes=%v-%v", i*b, (i+1)*b-1)}
+				return download(ctx, url, header, tmpFile)
+			})
+		}
+		if err := eg.Wait(); err != nil {
+			return errors.WithStack(err)
+		}
+
+		// Concatenate partial files
+		return concat(w, tmpFiles)
+	} else {
+		ctx, _ := context.WithCancel(context.Background())
+		return download(ctx, url, nil, w)
+	}
+}
+
+func acceptsRangeRequest(url string) (int, bool) {
+	resp, err := http.Head(url)
+	if err != nil {
+		return 0, false
+	}
+	defer resp.Body.Close()
+	unit := resp.Header.Get("Accept-Ranges")
+	length, err := strconv.Atoi(resp.Header.Get("Content-Length"))
+	if err != nil {
+		return 0, false
+	}
+	return length, unit == "bytes"
+}
+
+func download(ctx context.Context, url string, h map[string]string, w io.Writer) error {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	for k, v := range h {
+		req.Header.Set(k, v)
+	}
+	resp, err := http.DefaultClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	defer resp.Body.Close()
+
+	_, err = io.Copy(w, resp.Body)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	return nil
+}
+
+func concat(dst io.Writer, srcs []io.Reader) error {
+	for _, src := range srcs {
+		err := func() error {
+			_, err := io.Copy(dst, src)
+			if err != nil {
+				return errors.WithStack(err)
+			}
+			return nil
+		}()
+		if err != nil {
+			return errors.WithStack(err)
+		}
+	}
+	return nil
+}