Kadai3 2/yuonoda

.gitignore

@@ -0,0 +1 @@
+.idea

kadai3-2/yuonoda/downloader/downloader.go

@@ -0,0 +1,225 @@
+package downloader
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"github.com/yuonoda/gopherdojo-studyroom/kadai3-2/yuonoda/utilities"
+	"golang.org/x/sync/errgroup"
+	"io/ioutil"
+	"log"
+	"math"
+	"math/rand"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"time"
+)
+
+type partialContent struct {
+	StartByte int
+	EndByte   int
+	Body      []byte
+}
+
+type Downloader struct {
+	Url              string
+	Size             int
+	BatchSize        int
+	BatchCount       int
+	Content          []byte
+	PartialContentCh chan partialContent
+	Http             http.Client
+}
+
+func (d *Downloader) GetSize() error {
+	log.Printf("resource.getSize()\n")
+
+	// HEADでサイズを調べる
+	res, err := d.Http.Head(d.Url)
+	if err != nil {
+		return err
+	}
+
+	// データサイズを取得
+	header := res.Header
+	cl, ok := header["Content-Length"]
+	if !ok {
+		return errors.New("Content-Length couldn't be found")
+	}
+	d.Size, err = strconv.Atoi(cl[0])
+	if err != nil {
+		return err
+	}
+	return nil
+
+}
+
+func (d *Downloader) GetPartialContent(startByte int, endByte int, ctx context.Context, errCh chan error) {
+	log.Printf("resource.getPartialContent(%d, %d)\n", startByte, endByte)
+	// Rangeヘッダーを作成
+	rangeVal := fmt.Sprintf("bytes=%d-%d", startByte, endByte)
+
+	// リクエストとクライアントの作成
+	reader := bytes.NewReader([]byte{})
+	req, err := http.NewRequest("GET", d.Url, reader)
+	if err != nil {
+		errCh <- err
+		return
+	}
+	req.Header.Set("Range", rangeVal)
+	client := &http.Client{}
+
+	res := &http.Response{}
+	const retryCount = 3
+	for i := 0; i < retryCount; i++ {
+
+		// リクエストの実行
+		log.Printf("rangeVal[%d]:%s", i, rangeVal)
+		res, err = client.Do(req)
+		if err != nil {
+			errCh <- err
+			return
+		}
+
+		// ステータスが200系ならループを抜ける
+		log.Printf("res.StatusCode:%d\n", res.StatusCode)
+		if res.StatusCode >= 200 && res.StatusCode <= 299 {
+			break
+		}
+
+		// 乱数分スリープ
+		rand.Seed(time.Now().UnixNano())
+		randFloat := rand.Float64() + 1
+		randMs := math.Pow(randFloat, float64(i+1)) * 1000
+		sleepTime := time.Duration(randMs) * time.Millisecond
+		log.Printf("sleep:%v\n", sleepTime)
+		time.Sleep(sleepTime)
+	}
+
+	// 正常系レスポンスでないとき
+	if res.StatusCode < 200 || res.StatusCode > 299 {
+		errCh <- errors.New("status code is not 2xx, got " + res.Status)
+		return
+	}
+
+	// bodyの取得
+	log.Println("start reading")
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		errCh <- err
+		return
+	}
+	defer func() {
+		if err = res.Body.Close(); err != nil {
+			errCh <- err
+			return
+		}
+	}()
+
+	pc := partialContent{StartByte: startByte, EndByte: endByte, Body: body}
+	d.PartialContentCh <- pc
+	return
+}
+
+func (d *Downloader) GetContent(batchCount int, ctx context.Context) error {
+	log.Println("resource.getContent()")
+
+	// コンテンツのデータサイズを取得
+	err := d.GetSize()
+	if err != nil {
+		return err
+	}
+	log.Printf("d.size: %d\n", d.Size)
+
+	// batchCount分リクエスト
+	d.BatchCount = batchCount
+	d.BatchSize = int(math.Ceil(float64(d.Size) / float64(d.BatchCount)))
+	d.Content = make([]byte, d.Size)
+	var eg *errgroup.Group
+	eg, ctx = errgroup.WithContext(ctx)
+	errCh := make(chan error)
+	d.PartialContentCh = make(chan partialContent, d.BatchCount)
+	for i := 0; i < d.BatchCount; i++ {
+
+		// 担当する範囲を決定
+		startByte := d.BatchSize * i
+		endByte := d.BatchSize*(i+1) - 1
+		if endByte > d.Size {
+			endByte = d.Size
+		}
+
+		// レンジごとにリクエスト
+		go d.GetPartialContent(startByte, endByte, ctx, errCh)
+	}
+
+	// リクエスト回数分受け付けてマージ
+	var mu sync.Mutex
+	d.Content = make([]byte, d.Size)
+	for i := 0; i < d.BatchCount; i++ {
+		eg.Go(func() error {
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			case pc := <-d.PartialContentCh:
+				mu.Lock()
+				utilities.FillByteArr(d.Content[:], pc.StartByte, pc.Body)
+				mu.Unlock()
+			case err = <-errCh:
+				return err
+
+			}
+			return nil
+		})
+	}
+
+	// １リクエストでも失敗すれば終了
+	if err := eg.Wait(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (d Downloader) Download(ctx context.Context, batchCount int, dwDirPath string) error {
+	log.Println("Download")
+
+	// 一時ファイルの作成
+	_, filename := filepath.Split(d.Url)
+	dwFilePath := dwDirPath + "/" + filename + ".download"
+	finishedFilePath := dwDirPath + "/" + filename
+	dwFile, err := os.Create(dwFilePath)
+	if err != nil {
+		return err
+	}
+
+	// 関数の終了時に一時ファイルを削除
+	defer func() {
+		if _, err = os.Stat(dwFilePath); err != nil {
+			return
+		}
+		if err = os.Remove(dwFilePath); err != nil {
+			log.Fatalf("falid to remove .download file. %s", err)
+		}
+	}()
+
+	// ダウンロード実行
+	err = d.GetContent(batchCount, ctx)
+	if err != nil {
+		return err
+	}
+
+	// データの書き込み
+	_, err = dwFile.Write(d.Content)
+	if err != nil {
+		return err
+	}
+	if err = os.Rename(dwFilePath, finishedFilePath); err != nil {
+		return err
+	}
+	log.Println("download succeeded!")
+	return nil
+}

kadai3-2/yuonoda/downloader/downloader_test.go

@@ -0,0 +1,45 @@
+package downloader_test
+
+import (
+	"os"
+	"testing"
+
+	"github.com/yuonoda/gopherdojo-studyroom/kadai3-2/yuonoda/downloader"
+	"github.com/yuonoda/gopherdojo-studyroom/kadai3-2/yuonoda/terminate"
+)
+
+func TestDownload(t *testing.T) {
+	cases := []struct {
+		name         string
+		url          string
+		expectedSize int64
+		concurrency  int
+	}{
+		{
+			name:         "basic",
+			url:          "https://dumps.wikimedia.org/jawiki/20210101/jawiki-20210101-pages-articles-multistream-index.txt.bz2",
+			expectedSize: int64(25802009),
+			concurrency:  3,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			// ダウンロードパスを指定
+			homedir, err := os.UserHomeDir()
+			if err != nil {
+				t.Error(err)
+			}
+			dwDirPath := homedir + "/Downloads"
+
+			// ダウンロード
+			ctx := terminate.Listen()
+			d := downloader.Downloader{Url: c.url}
+			err = d.Download(ctx, c.concurrency, dwDirPath)
+			if err != nil {
+				t.Error(err)
+			}
+		})
+	}
+
+}

kadai3-2/yuonoda/go.mod

@@ -0,0 +1,11 @@
+module github.com/yuonoda/gopherdojo-studyroom/kadai3-2/yuonoda
+
+go 1.15
+
+require (
+	github.com/kisielk/errcheck v1.5.0 // indirect
+	golang.org/x/mod v0.4.1 // indirect
+	golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
+	golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect
+	golang.org/x/tools v0.1.0 // indirect
+)

kadai3-2/yuonoda/go.sum

@@ -0,0 +1,42 @@
+github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY=
+golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f h1:tuwaIjfUa6eI6REiNueIxvNm1popyPUnqWga83S7U0o=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

kadai3-2/yuonoda/main.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"flag"
+	downloader "github.com/yuonoda/gopherdojo-studyroom/kadai3-2/yuonoda/downloader"
+	"github.com/yuonoda/gopherdojo-studyroom/kadai3-2/yuonoda/terminate"
+	"log"
+)
+
+var url = flag.String("url", "https://dumps.wikimedia.org/jawiki/20210101/jawiki-20210101-pages-articles-multistream-index.txt.bz2", "URL to download")
+var batchCount = flag.Int("c", 1, "how many times you request content")
+var dwDirPath = flag.String("path", ".", "where to put a downloaded file")
+
+func main() {
+	flag.Parse()
+	ctx := terminate.Listen()
+	d := downloader.Downloader{Url: *url}
+	err := d.Download(ctx, *batchCount, *dwDirPath)
+	if err != nil {
+		log.Fatal(err)
+	}
+}