Kadai3 2/yuonoda

.gitignore

@@ -0,0 +1 @@
+.idea

kadai3-2/yuonoda/go.mod

@@ -0,0 +1,5 @@
+module github.com/yuonoda/gopherdojo-studyroom/kadai3-2/yuonoda
+
+go 1.15
+
+require golang.org/x/sync v0.0.0-20201207232520-09787c993a3a

kadai3-2/yuonoda/go.sum

@@ -0,0 +1,2 @@
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

kadai3-2/yuonoda/lib/export_test.go

@@ -0,0 +1,3 @@
+package splitDownload
+
+var ExportedFillByteArr = fillByteArr

kadai3-2/yuonoda/lib/resource.go

@@ -0,0 +1,157 @@
+package splitDownload
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"golang.org/x/sync/errgroup"
+	"io/ioutil"
+	"log"
+	"math"
+	"math/rand"
+	"net/http"
+	"strconv"
+	"time"
+)
+
+type partialContent struct {
+	StartByte int
+	EndByte   int
+	Body      []byte
+}
+
+type resource struct {
+	Url              string
+	Size             int
+	BatchSize        int
+	BatchCount       int
+	Content          []byte
+	PartialContentCh chan partialContent
+	Http             http.Client
+}
+
+func (r *resource) GetSize() error {
+	log.Printf("resource.getSize()\n")
+
+	// HEADでサイズを調べる
+	res, err := r.Http.Head(r.Url)
+	if err != nil {
+		return err
+	}
+
+	// データサイズを取得
+	header := res.Header
+	cl, ok := header["Content-Length"]
+	if !ok {
+		return errors.New("Content-Length couldn't be found")
+	}
+	r.Size, err = strconv.Atoi(cl[0])
+	if err != nil {
+		return err
+	}
+	return nil
+
+}
+
+func (r *resource) GetPartialContent(startByte int, endByte int) error {
+	log.Printf("resource.getPartialContent(%d, %d)\n", startByte, endByte)
+	// Rangeヘッダーを作成
+	rangeVal := fmt.Sprintf("bytes=%d-%d", startByte, endByte)
+
+	// リクエストとクライアントの作成
+	reader := bytes.NewReader([]byte{})
+	req, err := http.NewRequest("GET", r.Url, reader)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Range", rangeVal)
+	client := &http.Client{}
+
+	res := &http.Response{}
+	for i := 0; i < 3; i++ {
+		// リクエストの実行
+		log.Printf("rangeVal[%d]:%s", i, rangeVal)
+		res, err = client.Do(req)
+		if err != nil {
+			return err
+		}
+
+		// ステータスが200系ならループを抜ける
+		log.Printf("res.StatusCode:%d\n", res.StatusCode)
+		if res.StatusCode >= 200 && res.StatusCode <= 299 {
+			break
+		}
+
+		// 乱数分スリープ
+		rand.Seed(time.Now().UnixNano())
+		randFloat := rand.Float64() + 1
+		randMs := math.Pow(randFloat, float64(i+1)) * 1000
+		sleepTime := time.Duration(randMs) * time.Millisecond
+		log.Printf("sleep:%v\n", sleepTime)
+		time.Sleep(sleepTime)
+	}
+
+	// 正常系レスポンスでないとき
+	if res.StatusCode < 200 || res.StatusCode > 299 {
+		return errors.New("status code is not 2xx, got " + res.Status)
+	}
+
+	// bodyの取得
+	body, err := ioutil.ReadAll(res.Body)
+	defer res.Body.Close()
+	if err != nil {
+		return err
+	}
+
+	pc := partialContent{StartByte: startByte, EndByte: endByte, Body: body}
+	r.PartialContentCh <- pc
+	return nil
+}
+
+func (r *resource) GetContent(batchCount int) error {
+	log.Println("resource.getContent()")
+
+	// コンテンツのデータサイズを取得
+	err := r.GetSize()
+	if err != nil {
+		log.Fatal(err)
+	}
+	log.Printf("r.size: %d\n", r.Size)
+
+	// batchCount分リクエスト
+	r.BatchCount = batchCount
+	r.BatchSize = int(math.Ceil(float64(r.Size) / float64(r.BatchCount)))
+	r.Content = make([]byte, r.Size)
+	var eg errgroup.Group
+	r.PartialContentCh = make(chan partialContent, r.BatchCount)
+	for i := 0; i < r.BatchCount; i++ {
+
+		// 担当する範囲を決定
+		startByte := r.BatchSize * i
+		endByte := r.BatchSize*(i+1) - 1
+		if endByte > r.Size {
+			endByte = r.Size
+		}
+
+		// レンジごとにリクエスト
+		eg.Go(func() error {
+			return r.GetPartialContent(startByte, endByte)
+		})
+	}
+
+	// １リクエストでも失敗すれば終了
+	if err := eg.Wait(); err != nil {
+		return err
+	}
+
+	// 一つのバイト列にマージ
+	r.Content = make([]byte, r.Size)
+	for i := 0; i < r.BatchCount; i++ {
+		log.Println("merging...")
+		pc := <-r.PartialContentCh
+		log.Printf("pc.startByte: %v\n", pc.StartByte)
+		fillByteArr(r.Content[:], pc.StartByte, pc.Body)
+	}
+
+	return nil
+}

kadai3-2/yuonoda/lib/splitDownload.go

@@ -0,0 +1,48 @@
+package splitDownload
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func Run(url string, batchCount int, dwDirPath string) string {
+	log.Println("Run")
+
+	// ファイルの作成
+	_, filename := filepath.Split(url)
+	homedir, err := os.UserHomeDir()
+	if err != nil {
+		log.Fatal(err)
+	}
+	if dwDirPath == "" {
+		dwDirPath = homedir + "/Downloads"
+	}
+	dwFilePath := dwDirPath + "/" + filename + ".download"
+	log.Println(dwFilePath)
+	dwFile, err := os.Create(dwFilePath)
+	if err != nil {
+		os.Remove(dwFilePath)
+		log.Fatal(err)
+	}
+
+	// ダウンロード実行
+	r := &resource{Url: url}
+	err = r.GetContent(batchCount)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// データの書き込み
+	_, err = dwFile.Write(r.Content)
+	if err != nil {
+		os.Remove(dwFilePath)
+		log.Fatal(err)
+	}
+	finishedFilePath := strings.Trim(dwFilePath, ".download")
+	os.Rename(dwFilePath, finishedFilePath)
+
+	log.Println("download succeeded!")
+	return finishedFilePath
+}

kadai3-2/yuonoda/lib/splitDownload_test.go

@@ -0,0 +1,98 @@
+package splitDownload_test
+
+import (
+	"log"
+	"os"
+	"reflect"
+	"testing"
+
+	splitDownload "github.com/yuonoda/gopherdojo-studyroom/kadai3-2/yuonoda/lib"
+)
+
+func TestRun(t *testing.T) {
+	cases := []struct {
+		name         string
+		url          string
+		expectedSize int64
+		concurrency  int
+	}{
+		{
+			name:         "basic",
+			url:          "https://dumps.wikimedia.org/jawiki/20210101/jawiki-20210101-pages-articles-multistream-index.txt.bz2",
+			expectedSize: int64(25802009),
+			concurrency:  3,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			// ダウンロードパスを指定
+			homedir, err := os.UserHomeDir()
+			if err != nil {
+				log.Fatal(err)
+			}
+			dwDirPath := homedir + "/Downloads"
+
+			// ダウンロード
+			filePath := splitDownload.Run(c.url, c.concurrency, dwDirPath)
+			file, err := os.Open(filePath)
+			if err != nil {
+				t.Error(err)
+			}
+
+			// サイズを取得
+			info, err := file.Stat()
+			if err != nil {
+				t.Error(err)
+			}
+			dwSize := info.Size()
+
+			// サイズを比較
+			if dwSize != c.expectedSize {
+				t.Errorf("Size doesn't match, got %d but expexted %d", dwSize, c.expectedSize)
+			}
+
+			// ファイルを削除
+			err = os.Remove(filePath)
+			if err != nil {
+				t.Error(err)
+			}
+		})
+	}
+
+}
+
+func TestFillByteArr(t *testing.T) {
+	cases := []struct {
+		name        string
+		arr         []byte
+		startAt     int
+		partArr     []byte
+		expectedArr []byte
+	}{
+		{
+			name:        "basic",
+			arr:         []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+			startAt:     3,
+			partArr:     []byte{4, 5, 6},
+			expectedArr: []byte{0, 0, 0, 4, 5, 6, 0, 0, 0, 0},
+		},
+		{
+			name:        "basic2",
+			arr:         []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+			startAt:     8,
+			partArr:     []byte{9, 10},
+			expectedArr: []byte{0, 0, 0, 0, 0, 0, 0, 0, 9, 10},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			splitDownload.ExportedFillByteArr(c.arr[:], c.startAt, c.partArr)
+			if !reflect.DeepEqual(c.expectedArr, c.arr) {
+				t.Error("Array does not match")
+			}
+		})
+	}
+
+}

kadai3-2/yuonoda/lib/utilities.go

@@ -0,0 +1,9 @@
+package splitDownload
+
+// 配列の一部を別配列に置き換える
+func fillByteArr(arr []byte, startAt int, partArr []byte) {
+	for i := 0; i < len(partArr); i++ {
+		globalIndex := i + startAt
+		arr[globalIndex] = partArr[i]
+	}
+}

kadai3-2/yuonoda/main.go

@@ -0,0 +1,15 @@
+package main
+
+import (
+	"flag"
+	splitDownload "github.com/yuonoda/gopherdojo-studyroom/kadai3-2/yuonoda/lib"
+)
+
+var url = flag.String("url", "https://dumps.wikimedia.org/jawiki/20210101/jawiki-20210101-pages-articles-multistream-index.txt.bz2", "URL to download")
+var batchCount = flag.Int("c", 1, "how many times you request content")
+var dwDirPath = flag.String("path", "", "where to put a downloaded file")
+
+func main() {
+	flag.Parse()
+	splitDownload.Run(*url, *batchCount, *dwDirPath)
+}