diff --git a/kadai3-2/yoheimiyamoto/downloader/client.go b/kadai3-2/yoheimiyamoto/downloader/client.go new file mode 100644 index 0000000..e274b25 --- /dev/null +++ b/kadai3-2/yoheimiyamoto/downloader/client.go @@ -0,0 +1,150 @@ +package downloader + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "strconv" + + "golang.org/x/sync/errgroup" +) + +// Client ... +type Client struct { + *http.Client + url string +} + +// NewClient ... +func NewClient(url string) *Client { + return &Client{new(http.Client), url} +} + +// Download ... +func (c *Client) Download(ctx context.Context) error { + l, err := c.contentLength() + if err != nil { + return err + } + + fmt.Fprintf(os.Stdout, "contentLength: %d\n", l) + + eg, ctx := errgroup.WithContext(ctx) + + ps := newRangeProperties(l) + fmt.Fprintf(os.Stdout, "ps: %d\n", len(ps)) + + for _, p := range ps { + p := p + eg.Go(func() error { + return c.rangeDownload(ctx, p) + }) + } + + if err := eg.Wait(); err != nil { + return err + } + + defer removeFiles(ps) + + src := make([]string, len(ps)) + for i, p := range ps { + src[i] = p.path + } + err = mergeFiles(src, "output.jpg") + if err != nil { + return nil + } + return nil +} + +// contentLengthを取得 +func (c *Client) contentLength() (int, error) { + res, err := c.Head(c.url) + if err != nil { + return 0, err + } + return strconv.Atoi(res.Header.Get("Content-Length")) +} + +// ファイルの分割ダウンロード +// start -> 開始のbyte +// end -> 終了のbyte +// dst -> ダウンロード先のファイル名 +func (c *Client) rangeDownload(ctx context.Context, r *rangeProperty) error { + errCh := make(chan error, 1) + + go func() { + req, _ := http.NewRequest("GET", c.url, nil) + req.Header.Set("Authorization", "Bearer access-token") + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", r.start, r.end)) + + res, err := c.Do(req) + if err != nil { + errCh <- err + return + } + defer res.Body.Close() + + f, err := os.Create(r.path) + if err != nil { + errCh <- err + return + } + defer f.Close() + + _, err = io.Copy(f, res.Body) + if err != nil { + errCh <- err + return + } + errCh <- nil + }() + + select { + case err := <-errCh: + if err != nil { + return err + } + case <-ctx.Done(): + return ctx.Err() + } + + return nil +} + +// ダウンロードした分割ファイルのマージ +// src -> 元の分割ファイル名 +// dst -> マージ先のファイル名 +func mergeFiles(src []string, dst string) error { + f, err := os.Create(dst) + if err != nil { + return err + } + defer f.Close() + for _, s := range src { + _f, err := os.Open(s) + if err != nil { + return err + } + _, err = io.Copy(f, _f) + if err != nil { + return err + } + _f.Close() + } + return nil +} + +// 一時保存ファイルの全削除 +func removeFiles(ps []*rangeProperty) error { + for _, p := range ps { + err := os.Remove(p.path) + if err != nil { + return err + } + } + return nil +} diff --git a/kadai3-2/yoheimiyamoto/downloader/client_test.go b/kadai3-2/yoheimiyamoto/downloader/client_test.go new file mode 100644 index 0000000..8a03437 --- /dev/null +++ b/kadai3-2/yoheimiyamoto/downloader/client_test.go @@ -0,0 +1,41 @@ +package downloader + +import "testing" + +const URL = "https://images.pexels.com/photos/248304/pexels-photo-248304.jpeg" + +func TestContentLength(t *testing.T) { + c := NewClient(URL) + l, err := c.contentLength() + if err != nil { + t.Error(err) + } + expect := 6480509 + if l != expect { + t.Errorf("expect: %d, actual: %d", expect, l) + } +} + +func TestNewRangeProperties(t *testing.T) { + // ps := newRangeProperties(40001) + // t.Logf("%v", ps[0]) + // t.Logf("%v", ps[1]) + // t.Logf("%v", ps[2]) + // t.Logf("%v", ps[3]) + // t.Logf("%v", ps[4]) +} + +func TestRangeDownload(t *testing.T) { + // c := NewClient(URL) + // c.rangeDownload("1.jpg", 0, 5000) + // c.rangeDownload("2.jpg", 5001, 6480509) +} + +func TestMergeFiles(t *testing.T) { + // c := NewClient(URL) + // src := []string{"1.jpg", "2.jpg"} + // err := c.mergeFiles(src, "image.jpg") + // if err != nil { + // t.Error(err) + // } +} diff --git a/kadai3-2/yoheimiyamoto/downloader/image.jpg b/kadai3-2/yoheimiyamoto/downloader/image.jpg new file mode 100644 index 0000000..e69de29 diff --git a/kadai3-2/yoheimiyamoto/downloader/range_property.go b/kadai3-2/yoheimiyamoto/downloader/range_property.go new file mode 100644 index 0000000..276eb18 --- /dev/null +++ b/kadai3-2/yoheimiyamoto/downloader/range_property.go @@ -0,0 +1,48 @@ +package downloader + +import ( + "fmt" + "math" +) + +type rangeProperty struct { + path string // 一時保存先のファイル名 + start int // 開始のバイト数 + end int // 終了のバイト数 +} + +// rangeDownloadの引数として必要なrangePropertyを生成 +func newRangeProperties(contentLength int) []*rangeProperty { + num := 1000000 + + maxIndex := int(math.Ceil(float64(contentLength) / float64(num))) + + f := func(i int) *rangeProperty { + start := 0 + if i != 0 { + start = i*num + 1 + } + end := (i + 1) * num + if end > contentLength { + end = contentLength + } + return &rangeProperty{ + path: fmt.Sprintf("file%d.jpg", i), + start: start, + end: end, + } + } + + var out []*rangeProperty + + for i := 0; i < maxIndex; i++ { + start := i + num + end := start + num + if end > contentLength { + end = contentLength + } + out = append(out, f(i)) + } + + return out +} diff --git a/kadai3-2/yoheimiyamoto/main.go b/kadai3-2/yoheimiyamoto/main.go new file mode 100644 index 0000000..051925d --- /dev/null +++ b/kadai3-2/yoheimiyamoto/main.go @@ -0,0 +1,17 @@ +package main + +import ( + "context" + "fmt" + "os" + + "github.com/YoheiMiyamoto/dojo4/kadai3-2/yoheimiyamoto/downloader" +) + +func main() { + c := downloader.NewClient("https://images.pexels.com/photos/248304/pexels-photo-248304.jpeg") + err := c.Download(context.Background()) + if err != nil { + fmt.Fprintln(os.Stdout, err.Error()) + } +}