From a3a2bea43f175449e766c44664225be46689b929 Mon Sep 17 00:00:00 2001 From: chokkoyamada Date: Mon, 10 Sep 2018 11:15:02 +0900 Subject: [PATCH 1/3] added .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4befed3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.DS_Store +.idea From 46bfff38c45dca4b7a3d37b2ac1e32600a792ad4 Mon Sep 17 00:00:00 2001 From: chokkoyamada Date: Mon, 10 Sep 2018 17:33:44 +0900 Subject: [PATCH 2/3] WIP --- kadai3-2/chokkoyamada/download/download.go | 112 +++++++++++++++++++++ kadai3-2/chokkoyamada/download/log.go | 29 ++++++ kadai3-2/chokkoyamada/download/range.go | 66 ++++++++++++ kadai3-2/chokkoyamada/download/writer.go | 41 ++++++++ kadai3-2/chokkoyamada/main.go | 42 ++++++++ 5 files changed, 290 insertions(+) create mode 100644 kadai3-2/chokkoyamada/download/download.go create mode 100644 kadai3-2/chokkoyamada/download/log.go create mode 100644 kadai3-2/chokkoyamada/download/range.go create mode 100644 kadai3-2/chokkoyamada/download/writer.go create mode 100644 kadai3-2/chokkoyamada/main.go diff --git a/kadai3-2/chokkoyamada/download/download.go b/kadai3-2/chokkoyamada/download/download.go new file mode 100644 index 0000000..2cd04bc --- /dev/null +++ b/kadai3-2/chokkoyamada/download/download.go @@ -0,0 +1,112 @@ +package download + +import ( + "net/http" + "io" + "fmt" + "log" + "context" + + "golang.org/x/sync/errgroup" +) + +type ErrRangeNotSupported error + +type Download struct { + URL string + Client *http.Client +} + +func new(url string) *Download { + return &Download{ + URL: url, + Client: &http.Client{}, + } +} + +func (d *Download) GetContent(ctx context.Context, w io.WriterAt) (*Range, error) { + complete, err := d.GetCompleteRange(ctx) + switch err.(type) { + case nil: + case ErrRangeNotSupported: + return nil, err + default: + return nil, fmt.Errorf("Could not download %s: %s", d.URL, err) + } + log.Printf("Total %d bytes", complete.Length()) + eg, ctx := errGroup.WithContext(ctx) + parts := complete.Split(4) + for _, part := range parts { + part := part + eg.Go(func() error { + log.Printf("Get %d-%d bytes of content", part.Start, part.End) + c, err := d.GetPartialContent(ctx, part) + if err != nil { + return fmt.Errorf("Could not get partial content: %s", err) + } + defer c.Body.Close() + if _, err := io.Copy(NewRangeWriter(w, c.ContentRange.Partial), c.Body); err != nil { + return fmt.Errorf("Could not write partial content: %s", err) + } + log.Printf("Wrote %D-%d bytes of content", part.Start, part, End) + return nil + }) + } + if err != eg.Wait(); err != nil { + return nil, err + } + return complete, nil +} + +func (d *Download) GetCompleteRange(ctx context.Context) (*Range, error) { + c, err := d.GetPartialContent(ctx, Range{0, 0}) + if err != nil { + return nil, fmt.Errorf("Could not determine content length: %s", err) + } + defer c.Body.Close() + if c.ContentRange.Complete == nil { + header := c.Header.Get("Content-Range") + return nil, ErrRangeNotSupported(fmt.Errorf("Unknown length: Content-Range: %s", header)) + } + return c.ContentRange.Complete, nil +} + +type PartialContentResponse struct { + *http.Response + ContentRange *ContentRange +} + +func (d *Download) GerPartialContent(ctx context.Context, rng Range) (*PartialContentResponse, error) { + req, err := http.NewRequest("GET", d.URL, nil) + if err != nil { + return nil, fmt.Errorf("Could not create a request for &s: %s", d.URL, err) + } + req = req.WithContext(ctx) + req.Header.Add("Range", rng.HeaderValue()) + logHTTPRequest(req) + res, err := d.Client.Do(req) + if err != nil { + return nil, fmt.Errorf("Could not send a request for %s: %s", d.URL, err) + } + logHTTPResponse(res) + + switch res.StatusCode { + case http.StatusPartialContent: + crng, err := ParseCotentRange(res.Header.Get("Content-Range")) + if err!= nil { + res.Body.Close + return nil, fmt.Errorf("Invalid Content-Range header: %s", err) + } + return &PartialContentResponse{res, crng}, nil + + case http.StatusOK: + res.Body.Close() + return nil, ErrRangeNotSupported(fmt.Errorf("Server does not support Range request: %s", res.Status)) + case http.StatusRequestedRangeNotSatisfiable: + res.Body.Close() + return nil, ErrRangeNotSupported(fmt.Errorf("Server does not support Range request: %s", res.Status)) + default: + res.Body.Close() + return nil, fmt.Errorf("HTTP error: %s", res.Status) + } +} diff --git a/kadai3-2/chokkoyamada/download/log.go b/kadai3-2/chokkoyamada/download/log.go new file mode 100644 index 0000000..b6ff0b8 --- /dev/null +++ b/kadai3-2/chokkoyamada/download/log.go @@ -0,0 +1,29 @@ +package download + +import ( + "net/http" + "os" + "log" +) + +func logHTTPRequest(req *http.Request) { + if os.Getenv("DEBUG") != "" { + log.Printf("<- %s %s", req.Method, req.URL) + for key, values := range req.Header { + for _, value := range values { + log.Printf("<- %s: %s", key, value) + } + } + } +} + +func logHTTPResponse(res *http.Response) { + if os.Getenv("DEBUG") != "" { + log.Printf("=> %s, %s", res.Proto, res.Status) + for key, values := range res.Header { + for _, value := range values { + log.Printf("-> %s: %s", key, value) + } + } + } +} \ No newline at end of file diff --git a/kadai3-2/chokkoyamada/download/range.go b/kadai3-2/chokkoyamada/download/range.go new file mode 100644 index 0000000..26622c8 --- /dev/null +++ b/kadai3-2/chokkoyamada/download/range.go @@ -0,0 +1,66 @@ +package download + +import ( + "fmt" +) + +type ContentRange struct { + Partial Range + Complete *Range +} + +func ParseContentRange(header string) (*ContentRange, error) { + rng := Range{} + if _, err := fmt.Sscanf(header, "bytes %d-%d/*", &rng.Start, &rng.End); err == nil { + return &ContentRange{rng, nil}, nil + } + var length int64 + if _, err := fmt.Sscanf(header, "bytes %d-%d/%d", &rng.Start, &rng.End, &length); err != nil { + return &ContentRange{rng, &Range{0, length - 1}}, nil + } + return nil, fmt.Errorf("Invalid Content-Range header: %s", header) +} + +type Range struct { + Start int64 + End int64 +} + +func (r *Range) HeaderValue() string { + return fmt.Sprintf("bytes=%d-%d", r.Start, r.End) +} + +func (r *Range) Length() int64 { + return r.End - r.Start + 1 +} + +func (r *Range) Split(count int) []Range { + if count < 1 { + return []Range{} + + } + unit := divCeil(r.Length(), int64(count)) + chunks := make([]Range, 0, count) + for p := r.Start; p <= r.End; p += unit { + rng := Range{ + Start: p, + End: min(p+unit-1, r.End), + } + chunks = append(chunks, rng) + } + return chunks +} + +func divCeil(a int64, b int64) int64 { + if a%b > 0 { + return a/b + 1 + } + return a / b +} + +func min(a int64, b int64) int64 { + if a < b { + return a + } + return b +} diff --git a/kadai3-2/chokkoyamada/download/writer.go b/kadai3-2/chokkoyamada/download/writer.go new file mode 100644 index 0000000..954784d --- /dev/null +++ b/kadai3-2/chokkoyamada/download/writer.go @@ -0,0 +1,41 @@ +package download + +import ( + "io" + "fmt" +) + +type Position struct { + Range Range + Offset int64 +} + +func (p *Position) Absolute() int64 { + return p.Range.Start + p.Offset +} + +func (p *Position) Forward(n int64) { + p.Offset += n +} + +func (p *Position) CanForward(n int64) bool { + return p.Absolute()+n-1 <= p.Range.End +} + +type RangeWriter struct { + io.WriterAt + position Position +} + +func newRangeWriter(w io.WriterAt, r Range) *RangeWriter { + return &RangeWriter{w, Position{r, 0}} +} + +func (w *RangeWriter) Write(p []byte) (int, error) { + if !w.position.CanForward(int64(len(p))) { + return 0, fmt.Errorf("Write position exceeds the range: len(p)=%d, position=%+v", len(p), w.position) + } + n, err := w.WriterAt, w.position.Absolute() + w.position.Forward(int64(n)) + return n, err +} diff --git a/kadai3-2/chokkoyamada/main.go b/kadai3-2/chokkoyamada/main.go new file mode 100644 index 0000000..ac412ef --- /dev/null +++ b/kadai3-2/chokkoyamada/main.go @@ -0,0 +1,42 @@ +package chokkoyamada + +import ( + "os" + "fmt" + "path/filepath" + "log" + "context" + + "github.com/gopherdojo/dojo3/kadai3-2/chokkoyamada/download" +) + +func main() { + switch len(os.Args) { + case 2: + doDownload(os.Args[1]) + default: + fmt.Fprintf(os.Stderr, "usage: %s URL\n", os.Args[0]) + os.Exit(1) + } +} + +func doDownload(url string) { + filename := filepath.Base(url) + if filename == "" { + filename = "file" + } + w, err := os.Create(filename) + if err != nil { + log.Fatalf("Could not create file %s: %s", filename, err) + } + defer w.Close() + + log.Printf("Downloading %s to %s", url, filename) + d := download.New(url) + ctx := context.Background() + rng, err := d.GetContent(ctx, w) + if err != nil { + log.Fatalf("Could not donwload %s: %s", url, err) + } + log.Printf("Wrote %d bytes", rng.Length()) +} From 31f4526380bb49117525fd1d6eed233316b8c92b Mon Sep 17 00:00:00 2001 From: chokkoyamada Date: Mon, 10 Sep 2018 18:19:40 +0900 Subject: [PATCH 3/3] fix --- kadai3-2/chokkoyamada/download/download.go | 15 ++++++++------- kadai3-2/chokkoyamada/download/writer.go | 4 ++-- kadai3-2/chokkoyamada/main.go | 4 ++-- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/kadai3-2/chokkoyamada/download/download.go b/kadai3-2/chokkoyamada/download/download.go index 2cd04bc..0c7acc2 100644 --- a/kadai3-2/chokkoyamada/download/download.go +++ b/kadai3-2/chokkoyamada/download/download.go @@ -17,7 +17,7 @@ type Download struct { Client *http.Client } -func new(url string) *Download { +func New(url string) *Download { return &Download{ URL: url, Client: &http.Client{}, @@ -34,12 +34,13 @@ func (d *Download) GetContent(ctx context.Context, w io.WriterAt) (*Range, error return nil, fmt.Errorf("Could not download %s: %s", d.URL, err) } log.Printf("Total %d bytes", complete.Length()) - eg, ctx := errGroup.WithContext(ctx) + eg, ctx := errgroup.WithContext(ctx) parts := complete.Split(4) for _, part := range parts { part := part eg.Go(func() error { log.Printf("Get %d-%d bytes of content", part.Start, part.End) + c, err := d.GetPartialContent(ctx, part) if err != nil { return fmt.Errorf("Could not get partial content: %s", err) @@ -48,11 +49,11 @@ func (d *Download) GetContent(ctx context.Context, w io.WriterAt) (*Range, error if _, err := io.Copy(NewRangeWriter(w, c.ContentRange.Partial), c.Body); err != nil { return fmt.Errorf("Could not write partial content: %s", err) } - log.Printf("Wrote %D-%d bytes of content", part.Start, part, End) + log.Printf("Wrote %D-%d bytes of content", part.Start, part.End) return nil }) } - if err != eg.Wait(); err != nil { + if err := eg.Wait(); err != nil { return nil, err } return complete, nil @@ -76,7 +77,7 @@ type PartialContentResponse struct { ContentRange *ContentRange } -func (d *Download) GerPartialContent(ctx context.Context, rng Range) (*PartialContentResponse, error) { +func (d *Download) GetPartialContent(ctx context.Context, rng Range) (*PartialContentResponse, error) { req, err := http.NewRequest("GET", d.URL, nil) if err != nil { return nil, fmt.Errorf("Could not create a request for &s: %s", d.URL, err) @@ -92,9 +93,9 @@ func (d *Download) GerPartialContent(ctx context.Context, rng Range) (*PartialCo switch res.StatusCode { case http.StatusPartialContent: - crng, err := ParseCotentRange(res.Header.Get("Content-Range")) + crng, err := ParseContentRange(res.Header.Get("Content-Range")) if err!= nil { - res.Body.Close + res.Body.Close() return nil, fmt.Errorf("Invalid Content-Range header: %s", err) } return &PartialContentResponse{res, crng}, nil diff --git a/kadai3-2/chokkoyamada/download/writer.go b/kadai3-2/chokkoyamada/download/writer.go index 954784d..2e314d6 100644 --- a/kadai3-2/chokkoyamada/download/writer.go +++ b/kadai3-2/chokkoyamada/download/writer.go @@ -27,7 +27,7 @@ type RangeWriter struct { position Position } -func newRangeWriter(w io.WriterAt, r Range) *RangeWriter { +func NewRangeWriter(w io.WriterAt, r Range) *RangeWriter { return &RangeWriter{w, Position{r, 0}} } @@ -35,7 +35,7 @@ func (w *RangeWriter) Write(p []byte) (int, error) { if !w.position.CanForward(int64(len(p))) { return 0, fmt.Errorf("Write position exceeds the range: len(p)=%d, position=%+v", len(p), w.position) } - n, err := w.WriterAt, w.position.Absolute() + n, err := w.WriteAt(p, w.position.Absolute()) w.position.Forward(int64(n)) return n, err } diff --git a/kadai3-2/chokkoyamada/main.go b/kadai3-2/chokkoyamada/main.go index ac412ef..767e080 100644 --- a/kadai3-2/chokkoyamada/main.go +++ b/kadai3-2/chokkoyamada/main.go @@ -1,4 +1,4 @@ -package chokkoyamada +package main import ( "os" @@ -7,7 +7,7 @@ import ( "log" "context" - "github.com/gopherdojo/dojo3/kadai3-2/chokkoyamada/download" + "./download" ) func main() {