From a3900075f32646a809ba33e827ffecf18810132f Mon Sep 17 00:00:00 2001 From: Jaime Pillora Date: Fri, 24 Jun 2016 12:45:51 +1000 Subject: [PATCH 1/5] implement optionally gzip compression --- DOCS.md | 2 ++ README.md | 2 +- main.go | 6 ++++++ plugin.go | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 60 insertions(+), 4 deletions(-) diff --git a/DOCS.md b/DOCS.md index b00ed72..15dabfa 100644 --- a/DOCS.md +++ b/DOCS.md @@ -10,6 +10,7 @@ Use the S3 plugin to upload files and build artifacts to an S3 bucket. The follo * **target** - target location of files in the bucket * **exclude** - glob exclusion patterns * **path_style** - whether path style URLs should be used (true for minio, false for aws) +* **compress** - prior to upload, compress files and use gzip content-encoding The following is a sample S3 configuration in your .drone.yml file: @@ -26,4 +27,5 @@ publish: target: /target/location exclude: - **/*.xml + compress: false ``` diff --git a/README.md b/README.md index a69d716..33e172e 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # drone-s3 [![Build Status](http://beta.drone.io/api/badges/drone-plugins/drone-s3/status.svg)](http://beta.drone.io/drone-plugins/drone-s3) -[![Image Size](https://badge.imagelayers.io/plugins/drone-s3:latest.svg)](https://imagelayers.io/?images=plugins/drone-s3:latest 'Get your own badge on imagelayers.io') +[![Image Size](https://badge.imagelayers.io/plugins/s3:latest.svg)](https://imagelayers.io/?images=plugins/s3:latest 'Get your own badge on imagelayers.io') Drone plugin to publish files and artifacts to Amazon S3. For the usage information and a listing of the available options please take a look at [the docs](DOCS.md). diff --git a/main.go b/main.go index 6919a1f..2aa0688 100644 --- a/main.go +++ b/main.go @@ -82,6 +82,11 @@ func main() { Usage: "use path style for bucket paths", EnvVar: "PLUGIN_PATH_STYLE", }, + cli.BoolFlag{ + Name: "compress", + Usage: "prior to upload, compress files and use gzip content-encoding", + EnvVar: "PLUGIN_COMPRESS", + }, } if err := app.Run(os.Args); err != nil { @@ -103,6 +108,7 @@ func run(c *cli.Context) error { Exclude: c.StringSlice("exclude"), PathStyle: c.Bool("path-style"), DryRun: c.Bool("dry-run"), + Compress: c.Bool("compress"), } // normalize the target URL diff --git a/plugin.go b/plugin.go index 866de6f..fe053bd 100644 --- a/plugin.go +++ b/plugin.go @@ -1,6 +1,9 @@ package main import ( + "compress/gzip" + "errors" + "io" "mime" "os" "path/filepath" @@ -65,6 +68,8 @@ type Plugin struct { PathStyle bool // Dry run without uploading/ DryRun bool + // Compress objects and upload with Content-Encoding: gzip + Compress bool } // Exec runs the plugin @@ -138,13 +143,34 @@ func (p *Plugin) Exec() error { } defer f.Close() - _, err = client.PutObject(&s3.PutObjectInput{ - Body: f, + //prepare upload + input := &s3.PutObjectInput{ Bucket: &(p.Bucket), Key: &target, ACL: &(p.Access), ContentType: &content, - }) + } + + //optionally compress + if p.Compress { + gr, err := gzip.NewReader(f) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "file": match, + }).Error("Problem gzipping file") + return err + } + //wrap with gzip + input.Body = &gzipReadSeeker{f, gr} + //set encoding + input.ContentEncoding = aws.String("gzip") + } else { + input.Body = f + } + + //upload + _, err = client.PutObject(input) if err != nil { log.WithFields(log.Fields{ @@ -162,6 +188,28 @@ func (p *Plugin) Exec() error { return nil } +//gzipReadSeeker implements Seek over gzip.Reader +type gzipReadSeeker struct { + rs io.ReadSeeker + z *gzip.Reader +} + +func (grs *gzipReadSeeker) Read(p []byte) (n int, err error) { + return grs.z.Read(p) +} + +func (grs *gzipReadSeeker) Seek(offset int64, whence int) (int64, error) { + //only zero (reset) seeks are supported, in this case, this should be fine + //since AWS will rarely seek mid-file + if offset == 0 && whence == 0 { + if err := grs.z.Reset(grs.rs); err != nil { + return 0, err + } + return grs.rs.Seek(0, 0) + } + return 0, errors.New("Non-zero seek not supported") +} + // matches is a helper function that returns a list of all files matching the // included Glob pattern, while excluding all files that matche the exclusion // Glob pattners. From b8685f4c8fc048e743d1691b38f9f49f5c869c41 Mon Sep 17 00:00:00 2001 From: Jaime Pillora Date: Fri, 24 Jun 2016 13:07:25 +1000 Subject: [PATCH 2/5] add self-build to dockerfile --- Dockerfile | 16 ++++++++++++++-- plugin.go | 44 ++++++++++++++++++++++---------------------- 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/Dockerfile b/Dockerfile index ddcfd79..e608f7f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,18 @@ # docker build --rm=true -t plugins/s3 . FROM alpine:3.3 -RUN apk update && apk add ca-certificates mailcap && rm -rf /var/cache/apk/* -ADD drone-s3 /bin/ + +ENV GOPATH /root/go +ENV CGO_ENABLED 0 +ENV PKG org/user/drone-s3 + +RUN apk update && \ + apk add ca-certificates mailcap go && \ + mkdir -p $GOPATH/src/$PKG && \ + mv * $GOPATH/src/$PKG/ && \ + go build -a -tags netgo -o /bin/drone-s3 $PKG && \ + apk del go && \ + rm -rf /var/cache/apk/* && \ + echo "built drone-s3" + ENTRYPOINT ["/bin/drone-s3"] diff --git a/plugin.go b/plugin.go index fe053bd..6e26100 100644 --- a/plugin.go +++ b/plugin.go @@ -188,28 +188,6 @@ func (p *Plugin) Exec() error { return nil } -//gzipReadSeeker implements Seek over gzip.Reader -type gzipReadSeeker struct { - rs io.ReadSeeker - z *gzip.Reader -} - -func (grs *gzipReadSeeker) Read(p []byte) (n int, err error) { - return grs.z.Read(p) -} - -func (grs *gzipReadSeeker) Seek(offset int64, whence int) (int64, error) { - //only zero (reset) seeks are supported, in this case, this should be fine - //since AWS will rarely seek mid-file - if offset == 0 && whence == 0 { - if err := grs.z.Reset(grs.rs); err != nil { - return 0, err - } - return grs.rs.Seek(0, 0) - } - return 0, errors.New("Non-zero seek not supported") -} - // matches is a helper function that returns a list of all files matching the // included Glob pattern, while excluding all files that matche the exclusion // Glob pattners. @@ -257,3 +235,25 @@ func contentType(path string) string { } return typ } + +//gzipReadSeeker implements Seek over gzip.Reader +type gzipReadSeeker struct { + rs io.ReadSeeker + z *gzip.Reader +} + +func (grs *gzipReadSeeker) Read(p []byte) (n int, err error) { + return grs.z.Read(p) +} + +func (grs *gzipReadSeeker) Seek(offset int64, whence int) (int64, error) { + //only zero (reset) seeks are supported, in this case, this should be fine + //since AWS will rarely seek mid-file + if offset == 0 && whence == 0 { + if err := grs.z.Reset(grs.rs); err != nil { + return 0, err + } + return grs.rs.Seek(0, 0) + } + return 0, errors.New("Non-zero seek not supported") +} From bca77b06290be9912f62ce94aa75c3dd56266818 Mon Sep 17 00:00:00 2001 From: Jaime Pillora Date: Fri, 24 Jun 2016 13:42:32 +1000 Subject: [PATCH 3/5] fix docker build --- Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e608f7f..62acd30 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,14 +7,17 @@ FROM alpine:3.3 ENV GOPATH /root/go ENV CGO_ENABLED 0 +ENV GO15VENDOREXPERIMENT 1 ENV PKG org/user/drone-s3 +ADD vendor $GOPATH/src/$PKG/vendor +ADD *.go $GOPATH/src/$PKG/ + RUN apk update && \ apk add ca-certificates mailcap go && \ - mkdir -p $GOPATH/src/$PKG && \ - mv * $GOPATH/src/$PKG/ && \ go build -a -tags netgo -o /bin/drone-s3 $PKG && \ apk del go && \ + rm -rf $GOPATH && \ rm -rf /var/cache/apk/* && \ echo "built drone-s3" From f845a2bf6c89545b9e7d720496b320790d4728a2 Mon Sep 17 00:00:00 2001 From: Jaime Pillora Date: Fri, 24 Jun 2016 15:01:10 +1000 Subject: [PATCH 4/5] revert dockerfile --- Dockerfile | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index 62acd30..ddcfd79 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,21 +4,6 @@ # docker build --rm=true -t plugins/s3 . FROM alpine:3.3 - -ENV GOPATH /root/go -ENV CGO_ENABLED 0 -ENV GO15VENDOREXPERIMENT 1 -ENV PKG org/user/drone-s3 - -ADD vendor $GOPATH/src/$PKG/vendor -ADD *.go $GOPATH/src/$PKG/ - -RUN apk update && \ - apk add ca-certificates mailcap go && \ - go build -a -tags netgo -o /bin/drone-s3 $PKG && \ - apk del go && \ - rm -rf $GOPATH && \ - rm -rf /var/cache/apk/* && \ - echo "built drone-s3" - +RUN apk update && apk add ca-certificates mailcap && rm -rf /var/cache/apk/* +ADD drone-s3 /bin/ ENTRYPOINT ["/bin/drone-s3"] From cd157323a8220f33314270dcd031df430aa8dd9d Mon Sep 17 00:00:00 2001 From: Jaime Pillora Date: Fri, 24 Jun 2016 15:56:35 +1000 Subject: [PATCH 5/5] remove gzipReadSeeker, buffer file into memory instead --- plugin.go | 35 ++++++++--------------------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/plugin.go b/plugin.go index 6e26100..964ac75 100644 --- a/plugin.go +++ b/plugin.go @@ -1,8 +1,8 @@ package main import ( + "bytes" "compress/gzip" - "errors" "io" "mime" "os" @@ -153,16 +153,19 @@ func (p *Plugin) Exec() error { //optionally compress if p.Compress { - gr, err := gzip.NewReader(f) - if err != nil { + //currently buffers entire file into memory + //TODO: convert to on-demand gzip + b := bytes.Buffer{} + gw := gzip.NewWriter(&b) + if _, err := io.Copy(gw, f); err != nil { log.WithFields(log.Fields{ "error": err, "file": match, }).Error("Problem gzipping file") return err } - //wrap with gzip - input.Body = &gzipReadSeeker{f, gr} + gw.Close() + input.Body = bytes.NewReader(b.Bytes()) //set encoding input.ContentEncoding = aws.String("gzip") } else { @@ -235,25 +238,3 @@ func contentType(path string) string { } return typ } - -//gzipReadSeeker implements Seek over gzip.Reader -type gzipReadSeeker struct { - rs io.ReadSeeker - z *gzip.Reader -} - -func (grs *gzipReadSeeker) Read(p []byte) (n int, err error) { - return grs.z.Read(p) -} - -func (grs *gzipReadSeeker) Seek(offset int64, whence int) (int64, error) { - //only zero (reset) seeks are supported, in this case, this should be fine - //since AWS will rarely seek mid-file - if offset == 0 && whence == 0 { - if err := grs.z.Reset(grs.rs); err != nil { - return 0, err - } - return grs.rs.Seek(0, 0) - } - return 0, errors.New("Non-zero seek not supported") -}