11package main
22
33import (
4+ "io"
45 "mime"
56 "os"
67 "path/filepath"
@@ -15,6 +16,7 @@ import (
1516 "github.com/aws/aws-sdk-go/service/s3"
1617 "github.com/aws/aws-sdk-go/service/sts"
1718 "github.com/mattn/go-zglob"
19+ "github.com/pkg/errors"
1820 log "github.com/sirupsen/logrus"
1921)
2022
@@ -44,6 +46,9 @@ type Plugin struct {
4446 // sa-east-1
4547 Region string
4648
49+ // if true, plugin is set to download mode, which means `source` from the bucket will be downloaded
50+ Download bool
51+
4752 // Indicates the files ACL, which should be one
4853 // of the following:
4954 // private
@@ -97,42 +102,21 @@ type Plugin struct {
97102
98103// Exec runs the plugin
99104func (p * Plugin ) Exec () error {
100- // normalize the target URL
101- p .Target = strings .TrimPrefix (p .Target , "/" )
102-
103- // create the client
104- conf := & aws.Config {
105- Region : aws .String (p .Region ),
106- Endpoint : & p .Endpoint ,
107- DisableSSL : aws .Bool (strings .HasPrefix (p .Endpoint , "http://" )),
108- S3ForcePathStyle : aws .Bool (p .PathStyle ),
109- }
110-
111- if p .Key != "" && p .Secret != "" {
112- conf .Credentials = credentials .NewStaticCredentials (p .Key , p .Secret , "" )
113- } else if p .AssumeRole != "" {
114- conf .Credentials = assumeRole (p .AssumeRole , p .AssumeRoleSessionName , p .ExternalID )
105+ if p .Download {
106+ p .Source = normalizePath (p .Source )
107+ p .Target = normalizePath (p .Target )
115108 } else {
116- log . Warn ( "AWS Key and/or Secret not provided (falling back to ec2 instance profile) "
109+ p . Target = strings . TrimPrefix ( p . Target , "/ "
117110 }
118111
119- var client * s3.S3
120- sess , err := session .NewSession (conf )
121- if err != nil {
122- log .WithError (err ).Errorln ("could not instantiate session" )
123- return err
124- }
112+ // create the client
113+ client := p .createS3Client ()
125114
126- // If user role ARN is set then assume role here
127- if len (p .UserRoleArn ) > 0 {
128- confRoleArn := aws.Config {
129- Region : aws .String (p .Region ),
130- Credentials : stscreds .NewCredentials (sess , p .UserRoleArn ),
131- }
115+ // If in download mode, call the downloadS3Objects method
116+ if p .Download {
117+ sourceDir := normalizePath (p .Source )
132118
133- client = s3 .New (sess , & confRoleArn )
134- } else {
135- client = s3 .New (sess )
119+ return p .downloadS3Objects (client , sourceDir )
136120 }
137121
138122 // find the bucket
@@ -322,6 +306,14 @@ func resolveKey(target, srcPath, stripPrefix string) string {
322306 return key
323307}
324308
309+ func resolveSource (sourceDir , source , stripPrefix string ) string {
310+ // Remove the leading sourceDir from the source path
311+ path := strings .TrimPrefix (strings .TrimPrefix (source , sourceDir ), "/" )
312+
313+ // Add the specified stripPrefix to the resulting path
314+ return stripPrefix + path
315+ }
316+
325317// checks if the source path is a dir
326318func isDir (source string , matches []string ) bool {
327319 stat , err := os .Stat (source )
@@ -342,3 +334,128 @@ func isDir(source string, matches []string) bool {
342334 }
343335 return false
344336}
337+
338+ // normalizePath converts the path to a forward slash format and trims the prefix.
339+ func normalizePath (path string ) string {
340+ return strings .TrimPrefix (filepath .ToSlash (path ), "/" )
341+ }
342+
343+ // downloadS3Object downloads a single object from S3
344+ func (p * Plugin ) downloadS3Object (client * s3.S3 , sourceDir , key , target string ) error {
345+ log .WithFields (log.Fields {
346+ "bucket" : p .Bucket ,
347+ "key" : key ,
348+ }).Info ("Getting S3 object" )
349+
350+ obj , err := client .GetObject (& s3.GetObjectInput {
351+ Bucket : & p .Bucket ,
352+ Key : & key ,
353+ })
354+ if err != nil {
355+ log .WithFields (log.Fields {
356+ "error" : err ,
357+ "bucket" : p .Bucket ,
358+ "key" : key ,
359+ }).Error ("Cannot get S3 object" )
360+ return err
361+ }
362+ defer obj .Body .Close ()
363+
364+ // Create the destination file path
365+ destination := filepath .Join (p .Target , target )
366+ log .Println ("Destination: " , destination )
367+
368+ // Extract the directory from the destination path
369+ dir := filepath .Dir (destination )
370+
371+ // Create the directory and any necessary parent directories
372+ if err := os .MkdirAll (dir , os .ModePerm ); err != nil {
373+ return errors .Wrap (err , "error creating directories" )
374+ }
375+
376+ f , err := os .Create (destination )
377+ if err != nil {
378+ log .WithFields (log.Fields {
379+ "error" : err ,
380+ "file" : destination ,
381+ }).Error ("Failed to create file" )
382+ return err
383+ }
384+ defer f .Close ()
385+
386+ _ , err = io .Copy (f , obj .Body )
387+ if err != nil {
388+ log .WithFields (log.Fields {
389+ "error" : err ,
390+ "file" : destination ,
391+ }).Error ("Failed to write file" )
392+ return err
393+ }
394+
395+ return nil
396+ }
397+
398+ // downloadS3Objects downloads all objects in the specified S3 bucket path
399+ func (p * Plugin ) downloadS3Objects (client * s3.S3 , sourceDir string ) error {
400+ log .WithFields (log.Fields {
401+ "bucket" : p .Bucket ,
402+ "dir" : sourceDir ,
403+ }).Info ("Listing S3 directory" )
404+
405+ list , err := client .ListObjectsV2 (& s3.ListObjectsV2Input {
406+ Bucket : & p .Bucket ,
407+ Prefix : & sourceDir ,
408+ })
409+ if err != nil {
410+ log .WithFields (log.Fields {
411+ "error" : err ,
412+ "bucket" : p .Bucket ,
413+ "dir" : sourceDir ,
414+ }).Error ("Cannot list S3 directory" )
415+ return err
416+ }
417+
418+ for _ , item := range list .Contents {
419+ // resolveSource takes a source directory, a source path, and a prefix to strip,
420+ // and returns a resolved target path by removing the sourceDir from the source
421+ // and appending the stripPrefix.
422+ target := resolveSource (sourceDir , * item .Key , p .StripPrefix )
423+
424+ if err := p .downloadS3Object (client , sourceDir , * item .Key , target ); err != nil {
425+ return err
426+ }
427+ }
428+
429+ return nil
430+ }
431+
432+ // createS3Client creates and returns an S3 client based on the plugin configuration
433+ func (p * Plugin ) createS3Client () * s3.S3 {
434+ conf := & aws.Config {
435+ Region : aws .String (p .Region ),
436+ Endpoint : & p .Endpoint ,
437+ DisableSSL : aws .Bool (strings .HasPrefix (p .Endpoint , "http://" )),
438+ S3ForcePathStyle : aws .Bool (p .PathStyle ),
439+ }
440+
441+ if p .Key != "" && p .Secret != "" {
442+ conf .Credentials = credentials .NewStaticCredentials (p .Key , p .Secret , "" )
443+ } else if p .AssumeRole != "" {
444+ conf .Credentials = assumeRole (p .AssumeRole , p .AssumeRoleSessionName , p .ExternalID )
445+ } else {
446+ log .Warn ("AWS Key and/or Secret not provided (falling back to ec2 instance profile)" )
447+ }
448+
449+ sess , _ := session .NewSession (conf )
450+ client := s3 .New (sess )
451+
452+ if len (p .UserRoleArn ) > 0 {
453+ confRoleArn := aws.Config {
454+ Region : aws .String (p .Region ),
455+ Credentials : stscreds .NewCredentials (sess , p .UserRoleArn ),
456+ }
457+ client = s3 .New (sess , & confRoleArn )
458+ }
459+
460+ return client
461+ }
0 commit comments