Skip to content

Commit a18136d

Browse files
committed
Minor optimizations
1 parent f4ec4fd commit a18136d

File tree

1 file changed

+69
-73
lines changed

1 file changed

+69
-73
lines changed

plugin.go

Lines changed: 69 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ type Plugin struct {
103103
// Exec runs the plugin
104104
func (p *Plugin) Exec() error {
105105
// normalize the target URL
106-
p.Target = strings.TrimPrefix(p.Target, "/")
106+
p.Target = resolveTargetDir(p.Target)
107107

108108
// create the client
109109
conf := &aws.Config{
@@ -141,7 +141,74 @@ func (p *Plugin) Exec() error {
141141
}
142142

143143
if p.Download {
144-
downloadFromS3(p, client)
144+
targetDir := strings.TrimPrefix(filepath.ToSlash(p.Target), "/")
145+
log.WithFields(log.Fields{
146+
"bucket": p.Bucket,
147+
"dir": targetDir,
148+
}).Info("Listing S3 directory")
149+
150+
list, err := client.ListObjectsV2(&s3.ListObjectsV2Input{
151+
Bucket: &p.Bucket,
152+
Prefix: &targetDir,
153+
})
154+
if err != nil {
155+
log.WithFields(log.Fields{
156+
"error": err,
157+
"bucket": p.Bucket,
158+
"dir": targetDir,
159+
}).Error("Cannot list S3 directory")
160+
return err
161+
}
162+
163+
g := errgroup.Group{}
164+
165+
for _, item := range list.Contents {
166+
log.WithFields(log.Fields{
167+
"bucket": p.Bucket,
168+
"key": *item.Key,
169+
}).Info("Getting S3 object")
170+
171+
item := item
172+
g.Go(func() error {
173+
obj, err := client.GetObject(&s3.GetObjectInput{
174+
Bucket: &p.Bucket,
175+
Key: item.Key,
176+
})
177+
if err != nil {
178+
log.WithFields(log.Fields{
179+
"error": err,
180+
"bucket": p.Bucket,
181+
"key": *item.Key,
182+
}).Error("Cannot get S3 object")
183+
return err
184+
}
185+
186+
source := resolveSource(targetDir, *item.Key, p.StripPrefix)
187+
188+
f, err := os.Create(source)
189+
if err != nil {
190+
log.WithFields(log.Fields{
191+
"error": err,
192+
"file": source,
193+
}).Error("Problem opening file for writing")
194+
return err
195+
}
196+
defer f.Close()
197+
198+
_, err = io.Copy(f, obj.Body)
199+
if err != nil {
200+
log.WithFields(log.Fields{
201+
"error": err,
202+
"file": source,
203+
}).Error("Failed to write file")
204+
return err
205+
}
206+
207+
return nil
208+
})
209+
}
210+
211+
return g.Wait()
145212
}
146213

147214
// find the bucket
@@ -361,74 +428,3 @@ func resolveTargetDir(s string) string {
361428
res := strings.TrimPrefix(filepath.ToSlash(s), "/")
362429
return res
363430
}
364-
365-
func downloadFromS3(p *Plugin, client *s3.S3) error {
366-
targetDir := resolveTargetDir(p.Target)
367-
log.WithFields(log.Fields{
368-
"bucket": p.Bucket,
369-
"dir": targetDir,
370-
}).Info("Listing S3 directory")
371-
372-
list, err := client.ListObjectsV2(&s3.ListObjectsV2Input{
373-
Bucket: &p.Bucket,
374-
Prefix: &targetDir,
375-
})
376-
if err != nil {
377-
log.WithFields(log.Fields{
378-
"error": err,
379-
"bucket": p.Bucket,
380-
"dir": targetDir,
381-
}).Error("Cannot list S3 directory")
382-
return err
383-
}
384-
385-
g := errgroup.Group{}
386-
387-
for _, item := range list.Contents {
388-
log.WithFields(log.Fields{
389-
"bucket": p.Bucket,
390-
"key": *item.Key,
391-
}).Info("Getting S3 object")
392-
393-
item := item
394-
g.Go(func() error {
395-
obj, err := client.GetObject(&s3.GetObjectInput{
396-
Bucket: &p.Bucket,
397-
Key: item.Key,
398-
})
399-
if err != nil {
400-
log.WithFields(log.Fields{
401-
"error": err,
402-
"bucket": p.Bucket,
403-
"key": *item.Key,
404-
}).Error("Cannot get S3 object")
405-
return err
406-
}
407-
408-
source := resolveSource(targetDir, *item.Key, p.StripPrefix)
409-
410-
f, err := os.Create(source)
411-
if err != nil {
412-
log.WithFields(log.Fields{
413-
"error": err,
414-
"file": source,
415-
}).Error("Problem opening file for writing")
416-
return err
417-
}
418-
defer f.Close()
419-
420-
_, err = io.Copy(f, obj.Body)
421-
if err != nil {
422-
log.WithFields(log.Fields{
423-
"error": err,
424-
"file": source,
425-
}).Error("Failed to write file")
426-
return err
427-
}
428-
429-
return nil
430-
})
431-
}
432-
433-
return g.Wait()
434-
}

0 commit comments

Comments
 (0)