@@ -6,6 +6,7 @@ package signatures
66import (
77 "bytes"
88 "fmt"
9+ "net/http"
910 "strings"
1011 "sync"
1112
@@ -29,7 +30,7 @@ import (
2930
3031// constants
3132const (
32- ParallelDownloader = 100
33+ ParallelDownloader = 10
3334)
3435
3536// Zipper implements ZipBuilder interface
@@ -111,8 +112,9 @@ func (z *Zipper) buildPDFZip(claType string, claGroupID string) error {
111112 }
112113 var zipUpdated bool
113114 log .WithFields (f ).Debug ("getting s3 files" )
114- downloaderInputChan := make (chan * DownloadFileInput )
115- downloaderOutputChan := make (chan * FileContent )
115+ downloaderInputChan := make (chan * DownloadFileInput , 16 )
116+ downloaderOutputChan := make (chan * FileContent , 16 )
117+ listErrCh := make (chan error , 1 )
116118 var wg sync.WaitGroup
117119 wg .Add (ParallelDownloader )
118120 for i := 1 ; i <= ParallelDownloader ; i ++ {
@@ -123,7 +125,7 @@ func (z *Zipper) buildPDFZip(claType string, claGroupID string) error {
123125 close (downloaderOutputChan )
124126 }()
125127 go func () {
126- err = z .s3 .ListObjectsPages (& s3.ListObjectsInput {
128+ localErr : = z .s3 .ListObjectsPages (& s3.ListObjectsInput {
127129 Bucket : aws .String (z .bucketName ),
128130 Prefix : aws .String (s3ZipPrefix (claType , claGroupID )),
129131 }, func (output * s3.ListObjectsOutput , b bool ) bool {
@@ -147,12 +149,15 @@ func (z *Zipper) buildPDFZip(claType string, claGroupID string) error {
147149 return true
148150 })
149151 close (downloaderInputChan )
152+ listErrCh <- localErr
150153 }()
151154 zipUpdated = writeFileToZip (writer , downloaderOutputChan )
152- if err != nil {
153- return err
155+ if listErr := <- listErrCh ; listErr != nil {
156+ return listErr
157+ }
158+ if cerr := writer .Close (); cerr != nil {
159+ return cerr
154160 }
155- writer .Close ()
156161 if zipUpdated {
157162 remoteZipFileKey := s3ZipFilepath (claType , claGroupID )
158163 log .Debugf ("Uploading zip file %s" , remoteZipFileKey )
@@ -264,13 +269,16 @@ func getZipWriter(buff *bytes.Buffer) (*zip.Writer, error) {
264269func (z * Zipper ) getZipFileFromS3 (claType string , claGroupID string ) (* bytes.Buffer , error ) {
265270 var buff aws.WriteAtBuffer
266271 remoteFileKey := s3ZipFilepath (claType , claGroupID )
267- _ , err := z .s3 .GetObject (& s3.GetObjectInput {
272+ _ , err := z .s3 .HeadObject (& s3.HeadObjectInput {
268273 Bucket : aws .String (z .bucketName ),
269274 Key : aws .String (remoteFileKey ),
270275 })
271276 if err != nil {
272- aerr , ok := err .(awserr.Error )
273- if ok && aerr .Code () == s3 .ErrCodeNoSuchKey {
277+ if rf , ok := err .(awserr.RequestFailure ); ok && rf .StatusCode () == http .StatusNotFound {
278+ log .Debugf ("zip file %s does not exist on s3" , remoteFileKey )
279+ return bytes .NewBuffer (buff .Bytes ()), nil
280+ }
281+ if aerr , ok := err .(awserr.Error ); ok && (aerr .Code () == s3 .ErrCodeNoSuchKey || aerr .Code () == "NotFound" ) {
274282 log .Debugf ("zip file %s does not exist on s3" , remoteFileKey )
275283 return bytes .NewBuffer (buff .Bytes ()), nil
276284 }
0 commit comments