OTHER: improving error handling when object doesn't exist on BP in helper bulk get. (#119)

Author: RachelTucker

helpers/getTransfernator.go

@@ -1,6 +1,7 @@
 package helpers
 
 import (
+    "fmt"
     "github.com/SpectraLogic/ds3_go_sdk/ds3"
     ds3Models "github.com/SpectraLogic/ds3_go_sdk/ds3/models"
     helperModels "github.com/SpectraLogic/ds3_go_sdk/helpers/models"
@@ -86,8 +87,10 @@ func (transceiver *getTransceiver) createBulkGetJob() (*ds3Models.GetBulkJobSpec
     for _, obj := range *transceiver.ReadObjects {
         _, err := transceiver.Client.HeadObject(ds3Models.NewHeadObjectRequest(transceiver.BucketName, obj.Name))
         if err != nil {
-            //mark file as having a fatal error
-            obj.ChannelBuilder.SetFatalError(err)
+            // mark file as having a fatal error
+            readableErr := fmt.Errorf("failed HeadObject call on %s: %v", obj.Name, err)
+            obj.ChannelBuilder.SetFatalError(readableErr)
+            transceiver.Strategy.Listeners.Errored(obj.Name, readableErr)
         } else {
             objectsThatExist = append(objectsThatExist, obj)
         }

helpers_integration/helpersImpl_test.go

@@ -16,6 +16,7 @@ import (
     "log"
     "os"
     "sync"
+    "sync/atomic"
     "testing"
     "time"
 )
@@ -629,10 +630,17 @@ func TestRetryGetObjectsWhenSomeObjectsDoNotExist(t *testing.T) {
 
     helper := helpers.NewHelpers(client)
 
+    perObjectErrCount := int64(0)
+    errListener := helpers.ListenerStrategy{
+        ErrorCallback: func(objectName string, err error) {
+            atomic.AddInt64(&perObjectErrCount, 1)
+        },
+    }
+
     strategy := helpers.ReadTransferStrategy{
         Options: helpers.ReadBulkJobOptions{}, // use default job options
         BlobStrategy: newTestBlobStrategy(),
-        Listeners: newErrorOnErrorListenerStrategy(t),
+        Listeners: errListener,
     }
 
     file0, err := ioutil.TempFile(os.TempDir(), "goTest")
@@ -687,10 +695,16 @@ func TestRetryGetObjectsWhenSomeObjectsDoNotExist(t *testing.T) {
     utils.VerifyBookContent(testutils.BookTitles[2], file2)
     utils.VerifyBookContent(testutils.BookTitles[3], file3)
 
+    ds3Testing.AssertInt64(t, "per-object error count", 2, perObjectErrCount)
+
     doesNotExistStat, err := os.Stat(doesNotExist1.Name())
     ds3Testing.AssertNilError(t, err)
     ds3Testing.AssertInt64(t, "size of file", 0, doesNotExistStat.Size())
 
+    doesNotExistStat, err = os.Stat(doesNotExist2.Name())
+    ds3Testing.AssertNilError(t, err)
+    ds3Testing.AssertInt64(t, "size of file", 0, doesNotExistStat.Size())
+
     ds3Testing.AssertBool(t, "has fatal error", true, doesNotExistChannelBuilder1.HasFatalError())
     ds3Testing.AssertBool(t, "has fatal error", true, doesNotExistChannelBuilder2.HasFatalError())
 }