aws: Fixes bug in calculating throttled retry delay (#373)

The `Retry-After` duration specified in the request is now added to the Retry delay for throttled exception. Adds test for retry delays for throttled exceptions. Fixes bug where the throttled retry's math was off.

Fixes #45

Author: skotambkar

CHANGELOG_PENDING.md

@@ -9,4 +9,7 @@
   * Fixes [#365](https://github.com/aws/aws-sdk-go-v2/issues/365)
 
 ### SDK Bugs
-
+* `aws`: Fixes bug in calculating throttled retry delay ([#373](https://github.com/aws/aws-sdk-go/pull/373))
+  * The `Retry-After` duration specified in the request is now added to the Retry delay for throttled exception. Adds test for retry delays for throttled exceptions. Fixes bug where the throttled retry's math was off.
+  * Fixes [#45](https://github.com/aws/aws-sdk-go/issues/45)
+  

aws/default_retryer.go

@@ -34,25 +34,27 @@ var seededRand = rand.New(&lockedSource{src: rand.NewSource(time.Now().UnixNano(
 // RetryRules returns the delay duration before retrying this request again
 func (d DefaultRetryer) RetryRules(r *Request) time.Duration {
 	// Set the upper limit of delay in retrying at ~five minutes
-	minTime := 30
+	var minTime int64 = 30
+	var initialDelay time.Duration
+
 	throttle := d.shouldThrottle(r)
 	if throttle {
-		if delay, ok := getRetryDelay(r); ok {
-			return delay
+		if delay, ok := getRetryAfterDelay(r); ok {
+			initialDelay = delay
 		}
 
 		minTime = 500
 	}
 
 	retryCount := r.RetryCount
-	if retryCount > 13 {
-		retryCount = 13
-	} else if throttle && retryCount > 8 {
+	if throttle && retryCount > 8 {
 		retryCount = 8
+	} else if retryCount > 12 {
+		retryCount = 12
 	}
 
-	delay := (1 << uint(retryCount)) * (seededRand.Intn(minTime) + minTime)
-	return time.Duration(delay) * time.Millisecond
+	delay := (1 << uint(retryCount)) * (seededRand.Int63n(minTime) + minTime)
+	return (time.Duration(delay) * time.Millisecond) + initialDelay
 }
 
 // ShouldRetry returns true if the request should be retried.
@@ -85,7 +87,7 @@ func (d DefaultRetryer) shouldThrottle(r *Request) bool {
 
 // This will look in the Retry-After header, RFC 7231, for how long
 // it will wait before attempting another request
-func getRetryDelay(r *Request) (time.Duration, bool) {
+func getRetryAfterDelay(r *Request) (time.Duration, bool) {
 	if !canUseRetryAfterHeader(r) {
 		return 0, false
 	}

aws/default_retryer_test.go

@@ -152,7 +152,7 @@ func TestGetRetryDelay(t *testing.T) {
 	}
 
 	for i, c := range cases {
-		a, ok := getRetryDelay(&c.r)
+		a, ok := getRetryAfterDelay(&c.r)
 		if c.ok != ok {
 			t.Errorf("%d: expected %v, but received %v", i, c.ok, ok)
 		}
@@ -162,3 +162,35 @@ func TestGetRetryDelay(t *testing.T) {
 		}
 	}
 }
+
+func TestRetryDelay(t *testing.T) {
+	d := DefaultRetryer{100}
+	r := Request{}
+	for i := 0; i < 100; i++ {
+		rTemp := r
+		rTemp.HTTPResponse = &http.Response{StatusCode: 500, Header: http.Header{"Retry-After": []string{"299"}}}
+		rTemp.RetryCount = i
+		a := d.RetryRules(&rTemp)
+		if a > 5*time.Minute {
+			t.Errorf("retry delay should never be greater than five minutes, received %s for retrycount %d", a, i)
+		}
+	}
+
+	for i := 0; i < 100; i++ {
+		rTemp := r
+		rTemp.RetryCount = i
+		rTemp.HTTPResponse = &http.Response{StatusCode: 503, Header: http.Header{"Retry-After": []string{""}}}
+		a := d.RetryRules(&rTemp)
+		if a > 5*time.Minute {
+			t.Errorf("retry delay should not be greater than five minutes, received %s for retrycount %d", a, i)
+		}
+	}
+
+	rTemp := r
+	rTemp.RetryCount = 1
+	rTemp.HTTPResponse = &http.Response{StatusCode: 503, Header: http.Header{"Retry-After": []string{"300"}}}
+	a := d.RetryRules(&rTemp)
+	if a < 5*time.Minute{
+		t.Errorf("retry delay should not be less than retry-after duration, received %s for retrycount %d", a, 1)
+	}
+}