improvising retry mechanism and introducing max attempts for retry

Author: ganbaras

internal/tfresource/retry.go

@@ -65,7 +65,35 @@ func init() {
 }
 
 func GetRetryBackoffDuration(response oci_common.OCIOperationResponse, disableNotFoundRetries bool, service string, startTime time.Time, optionals ...interface{}) time.Duration {
-	return getRetryBackoffDurationWithExpectedRetryDurationFn(response, disableNotFoundRetries, service, startTime, getExpectedRetryDuration, optionals...)
+	backoffDuration := getRetryBackoffDurationWithExpectedRetryDurationFn(response, disableNotFoundRetries, service, startTime, getExpectedRetryDuration, optionals...)
+	// Do nothing if service specific retry duration exists
+	if _, ok := serviceExpectedRetryDurationMap[service]; ok {
+		return backoffDuration
+	}
+	if response.Response != nil && response.Response.HTTPResponse() != nil {
+		statusCode := response.Response.HTTPResponse().StatusCode
+		switch statusCode {
+		case 429, 503:
+			utils.Logf("[DEGUG] Handling Retry Timeout for API Response Error Code %d", statusCode)
+			rawResponse := response.Response.HTTPResponse()
+			retryAfterVal := rawResponse.Header.Get("Retry-After")
+			if retryAfterVal == "" {
+				retryAfterVal = rawResponse.Header.Get("retry-after")
+			}
+			if retryAfterVal != "" {
+				if waitTime, err := strconv.Atoi(retryAfterVal); err == nil {
+					backoffDuration = time.Duration(waitTime) * time.Second
+					utils.Logf("[DEGUG] Retry Timeout obtained from response header is %s", backoffDuration)
+
+				} else {
+					utils.Logf("[DEGUG] Error while reading retry-after from response header. Obtained retry-after value is %s. Proceeding with default retry time %s", retryAfterVal, backoffDuration)
+				}
+			} else {
+				utils.Logf("[DEGUG] retry-after value not found in header to handle API Response Error Code %d. Proceeding with default retry time %s", statusCode, backoffDuration)
+			}
+		}
+	}
+	return backoffDuration
 }
 
 func getRetryBackoffDurationWithExpectedRetryDurationFn(response oci_common.OCIOperationResponse, disableNotFoundRetries bool, service string, startTime time.Time, expectedRetryDurationFn expectedRetryDurationFn, optionals ...interface{}) time.Duration {
@@ -353,7 +381,7 @@ func GetRetryPolicy(disableNotFoundRetries bool, service string, optionals ...in
 func getDefaultRetryPolicy(disableNotFoundRetries bool, service string, optionals ...interface{}) *oci_common.RetryPolicy {
 	startTime := time.Now()
 	retryPolicy := &oci_common.RetryPolicy{
-		MaximumNumberAttempts: 0,
+		MaximumNumberAttempts: 10,
 		ShouldRetryOperation: func(response oci_common.OCIOperationResponse) bool {
 			return ShouldRetry(response, disableNotFoundRetries, service, startTime, optionals...)
 		},