Add Amazon exception map to CFN exception for throttling issues (#374)

* Add Amazon exception map to CFN exception for throttling issues

* Fix unit tests for Wrapper class

* Use RetryUtils for throttling pending

Author: Yunhao-Jiao

src/main/java/software/amazon/cloudformation/AbstractWrapper.java

@@ -15,6 +15,7 @@
 package software.amazon.cloudformation;
 
 import com.amazonaws.AmazonServiceException;
+import com.amazonaws.retry.RetryUtils;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.exc.MismatchedInputException;
 import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException;
@@ -373,9 +374,16 @@ private void logUnhandledError(final String errorDescription,
             logUnhandledError(e.getMessage(), request, e);
             return ProgressEvent.defaultFailureHandler(e, e.getErrorCode());
         } catch (final AmazonServiceException | AwsServiceException e) {
-            publishExceptionMetric(request.getAction(), e, HandlerErrorCode.GeneralServiceException);
-            logUnhandledError("A downstream service error occurred", request, e);
-            return ProgressEvent.defaultFailureHandler(e, HandlerErrorCode.GeneralServiceException);
+            if ((e instanceof AwsServiceException && ((AwsServiceException) e).isThrottlingException()) ||
+                (e instanceof AmazonServiceException && RetryUtils.isThrottlingException((AmazonServiceException) e))) {
+                this.log(String.format("%s [%s] call throttled by downstream service", request.getResourceType(), request.getAction()));
+                publishExceptionMetric(request.getAction(), e, HandlerErrorCode.Throttling);
+                return ProgressEvent.defaultFailureHandler(e, HandlerErrorCode.Throttling);
+            } else {
+                publishExceptionMetric(request.getAction(), e, HandlerErrorCode.GeneralServiceException);
+                logUnhandledError("A downstream service error occurred", request, e);
+                return ProgressEvent.defaultFailureHandler(e, HandlerErrorCode.GeneralServiceException);
+            }
         } catch (final Throwable e) {
             publishExceptionMetric(request.getAction(), e, HandlerErrorCode.InternalFailure);
             logUnhandledError("An unknown error occurred ", request, e);

src/test/java/software/amazon/cloudformation/WrapperTest.java

@@ -785,6 +785,41 @@ public void invokeHandler_throwsSDK2ServiceException_returnsServiceException() t
         }
     }
 
+    @Test
+    public void invokeHandler_throwsThrottlingException_returnsCFNThrottlingException() throws IOException {
+        AmazonServiceException exception = new AmazonServiceException("Rate Exceed ...");
+        exception.setErrorCode("Throttling");
+        wrapper.setInvokeHandlerException(exception);
+
+        wrapper.setTransformResponse(resourceHandlerRequest);
+
+        try (final InputStream in = loadRequestStream("create.request.json");
+             final OutputStream out = new ByteArrayOutputStream()) {
+
+            wrapper.processRequest(in, out);
+
+            // verify initialiseRuntime was called and initialised dependencies
+            verifyInitialiseRuntime();
+
+            // all metrics should be published, once for a single invocation
+            verify(providerMetricsPublisher, times(1)).publishInvocationMetric(any(Instant.class), eq(Action.CREATE));
+            verify(providerMetricsPublisher, times(1)).publishDurationMetric(any(Instant.class), eq(Action.CREATE), anyLong());
+
+            // failure metric should be published
+            verify(providerMetricsPublisher, times(1)).publishExceptionMetric(any(Instant.class), any(),
+                any(AmazonServiceException.class), any(HandlerErrorCode.class));
+
+            // verify that model validation occurred for CREATE/UPDATE/DELETE
+            verify(validator).validateObject(any(JSONObject.class), any(JSONObject.class));
+
+            // verify output response
+            verifyHandlerResponse(out,
+                ProgressEvent.<TestModel, TestContext>builder().errorCode(HandlerErrorCode.Throttling)
+                    .status(OperationStatus.FAILED)
+                    .message("some error (Service: null; Status Code: 0; Error Code: null; Request ID: null)").build());
+        }
+    }
+
     @Test
     public void invokeHandler_throwsResourceAlreadyExistsException_returnsAlreadyExists() throws IOException {
         // exceptions are caught consistently by LambdaWrapper