Skip to content
This repository was archived by the owner on Oct 23, 2025. It is now read-only.

Commit 54a930c

Browse files
gabrixdumtz-az
gabrixdu
and
mtz-az
authored
Adding resource stabilization time and improved UnexpectedErrorStatus (#532)
* Adding resource stabilization time and improved UnexpectedErrorStatus message. * Adding interface for timestamp. --------- Co-authored-by: moataz-mhmd <[email protected]>
1 parent e3cb5bd commit 54a930c

File tree

26 files changed

+811
-65
lines changed

26 files changed

+811
-65
lines changed

‎aws-rds-cfn-common/src/main/java/software/amazon/rds/common/handler/Commons.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
import java.time.Instant;
44
import java.time.ZonedDateTime;
5-
import java.util.Arrays;
65
import java.util.Map;
76
import java.util.function.Function;
87

@@ -72,8 +71,9 @@ public static <M, C> ProgressEvent<M, C> handleException(
7271
final M model = progress.getResourceModel();
7372
final C context = progress.getCallbackContext();
7473
final ErrorStatus errorStatus = errorRuleSet.handle(exception);
75-
final Throwable rootCause = ExceptionUtils.getRootCause(exception);
74+
final String exceptionMessage = getRootCauseExceptionMessage(exception);
7675
final String exceptionClass = exception.getClass().getCanonicalName();
76+
final StringBuilder messageBuilder = new StringBuilder();
7777

7878
if (errorStatus instanceof IgnoreErrorStatus) {
7979
switch (((IgnoreErrorStatus) errorStatus).getStatus()) {
@@ -95,7 +95,10 @@ public static <M, C> ProgressEvent<M, C> handleException(
9595
}
9696
return ProgressEvent.failed(model, context, handlerErrorStatus.getHandlerErrorCode(), exception.getMessage());
9797
} if (errorStatus instanceof UnexpectedErrorStatus && requestLogger != null) {
98-
requestLogger.log("UnexpectedErrorStatus: " + getRootCauseExceptionMessage(rootCause) + " " + exceptionClass);
98+
messageBuilder.append(exceptionClass).append("\n")
99+
.append(exceptionMessage).append("\n")
100+
.append(ExceptionUtils.getStackTrace(exception));
101+
requestLogger.log("UnexpectedErrorStatus", messageBuilder.toString());
99102
}
100103
return ProgressEvent.failed(model, context, HandlerErrorCode.InternalFailure, exception.getMessage());
101104
}
@@ -144,7 +147,7 @@ public static String getRootCauseExceptionMessage(final Throwable t)
144147
return null;
145148
}
146149
String statusMessage = t.getMessage();
147-
if(statusMessage != null) {
150+
if(statusMessage != null) {
148151
statusMessage = statusMessage.substring(0, Math.min(statusMessage.length(), STATUS_MESSAGE_MAXIMUM_LENGTH));
149152
}
150153
return statusMessage;

‎aws-rds-cfn-common/src/main/java/software/amazon/rds/common/handler/TimestampContext.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,5 +15,7 @@ public interface Provider {
1515
void timestampOnce(final String label, final Instant instant);
1616

1717
Instant getTimestamp(final String label);
18+
19+
void calculateTimeDeltaInMinutes(final String label, final Instant currentTime, final Instant startTime);
1820
}
1921
}

‎aws-rds-cfn-common/src/test/java/software/amazon/rds/common/handler/CommonsTest.java‎

Lines changed: 52 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -157,33 +157,27 @@ void test_RootCauseMessage_UnexpectedErrorStatus_Without_text()
157157
{
158158
final ProgressEvent<Void, Void> progressEvent = new ProgressEvent<>();
159159
final Exception ex = new RuntimeException();
160-
final Throwable exRootCause = ExceptionUtils.getRootCause(ex);
160+
final String rootCauseExceptionMessage = getRootCauseExceptionMessage(ex);
161161
final String exceptionClass = ex.getClass().getCanonicalName();
162162
final ErrorRuleSet ruleSet = ErrorRuleSet.extend(ErrorRuleSet.EMPTY_RULE_SET).build();
163+
final StringBuilder test_messageBuilder = new StringBuilder();
163164
Mockito.doAnswer(invocationOnMock
164165
->{Object arg0 = invocationOnMock.getArgument(0);
165-
return null;}).when(requestLogger).log(any(String.class));
166+
return null;}).when(requestLogger).log(any(String.class),any(String.class));
166167

167168
final ProgressEvent <Void, Void> resultEvent = Commons.handleException(progressEvent, ex, ruleSet, requestLogger);
168169

169170
ArgumentCaptor<String> captor = ArgumentCaptor.forClass(String.class);
170-
verify(requestLogger).log(captor.capture());
171+
verify(requestLogger).log(captor.capture(), captor.capture());
171172

172-
final String logLine = captor.getValue();
173-
assertThat(resultEvent).isNotNull();
174-
assertThat(resultEvent.isFailed()).isTrue();
175-
assertThat(logLine).contains("UnexpectedErrorStatus: " + getRootCauseExceptionMessage(exRootCause) + " " + exceptionClass);
176-
}
173+
test_messageBuilder.append(exceptionClass).append("\n")
174+
.append(rootCauseExceptionMessage).append("\n")
175+
.append(ExceptionUtils.getStackTrace(ex));
177176

178-
@Test
179-
void test_handleException_when_logger_is_null()
180-
{
181-
final ProgressEvent<Void, Void> progressEvent = new ProgressEvent<>();
182-
final Exception ex = new RuntimeException();
183-
final ErrorRuleSet ruleSet = ErrorRuleSet.extend(ErrorRuleSet.EMPTY_RULE_SET).build();
184-
final ProgressEvent <Void, Void> resultEvent = Commons.handleException(progressEvent, ex, ruleSet, null);
177+
final String logLine = captor.getValue();
185178
assertThat(resultEvent).isNotNull();
186179
assertThat(resultEvent.isFailed()).isTrue();
180+
assertThat(logLine).contains("UnexpectedErrorStatus", test_messageBuilder.toString());
187181
}
188182

189183
@Test
@@ -192,18 +186,34 @@ void test_RootCauseMessage_UnexpectedErrorStatus_With_text()
192186
final ProgressEvent<Void, Void> progressEvent = new ProgressEvent<>();
193187
final Exception ex = new RuntimeException("Runtime exception test");
194188
final String exceptionClass = ex.getClass().getCanonicalName();
189+
final String rootCauseExceptionMessage = getRootCauseExceptionMessage(ex);
190+
final StringBuilder test_messageBuilder = new StringBuilder();
195191
final ErrorRuleSet ruleSet = ErrorRuleSet.extend(ErrorRuleSet.EMPTY_RULE_SET).build();
196-
Mockito.doNothing().when(requestLogger).log(any(String.class));
192+
Mockito.doNothing().when(requestLogger).log(any(String.class), any((String.class)));
197193
final ProgressEvent <Void, Void> resultEvent = Commons.handleException(progressEvent, ex, ruleSet, requestLogger);
198194

199195
ArgumentCaptor<String> captor = ArgumentCaptor.forClass(String.class);
200-
verify(requestLogger).log(captor.capture());
196+
verify(requestLogger).log(captor.capture(), captor.capture());
197+
198+
test_messageBuilder.append(exceptionClass).append("\n")
199+
.append(rootCauseExceptionMessage).append("\n")
200+
.append(ExceptionUtils.getStackTrace(ex));
201201

202202
final String logLine = captor.getValue();
203203
assertThat(resultEvent).isNotNull();
204204
assertThat(resultEvent.isFailed()).isTrue();
205+
assertThat(logLine).contains("UnexpectedErrorStatus",test_messageBuilder.toString());
206+
}
205207

206-
assertThat(logLine).contains("UnexpectedErrorStatus: Runtime exception test" + " " + exceptionClass);
208+
@Test
209+
void test_handleException_when_logger_is_null()
210+
{
211+
final ProgressEvent<Void, Void> progressEvent = new ProgressEvent<>();
212+
final Exception ex = new RuntimeException();
213+
final ErrorRuleSet ruleSet = ErrorRuleSet.extend(ErrorRuleSet.EMPTY_RULE_SET).build();
214+
final ProgressEvent <Void, Void> resultEvent = Commons.handleException(progressEvent, ex, ruleSet, null);
215+
assertThat(resultEvent).isNotNull();
216+
assertThat(resultEvent.isFailed()).isTrue();
207217
}
208218

209219
@Test
@@ -233,6 +243,30 @@ void test_UnexpectedErrorStatus_Returns_Internal_Failure()
233243
assertThat(resultEvent.getErrorCode()).isEqualTo(HandlerErrorCode.InternalFailure);
234244
}
235245

246+
@Test
247+
void test_handleException_UnexpectedErrorStatus_getStackTrace(){
248+
final ProgressEvent<Void, Void> progressEvent = new ProgressEvent<>();
249+
final Exception ex = new RuntimeException("Runtime exception test");
250+
final String exRootCause = getRootCauseExceptionMessage(ex);
251+
final String exceptionClass = ex.getClass().getCanonicalName();
252+
final String exceptionStackTrace = ExceptionUtils.getStackTrace(ex);
253+
final ErrorRuleSet ruleSet = ErrorRuleSet.extend(ErrorRuleSet.EMPTY_RULE_SET).build();
254+
final StringBuilder test_messageBuilder = new StringBuilder();
255+
test_messageBuilder.append(exceptionClass).append("\n")
256+
.append(exRootCause).append("\n")
257+
.append(exceptionStackTrace);
258+
259+
Mockito.doNothing().when(requestLogger).log(any(String.class), any((String.class)));
260+
final ProgressEvent <Void, Void> resultEvent = Commons.handleException(progressEvent, ex, ruleSet, requestLogger);
261+
ArgumentCaptor<String> captor = ArgumentCaptor.forClass(String.class);
262+
verify(requestLogger).log(captor.capture(), captor.capture());
263+
264+
final String logLine = captor.getValue();
265+
assertThat(resultEvent).isNotNull();
266+
assertThat(resultEvent.isFailed()).isTrue();
267+
assertThat(logLine).contains("UnexpectedErrorStatus",test_messageBuilder.toString());
268+
}
269+
236270
@Test
237271
void test_execOnce_invoke() {
238272
AtomicReference<Boolean> flag = new AtomicReference<>(false);

‎aws-rds-customdbengineversion/src/main/java/software/amazon/rds/customdbengineversion/BaseHandlerStd.java‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package software.amazon.rds.customdbengineversion;
22

33
import java.time.Duration;
4+
import java.time.Instant;
45
import java.util.Optional;
56
import java.util.function.BiFunction;
67
import java.util.function.Function;
@@ -39,6 +40,9 @@ public abstract class BaseHandlerStd extends BaseHandler<CallbackContext> {
3940

4041
protected static final String STACK_NAME = "rds";
4142
protected static final String RESOURCE_IDENTIFIER = "customdbengineversion";
43+
protected static final String CUSTOM_DB_ENGINE_VERSION_REQUEST_STARTED_AT = "customdbengineversion-request-started-at";
44+
protected static final String CUSTOM_DB_ENGINE_VERSION_REQUEST_IN_PROGRESS_AT = "customdbengineversion-request-in-progress-at";
45+
protected static final String CUSTOM_DB_ENGINE_VERSION_RESOURCE_STABILIZATION_TIME = "customdbengineversion-stabilization-time";
4246
protected static final int RESOURCE_ID_MAX_LENGTH = 50;
4347
protected static final String IS_ALREADY_BEING_DELETED_ERROR_FRAGMENT = "is already being deleted";
4448
protected static final String SQL_SERVER_ENGINES = "custom-sqlserver";
@@ -123,6 +127,7 @@ protected ProgressEvent<ResourceModel, CallbackContext> handleRequest(
123127
final RequestLogger requestLogger)
124128
{
125129
this.requestLogger = requestLogger;
130+
resourceStabilizationTime(callbackContext);
126131
return handleRequest(proxy, request, callbackContext, proxyClient);
127132
}
128133

@@ -132,9 +137,9 @@ protected boolean isStabilized(final ResourceModel model, final ProxyClient<RdsC
132137
final String status = fetchDBEngineVersion(model, proxyClient).status();
133138
assertNoCustomDbEngineVersionTerminalStatus(status);
134139
return status != null && (CustomDBEngineVersionStatus.fromString(status).isStable() ||
135-
// SQL Server CEVs will remain in PendingValidation state until a new RDS Custom for SQL Server DB instance using the CEV is created.
136-
model.getEngine().contains(SQL_SERVER_ENGINES) &&
137-
CustomDBEngineVersionStatus.fromString(status) == CustomDBEngineVersionStatus.PendingValidation
140+
// SQL Server CEVs will remain in PendingValidation state until a new RDS Custom for SQL Server DB instance using the CEV is created.
141+
model.getEngine().contains(SQL_SERVER_ENGINES) &&
142+
CustomDBEngineVersionStatus.fromString(status) == CustomDBEngineVersionStatus.PendingValidation
138143
);
139144
} catch (CustomDbEngineVersionNotFoundException exception) {
140145
return false;
@@ -148,6 +153,12 @@ private void assertNoCustomDbEngineVersionTerminalStatus(final String source) th
148153
}
149154
}
150155

156+
private void resourceStabilizationTime(final CallbackContext callbackContext) {
157+
callbackContext.timestampOnce(CUSTOM_DB_ENGINE_VERSION_REQUEST_STARTED_AT, Instant.now());
158+
callbackContext.timestamp(CUSTOM_DB_ENGINE_VERSION_REQUEST_IN_PROGRESS_AT, Instant.now());
159+
callbackContext.calculateTimeDeltaInMinutes(CUSTOM_DB_ENGINE_VERSION_RESOURCE_STABILIZATION_TIME, callbackContext.getTimestamp(CUSTOM_DB_ENGINE_VERSION_REQUEST_IN_PROGRESS_AT), callbackContext.getTimestamp(CUSTOM_DB_ENGINE_VERSION_REQUEST_STARTED_AT));
160+
}
161+
151162
protected DBEngineVersion fetchDBEngineVersion(final ResourceModel model,
152163
final ProxyClient<RdsClient> proxyClient) {
153164
DescribeDbEngineVersionsResponse response = proxyClient.injectCredentialsAndInvokeV2(

‎aws-rds-customdbengineversion/src/main/java/software/amazon/rds/customdbengineversion/CallbackContext.java‎

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,28 @@
22

33
import software.amazon.cloudformation.proxy.StdCallbackContext;
44
import software.amazon.rds.common.handler.TaggingContext;
5+
import software.amazon.rds.common.handler.TimestampContext;
6+
7+
import java.time.Duration;
8+
import java.time.Instant;
9+
import java.util.HashMap;
10+
import java.util.Map;
511

612
@lombok.Getter
713
@lombok.Setter
814
@lombok.ToString
915
@lombok.EqualsAndHashCode(callSuper = true)
10-
public class CallbackContext extends StdCallbackContext implements TaggingContext.Provider {
16+
public class CallbackContext extends StdCallbackContext implements TaggingContext.Provider, TimestampContext.Provider {
1117
private boolean modified;
12-
1318
private TaggingContext taggingContext;
19+
private Map<String, Long> timestamps;
20+
private Map<String, Double> timeDelta;
1421

1522
public CallbackContext() {
1623
super();
1724
this.taggingContext = new TaggingContext();
25+
this.timestamps = new HashMap<>();
26+
this.timeDelta = new HashMap<>();
1827
}
1928

2029
@Override
@@ -29,4 +38,28 @@ public boolean isAddTagsComplete() {
2938
public void setAddTagsComplete(final boolean addTagsComplete) {
3039
this.taggingContext.setAddTagsComplete(addTagsComplete);
3140
}
41+
42+
@Override
43+
public void timestamp(final String label, final Instant instant) {
44+
timestamps.put(label, instant.getEpochSecond());
45+
}
46+
47+
@Override
48+
public void timestampOnce(final String label, final Instant instant) {
49+
timestamps.computeIfAbsent(label, s -> instant.getEpochSecond());
50+
}
51+
52+
@Override
53+
public Instant getTimestamp(final String label) {
54+
if (timestamps.containsKey(label)) {
55+
return Instant.ofEpochSecond(timestamps.get(label));
56+
}
57+
return null;
58+
}
59+
60+
@Override
61+
public void calculateTimeDeltaInMinutes(final String label, final Instant currentTime, final Instant startTime){
62+
double delta = Duration.between(currentTime, startTime).toMinutes();
63+
timeDelta.put(label, delta);
64+
}
3265
}

‎aws-rds-dbcluster/src/main/java/software/amazon/rds/dbcluster/BaseHandlerStd.java‎

Lines changed: 28 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import static software.amazon.rds.dbcluster.Translator.removeRoleFromDbClusterRequest;
55

66
import java.time.Duration;
7+
import java.time.Instant;
78
import java.util.Collection;
89
import java.util.Collections;
910
import java.util.LinkedHashSet;
@@ -92,6 +93,9 @@ public abstract class BaseHandlerStd extends BaseHandler<CallbackContext> {
9293
public static final String RESOURCE_IDENTIFIER = "dbcluster";
9394
public static final String ENGINE_AURORA_POSTGRESQL = "aurora-postgresql";
9495
private static final String MASTER_USER_SECRET_ACTIVE = "active";
96+
protected static final String DB_CLUSTER_REQUEST_STARTED_AT = "dbcluster-request-started-at";
97+
protected static final String DB_CLUSTER_REQUEST_IN_PROGRESS_AT = "dbcluster-request-in-progress-at";
98+
protected static final String DB_CLUSTER_STABILIZATION_TIME = "dbcluster-stabilization-time";
9599
public static final String STACK_NAME = "rds";
96100
protected static final int RESOURCE_ID_MAX_LENGTH = 63;
97101

@@ -210,6 +214,7 @@ protected ProgressEvent<ResourceModel, CallbackContext> handleRequest(
210214
final RequestLogger requestLogger
211215
) {
212216
this.requestLogger = requestLogger;
217+
resourceStabilizationTime(callbackContext);
213218
try {
214219
validateRequest(request);
215220
} catch (RequestValidationException exception) {
@@ -339,6 +344,12 @@ protected boolean isDBClusterStabilized(
339344
return isDBClusterStabilizedResult && isNoPendingChangesResult && isMasterUserSecretStabilizedResult && isGlobalWriteForwardingStabilizedResult;
340345
}
341346

347+
private void resourceStabilizationTime(final CallbackContext context) {
348+
context.timestampOnce(DB_CLUSTER_REQUEST_STARTED_AT, Instant.now());
349+
context.timestamp(DB_CLUSTER_REQUEST_IN_PROGRESS_AT, Instant.now());
350+
context.calculateTimeDeltaInMinutes(DB_CLUSTER_STABILIZATION_TIME, context.getTimestamp(DB_CLUSTER_REQUEST_IN_PROGRESS_AT), context.getTimestamp(DB_CLUSTER_REQUEST_STARTED_AT));
351+
}
352+
342353
protected static boolean isMasterUserSecretStabilized(DBCluster dbCluster) {
343354
if (dbCluster.masterUserSecret() == null ||
344355
CollectionUtils.isEmpty(dbCluster.dbClusterMembers())) {
@@ -351,7 +362,7 @@ protected static boolean isGlobalWriteForwardingStabilized(DBCluster dbCluster)
351362
return BooleanUtils.isNotTrue(dbCluster.globalWriteForwardingRequested()) ||
352363
// Even if GWF is requested the WF will not start until a replica is created by customers
353364
(dbCluster.globalWriteForwardingStatus() != WriteForwardingStatus.ENABLING &&
354-
dbCluster.globalWriteForwardingStatus() != WriteForwardingStatus.DISABLING);
365+
dbCluster.globalWriteForwardingStatus() != WriteForwardingStatus.DISABLING);
355366
}
356367

357368
protected boolean isClusterRemovedFromGlobalCluster(
@@ -492,22 +503,22 @@ protected ProgressEvent<ResourceModel, CallbackContext> enableHttpEndpointV2(
492503
final String dbClusterArn = fetchDBCluster(proxyClient, model).dbClusterArn();
493504

494505
return proxy.initiate("rds::enable-http-endpoint-v2", proxyClient, model, callbackContext)
495-
.translateToServiceRequest(modelRequest -> Translator.enableHttpEndpointRequest(dbClusterArn))
496-
.backoffDelay(config.getBackoff())
497-
.makeServiceCall((enableRequest, proxyInvocation) -> proxyInvocation.injectCredentialsAndInvokeV2(
498-
enableRequest,
499-
proxyInvocation.client()::enableHttpEndpoint
500-
))
501-
.stabilize((enableHttpRequest, enableHttpResponse, client, resourceModel, context) ->
502-
isHttpEndpointV2Set(client, resourceModel, true)
503-
)
504-
.handleError((enableHttpRequest, exception, client, resourceModel, callbackCtxt) -> Commons.handleException(
505-
ProgressEvent.progress(resourceModel, callbackCtxt),
506-
exception,
507-
DEFAULT_DB_CLUSTER_ERROR_RULE_SET,
508-
requestLogger
509-
))
510-
.progress();
506+
.translateToServiceRequest(modelRequest -> Translator.enableHttpEndpointRequest(dbClusterArn))
507+
.backoffDelay(config.getBackoff())
508+
.makeServiceCall((enableRequest, proxyInvocation) -> proxyInvocation.injectCredentialsAndInvokeV2(
509+
enableRequest,
510+
proxyInvocation.client()::enableHttpEndpoint
511+
))
512+
.stabilize((enableHttpRequest, enableHttpResponse, client, resourceModel, context) ->
513+
isHttpEndpointV2Set(client, resourceModel, true)
514+
)
515+
.handleError((enableHttpRequest, exception, client, resourceModel, callbackCtxt) -> Commons.handleException(
516+
ProgressEvent.progress(resourceModel, callbackCtxt),
517+
exception,
518+
DEFAULT_DB_CLUSTER_ERROR_RULE_SET,
519+
requestLogger
520+
))
521+
.progress();
511522
}
512523

513524
protected ProgressEvent<ResourceModel, CallbackContext> disableHttpEndpointV2(

0 commit comments

Comments
 (0)