Skip to content

Commit 8ed1680

Browse files
ianbotsfianbotsf
authored
fix: include more info for retry token bucket capacity errors (#1217)
1 parent 46c2683 commit 8ed1680

File tree

9 files changed

+204
-18
lines changed

9 files changed

+204
-18
lines changed

‎.changes/3456d00f-1b29-4d88-ab02-045db1b1ebce.json‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"id": "3456d00f-1b29-4d88-ab02-045db1b1ebce",
3+
"type": "bugfix",
4+
"description": "Include more information when retry strategy halts early due to token bucket capacity errors",
5+
"issues": [
6+
"awslabs/aws-sdk-kotlin#1321"
7+
]
8+
}

‎runtime/runtime-core/api/runtime-core.api‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,13 @@
1+
public final class aws/smithy/kotlin/runtime/ClientErrorContext {
2+
public fun <init> (Ljava/lang/String;Ljava/lang/String;)V
3+
public fun equals (Ljava/lang/Object;)Z
4+
public final fun getFormatted ()Ljava/lang/String;
5+
public final fun getKey ()Ljava/lang/String;
6+
public final fun getValue ()Ljava/lang/String;
7+
public fun hashCode ()I
8+
public fun toString ()Ljava/lang/String;
9+
}
10+
111
public class aws/smithy/kotlin/runtime/ClientException : aws/smithy/kotlin/runtime/SdkBaseException {
212
public fun <init> ()V
313
public fun <init> (Ljava/lang/String;)V
@@ -9,11 +19,13 @@ public class aws/smithy/kotlin/runtime/ErrorMetadata {
919
public static final field Companion Laws/smithy/kotlin/runtime/ErrorMetadata$Companion;
1020
public fun <init> ()V
1121
public final fun getAttributes ()Laws/smithy/kotlin/runtime/collections/MutableAttributes;
22+
public final fun getClientContext ()Ljava/util/List;
1223
public final fun isRetryable ()Z
1324
public final fun isThrottling ()Z
1425
}
1526

1627
public final class aws/smithy/kotlin/runtime/ErrorMetadata$Companion {
28+
public final fun getClientContext ()Laws/smithy/kotlin/runtime/collections/AttributeKey;
1729
public final fun getRetryable ()Laws/smithy/kotlin/runtime/collections/AttributeKey;
1830
public final fun getThrottlingError ()Laws/smithy/kotlin/runtime/collections/AttributeKey;
1931
}
@@ -62,7 +74,6 @@ public class aws/smithy/kotlin/runtime/ServiceException : aws/smithy/kotlin/runt
6274
public fun <init> (Ljava/lang/String;)V
6375
public fun <init> (Ljava/lang/String;Ljava/lang/Throwable;)V
6476
public fun <init> (Ljava/lang/Throwable;)V
65-
protected fun getDisplayMetadata ()Ljava/util/List;
6677
public fun getMessage ()Ljava/lang/String;
6778
public synthetic fun getSdkErrorMetadata ()Laws/smithy/kotlin/runtime/ErrorMetadata;
6879
public fun getSdkErrorMetadata ()Laws/smithy/kotlin/runtime/ServiceErrorMetadata;
@@ -134,6 +145,7 @@ public final class aws/smithy/kotlin/runtime/collections/AttributesBuilder {
134145
}
135146

136147
public final class aws/smithy/kotlin/runtime/collections/AttributesKt {
148+
public static final fun appendValue (Laws/smithy/kotlin/runtime/collections/MutableAttributes;Laws/smithy/kotlin/runtime/collections/AttributeKey;Ljava/lang/Object;)V
137149
public static final fun attributesOf (Lkotlin/jvm/functions/Function1;)Laws/smithy/kotlin/runtime/collections/Attributes;
138150
public static final fun emptyAttributes ()Laws/smithy/kotlin/runtime/collections/Attributes;
139151
public static final fun get (Laws/smithy/kotlin/runtime/collections/Attributes;Laws/smithy/kotlin/runtime/collections/AttributeKey;)Ljava/lang/Object;
@@ -1685,6 +1697,7 @@ public abstract class aws/smithy/kotlin/runtime/retries/RetryException : aws/smi
16851697
public final fun getAttempts ()I
16861698
public final fun getLastException ()Ljava/lang/Throwable;
16871699
public final fun getLastResponse ()Ljava/lang/Object;
1700+
public fun toString ()Ljava/lang/String;
16881701
}
16891702

16901703
public final class aws/smithy/kotlin/runtime/retries/RetryFailureException : aws/smithy/kotlin/runtime/retries/RetryException {

‎runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/Exceptions.kt‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,41 @@ import aws.smithy.kotlin.runtime.collections.AttributeKey
88
import aws.smithy.kotlin.runtime.collections.MutableAttributes
99
import aws.smithy.kotlin.runtime.collections.mutableAttributes
1010

11+
/**
12+
* Describes additional context about an error which may be useful in client-side debugging. This information will be
13+
* included in exception messages. This contrasts with [ErrorMetadata] which is not _necessarily_ included in messages
14+
* and not _necessarily_ client-related.
15+
* @param key A header or key for the information
16+
* @param value A value for the information
17+
*/
18+
public class ClientErrorContext(public val key: String, public val value: String) {
19+
override fun equals(other: Any?): Boolean {
20+
if (this === other) return true
21+
if (other == null || this::class != other::class) return false
22+
23+
other as ClientErrorContext
24+
25+
if (key != other.key) return false
26+
if (value != other.value) return false
27+
28+
return true
29+
}
30+
31+
/**
32+
* Gets a formatted representation of this error context suitable for inclusion in a message. This format is
33+
* generally `"$key: $value"`.
34+
*/
35+
public val formatted: String = "$key: $value"
36+
37+
override fun hashCode(): Int {
38+
var result = key.hashCode()
39+
result = 31 * result + value.hashCode()
40+
return result
41+
}
42+
43+
override fun toString(): String = "ClientErrorContext(key='$key', value='$value')"
44+
}
45+
1146
/**
1247
* Additional metadata about an error
1348
*/
@@ -16,6 +51,12 @@ public open class ErrorMetadata {
1651
public val attributes: MutableAttributes = mutableAttributes()
1752

1853
public companion object {
54+
/**
55+
* Set if there are additional context elements about the error
56+
*/
57+
public val ClientContext: AttributeKey<List<ClientErrorContext>> =
58+
AttributeKey("aws.smithy.kotlin#ClientContext")
59+
1960
/**
2061
* Set if an error is retryable
2162
*/
@@ -32,6 +73,9 @@ public open class ErrorMetadata {
3273

3374
public val isThrottling: Boolean
3475
get() = attributes.getOrNull(ThrottlingError) ?: false
76+
77+
public val clientContext: List<ClientErrorContext>
78+
get() = attributes.getOrNull(ClientContext).orEmpty()
3579
}
3680

3781
/**
@@ -156,7 +200,7 @@ public open class ServiceException : SdkBaseException {
156200

157201
public constructor(cause: Throwable?) : super(cause)
158202

159-
protected open val displayMetadata: List<String>
203+
private val displayMetadata: List<String>
160204
get() = buildList {
161205
val serviceProvidedMessage = super.message ?: sdkErrorMetadata.errorMessage
162206
if (serviceProvidedMessage == null) {
@@ -166,7 +210,10 @@ public open class ServiceException : SdkBaseException {
166210
} else {
167211
add(serviceProvidedMessage)
168212
}
213+
169214
sdkErrorMetadata.requestId?.let { add("Request ID: $it") }
215+
216+
sdkErrorMetadata.clientContext.mapTo(this@buildList) { it.formatted }
170217
}
171218

172219
override val message: String

‎runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/collections/Attributes.kt‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,17 @@ public fun MutableAttributes.merge(other: Attributes) {
115115
}
116116
}
117117

118+
/**
119+
* Appends a value to a list-typed attribute. If the attribute does not exist, it will be created.
120+
* @param key The key for the attribute
121+
* @param element The element to append to the existing (or new) list value of the attribute
122+
*/
123+
public fun <E> MutableAttributes.appendValue(key: AttributeKey<List<E>>, element: E) {
124+
val existingList = getOrNull(key).orEmpty()
125+
val newList = existingList + element
126+
set(key, newList)
127+
}
128+
118129
private class AttributesImpl constructor(seed: Attributes) : MutableAttributes {
119130
private val map: MutableMap<AttributeKey<*>, Any> = mutableMapOf()
120131
constructor() : this(emptyAttributes())

‎runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/retries/Exceptions.kt‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,28 @@ public sealed class RetryException(
2222
public val attempts: Int,
2323
public val lastResponse: Any?,
2424
public val lastException: Throwable?,
25-
) : ClientException(message, cause)
25+
) : ClientException(message, cause) {
26+
override fun toString(): String = buildString {
27+
append(this@RetryException::class.simpleName)
28+
append("(")
29+
30+
append("message=")
31+
append(message)
32+
33+
append(",attempts=")
34+
append(attempts)
35+
36+
if (lastException != null) {
37+
append(",lastException=")
38+
append(lastException)
39+
} else if (lastResponse != null) {
40+
append(",lastResponse=")
41+
append(lastResponse)
42+
}
43+
44+
append(")")
45+
}
46+
}
2647

2748
/**
2849
* Indicates that retrying has failed because too many attempts have completed unsuccessfully.

‎runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/retries/StandardRetryStrategy.kt‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,10 @@
55

66
package aws.smithy.kotlin.runtime.retries
77

8+
import aws.smithy.kotlin.runtime.ClientErrorContext
9+
import aws.smithy.kotlin.runtime.ErrorMetadata
10+
import aws.smithy.kotlin.runtime.ServiceException
11+
import aws.smithy.kotlin.runtime.collections.appendValue
812
import aws.smithy.kotlin.runtime.retries.delay.*
913
import aws.smithy.kotlin.runtime.retries.policy.RetryDirective
1014
import aws.smithy.kotlin.runtime.retries.policy.RetryPolicy
@@ -129,17 +133,35 @@ public open class StandardRetryStrategy(override val config: Config = Config.def
129133
}
130134
}
131135

132-
private fun <R> throwCapacityExceeded(cause: Throwable, attempt: Int, result: Result<R>?): Nothing =
133-
when (val ex = result?.exceptionOrNull()) {
136+
private fun <R> throwCapacityExceeded(
137+
cause: RetryCapacityExceededException,
138+
attempt: Int,
139+
result: Result<R>?,
140+
): Nothing {
141+
val capacityMessage = buildString {
142+
append("Insufficient client capacity to attempt retry, halting on attempt ")
143+
append(attempt)
144+
append(" of ")
145+
append(config.maxAttempts)
146+
}
147+
148+
throw when (val retryableException = result?.exceptionOrNull()) {
134149
null -> throw TooManyAttemptsException(
135-
cause.message!!,
150+
capacityMessage,
136151
cause,
137152
attempt,
138153
result?.getOrNull(),
139154
result?.exceptionOrNull(),
140155
)
141-
else -> throw ex
156+
157+
is ServiceException -> retryableException.apply {
158+
val addCtx = ClientErrorContext("Early retry termination", capacityMessage)
159+
sdkErrorMetadata.attributes.appendValue(ErrorMetadata.ClientContext, addCtx)
160+
}
161+
162+
else -> retryableException
142163
}
164+
}
143165

144166
/**
145167
* Handles the termination of the retry loop because of a non-retryable failure by throwing a

‎runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/retries/delay/StandardRetryTokenBucket.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ public class StandardRetryTokenBucket internal constructor(
6060
capacity -= size
6161
} else {
6262
if (config.useCircuitBreakerMode) {
63-
throw RetryCapacityExceededException("Insufficient capacity to attempt another retry")
63+
throw RetryCapacityExceededException("Insufficient capacity to attempt retry")
6464
}
6565

6666
val extraRequiredCapacity = size - capacity

‎runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/ExceptionsTest.kt‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,14 @@
55
package aws.smithy.kotlin.runtime
66

77
import aws.smithy.kotlin.runtime.collections.MutableAttributes
8+
import aws.smithy.kotlin.runtime.collections.appendValue
89
import kotlin.test.Test
910
import kotlin.test.assertEquals
1011

12+
private const val CTX_KEY_1 = "Color"
13+
private const val CTX_VALUE_1 = "blue"
14+
private const val CTX_KEY_2 = "Shape"
15+
private const val CTX_VALUE_2 = "square"
1116
private const val ERROR_CODE = "ErrorWithNoMessage"
1217
private const val METADATA_MESSAGE = "This is a message included in metadata but not the regular response"
1318
private const val PROTOCOL_RESPONSE_SUMMARY = "HTTP 418 I'm a teapot"
@@ -104,6 +109,35 @@ class ExceptionsTest {
104109
e.message,
105110
)
106111
}
112+
113+
@Test
114+
fun testNoMessageWithClientContext() {
115+
val e = FooServiceException {
116+
appendValue(ErrorMetadata.ClientContext, ClientErrorContext(CTX_KEY_1, CTX_VALUE_1))
117+
appendValue(ErrorMetadata.ClientContext, ClientErrorContext(CTX_KEY_2, CTX_VALUE_2))
118+
}
119+
assertEquals(
120+
buildList {
121+
add("Error type: Unknown")
122+
add("Protocol response: (empty response)")
123+
add("$CTX_KEY_1: $CTX_VALUE_1")
124+
add("$CTX_KEY_2: $CTX_VALUE_2")
125+
}.joinToString(),
126+
e.message,
127+
)
128+
}
129+
130+
@Test
131+
fun testMessageWithClientContext() {
132+
val e = FooServiceException(SERVICE_MESSAGE) {
133+
appendValue(ErrorMetadata.ClientContext, ClientErrorContext(CTX_KEY_1, CTX_VALUE_1))
134+
appendValue(ErrorMetadata.ClientContext, ClientErrorContext(CTX_KEY_2, CTX_VALUE_2))
135+
}
136+
assertEquals(
137+
"$SERVICE_MESSAGE, $CTX_KEY_1: $CTX_VALUE_1, $CTX_KEY_2: $CTX_VALUE_2",
138+
e.message,
139+
)
140+
}
107141
}
108142

109143
private class FooServiceException(

‎runtime/runtime-core/jvm/test/aws/smithy/kotlin/runtime/retries/impl/StandardRetryIntegrationTest.kt‎

Lines changed: 40 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@
55

66
package aws.smithy.kotlin.runtime.retries.impl
77

8+
import aws.smithy.kotlin.runtime.ServiceException
89
import aws.smithy.kotlin.runtime.retries.StandardRetryStrategy
9-
import aws.smithy.kotlin.runtime.retries.TooManyAttemptsException
1010
import aws.smithy.kotlin.runtime.retries.delay.StandardRetryTokenBucket
1111
import aws.smithy.kotlin.runtime.retries.getOrThrow
1212
import aws.smithy.kotlin.runtime.retries.policy.RetryDirective
@@ -15,6 +15,7 @@ import aws.smithy.kotlin.runtime.retries.policy.RetryPolicy
1515
import kotlinx.coroutines.ExperimentalCoroutinesApi
1616
import kotlinx.coroutines.test.currentTime
1717
import kotlinx.coroutines.test.runTest
18+
import org.junit.jupiter.api.assertThrows
1819
import kotlin.test.*
1920
import kotlin.time.Duration.Companion.milliseconds
2021

@@ -38,7 +39,7 @@ class StandardRetryIntegrationTest {
3839

3940
val block = object {
4041
var index = 0
41-
suspend fun doIt() = tc.responses[index++].response.statusCode
42+
suspend fun doIt() = tc.responses[index++].response.getOrThrow()
4243
}::doIt
4344

4445
val startTimeMs = currentTime
@@ -47,9 +48,29 @@ class StandardRetryIntegrationTest {
4748

4849
val finalState = tc.responses.last().expected
4950
when (finalState.outcome) {
50-
TestOutcome.Success -> assertEquals(200, result.getOrNull()?.getOrThrow(), "Unexpected outcome for $name")
51-
TestOutcome.MaxAttemptsExceeded -> assertIs<TooManyAttemptsException>(result.exceptionOrNull())
52-
TestOutcome.RetryQuotaExceeded -> assertIs<TooManyAttemptsException>(result.exceptionOrNull())
51+
TestOutcome.Success ->
52+
assertEquals(Ok, result.getOrThrow().getOrThrow(), "Unexpected outcome for $name")
53+
54+
TestOutcome.MaxAttemptsExceeded -> {
55+
val e = assertThrows<HttpCodeException>("Expected exception for $name") {
56+
result.getOrThrow()
57+
}
58+
59+
assertEquals(tc.responses.last().response.statusCode, e.code, "Unexpected error code for $name")
60+
}
61+
62+
TestOutcome.RetryQuotaExceeded -> {
63+
val e = assertThrows<HttpCodeException>("Expected exception for $name") {
64+
result.getOrThrow()
65+
}
66+
67+
assertEquals(tc.responses.last().response.statusCode, e.code, "Unexpected error code for $name")
68+
69+
assertTrue("Expected retry capacity message in exception for $name") {
70+
"Insufficient client capacity to attempt retry" in e.message
71+
}
72+
}
73+
5374
else -> fail("Unexpected outcome for $name: ${finalState.outcome}")
5475
}
5576

@@ -72,10 +93,19 @@ class StandardRetryIntegrationTest {
7293
}
7394
}
7495

75-
object IntegrationTestPolicy : RetryPolicy<Int> {
76-
override fun evaluate(result: Result<Int>): RetryDirective = when (val code = result.getOrNull()!!) {
77-
200 -> RetryDirective.TerminateAndSucceed
78-
500, 502 -> RetryDirective.RetryError(RetryErrorType.ServerSide)
79-
else -> fail("Unexpected status code: $code")
96+
object IntegrationTestPolicy : RetryPolicy<Ok> {
97+
override fun evaluate(result: Result<Ok>): RetryDirective = when {
98+
result.isSuccess -> RetryDirective.TerminateAndSucceed
99+
result.isFailure -> RetryDirective.RetryError(RetryErrorType.ServerSide)
100+
else -> fail("Unexpected result condition")
80101
}
81102
}
103+
104+
data object Ok
105+
106+
class HttpCodeException(val code: Int) : ServiceException()
107+
108+
fun Response.getOrThrow() = when (statusCode) {
109+
200 -> Ok
110+
else -> throw HttpCodeException(statusCode)
111+
}

0 commit comments

Comments
 (0)