Skip to content

Commit d9aaf2d

Browse files
scaugratedscaugrated
authored
generate error/fault metrics by aws sdk status code (#924)
1 parent 17d03c8 commit d9aaf2d

File tree

2 files changed

+160
-5
lines changed

2 files changed

+160
-5
lines changed

‎aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/AwsSpanMetricsProcessor.java‎

Lines changed: 58 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,11 @@
1515
import io.opentelemetry.sdk.trace.ReadWriteSpan;
1616
import io.opentelemetry.sdk.trace.ReadableSpan;
1717
import io.opentelemetry.sdk.trace.SpanProcessor;
18+
import io.opentelemetry.sdk.trace.data.EventData;
1819
import io.opentelemetry.sdk.trace.data.SpanData;
20+
import io.opentelemetry.sdk.trace.internal.data.ExceptionEventData;
21+
import java.lang.reflect.Method;
22+
import javax.annotation.Nullable;
1923
import javax.annotation.concurrent.Immutable;
2024

2125
/**
@@ -92,7 +96,7 @@ public void onEnd(ReadableSpan span) {
9296

9397
// Only record metrics if non-empty attributes are returned.
9498
if (!attributes.isEmpty()) {
95-
recordErrorOrFault(span, attributes);
99+
recordErrorOrFault(spanData, attributes);
96100
recordLatency(span, attributes);
97101
}
98102
}
@@ -102,10 +106,14 @@ public boolean isEndRequired() {
102106
return true;
103107
}
104108

105-
private void recordErrorOrFault(ReadableSpan span, Attributes attributes) {
106-
Long httpStatusCode = span.getAttribute(HTTP_STATUS_CODE);
109+
private void recordErrorOrFault(SpanData spanData, Attributes attributes) {
110+
Long httpStatusCode = spanData.getAttributes().get(HTTP_STATUS_CODE);
107111
if (httpStatusCode == null) {
108-
return;
112+
httpStatusCode = getAwsStatusCode(spanData);
113+
114+
if (httpStatusCode == null || httpStatusCode < 100L || httpStatusCode > 599L) {
115+
return;
116+
}
109117
}
110118

111119
if (httpStatusCode >= ERROR_CODE_LOWER_BOUND && httpStatusCode <= ERROR_CODE_UPPER_BOUND) {
@@ -116,6 +124,52 @@ private void recordErrorOrFault(ReadableSpan span, Attributes attributes) {
116124
}
117125
}
118126

127+
/**
128+
* Attempt to pull status code from spans produced by AWS SDK instrumentation (both v1 and v2).
129+
* AWS SDK instrumentation does not populate http.status_code when non-200 status codes are
130+
* returned, as the AWS SDK throws exceptions rather than returning responses with status codes.
131+
* To work around this, we are attempting to get the exception out of the events, then calling
132+
* getStatusCode (for AWS SDK V1) and statusCode (for AWS SDK V2) to get the status code fromt the
133+
* exception. We rely on reflection here because we cannot cast the throwable to
134+
* AmazonServiceExceptions (V1) or AwsServiceExceptions (V2) because the throwable comes from a
135+
* separate class loader and attempts to cast will fail with ClassCastException.
136+
*
137+
* <p>TODO: Short term workaround. This can be completely removed once
138+
* https://github.com/open-telemetry/opentelemetry-java-contrib/issues/919 is resolved.
139+
*/
140+
@Nullable
141+
private static Long getAwsStatusCode(SpanData spanData) {
142+
String scopeName = spanData.getInstrumentationScopeInfo().getName();
143+
if (!scopeName.contains("aws-sdk")) {
144+
return null;
145+
}
146+
147+
for (EventData event : spanData.getEvents()) {
148+
if (event instanceof ExceptionEventData) {
149+
ExceptionEventData exceptionEvent = (ExceptionEventData) event;
150+
Throwable throwable = exceptionEvent.getException();
151+
152+
try {
153+
Method method = throwable.getClass().getMethod("getStatusCode", new Class<?>[] {});
154+
Object code = method.invoke(throwable, new Object[] {});
155+
return Long.valueOf((Integer) code);
156+
} catch (Exception e) {
157+
// Take no action
158+
}
159+
160+
try {
161+
Method method = throwable.getClass().getMethod("statusCode", new Class<?>[] {});
162+
Object code = method.invoke(throwable, new Object[] {});
163+
return Long.valueOf((Integer) code);
164+
} catch (Exception e) {
165+
// Take no action
166+
}
167+
}
168+
}
169+
170+
return null;
171+
}
172+
119173
private void recordLatency(ReadableSpan span, Attributes attributes) {
120174
long nanos = span.getLatencyNanos();
121175
double millis = nanos / NANOS_TO_MILLIS;

‎aws-xray/src/test/java/io/opentelemetry/contrib/awsxray/AwsSpanMetricsProcessorTest.java‎

Lines changed: 102 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,16 +22,21 @@
2222
import io.opentelemetry.api.metrics.LongCounter;
2323
import io.opentelemetry.context.Context;
2424
import io.opentelemetry.sdk.common.CompletableResultCode;
25+
import io.opentelemetry.sdk.common.InstrumentationScopeInfo;
2526
import io.opentelemetry.sdk.resources.Resource;
2627
import io.opentelemetry.sdk.trace.ReadWriteSpan;
2728
import io.opentelemetry.sdk.trace.ReadableSpan;
29+
import io.opentelemetry.sdk.trace.data.EventData;
2830
import io.opentelemetry.sdk.trace.data.SpanData;
31+
import io.opentelemetry.sdk.trace.internal.data.ExceptionEventData;
32+
import java.util.ArrayList;
33+
import java.util.Arrays;
34+
import java.util.List;
2935
import org.junit.jupiter.api.BeforeEach;
3036
import org.junit.jupiter.api.Test;
3137

3238
/** Unit tests for {@link AwsSpanMetricsProcessor}. */
3339
class AwsSpanMetricsProcessorTest {
34-
3540
// Test constants
3641
private static final boolean CONTAINS_ATTRIBUTES = true;
3742
private static final boolean CONTAINS_NO_ATTRIBUTES = false;
@@ -56,6 +61,32 @@ private enum ExpectedStatusMetric {
5661

5762
private AwsSpanMetricsProcessor awsSpanMetricsProcessor;
5863

64+
static class ThrowableWithMethodGetStatusCode extends Throwable {
65+
private final int httpStatusCode;
66+
67+
ThrowableWithMethodGetStatusCode(int httpStatusCode) {
68+
this.httpStatusCode = httpStatusCode;
69+
}
70+
71+
public int getStatusCode() {
72+
return this.httpStatusCode;
73+
}
74+
}
75+
76+
static class ThrowableWithMethodStatusCode extends Throwable {
77+
private final int httpStatusCode;
78+
79+
ThrowableWithMethodStatusCode(int httpStatusCode) {
80+
this.httpStatusCode = httpStatusCode;
81+
}
82+
83+
public int statusCode() {
84+
return this.httpStatusCode;
85+
}
86+
}
87+
88+
static class ThrowableWithoutStatusCode extends Throwable {}
89+
5990
@BeforeEach
6091
public void setUpMocks() {
6192
errorCounterMock = mock(LongCounter.class);
@@ -149,6 +180,16 @@ public void testOnEndMetricsGenerationWithLatency() {
149180
verify(latencyHistogramMock, times(1)).record(eq(5.5), eq(metricAttributes));
150181
}
151182

183+
@Test
184+
public void testOnEndMetricsGenerationWithAwsStatusCodes() {
185+
validateMetricsGeneratedForAwsStatusCode(399L, ExpectedStatusMetric.NEITHER);
186+
validateMetricsGeneratedForAwsStatusCode(400L, ExpectedStatusMetric.ERROR);
187+
validateMetricsGeneratedForAwsStatusCode(499L, ExpectedStatusMetric.ERROR);
188+
validateMetricsGeneratedForAwsStatusCode(500L, ExpectedStatusMetric.FAULT);
189+
validateMetricsGeneratedForAwsStatusCode(599L, ExpectedStatusMetric.FAULT);
190+
validateMetricsGeneratedForAwsStatusCode(600L, ExpectedStatusMetric.NEITHER);
191+
}
192+
152193
@Test
153194
public void testOnEndMetricsGenerationWithStatusCodes() {
154195
// Invalid HTTP status codes
@@ -192,13 +233,44 @@ private static ReadableSpan buildReadableSpanMock(Attributes spanAttributes) {
192233

193234
// Configure spanData
194235
SpanData mockSpanData = mock(SpanData.class);
236+
InstrumentationScopeInfo awsSdkScopeInfo =
237+
InstrumentationScopeInfo.builder("aws-sdk").setVersion("version").build();
238+
when(mockSpanData.getInstrumentationScopeInfo()).thenReturn(awsSdkScopeInfo);
195239
when(mockSpanData.getAttributes()).thenReturn(spanAttributes);
196240
when(mockSpanData.getTotalAttributeCount()).thenReturn(spanAttributes.size());
197241
when(readableSpanMock.toSpanData()).thenReturn(mockSpanData);
198242

199243
return readableSpanMock;
200244
}
201245

246+
private static ReadableSpan buildReadableSpanWithThrowableMock(Throwable throwable) {
247+
// config http status code as null
248+
Attributes spanAttributes = Attributes.of(HTTP_STATUS_CODE, null);
249+
ReadableSpan readableSpanMock = mock(ReadableSpan.class);
250+
SpanData mockSpanData = mock(SpanData.class);
251+
InstrumentationScopeInfo awsSdkScopeInfo =
252+
InstrumentationScopeInfo.builder("aws-sdk").setVersion("version").build();
253+
ExceptionEventData mockEventData = mock(ExceptionEventData.class);
254+
List<EventData> events = new ArrayList<>(Arrays.asList(mockEventData));
255+
256+
// Configure latency
257+
when(readableSpanMock.getLatencyNanos()).thenReturn(TEST_LATENCY_NANOS);
258+
259+
// Configure attributes
260+
when(readableSpanMock.getAttribute(any()))
261+
.thenAnswer(invocation -> spanAttributes.get(invocation.getArgument(0)));
262+
263+
// Configure spanData
264+
when(mockSpanData.getInstrumentationScopeInfo()).thenReturn(awsSdkScopeInfo);
265+
when(mockSpanData.getAttributes()).thenReturn(spanAttributes);
266+
when(mockSpanData.getTotalAttributeCount()).thenReturn(spanAttributes.size());
267+
when(mockSpanData.getEvents()).thenReturn(events);
268+
when(mockEventData.getException()).thenReturn(throwable);
269+
when(readableSpanMock.toSpanData()).thenReturn(mockSpanData);
270+
271+
return readableSpanMock;
272+
}
273+
202274
private void configureMocksForOnEnd(ReadableSpan readableSpanMock, Attributes metricAttributes) {
203275
// Configure generated attributes
204276
when(generatorMock.generateMetricAttributesFromSpan(
@@ -214,6 +286,35 @@ private void validateMetricsGeneratedForHttpStatusCode(
214286
configureMocksForOnEnd(readableSpanMock, metricAttributes);
215287

216288
awsSpanMetricsProcessor.onEnd(readableSpanMock);
289+
validateMetrics(metricAttributes, expectedStatusMetric);
290+
}
291+
292+
private void validateMetricsGeneratedForAwsStatusCode(
293+
Long awsStatusCode, ExpectedStatusMetric expectedStatusMetric) {
294+
Throwable throwableWithMethodGetStatusCode =
295+
new ThrowableWithMethodGetStatusCode(awsStatusCode.intValue());
296+
validateMetricsGeneratedByThrowable(throwableWithMethodGetStatusCode, expectedStatusMetric);
297+
298+
Throwable throwableWithMethodStatusCode =
299+
new ThrowableWithMethodGetStatusCode(awsStatusCode.intValue());
300+
validateMetricsGeneratedByThrowable(throwableWithMethodStatusCode, expectedStatusMetric);
301+
302+
Throwable throwableWithoutStatusCode = new ThrowableWithoutStatusCode();
303+
validateMetricsGeneratedByThrowable(throwableWithoutStatusCode, ExpectedStatusMetric.NEITHER);
304+
}
305+
306+
private void validateMetricsGeneratedByThrowable(
307+
Throwable throwable, ExpectedStatusMetric expectedStatusMetric) {
308+
ReadableSpan readableSpanMock = buildReadableSpanWithThrowableMock(throwable);
309+
Attributes metricAttributes = buildMetricAttributes(CONTAINS_ATTRIBUTES);
310+
configureMocksForOnEnd(readableSpanMock, metricAttributes);
311+
312+
awsSpanMetricsProcessor.onEnd(readableSpanMock);
313+
validateMetrics(metricAttributes, expectedStatusMetric);
314+
}
315+
316+
private void validateMetrics(
317+
Attributes metricAttributes, ExpectedStatusMetric expectedStatusMetric) {
217318
switch (expectedStatusMetric) {
218319
case ERROR:
219320
verify(errorCounterMock, times(1)).add(eq(1L), eq(metricAttributes));

0 commit comments

Comments
 (0)