Skip to content

Commit 76c7493

Browse files
onurkybsilaurit
onurkybsi
and
laurit
authored
Failsafe RetryPolicy instrumentation added (#15255)
Co-authored-by: Lauri Tulmin <[email protected]>
1 parent 8481d9a commit 76c7493

File tree

3 files changed

+183
-12
lines changed

3 files changed

+183
-12
lines changed

‎instrumentation/failsafe-3.0/library/src/main/java/io/opentelemetry/instrumentation/failsafe/v3_0/FailsafeTelemetry.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,18 +13,24 @@
1313

1414
import dev.failsafe.CircuitBreaker;
1515
import dev.failsafe.CircuitBreakerConfig;
16+
import dev.failsafe.RetryPolicy;
17+
import dev.failsafe.RetryPolicyConfig;
1618
import io.opentelemetry.api.OpenTelemetry;
1719
import io.opentelemetry.api.common.AttributeKey;
1820
import io.opentelemetry.api.common.Attributes;
1921
import io.opentelemetry.api.metrics.LongCounter;
22+
import io.opentelemetry.api.metrics.LongHistogram;
2023
import io.opentelemetry.api.metrics.Meter;
24+
import java.util.Arrays;
2125

2226
/** Entrypoint for instrumenting Failsafe components. */
2327
public final class FailsafeTelemetry {
2428
private static final String INSTRUMENTATION_NAME = "io.opentelemetry.failsafe-3.0";
2529

2630
private static final AttributeKey<String> CIRCUIT_BREAKER_NAME =
2731
AttributeKey.stringKey("failsafe.circuit_breaker.name");
32+
private static final AttributeKey<String> RETRY_POLICY_NAME =
33+
AttributeKey.stringKey("failsafe.retry_policy.name");
2834

2935
/** Returns a new {@link FailsafeTelemetry} configured with the given {@link OpenTelemetry}. */
3036
public static FailsafeTelemetry create(OpenTelemetry openTelemetry) {
@@ -70,4 +76,41 @@ public <R> CircuitBreaker<R> createCircuitBreaker(
7076
.onClose(buildInstrumentedCloseListener(userConfig, stateChangesCounter, attributes))
7177
.build();
7278
}
79+
80+
/**
81+
* Returns an instrumented {@link RetryPolicy} by given values.
82+
*
83+
* @param delegate user configured {@link RetryPolicy} to be instrumented
84+
* @param retryPolicyName identifier of given {@link RetryPolicy}
85+
* @param <R> {@link RetryPolicy}'s result type
86+
* @return instrumented {@link RetryPolicy}
87+
*/
88+
public <R> RetryPolicy<R> createRetryPolicy(RetryPolicy<R> delegate, String retryPolicyName) {
89+
RetryPolicyConfig<R> userConfig = delegate.getConfig();
90+
Meter meter = openTelemetry.getMeter(INSTRUMENTATION_NAME);
91+
LongCounter executionCounter =
92+
meter
93+
.counterBuilder("failsafe.retry_policy.execution.count")
94+
.setDescription(
95+
"Count of execution attempts processed by the retry policy, "
96+
+ "where one execution represents the total number of attempts.")
97+
.setUnit("{execution}")
98+
.build();
99+
LongHistogram attemptsHistogram =
100+
meter
101+
.histogramBuilder("failsafe.retry_policy.attempts")
102+
.setDescription("Number of attempts for each execution.")
103+
.ofLongs()
104+
.setExplicitBucketBoundariesAdvice(Arrays.asList(1L, 2L, 3L, 5L))
105+
.build();
106+
Attributes attributes = Attributes.of(RETRY_POLICY_NAME, retryPolicyName);
107+
return RetryPolicy.builder(userConfig)
108+
.onFailure(
109+
RetryPolicyEventListenerBuilders.buildInstrumentedFailureListener(
110+
userConfig, executionCounter, attemptsHistogram, attributes))
111+
.onSuccess(
112+
RetryPolicyEventListenerBuilders.buildInstrumentedSuccessListener(
113+
userConfig, executionCounter, attemptsHistogram, attributes))
114+
.build();
115+
}
73116
}

‎instrumentation/failsafe-3.0/library/src/main/java/io/opentelemetry/instrumentation/failsafe/v3_0/RetryPolicyEventListenerBuilders.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/*
2+
* Copyright The OpenTelemetry Authors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package io.opentelemetry.instrumentation.failsafe.v3_0;
7+
8+
import static io.opentelemetry.api.common.AttributeKey.stringKey;
9+
10+
import dev.failsafe.RetryPolicyConfig;
11+
import dev.failsafe.event.EventListener;
12+
import dev.failsafe.event.ExecutionCompletedEvent;
13+
import io.opentelemetry.api.common.AttributeKey;
14+
import io.opentelemetry.api.common.Attributes;
15+
import io.opentelemetry.api.metrics.LongCounter;
16+
import io.opentelemetry.api.metrics.LongHistogram;
17+
18+
final class RetryPolicyEventListenerBuilders {
19+
private static final AttributeKey<String> OUTCOME_KEY =
20+
stringKey("failsafe.retry_policy.outcome");
21+
22+
private RetryPolicyEventListenerBuilders() {}
23+
24+
static <R> EventListener<ExecutionCompletedEvent<R>> buildInstrumentedFailureListener(
25+
RetryPolicyConfig<R> userConfig,
26+
LongCounter executionCounter,
27+
LongHistogram attemptsHistogram,
28+
Attributes commonAttributes) {
29+
Attributes attributes = commonAttributes.toBuilder().put(OUTCOME_KEY, "failure").build();
30+
EventListener<ExecutionCompletedEvent<R>> userFailureListener = userConfig.getFailureListener();
31+
return e -> {
32+
executionCounter.add(1, attributes);
33+
attemptsHistogram.record(e.getAttemptCount(), attributes);
34+
if (userFailureListener != null) {
35+
userFailureListener.accept(e);
36+
}
37+
};
38+
}
39+
40+
static <R> EventListener<ExecutionCompletedEvent<R>> buildInstrumentedSuccessListener(
41+
RetryPolicyConfig<R> userConfig,
42+
LongCounter executionCounter,
43+
LongHistogram attemptsHistogram,
44+
Attributes commonAttributes) {
45+
Attributes attributes = commonAttributes.toBuilder().put(OUTCOME_KEY, "success").build();
46+
EventListener<ExecutionCompletedEvent<R>> userSuccessListener = userConfig.getSuccessListener();
47+
return e -> {
48+
executionCounter.add(1, attributes);
49+
attemptsHistogram.record(e.getAttemptCount(), attributes);
50+
if (userSuccessListener != null) {
51+
userSuccessListener.accept(e);
52+
}
53+
};
54+
}
55+
}

‎instrumentation/failsafe-3.0/library/src/test/java/io/opentelemetry/instrumentation/failsafe/v3_0/FailsafeTelemetryTest.java‎

Lines changed: 85 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,21 @@
55

66
package io.opentelemetry.instrumentation.failsafe.v3_0;
77

8+
import static io.opentelemetry.api.common.AttributeKey.stringKey;
9+
import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo;
810
import static org.assertj.core.api.Assertions.assertThat;
9-
import static org.junit.jupiter.api.Assertions.assertEquals;
1011

1112
import dev.failsafe.CircuitBreaker;
1213
import dev.failsafe.CircuitBreakerOpenException;
1314
import dev.failsafe.Failsafe;
15+
import dev.failsafe.RetryPolicy;
1416
import io.opentelemetry.api.common.Attributes;
1517
import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension;
1618
import io.opentelemetry.instrumentation.testing.junit.LibraryInstrumentationExtension;
1719
import io.opentelemetry.sdk.testing.assertj.LongPointAssert;
1820
import java.time.Duration;
1921
import java.util.Objects;
22+
import java.util.concurrent.atomic.AtomicInteger;
2023
import java.util.function.Consumer;
2124
import org.junit.jupiter.api.Test;
2225
import org.junit.jupiter.api.extension.RegisterExtension;
@@ -62,9 +65,7 @@ void captureCircuitBreakerMetrics() {
6265
buildCircuitBreakerAssertion(
6366
2, "failsafe.circuit_breaker.outcome", "failure"),
6467
buildCircuitBreakerAssertion(
65-
3, "failsafe.circuit_breaker.outcome", "success"))));
66-
testing.waitAndAssertMetrics(
67-
"io.opentelemetry.failsafe-3.0",
68+
3, "failsafe.circuit_breaker.outcome", "success"))),
6869
metricAssert ->
6970
metricAssert
7071
.hasName("failsafe.circuit_breaker.state_change.count")
@@ -80,18 +81,90 @@ void captureCircuitBreakerMetrics() {
8081
1, "failsafe.circuit_breaker.state", "closed"))));
8182
}
8283

84+
@Test
85+
void captureRetryPolicyMetrics() {
86+
// given
87+
RetryPolicy<Object> userRetryPolicy =
88+
dev.failsafe.RetryPolicy.builder()
89+
.handleResultIf(Objects::isNull)
90+
.withMaxAttempts(3)
91+
.build();
92+
FailsafeTelemetry failsafeTelemetry = FailsafeTelemetry.create(testing.getOpenTelemetry());
93+
RetryPolicy<Object> instrumentedRetryPolicy =
94+
failsafeTelemetry.createRetryPolicy(userRetryPolicy, "testing");
95+
96+
// when
97+
for (int i = 0; i <= 4; i++) {
98+
int temp = i;
99+
AtomicInteger retry = new AtomicInteger(0);
100+
Failsafe.with(instrumentedRetryPolicy)
101+
.get(
102+
() -> {
103+
if (retry.get() < temp) {
104+
retry.incrementAndGet();
105+
return null;
106+
} else {
107+
return new Object();
108+
}
109+
});
110+
}
111+
112+
// then
113+
testing.waitAndAssertMetrics(
114+
"io.opentelemetry.failsafe-3.0",
115+
metricAssert ->
116+
metricAssert
117+
.hasName("failsafe.retry_policy.execution.count")
118+
.hasLongSumSatisfying(
119+
sum ->
120+
sum.isMonotonic()
121+
.hasPointsSatisfying(
122+
buildRetryPolicyAssertion(2, "failure"),
123+
buildRetryPolicyAssertion(3, "success"))),
124+
metricAssert ->
125+
metricAssert
126+
.hasName("failsafe.retry_policy.attempts")
127+
.hasHistogramSatisfying(
128+
histogramAssert ->
129+
histogramAssert.hasPointsSatisfying(
130+
histogramPointAssert ->
131+
histogramPointAssert
132+
.hasCount(3)
133+
.hasMin(1)
134+
.hasMax(3)
135+
.hasAttributes(buildExpectedRetryPolicyAttributes("success"))
136+
.hasBucketCounts(1L, 1L, 1L, 0L, 0L),
137+
histogramPointAssert ->
138+
histogramPointAssert
139+
.hasCount(2)
140+
.hasMin(3)
141+
.hasMax(3)
142+
.hasAttributes(buildExpectedRetryPolicyAttributes("failure"))
143+
.hasBucketCounts(0L, 0L, 2L, 0L, 0L))));
144+
}
145+
83146
private static Consumer<LongPointAssert> buildCircuitBreakerAssertion(
84147
long expectedValue, String expectedAttributeKey, String expectedAttributeValue) {
85148
return longSumAssert ->
86149
longSumAssert
87150
.hasValue(expectedValue)
88-
.hasAttributesSatisfying(
89-
attributes ->
90-
assertEquals(
91-
Attributes.builder()
92-
.put("failsafe.circuit_breaker.name", "testing")
93-
.put(expectedAttributeKey, expectedAttributeValue)
94-
.build(),
95-
attributes));
151+
.hasAttributesSatisfyingExactly(
152+
equalTo(stringKey("failsafe.circuit_breaker.name"), "testing"),
153+
equalTo(stringKey(expectedAttributeKey), expectedAttributeValue));
154+
}
155+
156+
private static Consumer<LongPointAssert> buildRetryPolicyAssertion(
157+
long expectedValue, String expectedOutcomeValue) {
158+
return longSumAssert ->
159+
longSumAssert
160+
.hasValue(expectedValue)
161+
.hasAttributes(buildExpectedRetryPolicyAttributes(expectedOutcomeValue));
162+
}
163+
164+
private static Attributes buildExpectedRetryPolicyAttributes(String expectedOutcome) {
165+
return Attributes.builder()
166+
.put("failsafe.retry_policy.name", "testing")
167+
.put("failsafe.retry_policy.outcome", expectedOutcome)
168+
.build();
96169
}
97170
}

0 commit comments

Comments
 (0)