Skip to content

Commit 019a5a6

Browse files
michaelraneymichaelraney
committed
Updated Exponential Retry policy
Exponential retry policy is configurable for min and max duration times.
1 parent e7e2121 commit 019a5a6

File tree

5 files changed

+129
-46
lines changed

5 files changed

+129
-46
lines changed

‎src/main/java/com/aws/ssa/keyspaces/retry/AmazonKeyspacesExponentialRetryPolicy.java‎

Lines changed: 43 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
package com.aws.ssa.keyspaces.retry;
22

33
import com.datastax.oss.driver.api.core.ConsistencyLevel;
4+
import com.datastax.oss.driver.api.core.config.DefaultDriverOption;
45
import com.datastax.oss.driver.api.core.config.DriverExecutionProfile;
6+
import com.datastax.oss.driver.api.core.config.DriverOption;
57
import com.datastax.oss.driver.api.core.context.DriverContext;
68
import com.datastax.oss.driver.api.core.retry.RetryDecision;
79
import com.datastax.oss.driver.api.core.retry.RetryPolicy;
@@ -10,13 +12,19 @@
1012
import com.datastax.oss.driver.api.core.servererrors.WriteTimeoutException;
1113
import com.datastax.oss.driver.api.core.servererrors.WriteType;
1214
import com.datastax.oss.driver.api.core.session.Request;
15+
import com.datastax.oss.driver.internal.core.retry.DefaultRetryPolicy;
1316
import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting;
1417
import com.datastax.oss.driver.shaded.guava.common.util.concurrent.Uninterruptibles;
1518
import edu.umd.cs.findbugs.annotations.NonNull;
19+
import jnr.ffi.annotations.In;
1620
import net.jcip.annotations.ThreadSafe;
1721
import org.slf4j.Logger;
1822
import org.slf4j.LoggerFactory;
1923

24+
import java.time.Duration;
25+
import java.time.Period;
26+
import java.time.temporal.ChronoUnit;
27+
import java.time.temporal.TemporalUnit;
2028
import java.util.concurrent.ThreadLocalRandom;
2129
import java.util.concurrent.TimeUnit;
2230

@@ -37,10 +45,13 @@
3745
*
3846
* <pre>
3947
* datastax-java-driver {
40-
* advanced.retry-policy {
41-
* class = com.aws.ssa.keyspaces.retry.AmazonKeyspacesRetryPolicy
42-
* max-attempts = 2
43-
* }
48+
* basic.request.default-idempotence = true
49+
* advanced.retry-policy{
50+
* class = com.aws.ssa.keyspaces.retry.AmazonKeyspacesExponentialRetryPolicy
51+
* max-attempts = 3
52+
* min-wait = 10 mills
53+
* max-wait = 100 mills
54+
* }
4455
* }
4556
* </pre>
4657
*/
@@ -64,6 +75,8 @@ public class AmazonKeyspacesExponentialRetryPolicy implements RetryPolicy {
6475
private final String logPrefix;
6576

6677
private final Integer maxRetryCount;
78+
private final Long minWaitTime;
79+
private final Long maxWaitTime;
6780

6881
//private final Integer maxTimeToWait;
6982

@@ -75,16 +88,40 @@ public AmazonKeyspacesExponentialRetryPolicy(DriverContext context, Integer maxR
7588

7689
String profileName = context.getConfig().getDefaultProfile().getName();
7790

91+
DriverExecutionProfile retryExecutionProfile = context.getConfig().getProfile(profileName);
92+
7893
this.maxRetryCount = maxRetryCount;
94+
Duration minWaitDuration = retryExecutionProfile.getDuration(KeyspacesRetryOption.KEYSPACES_RETRY_MIN_WAIT, KeyspacesRetryOption.DEFAULT_KEYSPACES_RETRY_MIN_WAIT);
95+
Duration maxWaitDuration = retryExecutionProfile.getDuration(KeyspacesRetryOption.KEYSPACES_RETRY_MAX_WAIT, KeyspacesRetryOption.DEFAULT_KEYSPACES_RETRY_MAX_WAIT);
96+
97+
this.minWaitTime = minWaitDuration.toMillis();
98+
this.maxWaitTime = maxWaitDuration.toMillis();
99+
100+
this.logPrefix = (context != null ? context.getSessionName() : null) + "|" + profileName;
101+
}
102+
public AmazonKeyspacesExponentialRetryPolicy(DriverContext context, Integer maxRetryCount, Duration minWaitTime, Duration maxWaitTime) {
103+
104+
String profileName = context.getConfig().getDefaultProfile().getName();
105+
106+
this.maxRetryCount = maxRetryCount;
107+
this.minWaitTime = minWaitTime.toMillis();
108+
this.maxWaitTime = maxWaitTime.toMillis();
79109

80110
this.logPrefix = (context != null ? context.getSessionName() : null) + "|" + profileName;
81111
}
82112

83113
public AmazonKeyspacesExponentialRetryPolicy(DriverContext context, String profileName) {
84114
DriverExecutionProfile retryExecutionProfile = context.getConfig().getProfile(profileName);
85115

116+
86117
maxRetryCount = retryExecutionProfile.getInt(KeyspacesRetryOption.KEYSPACES_RETRY_MAX_ATTEMPTS, KeyspacesRetryOption.DEFAULT_KEYSPACES_RETRY_MAX_ATTEMPTS);
87118

119+
Duration minWaitDuration = retryExecutionProfile.getDuration(KeyspacesRetryOption.KEYSPACES_RETRY_MIN_WAIT, KeyspacesRetryOption.DEFAULT_KEYSPACES_RETRY_MIN_WAIT);
120+
Duration maxWaitDuration = retryExecutionProfile.getDuration(KeyspacesRetryOption.KEYSPACES_RETRY_MAX_WAIT, KeyspacesRetryOption.DEFAULT_KEYSPACES_RETRY_MAX_WAIT);
121+
122+
this.minWaitTime = minWaitDuration.toMillis();
123+
this.maxWaitTime = maxWaitDuration.toMillis();
124+
88125
this.logPrefix = (context != null ? context.getSessionName() : null) + "|" + profileName;
89126
}
90127

@@ -103,9 +140,9 @@ protected RetryDecision determineRetryDecision(int retryCount) {
103140
}
104141
protected void timeToWait(int retryCount){
105142

106-
int timeToWaitCalculation = (retryCount + 1) * ThreadLocalRandom.current().nextInt(1, 20);
143+
long timeToWaitCalculation = minWaitTime + ThreadLocalRandom.current().nextInt(retryCount, Double.valueOf(Math.pow(2d, Integer.valueOf(retryCount).doubleValue())).intValue());
107144

108-
int timeToWaitFinal = Math.min(500, timeToWaitCalculation);
145+
long timeToWaitFinal = Math.min(maxWaitTime, timeToWaitCalculation);
109146

110147
Uninterruptibles.sleepUninterruptibly(timeToWaitFinal, TimeUnit.MILLISECONDS);
111148
}

‎src/main/java/com/aws/ssa/keyspaces/retry/KeyspacesRetryOption.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,21 @@
11
package com.aws.ssa.keyspaces.retry;
22

3+
import com.datastax.oss.driver.api.core.config.DefaultDriverOption;
34
import com.datastax.oss.driver.api.core.config.DriverOption;
45

6+
import java.time.Duration;
7+
58
public enum KeyspacesRetryOption implements DriverOption {
69

7-
KEYSPACES_RETRY_MAX_ATTEMPTS("advanced.retry-policy.max-attempts");
10+
11+
KEYSPACES_RETRY_MAX_ATTEMPTS("advanced.retry-policy.max-attempts"),
12+
KEYSPACES_RETRY_MIN_WAIT("advanced.retry-policy.min-wait"),
13+
KEYSPACES_RETRY_MAX_WAIT("advanced.retry-policy.max-wait");
14+
815

916
public static final Integer DEFAULT_KEYSPACES_RETRY_MAX_ATTEMPTS = 3;
17+
public static final Duration DEFAULT_KEYSPACES_RETRY_MIN_WAIT = Duration.ofMillis(10);
18+
public static final Duration DEFAULT_KEYSPACES_RETRY_MAX_WAIT = Duration.ofMillis(50);
1019

1120
private final String path;
1221

‎src/test/java/com/aws/ssa/keyspaces/retry/AmazonKeyspacesExponentialRetryPolicyTest.java‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,15 @@
1414
import org.junit.Test;
1515
import com.datastax.oss.driver.shaded.guava.common.base.Stopwatch;
1616

17+
import java.time.Duration;
18+
1719
public class AmazonKeyspacesExponentialRetryPolicyTest {
1820

1921
@Test
2022
public void determineRetryDecisionExceed() {
2123

2224
DriverContext context = new DefaultDriverContext(new DefaultProgrammaticDriverConfigLoaderBuilder().build(), ProgrammaticArguments.builder().build());
23-
AmazonKeyspacesExponentialRetryPolicy st = new AmazonKeyspacesExponentialRetryPolicy(context, 3);
25+
AmazonKeyspacesExponentialRetryPolicy st = new AmazonKeyspacesExponentialRetryPolicy(context, 3, Duration.ofMillis(10), Duration.ofMillis(50));
2426

2527
Assert.assertEquals(RetryDecision.RETHROW, st.determineRetryDecision(4));
2628

@@ -30,7 +32,7 @@ public void determineRetryTimeToMinWhenRetry() {
3032

3133
DriverContext context = new DefaultDriverContext(new DefaultProgrammaticDriverConfigLoaderBuilder().build(), ProgrammaticArguments.builder().build());
3234

33-
AmazonKeyspacesExponentialRetryPolicy st = new AmazonKeyspacesExponentialRetryPolicy(context, 1);
35+
AmazonKeyspacesExponentialRetryPolicy st = new AmazonKeyspacesExponentialRetryPolicy(context, 1, Duration.ofMillis(10), Duration.ofMillis(50));
3436

3537
Stopwatch stopwatch = Stopwatch.createStarted();
3638

@@ -40,15 +42,15 @@ public void determineRetryTimeToMinWhenRetry() {
4042

4143
long millsObserved = stopwatch.elapsed().toMillis();
4244

43-
Assert.assertTrue(millsObserved > 1 && millsObserved < 20);
45+
Assert.assertTrue(millsObserved > 1 && millsObserved < 21);
4446

4547
}
4648
@Test
4749
public void determineRetryTimeToMinWhenNotRetry() {
4850

4951
DriverContext context = new DefaultDriverContext(new DefaultProgrammaticDriverConfigLoaderBuilder().build(), ProgrammaticArguments.builder().build());
5052

51-
AmazonKeyspacesExponentialRetryPolicy st = new AmazonKeyspacesExponentialRetryPolicy(context, 0);
53+
AmazonKeyspacesExponentialRetryPolicy st= new AmazonKeyspacesExponentialRetryPolicy(context, 0, Duration.ofMillis(10), Duration.ofMillis(50));
5254

5355
Stopwatch stopwatch = Stopwatch.createStarted();
5456

@@ -78,7 +80,8 @@ public void determineRetryTimeToMinWait() {
7880

7981
long millsObserved = stopwatch.elapsed().toMillis();
8082

81-
Assert.assertTrue(millsObserved > 1 && millsObserved < 20);
83+
84+
Assert.assertTrue(String.format("Asset %s", millsObserved), millsObserved > 1 && millsObserved < 20);
8285

8386
}
8487
@Test
@@ -90,15 +93,13 @@ public void determineRetryTimeToWaitMax() {
9093

9194
Stopwatch stopwatch = Stopwatch.createStarted();
9295

93-
st.timeToWait(100000);
96+
st.timeToWait(10);
9497

9598
stopwatch.stop();
9699

97100
long millsObserved = stopwatch.elapsed().toMillis();
98101

99-
System.out.println(millsObserved);
100-
101-
Assert.assertTrue(millsObserved > 999 && millsObserved < 1010);
102+
Assert.assertTrue(String.format("millsObserved: %d",millsObserved), millsObserved > 10 && millsObserved < 100);
102103

103104
}
104105

@@ -195,7 +196,9 @@ public void onUnavailableExceed() {
195196

196197
@Test
197198
public void testConfig() {
198-
Assert.assertEquals(3, DriverConfigLoader.fromClasspath("retry-example").getInitialConfig().getDefaultProfile().getInt(KeyspacesRetryOption.KEYSPACES_RETRY_MAX_ATTEMPTS, KeyspacesRetryOption.DEFAULT_KEYSPACES_RETRY_MAX_ATTEMPTS));
199+
Assert.assertEquals(3, DriverConfigLoader.fromClasspath("exponential-retry-example").getInitialConfig().getDefaultProfile().getInt(KeyspacesRetryOption.KEYSPACES_RETRY_MAX_ATTEMPTS, KeyspacesRetryOption.DEFAULT_KEYSPACES_RETRY_MAX_ATTEMPTS));
200+
Assert.assertEquals(5, DriverConfigLoader.fromClasspath("exponential-retry-example").getInitialConfig().getDefaultProfile().getDuration(KeyspacesRetryOption.KEYSPACES_RETRY_MIN_WAIT, KeyspacesRetryOption.DEFAULT_KEYSPACES_RETRY_MIN_WAIT).toMillis());
201+
Assert.assertEquals(100, DriverConfigLoader.fromClasspath("exponential-retry-example").getInitialConfig().getDefaultProfile().getDuration(KeyspacesRetryOption.KEYSPACES_RETRY_MAX_WAIT, KeyspacesRetryOption.DEFAULT_KEYSPACES_RETRY_MAX_WAIT).toMillis());
199202
}
200203

201204
}

‎src/test/resources/exponential-retry-example.conf‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
2+
datastax-java-driver {
3+
basic.request.consistency = "LOCAL_QUORUM"
4+
basic.contact-points = [ "cassandra.us-east-1.amazonaws.com:9142"]
5+
6+
advanced.reconnect-on-init = true
7+
8+
basic.load-balancing-policy {
9+
local-datacenter = "us-east-1"
10+
}
11+
basic.request.timeout = 10 seconds
12+
13+
basic.request.default-idempotence = true
14+
advanced.retry-policy{
15+
class = com.aws.ssa.keyspaces.retry.AmazonKeyspacesExponentialRetryPolicy
16+
max-attempts = 3
17+
min-wait = 10 mills
18+
max-wait = 100 mills
19+
}
20+
advanced.auth-provider = {
21+
class = software.aws.mcs.auth.SigV4AuthProvider
22+
aws-region = us-east-1
23+
}
24+
25+
#advanced.auth-provider{
26+
# class = PlainTextAuthProvider
27+
# username = "mike-at-102"
28+
# password = "fakepassword"
29+
#}
30+
31+
32+
33+
advanced.ssl-engine-factory {
34+
class = DefaultSslEngineFactory
35+
#truststore-path = "/Users/user/.cassandra/cassandra_truststore.jks"
36+
#truststore-password = "amazon"
37+
hostname-validation = false
38+
}
39+
40+
advanced.connection.pool.local.size = 2
41+
42+
43+
}

‎src/test/resources/throttler-example.conf‎

Lines changed: 20 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,49 +1,40 @@
11

22
datastax-java-driver {
33
basic.request.consistency = "LOCAL_QUORUM"
4+
basic.request.default-idempotence = true
45
basic.contact-points = [ "cassandra.us-east-1.amazonaws.com:9142"]
5-
6-
advanced.metadata.token-map.enabled = false
7-
86
advanced.reconnect-on-init = true
97

108
basic.load-balancing-policy {
119
local-datacenter = "us-east-1"
10+
slow-replica-avoidance = false
1211
}
13-
basic.request.timeout = 4000 milliseconds
14-
15-
advanced.auth-provider = {
16-
class = software.aws.mcs.auth.SigV4AuthProvider
17-
aws-region = us-east-1
18-
}
1912

20-
advanced.throttler = {
21-
class = com.aws.ssa.keyspaces.throttler.AmazonKeyspacesFixedRateThrottler
22-
max-requests-per-second = 1000
23-
register-timeout = 3 seconds
24-
number-of-hosts = 1
25-
}
26-
basic.request.default-idempotence = true
27-
advanced.retry-policy{
28-
class = com.aws.ssa.keyspaces.retry.AmazonKeyspacesRetryPolicy
29-
max-attempts = 3
30-
}
31-
#advanced.auth-provider{
32-
# class = PlainTextAuthProvider
33-
# username = "mike-at-102"
34-
# password = "fakepassword"
35-
#}
3613

14+
advanced.auth-provider = {
15+
class = software.aws.mcs.auth.SigV4AuthProvider
16+
aws-region = us-east-1
17+
}
3718

3819

39-
advanced.ssl-engine-factory {
20+
advanced.ssl-engine-factory {
4021
class = DefaultSslEngineFactory
41-
#truststore-path = "/Users/user/.cassandra/cassandra_truststore.jks"
42-
#truststore-password = "amazon"
4322
hostname-validation = false
4423
}
4524

46-
advanced.connection.pool.local.size = 2
25+
advanced.netty {
26+
timer.tick-duration = 1000 milliseconds
27+
io-group.size = 64
28+
}
29+
4730

31+
advanced.retry-policy {
32+
class = com.aws.ssa.keyspaces.retry.AmazonKeyspacesRetryPolicy
33+
max-attempts = 1000
34+
}
35+
advanced.connection.pool.local.size = 3
36+
advanced.resolve-contact-points = false
37+
advanced.prepared-statements.prepare-on-all-nodes = false
38+
advanced.prepared-statements.reprepare-on-up.endabled = false
4839

4940
}

0 commit comments

Comments
 (0)