> fallbackEx
return this;
}
+ public Builder retryOnFailover(boolean retryOnFailover) {
+ this.retryOnFailover = retryOnFailover;
+ return this;
+ }
+
+ public Builder failbackSupported(boolean supported) {
+ this.isFailbackSupported = supported;
+ return this;
+ }
+
+ public Builder failbackCheckInterval(long failbackCheckInterval) {
+ this.failbackCheckInterval = failbackCheckInterval;
+ return this;
+ }
+
+ public Builder gracePeriod(long gracePeriod) {
+ this.gracePeriod = gracePeriod;
+ return this;
+ }
+
+ public Builder fastFailover(boolean fastFailover) {
+ this.fastFailover = fastFailover;
+ return this;
+ }
+
public MultiClusterClientConfig build() {
MultiClusterClientConfig config = new MultiClusterClientConfig(this.clusterConfigs);
@@ -373,6 +535,12 @@ public MultiClusterClientConfig build() {
config.fallbackExceptionList = this.fallbackExceptionList;
+ config.retryOnFailover = this.retryOnFailover;
+ config.isFailbackSupported = this.isFailbackSupported;
+ config.failbackCheckInterval = this.failbackCheckInterval;
+ config.gracePeriod = this.gracePeriod;
+ config.fastFailover = this.fastFailover;
+
return config;
}
}
diff --git a/src/main/java/redis/clients/jedis/UnifiedJedis.java b/src/main/java/redis/clients/jedis/UnifiedJedis.java
index 21a126eaad..73142909b9 100644
--- a/src/main/java/redis/clients/jedis/UnifiedJedis.java
+++ b/src/main/java/redis/clients/jedis/UnifiedJedis.java
@@ -30,7 +30,6 @@
import redis.clients.jedis.json.Path2;
import redis.clients.jedis.json.JsonObjectMapper;
import redis.clients.jedis.mcf.CircuitBreakerCommandExecutor;
-import redis.clients.jedis.mcf.FailoverOptions;
import redis.clients.jedis.mcf.MultiClusterPipeline;
import redis.clients.jedis.mcf.MultiClusterTransaction;
import redis.clients.jedis.params.*;
@@ -238,19 +237,7 @@ public UnifiedJedis(ConnectionProvider provider, int maxAttempts, Duration maxTo
*/
@Experimental
public UnifiedJedis(MultiClusterPooledConnectionProvider provider) {
- this(new CircuitBreakerCommandExecutor(provider, FailoverOptions.builder().build()), provider);
- }
-
- /**
- * Constructor which supports multiple cluster/database endpoints each with their own isolated connection pool.
- *
- * With this Constructor users can seamlessly failover to Disaster Recovery (DR), Backup, and Active-Active cluster(s)
- * by using simple configuration which is passed through from Resilience4j - https://resilience4j.readme.io/docs
- *
- */
- @Experimental
- public UnifiedJedis(MultiClusterPooledConnectionProvider provider, FailoverOptions failoverOptions) {
- this(new CircuitBreakerCommandExecutor(provider, failoverOptions), provider);
+ this(new CircuitBreakerCommandExecutor(provider), provider);
}
/**
@@ -354,6 +341,10 @@ public String ping() {
return checkAndBroadcastCommand(commandObjects.ping());
}
+ public String echo(String string) {
+ return executeCommand(commandObjects.echo(string));
+ }
+
public String flushDB() {
return checkAndBroadcastCommand(commandObjects.flushDB());
}
diff --git a/src/main/java/redis/clients/jedis/csc/CacheConnection.java b/src/main/java/redis/clients/jedis/csc/CacheConnection.java
index f157d95a94..14b7c66144 100644
--- a/src/main/java/redis/clients/jedis/csc/CacheConnection.java
+++ b/src/main/java/redis/clients/jedis/csc/CacheConnection.java
@@ -14,6 +14,32 @@
public class CacheConnection extends Connection {
+ public static class Builder extends Connection.Builder {
+ private Cache cache;
+
+ public Builder(Cache cache) {
+ this.cache = cache;
+ }
+
+ public Builder setCache(Cache cache) {
+ this.cache = cache;
+ return this;
+ }
+
+ public Cache getCache() {
+ return cache;
+ }
+
+ @Override
+ public CacheConnection build() {
+ return new CacheConnection(this);
+ }
+ }
+
+ public static Builder builder(Cache cache) {
+ return new Builder(cache);
+ }
+
private final Cache cache;
private ReentrantLock lock;
private static final String REDIS = "redis";
@@ -21,18 +47,13 @@ public class CacheConnection extends Connection {
public CacheConnection(final JedisSocketFactory socketFactory, JedisClientConfig clientConfig, Cache cache) {
super(socketFactory, clientConfig);
+ this.cache = cache;
+ initializeClientSideCache();
+ }
- if (protocol != RedisProtocol.RESP3) {
- throw new JedisException("Client side caching is only supported with RESP3.");
- }
- if (!cache.compatibilityMode()) {
- RedisVersion current = new RedisVersion(version);
- RedisVersion required = new RedisVersion(MIN_REDIS_VERSION);
- if (!REDIS.equals(server) || current.compareTo(required) < 0) {
- throw new JedisException(String.format("Client side caching is only supported with 'Redis %s' or later.", MIN_REDIS_VERSION));
- }
- }
- this.cache = Objects.requireNonNull(cache);
+ CacheConnection(Builder builder) {
+ super(builder);
+ this.cache = builder.getCache();
initializeClientSideCache();
}
@@ -102,6 +123,19 @@ public Cache getCache() {
}
private void initializeClientSideCache() {
+ if (protocol != RedisProtocol.RESP3) {
+ throw new JedisException("Client side caching is only supported with RESP3.");
+ }
+ Objects.requireNonNull(cache);
+ if (!cache.compatibilityMode()) {
+ RedisVersion current = new RedisVersion(version);
+ RedisVersion required = new RedisVersion(MIN_REDIS_VERSION);
+ if (!REDIS.equals(server) || current.compareTo(required) < 0) {
+ throw new JedisException(
+ String.format("Client side caching is only supported with 'Redis %s' or later.", MIN_REDIS_VERSION));
+ }
+ }
+
sendCommand(Protocol.Command.CLIENT, "TRACKING", "ON");
String reply = getStatusCodeReply();
if (!"OK".equals(reply)) {
diff --git a/src/main/java/redis/clients/jedis/mcf/CircuitBreakerCommandExecutor.java b/src/main/java/redis/clients/jedis/mcf/CircuitBreakerCommandExecutor.java
index cfde64ad94..e68d15bcac 100644
--- a/src/main/java/redis/clients/jedis/mcf/CircuitBreakerCommandExecutor.java
+++ b/src/main/java/redis/clients/jedis/mcf/CircuitBreakerCommandExecutor.java
@@ -22,11 +22,8 @@
@Experimental
public class CircuitBreakerCommandExecutor extends CircuitBreakerFailoverBase implements CommandExecutor {
- private final FailoverOptions options;
-
- public CircuitBreakerCommandExecutor(MultiClusterPooledConnectionProvider provider, FailoverOptions options) {
+ public CircuitBreakerCommandExecutor(MultiClusterPooledConnectionProvider provider) {
super(provider);
- this.options = options != null ? options : FailoverOptions.builder().build();
}
@Override
@@ -50,8 +47,7 @@ private T handleExecuteCommand(CommandObject commandObject, Cluster clust
try (Connection connection = cluster.getConnection()) {
return connection.executeCommand(commandObject);
} catch (Exception e) {
-
- if (retryOnFailover() && !isActiveCluster(cluster)
+ if (cluster.retryOnFailover() && !isActiveCluster(cluster)
&& isCircuitBreakerTrackedException(e, cluster.getCircuitBreaker())) {
throw new ConnectionFailoverException(
"Command failed during failover: " + cluster.getCircuitBreaker().getName(), e);
@@ -65,10 +61,6 @@ private boolean isCircuitBreakerTrackedException(Exception e, CircuitBreaker cb)
return cb.getCircuitBreakerConfig().getRecordExceptionPredicate().test(e);
}
- private boolean retryOnFailover() {
- return options.isRetryOnFailover();
- }
-
private boolean isActiveCluster(Cluster cluster) {
Cluster activeCluster = provider.getCluster();
return activeCluster != null && activeCluster.equals(cluster);
diff --git a/src/main/java/redis/clients/jedis/mcf/CircuitBreakerFailoverBase.java b/src/main/java/redis/clients/jedis/mcf/CircuitBreakerFailoverBase.java
index ec15a5ae98..974949e91c 100644
--- a/src/main/java/redis/clients/jedis/mcf/CircuitBreakerFailoverBase.java
+++ b/src/main/java/redis/clients/jedis/mcf/CircuitBreakerFailoverBase.java
@@ -6,6 +6,7 @@
import redis.clients.jedis.annots.Experimental;
import redis.clients.jedis.exceptions.JedisConnectionException;
import redis.clients.jedis.providers.MultiClusterPooledConnectionProvider;
+import redis.clients.jedis.providers.MultiClusterPooledConnectionProvider.Cluster;
import redis.clients.jedis.util.IOUtils;
/**
@@ -39,34 +40,40 @@ protected void clusterFailover(CircuitBreaker circuitBreaker) {
lock.lock();
try {
- // Check state to handle race conditions since incrementActiveMultiClusterIndex() is
+ // Check state to handle race conditions since iterateActiveCluster() is
// non-idempotent
if (!CircuitBreaker.State.FORCED_OPEN.equals(circuitBreaker.getState())) {
// Transitions state machine to a FORCED_OPEN state, stopping state transition, metrics and
// event publishing.
// To recover/transition from this forced state the user will need to manually failback
- circuitBreaker.transitionToForcedOpenState();
- // Incrementing the activeMultiClusterIndex will allow subsequent calls to the
- // executeCommand()
- // to use the next cluster's connection pool - according to the configuration's
- // prioritization/order
- int activeMultiClusterIndex = provider.incrementActiveMultiClusterIndex();
+ Cluster activeCluster = provider.getCluster();
+ // This should be possible only if active cluster is switched from by other reasons than circuit
+ // breaker, just before circuit breaker triggers
+ if (activeCluster.getCircuitBreaker() != circuitBreaker) {
+ return;
+ }
- // Implementation is optionally provided during configuration. Typically, used for
- // activeMultiClusterIndex persistence or custom logging
- provider.runClusterFailoverPostProcessor(activeMultiClusterIndex);
- }
+ activeCluster.setGracePeriod();
+ circuitBreaker.transitionToForcedOpenState();
- // Once the priority list is exhausted only a manual failback can open the circuit breaker so
- // all subsequent operations will fail
- else if (provider.isLastClusterCircuitBreakerForcedOpen()) {
+ // Iterating the active cluster will allow subsequent calls to the executeCommand() to use the next
+ // cluster's connection pool - according to the configuration's prioritization/order/weight
+ // int activeMultiClusterIndex = provider.incrementActiveMultiClusterIndex1();
+ provider.iterateActiveCluster(SwitchReason.CIRCUIT_BREAKER);
+ }
+ // this check relies on the fact that many failover attempts can hit with the same CB,
+ // only the first one will trigger a failover, and make the CB FORCED_OPEN.
+ // when the rest reaches here, the active cluster is already the next one, and should be different than
+ // active CB. If its the same one and there are no more clusters to failover to, then throw an exception
+ else if (circuitBreaker == provider.getCluster().getCircuitBreaker() && !provider.canIterateOnceMore()) {
throw new JedisConnectionException(
"Cluster/database endpoint could not failover since the MultiClusterClientConfig was not "
+ "provided with an additional cluster/database endpoint according to its prioritized sequence. "
+ "If applicable, consider failing back OR restarting with an available cluster/database endpoint");
}
+ // Ignore exceptions since we are already in a failure state
} finally {
lock.unlock();
}
diff --git a/src/main/java/redis/clients/jedis/mcf/ClusterSwitchEventArgs.java b/src/main/java/redis/clients/jedis/mcf/ClusterSwitchEventArgs.java
new file mode 100644
index 0000000000..b042952f4f
--- /dev/null
+++ b/src/main/java/redis/clients/jedis/mcf/ClusterSwitchEventArgs.java
@@ -0,0 +1,29 @@
+package redis.clients.jedis.mcf;
+
+import redis.clients.jedis.providers.MultiClusterPooledConnectionProvider.Cluster;
+
+public class ClusterSwitchEventArgs {
+
+ private final SwitchReason reason;
+ private final String ClusterName;
+ private final Endpoint Endpoint;
+
+ public ClusterSwitchEventArgs(SwitchReason reason, Endpoint endpoint, Cluster cluster) {
+ this.reason = reason;
+ this.ClusterName = cluster.getCircuitBreaker().getName();
+ this.Endpoint = endpoint;
+ }
+
+ public SwitchReason getReason() {
+ return reason;
+ }
+
+ public String getClusterName() {
+ return ClusterName;
+ }
+
+ public Endpoint getEndpoint() {
+ return Endpoint;
+ }
+
+}
diff --git a/src/main/java/redis/clients/jedis/mcf/EchoStrategy.java b/src/main/java/redis/clients/jedis/mcf/EchoStrategy.java
new file mode 100644
index 0000000000..e9265b88d6
--- /dev/null
+++ b/src/main/java/redis/clients/jedis/mcf/EchoStrategy.java
@@ -0,0 +1,48 @@
+package redis.clients.jedis.mcf;
+
+import redis.clients.jedis.HostAndPort;
+import redis.clients.jedis.JedisClientConfig;
+import redis.clients.jedis.UnifiedJedis;
+import redis.clients.jedis.MultiClusterClientConfig.StrategySupplier;
+
+public class EchoStrategy implements HealthCheckStrategy {
+
+ private int interval;
+ private int timeout;
+ private UnifiedJedis jedis;
+
+ public EchoStrategy(HostAndPort hostAndPort, JedisClientConfig jedisClientConfig) {
+ this(hostAndPort, jedisClientConfig, 1000, 1000);
+ }
+
+ public EchoStrategy(HostAndPort hostAndPort, JedisClientConfig jedisClientConfig, int interval, int timeout) {
+ this.interval = interval;
+ this.timeout = timeout;
+ this.jedis = new UnifiedJedis(hostAndPort, jedisClientConfig);
+ }
+
+ @Override
+ public int getInterval() {
+ return interval;
+ }
+
+ @Override
+ public int getTimeout() {
+ return timeout;
+ }
+
+ @Override
+ public HealthStatus doHealthCheck(Endpoint endpoint) {
+ return "HealthCheck".equals(jedis.echo("HealthCheck")) ? HealthStatus.HEALTHY : HealthStatus.UNHEALTHY;
+ }
+
+ @Override
+ public void close() {
+ jedis.close();
+ }
+
+ public static final StrategySupplier DEFAULT = (hostAndPort, jedisClientConfig) -> {
+ return new EchoStrategy(hostAndPort, jedisClientConfig);
+ };
+
+}
diff --git a/src/main/java/redis/clients/jedis/mcf/Endpoint.java b/src/main/java/redis/clients/jedis/mcf/Endpoint.java
new file mode 100644
index 0000000000..e50d8b21f2
--- /dev/null
+++ b/src/main/java/redis/clients/jedis/mcf/Endpoint.java
@@ -0,0 +1,9 @@
+package redis.clients.jedis.mcf;
+
+public interface Endpoint {
+
+ String getHost();
+
+ int getPort();
+
+}
diff --git a/src/main/java/redis/clients/jedis/mcf/FailoverOptions.java b/src/main/java/redis/clients/jedis/mcf/FailoverOptions.java
deleted file mode 100644
index 9b919fd134..0000000000
--- a/src/main/java/redis/clients/jedis/mcf/FailoverOptions.java
+++ /dev/null
@@ -1,59 +0,0 @@
-package redis.clients.jedis.mcf;
-
-import redis.clients.jedis.annots.Experimental;
-
-/**
- * Configuration options for CircuitBreakerCommandExecutor
- */
-@Experimental
-public class FailoverOptions {
- private final boolean retryOnFailover;
-
- private FailoverOptions(Builder builder) {
- this.retryOnFailover = builder.retryOnFailover;
- }
-
- /**
- * Gets whether to retry failed commands during failover
- * @return true if retry is enabled, false otherwise
- */
- public boolean isRetryOnFailover() {
- return retryOnFailover;
- }
-
- /**
- * Creates a new builder with default options
- * @return a new builder
- */
- public static Builder builder() {
- return new Builder();
- }
-
- /**
- * Builder for FailoverOptions
- */
- public static class Builder {
- private boolean retryOnFailover = false;
-
- private Builder() {
- }
-
- /**
- * Sets whether to retry failed commands during failover
- * @param retry true to retry, false otherwise
- * @return this builder for method chaining
- */
- public Builder retryOnFailover(boolean retry) {
- this.retryOnFailover = retry;
- return this;
- }
-
- /**
- * Builds a new FailoverOptions instance with the configured options
- * @return a new FailoverOptions instance
- */
- public FailoverOptions build() {
- return new FailoverOptions(this);
- }
- }
-}
\ No newline at end of file
diff --git a/src/main/java/redis/clients/jedis/mcf/HealthCheck.java b/src/main/java/redis/clients/jedis/mcf/HealthCheck.java
new file mode 100644
index 0000000000..dbf0558f54
--- /dev/null
+++ b/src/main/java/redis/clients/jedis/mcf/HealthCheck.java
@@ -0,0 +1,122 @@
+
+package redis.clients.jedis.mcf;
+
+import java.util.AbstractMap.SimpleEntry;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HealthCheck {
+
+ private static final Logger log = LoggerFactory.getLogger(HealthCheck.class);
+
+ private Endpoint endpoint;
+ private HealthCheckStrategy strategy;
+ private AtomicReference> statusRef = new AtomicReference>();
+ private Consumer statusChangeCallback;
+
+ private ScheduledExecutorService scheduler;
+ private ExecutorService executor = Executors.newCachedThreadPool();
+
+ HealthCheck(Endpoint endpoint, HealthCheckStrategy strategy,
+ Consumer statusChangeCallback) {
+ this.endpoint = endpoint;
+ this.strategy = strategy;
+ this.statusChangeCallback = statusChangeCallback;
+ statusRef.set(new SimpleEntry<>(0L, HealthStatus.UNKNOWN));
+
+ scheduler = Executors.newSingleThreadScheduledExecutor(r -> {
+ Thread t = new Thread(r, "jedis-healthcheck-" + this.endpoint);
+ t.setDaemon(true);
+ return t;
+ });
+ }
+
+ public Endpoint getEndpoint() {
+ return endpoint;
+ }
+
+ public HealthStatus getStatus() {
+ return statusRef.get().getValue();
+ }
+
+ public void start() {
+ scheduler.scheduleAtFixedRate(this::healthCheck, 0, strategy.getInterval(), TimeUnit.MILLISECONDS);
+ }
+
+ public void stop() {
+ strategy.close();
+ this.statusChangeCallback = null;
+ scheduler.shutdown();
+ executor.shutdown();
+ try {
+ // Wait for graceful shutdown then force if required
+ if (!scheduler.awaitTermination(1, TimeUnit.SECONDS)) {
+ scheduler.shutdownNow();
+ }
+ if (!executor.awaitTermination(1, TimeUnit.SECONDS)) {
+ executor.shutdownNow();
+ }
+ } catch (InterruptedException e) {
+ // Force shutdown immediately
+ scheduler.shutdownNow();
+ executor.shutdownNow();
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ private void healthCheck() {
+ long me = System.currentTimeMillis();
+ Future future = executor.submit(() -> {
+ HealthStatus newStatus = strategy.doHealthCheck(endpoint);
+ safeUpdate(me, newStatus);
+ log.trace("Health check completed for {} with status {}", endpoint, newStatus);
+ });
+
+ try {
+ future.get(strategy.getTimeout(), TimeUnit.MILLISECONDS);
+ } catch (TimeoutException | ExecutionException e) {
+ // Cancel immediately on timeout or exec exception
+ future.cancel(true);
+ safeUpdate(me, HealthStatus.UNHEALTHY);
+ log.warn("Health check timed out or failed for {}", endpoint, e);
+ } catch (InterruptedException e) {
+ // Health check thread was interrupted
+ future.cancel(true);
+ safeUpdate(me, HealthStatus.UNHEALTHY);
+ Thread.currentThread().interrupt(); // Restore interrupted status
+ log.warn("Health check interrupted for {}", endpoint, e);
+ }
+ }
+
+ // just to avoid to replace status with an outdated result from another healthCheck
+ private void safeUpdate(long owner, HealthStatus status) {
+ SimpleEntry newStatus = new SimpleEntry<>(owner, status);
+ SimpleEntry oldStatus = statusRef.getAndUpdate(current -> {
+ if (current.getKey() < owner) {
+ return newStatus;
+ }
+ return current;
+ });
+ if (oldStatus.getValue() != status) {
+ // notify listeners
+ notifyListeners(oldStatus.getValue(), status);
+ }
+ }
+
+ private void notifyListeners(HealthStatus oldStatus, HealthStatus newStatus) {
+ if (statusChangeCallback != null) {
+ statusChangeCallback.accept(new HealthStatusChangeEvent(endpoint, oldStatus, newStatus));
+ }
+ }
+
+}
diff --git a/src/main/java/redis/clients/jedis/mcf/HealthCheckCollection.java b/src/main/java/redis/clients/jedis/mcf/HealthCheckCollection.java
new file mode 100644
index 0000000000..9584a01c6a
--- /dev/null
+++ b/src/main/java/redis/clients/jedis/mcf/HealthCheckCollection.java
@@ -0,0 +1,46 @@
+package redis.clients.jedis.mcf;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class HealthCheckCollection {
+
+ private Map healthChecks = new ConcurrentHashMap();
+
+ public HealthCheck add(HealthCheck healthCheck) {
+ return healthChecks.put(healthCheck.getEndpoint(), healthCheck);
+ }
+
+ public HealthCheck[] addAll(HealthCheck[] healthChecks) {
+ HealthCheck[] old = new HealthCheck[healthChecks.length];
+ for (int i = 0; i < healthChecks.length; i++) {
+ old[i] = add(healthChecks[i]);
+ }
+ return old;
+ }
+
+ public HealthCheck remove(Endpoint endpoint) {
+ HealthCheck old = healthChecks.remove(endpoint);
+ if (old != null) {
+ old.stop();
+ }
+ return old;
+ }
+
+ public HealthCheck remove(HealthCheck healthCheck) {
+ HealthCheck[] temp = new HealthCheck[1];
+ healthChecks.computeIfPresent(healthCheck.getEndpoint(), (key, existing) -> {
+ if (existing == healthCheck) {
+ temp[0] = existing;
+ return null;
+ }
+ return existing;
+ });
+ return temp[0];
+ }
+
+ public HealthCheck get(Endpoint endpoint) {
+ return healthChecks.get(endpoint);
+ }
+
+}
diff --git a/src/main/java/redis/clients/jedis/mcf/HealthCheckStrategy.java b/src/main/java/redis/clients/jedis/mcf/HealthCheckStrategy.java
new file mode 100644
index 0000000000..9fe2029ec5
--- /dev/null
+++ b/src/main/java/redis/clients/jedis/mcf/HealthCheckStrategy.java
@@ -0,0 +1,16 @@
+package redis.clients.jedis.mcf;
+
+import java.io.Closeable;
+
+public interface HealthCheckStrategy extends Closeable {
+
+ int getInterval();
+
+ int getTimeout();
+
+ HealthStatus doHealthCheck(Endpoint endpoint);
+
+ default void close() {
+ }
+
+}
diff --git a/src/main/java/redis/clients/jedis/mcf/HealthStatus.java b/src/main/java/redis/clients/jedis/mcf/HealthStatus.java
new file mode 100644
index 0000000000..2620f4131e
--- /dev/null
+++ b/src/main/java/redis/clients/jedis/mcf/HealthStatus.java
@@ -0,0 +1,15 @@
+package redis.clients.jedis.mcf;
+
+public enum HealthStatus {
+ UNKNOWN(0x00), HEALTHY(0x01), UNHEALTHY(0x02);
+
+ private final int value;
+
+ HealthStatus(int val) {
+ this.value = val;
+ }
+
+ public boolean isHealthy() {
+ return (this.value & HEALTHY.value) != 0;
+ }
+}
diff --git a/src/main/java/redis/clients/jedis/mcf/HealthStatusChangeEvent.java b/src/main/java/redis/clients/jedis/mcf/HealthStatusChangeEvent.java
new file mode 100644
index 0000000000..aefa2e06cc
--- /dev/null
+++ b/src/main/java/redis/clients/jedis/mcf/HealthStatusChangeEvent.java
@@ -0,0 +1,26 @@
+package redis.clients.jedis.mcf;
+
+public class HealthStatusChangeEvent {
+
+ private final Endpoint endpoint;
+ private final HealthStatus oldStatus;
+ private final HealthStatus newStatus;
+
+ public HealthStatusChangeEvent(Endpoint endpoint, HealthStatus oldStatus, HealthStatus newStatus) {
+ this.endpoint = endpoint;
+ this.oldStatus = oldStatus;
+ this.newStatus = newStatus;
+ }
+
+ public Endpoint getEndpoint() {
+ return endpoint;
+ }
+
+ public HealthStatus getOldStatus() {
+ return oldStatus;
+ }
+
+ public HealthStatus getNewStatus() {
+ return newStatus;
+ }
+}
diff --git a/src/main/java/redis/clients/jedis/mcf/HealthStatusListener.java b/src/main/java/redis/clients/jedis/mcf/HealthStatusListener.java
new file mode 100644
index 0000000000..f01c2da563
--- /dev/null
+++ b/src/main/java/redis/clients/jedis/mcf/HealthStatusListener.java
@@ -0,0 +1,7 @@
+package redis.clients.jedis.mcf;
+
+public interface HealthStatusListener {
+
+ void onStatusChange(HealthStatusChangeEvent event);
+
+}
diff --git a/src/main/java/redis/clients/jedis/mcf/HealthStatusManager.java b/src/main/java/redis/clients/jedis/mcf/HealthStatusManager.java
new file mode 100644
index 0000000000..c3787e12a1
--- /dev/null
+++ b/src/main/java/redis/clients/jedis/mcf/HealthStatusManager.java
@@ -0,0 +1,81 @@
+package redis.clients.jedis.mcf;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+public class HealthStatusManager {
+
+ private HealthCheckCollection healthChecks = new HealthCheckCollection();
+ private final List listeners = new CopyOnWriteArrayList<>();
+ private final Map> endpointListeners = new ConcurrentHashMap>();
+
+ public void registerListener(HealthStatusListener listener) {
+ listeners.add(listener);
+ }
+
+ public void unregisterListener(HealthStatusListener listener) {
+ listeners.remove(listener);
+ }
+
+ public void registerListener(Endpoint endpoint, HealthStatusListener listener) {
+ endpointListeners.computeIfAbsent(endpoint, k -> new CopyOnWriteArrayList<>()).add(listener);
+ }
+
+ public void unregisterListener(Endpoint endpoint, HealthStatusListener listener) {
+ endpointListeners.computeIfPresent(endpoint, (k, v) -> {
+ v.remove(listener);
+ return v;
+ });
+ }
+
+ public void notifyListeners(HealthStatusChangeEvent eventArgs) {
+ endpointListeners.computeIfPresent(eventArgs.getEndpoint(), (k, v) -> {
+ for (HealthStatusListener listener : v) {
+ listener.onStatusChange(eventArgs);
+ }
+ return v;
+ });
+ for (HealthStatusListener listener : listeners) {
+ listener.onStatusChange(eventArgs);
+ }
+ }
+
+ public void add(Endpoint endpoint, HealthCheckStrategy strategy) {
+ HealthCheck hc = new HealthCheck(endpoint, strategy, this::notifyListeners);
+ HealthCheck old = healthChecks.add(hc);
+ hc.start();
+ if (old != null) {
+ old.stop();
+ }
+ }
+
+ public void addAll(Endpoint[] endpoints, HealthCheckStrategy strategy) {
+ for (Endpoint endpoint : endpoints) {
+ add(endpoint, strategy);
+ }
+ }
+
+ public void remove(Endpoint endpoint) {
+ HealthCheck old = healthChecks.remove(endpoint);
+ if (old != null) {
+ old.stop();
+ }
+ }
+
+ public void removeAll(Endpoint[] endpoints) {
+ for (Endpoint endpoint : endpoints) {
+ remove(endpoint);
+ }
+ }
+
+ public HealthStatus getHealthStatus(Endpoint endpoint) {
+ HealthCheck healthCheck = healthChecks.get(endpoint);
+ return healthCheck != null ? healthCheck.getStatus() : HealthStatus.UNKNOWN;
+ }
+
+ public boolean hasHealthCheck(Endpoint endpoint) {
+ return healthChecks.get(endpoint) != null;
+ }
+}
diff --git a/src/main/java/redis/clients/jedis/mcf/LagAwareStrategy.java b/src/main/java/redis/clients/jedis/mcf/LagAwareStrategy.java
new file mode 100644
index 0000000000..60124f0854
--- /dev/null
+++ b/src/main/java/redis/clients/jedis/mcf/LagAwareStrategy.java
@@ -0,0 +1,43 @@
+package redis.clients.jedis.mcf;
+
+import java.io.IOException;
+import java.util.List;
+
+public class LagAwareStrategy implements HealthCheckStrategy {
+
+ private int interval;
+ private int timeout;
+
+ public LagAwareStrategy(int healthCheckInterval, int healthCheckTimeout) {
+ this.interval = healthCheckInterval;
+ this.timeout = healthCheckTimeout;
+ }
+
+ @Override
+ public int getInterval() {
+ return interval;
+ }
+
+ @Override
+ public int getTimeout() {
+ return timeout;
+ }
+
+ @Override
+ public HealthStatus doHealthCheck(Endpoint endpoint) {
+ RedisRestAPIHelper helper = new RedisRestAPIHelper(endpoint.getHost(), String.valueOf(endpoint.getPort()),
+ "admin", "admin");
+ try {
+ List bdbs = helper.getBdbs();
+ if (bdbs.size() > 0) {
+ if ("available".equals(helper.checkBdbAvailability(bdbs.get(0)))) {
+ return HealthStatus.HEALTHY;
+ }
+ }
+ } catch (IOException e) {
+ // log error
+ return HealthStatus.UNHEALTHY;
+ }
+ return HealthStatus.UNHEALTHY;
+ }
+}
diff --git a/src/main/java/redis/clients/jedis/mcf/RedisRestAPIHelper.java b/src/main/java/redis/clients/jedis/mcf/RedisRestAPIHelper.java
new file mode 100644
index 0000000000..8863e49d46
--- /dev/null
+++ b/src/main/java/redis/clients/jedis/mcf/RedisRestAPIHelper.java
@@ -0,0 +1,95 @@
+package redis.clients.jedis.mcf;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+
+/**
+ * Helper class to check the availability of a Redis database
+ */
+class RedisRestAPIHelper {
+ // Required connection information - replace with your host, port, username, and
+ // password
+ private final String host;
+ private final String port;
+ private final String userName;
+ private final String password;
+
+ RedisRestAPIHelper(String host, String port, String userName, String password) {
+ this.host = host;
+ this.port = port;
+ this.userName = userName;
+ this.password = password;
+ }
+
+ private static final String BDBS_URL = "https://%s:%s/v1/bdbs";
+ private static final String AVAILABILITY_URL = "https://%s:%s/v1/bdbs/%d/availability";
+
+ public List getBdbs() throws IOException {
+ String bdbsUri = String.format(BDBS_URL, host, port);
+ HttpURLConnection getConnection = createConnection(bdbsUri, "GET");
+ getConnection.setRequestProperty("Accept", "application/json");
+
+ String responseBody = readResponse(getConnection);
+ return JsonParser.parseString(responseBody).getAsJsonArray().asList().stream().map(e -> e.getAsString())
+ .collect(Collectors.toList());
+ }
+
+ public String checkBdbAvailability(String uid) throws IOException {
+ String availabilityUri = String.format(AVAILABILITY_URL, host, port, uid);
+ HttpURLConnection availConnection = createConnection(availabilityUri, "GET");
+ availConnection.setRequestProperty("Accept", "application/json");
+
+ String availResponse = readResponse(availConnection);
+ JsonObject availJson = JsonParser.parseString(availResponse).getAsJsonObject();
+
+ return availJson.get("status").getAsString();
+
+ }
+
+ private HttpURLConnection createConnection(String urlString, String method) throws IOException {
+ URL url = new URL(urlString);
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setRequestMethod(method);
+
+ // Set basic authentication
+ String auth = userName + ":" + password;
+ String encodedAuth = Base64.getEncoder().encodeToString(auth.getBytes(StandardCharsets.UTF_8));
+ connection.setRequestProperty("Authorization", "Basic " + encodedAuth);
+
+ return connection;
+ }
+
+ private String readResponse(HttpURLConnection connection) throws IOException {
+ InputStream inputStream;
+ try {
+ inputStream = connection.getInputStream();
+ } catch (IOException e) {
+ // If there's an error, try to read from error stream
+ inputStream = connection.getErrorStream();
+ if (inputStream == null) {
+ return "";
+ }
+ }
+
+ StringBuilder response = new StringBuilder();
+ byte[] buffer = new byte[1024];
+ int bytesRead;
+
+ while ((bytesRead = inputStream.read(buffer)) != -1) {
+ response.append(new String(buffer, 0, bytesRead, StandardCharsets.UTF_8));
+ }
+
+ inputStream.close();
+ return response.toString();
+ }
+
+}
\ No newline at end of file
diff --git a/src/main/java/redis/clients/jedis/mcf/StatusTracker.java b/src/main/java/redis/clients/jedis/mcf/StatusTracker.java
new file mode 100644
index 0000000000..ed3d4ee8de
--- /dev/null
+++ b/src/main/java/redis/clients/jedis/mcf/StatusTracker.java
@@ -0,0 +1,77 @@
+package redis.clients.jedis.mcf;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import redis.clients.jedis.exceptions.JedisConnectionException;
+import redis.clients.jedis.exceptions.JedisValidationException;
+
+/**
+ * StatusTracker is responsible for tracking and waiting for health status changes for specific endpoints. It provides
+ * an event-driven approach to wait for health status transitions from UNKNOWN to either HEALTHY or UNHEALTHY.
+ */
+public class StatusTracker {
+
+ private final HealthStatusManager healthStatusManager;
+
+ public StatusTracker(HealthStatusManager healthStatusManager) {
+ this.healthStatusManager = healthStatusManager;
+ }
+
+ /**
+ * Waits for a specific endpoint's health status to be determined (not UNKNOWN). Uses event-driven approach with
+ * CountDownLatch to avoid polling.
+ * @param endpoint the endpoint to wait for
+ * @return the determined health status (HEALTHY or UNHEALTHY)
+ * @throws JedisConnectionException if interrupted while waiting
+ */
+ public HealthStatus waitForHealthStatus(Endpoint endpoint) {
+ // First check if status is already determined
+ HealthStatus currentStatus = healthStatusManager.getHealthStatus(endpoint);
+ if (currentStatus != HealthStatus.UNKNOWN) {
+ return currentStatus;
+ }
+
+ // Set up event-driven waiting
+ final CountDownLatch latch = new CountDownLatch(1);
+ final AtomicReference resultStatus = new AtomicReference<>();
+
+ // Create a temporary listener for this specific endpoint
+ HealthStatusListener tempListener = new HealthStatusListener() {
+ @Override
+ public void onStatusChange(HealthStatusChangeEvent event) {
+ if (event.getEndpoint().equals(endpoint) && event.getNewStatus() != HealthStatus.UNKNOWN) {
+ resultStatus.set(event.getNewStatus());
+ latch.countDown();
+ }
+ }
+ };
+
+ // Register the temporary listener
+ healthStatusManager.registerListener(endpoint, tempListener);
+
+ try {
+ // Double-check status after registering listener (race condition protection)
+ currentStatus = healthStatusManager.getHealthStatus(endpoint);
+ if (currentStatus != HealthStatus.UNKNOWN) {
+ return currentStatus;
+ }
+
+ // Wait for the health status change event
+ // just for safety to not block indefinitely
+ boolean completed = latch.await(60, TimeUnit.SECONDS);
+ if (!completed) {
+ throw new JedisValidationException("Timeout while waiting for health check result");
+ }
+ return resultStatus.get();
+
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new JedisConnectionException("Interrupted while waiting for health check result", e);
+ } finally {
+ // Clean up: unregister the temporary listener
+ healthStatusManager.unregisterListener(endpoint, tempListener);
+ }
+ }
+}
diff --git a/src/main/java/redis/clients/jedis/mcf/SwitchReason.java b/src/main/java/redis/clients/jedis/mcf/SwitchReason.java
new file mode 100644
index 0000000000..1cba831f8c
--- /dev/null
+++ b/src/main/java/redis/clients/jedis/mcf/SwitchReason.java
@@ -0,0 +1,5 @@
+package redis.clients.jedis.mcf;
+
+public enum SwitchReason {
+ HEALTH_CHECK, CIRCUIT_BREAKER, FAILBACK, FORCED
+}
diff --git a/src/main/java/redis/clients/jedis/mcf/TrackingConnectionPool.java b/src/main/java/redis/clients/jedis/mcf/TrackingConnectionPool.java
new file mode 100644
index 0000000000..51f6d77700
--- /dev/null
+++ b/src/main/java/redis/clients/jedis/mcf/TrackingConnectionPool.java
@@ -0,0 +1,155 @@
+package redis.clients.jedis.mcf;
+
+import java.util.Iterator;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import redis.clients.jedis.Connection;
+import redis.clients.jedis.ConnectionFactory;
+import redis.clients.jedis.ConnectionPool;
+import redis.clients.jedis.HostAndPort;
+import redis.clients.jedis.InitializationTracker;
+import redis.clients.jedis.JedisClientConfig;
+import redis.clients.jedis.exceptions.JedisConnectionException;
+
+public class TrackingConnectionPool extends ConnectionPool {
+
+ private static final Logger log = LoggerFactory.getLogger(TrackingConnectionPool.class);
+
+ private final InitializationTracker tracker;
+ private final GenericObjectPoolConfig poolConfig;
+ private final JedisClientConfig clientConfig;
+ private final AtomicInteger numWaiters = new AtomicInteger();
+
+ public static class Builder {
+ private HostAndPort hostAndPort;
+ private JedisClientConfig clientConfig;
+ private GenericObjectPoolConfig poolConfig;
+ private InitializationTracker tracker;
+
+ public Builder hostAndPort(HostAndPort hostAndPort) {
+ this.hostAndPort = hostAndPort;
+ return this;
+ }
+
+ public Builder clientConfig(JedisClientConfig clientConfig) {
+ this.clientConfig = clientConfig;
+ return this;
+ }
+
+ public Builder poolConfig(GenericObjectPoolConfig poolConfig) {
+ this.poolConfig = poolConfig;
+ return this;
+ }
+
+ public Builder tracker(InitializationTracker tracker) {
+ this.tracker = tracker;
+ return this;
+ }
+
+ public TrackingConnectionPool build() {
+ return new TrackingConnectionPool(this);
+ }
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public TrackingConnectionPool(HostAndPort hostAndPort, JedisClientConfig clientConfig,
+ GenericObjectPoolConfig poolConfig) {
+ this(builder().hostAndPort(hostAndPort).clientConfig(clientConfig).poolConfig(poolConfig)
+ .tracker(createSimpleTracker()));
+ }
+
+ private TrackingConnectionPool(Builder builder) {
+ super(
+ ConnectionFactory.builder().setHostAndPort(builder.hostAndPort).setClientConfig(builder.clientConfig)
+ .setTracker(builder.tracker).build(),
+ builder.poolConfig != null ? builder.poolConfig : new GenericObjectPoolConfig<>());
+
+ this.tracker = builder.tracker;
+ this.clientConfig = builder.clientConfig;
+ this.poolConfig = builder.poolConfig;
+ this.attachAuthenticationListener(builder.clientConfig.getAuthXManager());
+ }
+
+ public static TrackingConnectionPool from(TrackingConnectionPool pool) {
+ return builder().clientConfig(pool.clientConfig).poolConfig(pool.poolConfig).tracker(pool.tracker).build();
+ }
+
+ @Override
+ public Connection getResource() {
+ try {
+ numWaiters.incrementAndGet();
+ Connection conn = super.getResource();
+ tracker.add(conn);
+ return conn;
+ } catch (Exception e) {
+ if (this.isClosed()) {
+ throw new JedisConnectionException("Pool is closed", e);
+ }
+ throw e;
+ } finally {
+ numWaiters.decrementAndGet();
+ }
+ }
+
+ @Override
+ public void returnResource(final Connection resource) {
+ super.returnResource(resource);
+ tracker.remove(resource);
+ }
+
+ @Override
+ public void returnBrokenResource(final Connection resource) {
+ super.returnBrokenResource(resource);
+ tracker.remove(resource);
+ }
+
+ public void forceDisconnect() {
+ this.close();
+ while (numWaiters.get() > 0 || getNumWaiters() > 0 || getNumActive() > 0 || getNumIdle() > 0) {
+ this.clear();
+ for (Connection connection : tracker) {
+ try {
+ connection.forceDisconnect();
+ } catch (Exception e) {
+ log.warn("Error while force disconnecting connection: " + connection.toIdentityString());
+ }
+ }
+ }
+ }
+
+ private static InitializationTracker createSimpleTracker() {
+ return new InitializationTracker() {
+ private final Set allCreatedObjects = ConcurrentHashMap.newKeySet();
+
+ @Override
+ public void add(Connection target) {
+ allCreatedObjects.add(target);
+ }
+
+ @Override
+ public void remove(Connection target) {
+ allCreatedObjects.remove(target);
+ }
+
+ @Override
+ public Iterator iterator() {
+ return allCreatedObjects.iterator();
+ }
+ };
+ }
+
+ @Override
+ public void close() {
+ this.destroy();
+ this.detachAuthenticationListener();
+ }
+}
diff --git a/src/main/java/redis/clients/jedis/providers/MultiClusterPooledConnectionProvider.java b/src/main/java/redis/clients/jedis/providers/MultiClusterPooledConnectionProvider.java
index afa9cf6c37..d38cdf52b6 100644
--- a/src/main/java/redis/clients/jedis/providers/MultiClusterPooledConnectionProvider.java
+++ b/src/main/java/redis/clients/jedis/providers/MultiClusterPooledConnectionProvider.java
@@ -10,12 +10,20 @@
import io.github.resilience4j.retry.RetryRegistry;
import java.util.Collections;
+import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
import java.util.function.Consumer;
+import java.util.function.Predicate;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import org.slf4j.Logger;
@@ -26,8 +34,21 @@
import redis.clients.jedis.annots.Experimental;
import redis.clients.jedis.annots.VisibleForTesting;
import redis.clients.jedis.exceptions.JedisConnectionException;
+import redis.clients.jedis.exceptions.JedisException;
import redis.clients.jedis.exceptions.JedisValidationException;
+import redis.clients.jedis.mcf.HealthStatus;
+import redis.clients.jedis.mcf.HealthStatusChangeEvent;
+import redis.clients.jedis.mcf.HealthStatusManager;
+import redis.clients.jedis.mcf.StatusTracker;
+import redis.clients.jedis.mcf.SwitchReason;
+import redis.clients.jedis.mcf.TrackingConnectionPool;
+import redis.clients.jedis.MultiClusterClientConfig.StrategySupplier;
+
import redis.clients.jedis.util.Pool;
+import redis.clients.jedis.mcf.ClusterSwitchEventArgs;
+import redis.clients.jedis.mcf.Endpoint;
+
+import redis.clients.jedis.mcf.HealthCheckStrategy;
/**
* @author Allen Terleto (aterleto)
@@ -37,7 +58,7 @@
* Active-Active cluster(s) by using simple configuration which is passed through from Resilience4j -
* https://resilience4j.readme.io/docs
*
- * Support for manual failback is provided by way of {@link #setActiveMultiClusterIndex(int)}
+ * Support for manual failback is provided by way of {@link #setActiveCluster(Endpoint)}
*
*/
// TODO: move?
@@ -50,38 +71,50 @@ public class MultiClusterPooledConnectionProvider implements ConnectionProvider
* Ordered map of cluster/database endpoints which were provided at startup via the MultiClusterClientConfig. Users
* can move down (failover) or (up) failback the map depending on their availability and order.
*/
- private final Map multiClusterMap = new ConcurrentHashMap<>();
+ private final Map multiClusterMap = new ConcurrentHashMap<>();
/**
* Indicates the actively used cluster/database endpoint (connection pool) amongst the pre-configured list which
- * were provided at startup via the MultiClusterClientConfig. All traffic will be routed according to this index.
+ * were provided at startup via the MultiClusterClientConfig. All traffic will be routed with this cluster/database
*/
- private volatile Integer activeMultiClusterIndex = 1;
+ private volatile Cluster activeCluster;
private final Lock activeClusterIndexLock = new ReentrantLock(true);
/**
- * Indicates the final cluster/database endpoint (connection pool), according to the pre-configured list provided at
- * startup via the MultiClusterClientConfig, is unavailable and therefore no further failover is possible. Users can
- * manually failback to an available cluster which would reset this flag via
- * {@link #setActiveMultiClusterIndex(int)}
- */
- private volatile boolean lastClusterCircuitBreakerForcedOpen = false;
-
- /**
- * Functional interface typically used for activeMultiClusterIndex persistence or custom logging after a successful
- * failover of a cluster/database endpoint (connection pool). Cluster/database endpoint info is passed as the sole
- * parameter Example: cluster:2:redis-smart-cache.demo.com:12000
+ * Functional interface for listening to cluster switch events. The event args contain the reason for the switch,
+ * the endpoint, and the cluster.
*/
- private Consumer clusterFailoverPostProcessor;
+ private Consumer clusterSwitchListener;
private List> fallbackExceptionList;
+ private HealthStatusManager healthStatusManager = new HealthStatusManager();
+ private StatusTracker statusTracker;
+
+ // Flag to control when handleHealthStatusChange should process events (only after initialization)
+ private volatile boolean initializationComplete = false;
+
+ // Failback mechanism fields
+ private static final AtomicInteger failbackThreadCounter = new AtomicInteger(1);
+ private final ScheduledExecutorService failbackScheduler = Executors.newSingleThreadScheduledExecutor(r -> {
+ Thread t = new Thread(r, "jedis-failback-" + failbackThreadCounter.getAndIncrement());
+ t.setDaemon(true);
+ return t;
+ });
+
+ // Store retry and circuit breaker configs for dynamic cluster addition/removal
+ private RetryConfig retryConfig;
+ private CircuitBreakerConfig circuitBreakerConfig;
+ private MultiClusterClientConfig multiClusterClientConfig;
+
public MultiClusterPooledConnectionProvider(MultiClusterClientConfig multiClusterClientConfig) {
if (multiClusterClientConfig == null) throw new JedisValidationException(
"MultiClusterClientConfig must not be NULL for MultiClusterPooledConnectionProvider");
+ this.multiClusterClientConfig = multiClusterClientConfig;
+
////////////// Configure Retry ////////////////////
RetryConfig.Builder retryConfigBuilder = RetryConfig.custom();
@@ -97,7 +130,7 @@ public MultiClusterPooledConnectionProvider(MultiClusterClientConfig multiCluste
if (retryIgnoreExceptionList != null)
retryConfigBuilder.ignoreExceptions(retryIgnoreExceptionList.stream().toArray(Class[]::new));
- RetryConfig retryConfig = retryConfigBuilder.build();
+ this.retryConfig = retryConfigBuilder.build();
////////////// Configure Circuit Breaker ////////////////////
@@ -122,86 +155,298 @@ public MultiClusterPooledConnectionProvider(MultiClusterClientConfig multiCluste
if (circuitBreakerIgnoreExceptionList != null) circuitBreakerConfigBuilder
.ignoreExceptions(circuitBreakerIgnoreExceptionList.stream().toArray(Class[]::new));
- CircuitBreakerConfig circuitBreakerConfig = circuitBreakerConfigBuilder.build();
+ this.circuitBreakerConfig = circuitBreakerConfigBuilder.build();
////////////// Configure Cluster Map ////////////////////
ClusterConfig[] clusterConfigs = multiClusterClientConfig.getClusterConfigs();
+
+ // Now add clusters - health checks will start but events will be queued
for (ClusterConfig config : clusterConfigs) {
- GenericObjectPoolConfig poolConfig = config.getConnectionPoolConfig();
+ addClusterInternal(multiClusterClientConfig, config);
+ }
+
+ // Initialize StatusTracker for waiting on health check results
+ statusTracker = new StatusTracker(healthStatusManager);
+
+ // Wait for initial health check results and select active cluster based on weights
+ activeCluster = waitForInitialHealthyCluster();
+
+ // Mark initialization as complete - handleHealthStatusChange can now process events
+ initializationComplete = true;
+ if (!activeCluster.isHealthy()) {
+ activeCluster = waitForInitialHealthyCluster();
+ }
+ this.fallbackExceptionList = multiClusterClientConfig.getFallbackExceptionList();
- String clusterId = "cluster:" + config.getPriority() + ":" + config.getHostAndPort();
+ // Start periodic failback checker
+ if (multiClusterClientConfig.isFailbackSupported()) {
+ long failbackInterval = multiClusterClientConfig.getFailbackCheckInterval();
+ failbackScheduler.scheduleAtFixedRate(this::periodicFailbackCheck, failbackInterval, failbackInterval,
+ TimeUnit.MILLISECONDS);
+ }
+ }
+
+ /**
+ * Adds a new cluster endpoint to the provider.
+ * @param clusterConfig the configuration for the new cluster
+ * @throws JedisValidationException if the endpoint already exists
+ */
+ public void add(ClusterConfig clusterConfig) {
+ if (clusterConfig == null) {
+ throw new JedisValidationException("ClusterConfig must not be null");
+ }
+
+ Endpoint endpoint = clusterConfig.getHostAndPort();
+ if (multiClusterMap.containsKey(endpoint)) {
+ throw new JedisValidationException("Endpoint " + endpoint + " already exists in the provider");
+ }
- Retry retry = RetryRegistry.of(retryConfig).retry(clusterId);
+ activeClusterIndexLock.lock();
+ try {
+ addClusterInternal(multiClusterClientConfig, clusterConfig);
+ } finally {
+ activeClusterIndexLock.unlock();
+ }
+ }
- Retry.EventPublisher retryPublisher = retry.getEventPublisher();
- retryPublisher.onRetry(event -> log.warn(String.valueOf(event)));
- retryPublisher.onError(event -> log.error(String.valueOf(event)));
+ /**
+ * Removes a cluster endpoint from the provider.
+ * @param endpoint the endpoint to remove
+ * @throws JedisValidationException if the endpoint doesn't exist or is the last remaining endpoint
+ */
+ public void remove(Endpoint endpoint) {
+ if (endpoint == null) {
+ throw new JedisValidationException("Endpoint must not be null");
+ }
- CircuitBreaker circuitBreaker = CircuitBreakerRegistry.of(circuitBreakerConfig).circuitBreaker(clusterId);
+ if (!multiClusterMap.containsKey(endpoint)) {
+ throw new JedisValidationException("Endpoint " + endpoint + " does not exist in the provider");
+ }
- CircuitBreaker.EventPublisher circuitBreakerEventPublisher = circuitBreaker.getEventPublisher();
- circuitBreakerEventPublisher.onCallNotPermitted(event -> log.error(String.valueOf(event)));
- circuitBreakerEventPublisher.onError(event -> log.error(String.valueOf(event)));
- circuitBreakerEventPublisher.onFailureRateExceeded(event -> log.error(String.valueOf(event)));
- circuitBreakerEventPublisher.onSlowCallRateExceeded(event -> log.error(String.valueOf(event)));
- circuitBreakerEventPublisher.onStateTransition(event -> log.warn(String.valueOf(event)));
+ if (multiClusterMap.size() < 2) {
+ throw new JedisValidationException("Cannot remove the last remaining endpoint");
+ }
+ log.debug("Removing endpoint {}", endpoint);
- if (poolConfig != null) {
- multiClusterMap.put(config.getPriority(),
- new Cluster(new ConnectionPool(config.getHostAndPort(), config.getJedisClientConfig(), poolConfig),
- retry, circuitBreaker));
- } else {
- multiClusterMap.put(config.getPriority(), new Cluster(
- new ConnectionPool(config.getHostAndPort(), config.getJedisClientConfig()), retry, circuitBreaker));
+ activeClusterIndexLock.lock();
+ try {
+ Cluster clusterToRemove = multiClusterMap.get(endpoint);
+ boolean isActiveCluster = (activeCluster == clusterToRemove);
+
+ if (isActiveCluster) {
+ log.info("Active cluster is being removed. Finding a new active cluster...");
+ Map.Entry candidate = findWeightedHealthyClusterToIterate();
+ if (candidate != null) {
+ Cluster selectedCluster = candidate.getValue();
+ if (setActiveCluster(selectedCluster, true)) {
+ log.info("New active cluster set to {}", candidate.getKey());
+ onClusterSwitch(SwitchReason.FORCED, candidate.getKey(), selectedCluster);
+ }
+ } else {
+ throw new JedisException(
+ "Cluster can not be removed due to no healthy cluster available to switch!");
+ }
+ }
+
+ // Remove from health status manager first
+ healthStatusManager.unregisterListener(endpoint, this::onHealthStatusChange);
+ healthStatusManager.remove(endpoint);
+
+ // Remove from cluster map
+ multiClusterMap.remove(endpoint);
+
+ // Close the cluster resources
+ if (clusterToRemove != null) {
+ clusterToRemove.setDisabled(true);
+ clusterToRemove.close();
}
+ } finally {
+ activeClusterIndexLock.unlock();
+ }
+ }
+
+ /**
+ * Internal method to add a cluster configuration. This method is not thread-safe and should be called within
+ * appropriate locks.
+ */
+ private void addClusterInternal(MultiClusterClientConfig multiClusterClientConfig, ClusterConfig config) {
+ if (multiClusterMap.containsKey(config.getHostAndPort())) {
+ throw new JedisValidationException(
+ "Endpoint " + config.getHostAndPort() + " already exists in the provider");
}
+ GenericObjectPoolConfig poolConfig = config.getConnectionPoolConfig();
- /// --- ///
+ String clusterId = "cluster:" + config.getHostAndPort();
- this.fallbackExceptionList = multiClusterClientConfig.getFallbackExceptionList();
+ Retry retry = RetryRegistry.of(retryConfig).retry(clusterId);
+
+ Retry.EventPublisher retryPublisher = retry.getEventPublisher();
+ retryPublisher.onRetry(event -> log.warn(String.valueOf(event)));
+ retryPublisher.onError(event -> log.error(String.valueOf(event)));
+
+ CircuitBreaker circuitBreaker = CircuitBreakerRegistry.of(circuitBreakerConfig).circuitBreaker(clusterId);
+
+ CircuitBreaker.EventPublisher circuitBreakerEventPublisher = circuitBreaker.getEventPublisher();
+ circuitBreakerEventPublisher.onCallNotPermitted(event -> log.error(String.valueOf(event)));
+ circuitBreakerEventPublisher.onError(event -> log.error(String.valueOf(event)));
+ circuitBreakerEventPublisher.onFailureRateExceeded(event -> log.error(String.valueOf(event)));
+ circuitBreakerEventPublisher.onSlowCallRateExceeded(event -> log.error(String.valueOf(event)));
+
+ TrackingConnectionPool pool = new TrackingConnectionPool(config.getHostAndPort(), config.getJedisClientConfig(),
+ poolConfig);
+ Cluster cluster = new Cluster(pool, retry, circuitBreaker, config.getWeight(), multiClusterClientConfig);
+ multiClusterMap.put(config.getHostAndPort(), cluster);
+
+ StrategySupplier strategySupplier = config.getHealthCheckStrategySupplier();
+ if (strategySupplier != null) {
+ HealthCheckStrategy hcs = strategySupplier.get(config.getHostAndPort(), config.getJedisClientConfig());
+ // Register listeners BEFORE adding clusters to avoid missing events
+ healthStatusManager.registerListener(config.getHostAndPort(), this::onHealthStatusChange);
+ healthStatusManager.add(config.getHostAndPort(), hcs);
+ } else {
+ cluster.setHealthStatus(HealthStatus.HEALTHY);
+ }
}
/**
- * Increments the actively used cluster/database endpoint (connection pool) amongst the pre-configured list which
- * were provided at startup via the MultiClusterClientConfig. All traffic will be routed according to this index.
- * Only indexes within the pre-configured range (static) are supported otherwise an exception will be thrown. In the
- * event that the next prioritized connection has a forced open state, the method will recursively increment the
- * index in order to avoid a failed command.
+ * Handles health status changes for clusters. This method is called by the health status manager when the health
+ * status of a cluster changes.
*/
- public int incrementActiveMultiClusterIndex() {
+ @VisibleForTesting
+ void onHealthStatusChange(HealthStatusChangeEvent eventArgs) {
+ Endpoint endpoint = eventArgs.getEndpoint();
+ HealthStatus newStatus = eventArgs.getNewStatus();
+ log.debug("Health status changed for {} from {} to {}", endpoint, eventArgs.getOldStatus(), newStatus);
+ Cluster clusterWithHealthChange = multiClusterMap.get(endpoint);
+
+ if (clusterWithHealthChange == null) return;
+
+ clusterWithHealthChange.setHealthStatus(newStatus);
+ if (initializationComplete) {
+ if (!newStatus.isHealthy() && clusterWithHealthChange == activeCluster) {
+ clusterWithHealthChange.setGracePeriod();
+ iterateActiveCluster(SwitchReason.HEALTH_CHECK);
+ }
+ }
+ }
- // Field-level synchronization is used to avoid the edge case in which
- // setActiveMultiClusterIndex(int multiClusterIndex) is called at the same time
- activeClusterIndexLock.lock();
+ /**
+ * Waits for initial health check results and selects the first healthy cluster based on weight priority. Blocks
+ * until at least one cluster becomes healthy or all clusters are determined to be unhealthy.
+ * @return the first healthy cluster found, ordered by weight (highest first)
+ * @throws JedisConnectionException if all clusters are unhealthy
+ */
+ private Cluster waitForInitialHealthyCluster() {
+ // Sort clusters by weight in descending order
+ List> sortedClusters = multiClusterMap.entrySet().stream()
+ .sorted(Map.Entry. comparingByValue(Comparator.comparing(Cluster::getWeight).reversed()))
+ .collect(Collectors.toList());
- try {
- String originalClusterName = getClusterCircuitBreaker().getName();
+ log.info("Selecting initial cluster from {} configured clusters", sortedClusters.size());
- // Only increment if it can pass this validation otherwise we will need to check for NULL in the data path
- if (activeMultiClusterIndex + 1 > multiClusterMap.size()) {
+ // Select cluster in weight order
+ for (Map.Entry entry : sortedClusters) {
+ Endpoint endpoint = entry.getKey();
+ Cluster cluster = entry.getValue();
- lastClusterCircuitBreakerForcedOpen = true;
+ log.info("Evaluating cluster {} (weight: {})", endpoint, cluster.getWeight());
- throw new JedisConnectionException(
- "Cluster/database endpoint could not failover since the MultiClusterClientConfig was not "
- + "provided with an additional cluster/database endpoint according to its prioritized sequence. "
- + "If applicable, consider failing back OR restarting with an available cluster/database endpoint.");
- } else activeMultiClusterIndex++;
+ HealthStatus status;
- CircuitBreaker circuitBreaker = getClusterCircuitBreaker();
+ // Check if health checks are enabled for this endpoint
+ if (healthStatusManager.hasHealthCheck(endpoint)) {
+ log.info("Health checks enabled for {}, waiting for result", endpoint);
+ // Wait for this cluster's health status to be determined
+ status = statusTracker.waitForHealthStatus(endpoint);
+ } else {
+ // No health check configured - assume healthy
+ log.info("No health check configured for cluster {}, defaulting to HEALTHY", endpoint);
+ status = HealthStatus.HEALTHY;
+ }
- // Handles edge-case in which the user resets the activeMultiClusterIndex to a higher priority prematurely
- // which forces a failover to the next prioritized cluster that has potentially not yet recovered
- if (CircuitBreaker.State.FORCED_OPEN.equals(circuitBreaker.getState())) incrementActiveMultiClusterIndex();
+ cluster.setHealthStatus(status);
- else log.warn("Cluster/database endpoint successfully updated from '{}' to '{}'", originalClusterName,
- circuitBreaker.getName());
- } finally {
- activeClusterIndexLock.unlock();
+ if (status.isHealthy()) {
+ log.info("Found healthy cluster: {} (weight: {})", endpoint, cluster.getWeight());
+ return cluster;
+ } else {
+ log.info("Cluster {} is unhealthy, trying next cluster", endpoint);
+ }
+ }
+
+ // All clusters are unhealthy
+ throw new JedisConnectionException(
+ "All configured clusters are unhealthy. Cannot initialize MultiClusterPooledConnectionProvider.");
+ }
+
+ /**
+ * Periodic failback checker - runs at configured intervals to check for failback opportunities
+ */
+ @VisibleForTesting
+ void periodicFailbackCheck() {
+ try {
+ // Find the best candidate cluster for failback
+ Map.Entry bestCandidate = null;
+ float bestWeight = activeCluster.getWeight();
+
+ for (Map.Entry entry : multiClusterMap.entrySet()) {
+ Cluster cluster = entry.getValue();
+
+ // Skip if this is already the active cluster
+ if (cluster == activeCluster) {
+ continue;
+ }
+
+ // Skip if cluster is not healthy
+ if (!cluster.isHealthy()) {
+ continue;
+ }
+
+ // This cluster is a valid candidate
+ if (cluster.getWeight() > bestWeight) {
+ bestCandidate = entry;
+ bestWeight = cluster.getWeight();
+ }
+ }
+
+ // Perform failback if we found a better candidate
+ if (bestCandidate != null) {
+ Cluster selectedCluster = bestCandidate.getValue();
+ log.info("Performing failback from {} to {} (higher weight cluster available)",
+ activeCluster.getCircuitBreaker().getName(), selectedCluster.getCircuitBreaker().getName());
+ if (setActiveCluster(selectedCluster, true)) {
+ onClusterSwitch(SwitchReason.FAILBACK, bestCandidate.getKey(), selectedCluster);
+ }
+ }
+ } catch (Exception e) {
+ log.error("Error during periodic failback check", e);
}
+ }
+
+ public Endpoint iterateActiveCluster(SwitchReason reason) {
+ Map.Entry clusterToIterate = findWeightedHealthyClusterToIterate();
+ if (clusterToIterate == null) {
+ throw new JedisConnectionException(
+ "Cluster/database endpoint could not failover since the MultiClusterClientConfig was not "
+ + "provided with an additional cluster/database endpoint according to its prioritized sequence. "
+ + "If applicable, consider failing back OR restarting with an available cluster/database endpoint");
+ }
+ Cluster cluster = clusterToIterate.getValue();
+ boolean changed = setActiveCluster(cluster, false);
+ if (!changed) return null;
+ onClusterSwitch(reason, clusterToIterate.getKey(), cluster);
+ return clusterToIterate.getKey();
+ }
+
+ private static Comparator> maxByWeight = Map.Entry
+ . comparingByValue(Comparator.comparing(Cluster::getWeight));
- return activeMultiClusterIndex;
+ private static Predicate> filterByHealth = c -> c.getValue().isHealthy();
+
+ private Map.Entry findWeightedHealthyClusterToIterate() {
+ return multiClusterMap.entrySet().stream().filter(filterByHealth)
+ .filter(entry -> entry.getValue() != activeCluster).max(maxByWeight).orElse(null);
}
/**
@@ -209,9 +454,13 @@ public int incrementActiveMultiClusterIndex() {
* there was discussion to handle cross-cluster replication validation by setting a key/value pair per hashslot in
* the active connection (with a TTL) and subsequently reading it from the target connection.
*/
- public void validateTargetConnection(int multiClusterIndex) {
+ public void validateTargetConnection(Endpoint endpoint) {
+ Cluster cluster = multiClusterMap.get(endpoint);
+ validateTargetConnection(cluster);
+ }
- CircuitBreaker circuitBreaker = getClusterCircuitBreaker(multiClusterIndex);
+ private void validateTargetConnection(Cluster cluster) {
+ CircuitBreaker circuitBreaker = cluster.getCircuitBreaker();
State originalState = circuitBreaker.getState();
try {
@@ -220,113 +469,157 @@ public void validateTargetConnection(int multiClusterIndex) {
// yet
circuitBreaker.transitionToClosedState();
- try (Connection targetConnection = getConnection(multiClusterIndex)) {
+ try (Connection targetConnection = cluster.getConnection()) {
targetConnection.ping();
}
} catch (Exception e) {
// If the original state was FORCED_OPEN, then transition it back which stops state transition, metrics and
// event publishing
- if (CircuitBreaker.State.FORCED_OPEN.equals(originalState)) circuitBreaker.transitionToForcedOpenState();
+ if (State.FORCED_OPEN.equals(originalState)) circuitBreaker.transitionToForcedOpenState();
throw new JedisValidationException(
circuitBreaker.getName() + " failed to connect. Please check configuration and try again.", e);
}
}
- /**
- * Manually overrides the actively used cluster/database endpoint (connection pool) amongst the pre-configured list
- * which were provided at startup via the MultiClusterClientConfig. All traffic will be routed according to the
- * provided new index. Special care should be taken to confirm cluster/database availability AND potentially
- * cross-cluster replication BEFORE using this capability.
- */
- public void setActiveMultiClusterIndex(int multiClusterIndex) {
+ public void setActiveCluster(Endpoint endpoint) {
+ if (endpoint == null) {
+ throw new JedisValidationException("Provided endpoint is null. Please use one from the configuration");
+ }
+ Cluster cluster = multiClusterMap.get(endpoint);
+ if (cluster == null) {
+ throw new JedisValidationException("Provided endpoint: " + endpoint + " is not within "
+ + "the configured endpoints. Please use one from the configuration");
+ }
+ if (setActiveCluster(cluster, true)) {
+ onClusterSwitch(SwitchReason.FORCED, endpoint, cluster);
+ }
+ }
+
+ public void forceActiveCluster(Endpoint endpoint, long forcedActiveDuration) {
+ Cluster cluster = multiClusterMap.get(endpoint);
+ cluster.clearGracePeriod();
+ if (!cluster.isHealthy()) {
+ throw new JedisValidationException("Provided endpoint: " + endpoint
+ + " is not healthy. Please consider a healthy endpoint from the configuration");
+ }
+ multiClusterMap.entrySet().stream().forEach(entry -> {
+ if (entry.getKey() != endpoint) {
+ entry.getValue().setGracePeriod(forcedActiveDuration);
+ }
+ });
+ setActiveCluster(endpoint);
+ }
+ private boolean setActiveCluster(Cluster cluster, boolean validateConnection) {
+ // Cluster cluster = clusterEntry.getValue();
// Field-level synchronization is used to avoid the edge case in which
// incrementActiveMultiClusterIndex() is called at the same time
activeClusterIndexLock.lock();
-
+ Cluster oldCluster;
try {
// Allows an attempt to reset the current cluster from a FORCED_OPEN to CLOSED state in the event that no
// failover is possible
- if (activeMultiClusterIndex == multiClusterIndex
- && !CircuitBreaker.State.FORCED_OPEN.equals(getClusterCircuitBreaker(multiClusterIndex).getState()))
- return;
+ if (activeCluster == cluster && !cluster.isCBForcedOpen()) return false;
- if (multiClusterIndex < 1 || multiClusterIndex > multiClusterMap.size())
- throw new JedisValidationException("MultiClusterIndex: " + multiClusterIndex + " is not within "
- + "the configured range. Please choose an index between 1 and " + multiClusterMap.size());
-
- validateTargetConnection(multiClusterIndex);
+ if (validateConnection) validateTargetConnection(cluster);
String originalClusterName = getClusterCircuitBreaker().getName();
- if (activeMultiClusterIndex == multiClusterIndex)
+ if (activeCluster == cluster)
log.warn("Cluster/database endpoint '{}' successfully closed its circuit breaker", originalClusterName);
else log.warn("Cluster/database endpoint successfully updated from '{}' to '{}'", originalClusterName,
- getClusterCircuitBreaker(multiClusterIndex).getName());
-
- activeMultiClusterIndex = multiClusterIndex;
- lastClusterCircuitBreakerForcedOpen = false;
+ cluster.circuitBreaker.getName());
+ oldCluster = activeCluster;
+ activeCluster = cluster;
} finally {
activeClusterIndexLock.unlock();
}
+ boolean switched = oldCluster != cluster;
+ if (switched && this.multiClusterClientConfig.isFastFailover()) {
+ log.info("Forcing disconnect of all active connections in old cluster: {}",
+ oldCluster.circuitBreaker.getName());
+ oldCluster.forceDisconnect();
+ log.info("Disconnected all active connections in old cluster: {}", oldCluster.circuitBreaker.getName());
+
+ }
+ return switched;
+
}
@Override
public void close() {
- multiClusterMap.get(activeMultiClusterIndex).getConnectionPool().close();
+ // Shutdown the failback scheduler
+ failbackScheduler.shutdown();
+ try {
+ if (!failbackScheduler.awaitTermination(1, TimeUnit.SECONDS)) {
+ failbackScheduler.shutdownNow();
+ }
+ } catch (InterruptedException e) {
+ failbackScheduler.shutdownNow();
+ Thread.currentThread().interrupt();
+ }
+
+ // Close all cluster connection pools
+ for (Cluster cluster : multiClusterMap.values()) {
+ cluster.close();
+ }
}
@Override
public Connection getConnection() {
- return multiClusterMap.get(activeMultiClusterIndex).getConnection();
+ return activeCluster.getConnection();
}
- public Connection getConnection(int multiClusterIndex) {
- return multiClusterMap.get(multiClusterIndex).getConnection();
+ public Connection getConnection(Endpoint endpoint) {
+ return multiClusterMap.get(endpoint).getConnection();
}
@Override
public Connection getConnection(CommandArguments args) {
- return multiClusterMap.get(activeMultiClusterIndex).getConnection();
+ return activeCluster.getConnection();
}
@Override
public Map> getConnectionMap() {
- ConnectionPool connectionPool = multiClusterMap.get(activeMultiClusterIndex).getConnectionPool();
+ ConnectionPool connectionPool = activeCluster.connectionPool;
return Collections.singletonMap(connectionPool.getFactory(), connectionPool);
}
public Cluster getCluster() {
- return multiClusterMap.get(activeMultiClusterIndex);
+ return activeCluster;
}
@VisibleForTesting
- public Cluster getCluster(int multiClusterIndex) {
- return multiClusterMap.get(multiClusterIndex);
+ public Cluster getCluster(Endpoint endpoint) {
+ return multiClusterMap.get(endpoint);
}
public CircuitBreaker getClusterCircuitBreaker() {
- return multiClusterMap.get(activeMultiClusterIndex).getCircuitBreaker();
- }
-
- public CircuitBreaker getClusterCircuitBreaker(int multiClusterIndex) {
- return multiClusterMap.get(multiClusterIndex).getCircuitBreaker();
+ return activeCluster.getCircuitBreaker();
}
- public boolean isLastClusterCircuitBreakerForcedOpen() {
- return lastClusterCircuitBreakerForcedOpen;
+ /**
+ * Indicates the final cluster/database endpoint (connection pool), according to the pre-configured list provided at
+ * startup via the MultiClusterClientConfig, is unavailable and therefore no further failover is possible. Users can
+ * manually failback to an available cluster
+ */
+ public boolean canIterateOnceMore() {
+ Map.Entry e = findWeightedHealthyClusterToIterate();
+ return e != null;
}
- public void runClusterFailoverPostProcessor(Integer multiClusterIndex) {
- if (clusterFailoverPostProcessor != null)
- clusterFailoverPostProcessor.accept(getClusterCircuitBreaker(multiClusterIndex).getName());
+ public void onClusterSwitch(SwitchReason reason, Endpoint endpoint, Cluster cluster) {
+ if (clusterSwitchListener != null) {
+ ClusterSwitchEventArgs eventArgs = new ClusterSwitchEventArgs(reason, endpoint, cluster);
+ clusterSwitchListener.accept(eventArgs);
+ }
}
- public void setClusterFailoverPostProcessor(Consumer clusterFailoverPostProcessor) {
- this.clusterFailoverPostProcessor = clusterFailoverPostProcessor;
+ public void setClusterSwitchListener(Consumer clusterSwitchListener) {
+ this.clusterSwitchListener = clusterSwitchListener;
}
public List> getFallbackExceptionList() {
@@ -335,20 +628,37 @@ public List> getFallbackExceptionList() {
public static class Cluster {
- private final ConnectionPool connectionPool;
+ private TrackingConnectionPool connectionPool;
private final Retry retry;
private final CircuitBreaker circuitBreaker;
-
- public Cluster(ConnectionPool connectionPool, Retry retry, CircuitBreaker circuitBreaker) {
+ private final float weight;
+ // it starts its life with unknown health status, waiting for initial health check
+ private HealthStatus healthStatus = HealthStatus.UNKNOWN;
+ private MultiClusterClientConfig multiClusterClientConfig;
+ private boolean disabled = false;
+
+ // Grace period tracking
+ private volatile long gracePeriodEndsAt = 0;
+ private final Logger log = LoggerFactory.getLogger(getClass());
+
+ private Cluster(TrackingConnectionPool connectionPool, Retry retry, CircuitBreaker circuitBreaker, float weight,
+ MultiClusterClientConfig multiClusterClientConfig) {
this.connectionPool = connectionPool;
this.retry = retry;
this.circuitBreaker = circuitBreaker;
+ this.weight = weight;
+ this.multiClusterClientConfig = multiClusterClientConfig;
}
public Connection getConnection() {
+ if (!isHealthy()) throw new JedisConnectionException("Cluster is not healthy");
+ if (connectionPool.isClosed()) {
+ connectionPool = TrackingConnectionPool.from(connectionPool);
+ }
return connectionPool.getResource();
}
+ @VisibleForTesting
public ConnectionPool getConnectionPool() {
return connectionPool;
}
@@ -360,6 +670,91 @@ public Retry getRetry() {
public CircuitBreaker getCircuitBreaker() {
return circuitBreaker;
}
+
+ public HealthStatus getHealthStatus() {
+ return healthStatus;
+ }
+
+ public void setHealthStatus(HealthStatus healthStatus) {
+ this.healthStatus = healthStatus;
+ }
+
+ /**
+ * Assigned weight for this cluster
+ */
+ public float getWeight() {
+ return weight;
+ }
+
+ public boolean isCBForcedOpen() {
+ if (circuitBreaker.getState() == State.FORCED_OPEN && !isInGracePeriod()) {
+ log.info("Transitioning circuit breaker from FORCED_OPEN to CLOSED state due to end of grace period!");
+ circuitBreaker.transitionToClosedState();
+ }
+ return circuitBreaker.getState() == CircuitBreaker.State.FORCED_OPEN;
+ }
+
+ public boolean isHealthy() {
+ return healthStatus.isHealthy() && !isCBForcedOpen() && !disabled && !isInGracePeriod();
+ }
+
+ public boolean retryOnFailover() {
+ return multiClusterClientConfig.isRetryOnFailover();
+ }
+
+ public boolean isDisabled() {
+ return disabled;
+ }
+
+ public void setDisabled(boolean disabled) {
+ this.disabled = disabled;
+ }
+
+ /**
+ * Checks if the cluster is currently in grace period
+ */
+ public boolean isInGracePeriod() {
+ return System.currentTimeMillis() < gracePeriodEndsAt;
+ }
+
+ /**
+ * Sets the grace period for this cluster
+ */
+ public void setGracePeriod() {
+ setGracePeriod(multiClusterClientConfig.getGracePeriod());
+ }
+
+ public void setGracePeriod(long gracePeriod) {
+ long endTime = System.currentTimeMillis() + gracePeriod;
+ if (endTime < gracePeriodEndsAt) return;
+ gracePeriodEndsAt = endTime;
+ }
+
+ public void clearGracePeriod() {
+ gracePeriodEndsAt = 0;
+ }
+
+ /**
+ * Whether failback is supported by client
+ */
+ public boolean isFailbackSupported() {
+ return multiClusterClientConfig.isFailbackSupported();
+ }
+
+ public void forceDisconnect() {
+ connectionPool.forceDisconnect();
+ }
+
+ public void close() {
+ connectionPool.close();
+ }
+
+ @Override
+ public String toString() {
+ return circuitBreaker.getName() + "{" + "connectionPool=" + connectionPool + ", retry=" + retry
+ + ", circuitBreaker=" + circuitBreaker + ", weight=" + weight + ", healthStatus=" + healthStatus
+ + ", multiClusterClientConfig=" + multiClusterClientConfig + '}';
+ }
}
}
\ No newline at end of file
diff --git a/src/test/java/redis/clients/jedis/failover/FailoverIntegrationTest.java b/src/test/java/redis/clients/jedis/failover/FailoverIntegrationTest.java
index dcd5e681a9..b43676c88d 100644
--- a/src/test/java/redis/clients/jedis/failover/FailoverIntegrationTest.java
+++ b/src/test/java/redis/clients/jedis/failover/FailoverIntegrationTest.java
@@ -19,7 +19,7 @@
import redis.clients.jedis.MultiClusterClientConfig;
import redis.clients.jedis.UnifiedJedis;
import redis.clients.jedis.exceptions.JedisConnectionException;
-import redis.clients.jedis.mcf.FailoverOptions;
+
import redis.clients.jedis.providers.MultiClusterPooledConnectionProvider;
import redis.clients.jedis.scenario.RecommendedSettings;
@@ -109,7 +109,7 @@ public void setup() throws IOException {
// Create default provider and client for most tests
provider = createProvider();
- failoverClient = createClient(provider, null);
+ failoverClient = new UnifiedJedis(provider);
}
@AfterEach
@@ -143,7 +143,8 @@ public void testAutomaticFailoverWhenServerBecomesUnavailable() throws Exception
// 3. Subsequent calls should be routed to Endpoint 2
assertThrows(JedisConnectionException.class, () -> failoverClient.info("server"));
- assertThat(provider.getCluster(1).getCircuitBreaker().getState(), equalTo(CircuitBreaker.State.OPEN));
+ assertThat(provider.getCluster(endpoint1.getHostAndPort()).getCircuitBreaker().getState(),
+ equalTo(CircuitBreaker.State.OPEN));
// Check that the failoverClient is now using Endpoint 2
assertThat(getNodeId(failoverClient.info("server")), equalTo(JEDIS2_ID));
@@ -153,7 +154,8 @@ public void testAutomaticFailoverWhenServerBecomesUnavailable() throws Exception
// Endpoint1 and Endpoint2 are NOT available,
assertThrows(JedisConnectionException.class, () -> failoverClient.info("server"));
- assertThat(provider.getCluster(2).getCircuitBreaker().getState(), equalTo(CircuitBreaker.State.OPEN));
+ assertThat(provider.getCluster(endpoint2.getHostAndPort()).getCircuitBreaker().getState(),
+ equalTo(CircuitBreaker.State.OPEN));
// and since no other nodes are available, it should propagate the errors to the caller
// subsequent calls
@@ -164,7 +166,7 @@ public void testAutomaticFailoverWhenServerBecomesUnavailable() throws Exception
public void testManualFailoverNewCommandsAreSentToActiveCluster() throws InterruptedException {
assertThat(getNodeId(failoverClient.info("server")), equalTo(JEDIS1_ID));
- provider.setActiveMultiClusterIndex(2);
+ provider.setActiveCluster(endpoint2.getHostAndPort());
assertThat(getNodeId(failoverClient.info("server")), equalTo(JEDIS2_ID));
}
@@ -173,7 +175,7 @@ private List getClusterConfigs(JedisClie
EndpointConfig... endpoints) {
return Arrays.stream(endpoints)
- .map(e -> new MultiClusterClientConfig.ClusterConfig(e.getHostAndPort(), clientConfig))
+ .map(e -> MultiClusterClientConfig.ClusterConfig.builder(e.getHostAndPort(), clientConfig).build())
.collect(Collectors.toList());
}
@@ -186,14 +188,15 @@ public void testManualFailoverInflightCommandsCompleteGracefully() throws Execut
// We will trigger failover while this command is in-flight
Future> blpop = executor.submit(() -> failoverClient.blpop(1000, "test-list"));
- provider.setActiveMultiClusterIndex(2);
+ provider.setActiveCluster(endpoint2.getHostAndPort());
// After the manual failover, commands should be executed against Endpoint 2
assertThat(getNodeId(failoverClient.info("server")), equalTo(JEDIS2_ID));
// Failover was manually triggered, and there were no errors
// previous endpoint CB should still be in CLOSED state
- assertThat(provider.getCluster(1).getCircuitBreaker().getState(), equalTo(CircuitBreaker.State.CLOSED));
+ assertThat(provider.getCluster(endpoint1.getHostAndPort()).getCircuitBreaker().getState(),
+ equalTo(CircuitBreaker.State.CLOSED));
jedis1.rpush("test-list", "somevalue");
@@ -211,7 +214,7 @@ public void testManualFailoverInflightCommandsWithErrorsPropagateError() throws
Future> blpop = executor.submit(() -> failoverClient.blpop(10000, "test-list-1"));
// trigger failover manually
- provider.setActiveMultiClusterIndex(2);
+ provider.setActiveCluster(endpoint2.getHostAndPort());
Future infoCmd = executor.submit(() -> failoverClient.info("server"));
// After the manual failover, commands should be executed against Endpoint 2
@@ -225,7 +228,8 @@ public void testManualFailoverInflightCommandsWithErrorsPropagateError() throws
assertThat(exception.getCause(), instanceOf(JedisConnectionException.class));
// Check that the circuit breaker for Endpoint 1 is open after the error
- assertThat(provider.getCluster(1).getCircuitBreaker().getState(), equalTo(CircuitBreaker.State.OPEN));
+ assertThat(provider.getCluster(endpoint1.getHostAndPort()).getCircuitBreaker().getState(),
+ equalTo(CircuitBreaker.State.OPEN));
// Ensure that the active cluster is still Endpoint 2
assertThat(getNodeId(failoverClient.info("server")), equalTo(JEDIS2_ID));
@@ -272,7 +276,8 @@ public void testCircuitBreakerCountsEachConnectionErrorSeparately() throws IOExc
assertThrows(JedisConnectionException.class, () -> client.info("server"));
// Circuit breaker should be open after just one command with retries
- assertThat(provider.getCluster(1).getCircuitBreaker().getState(), equalTo(CircuitBreaker.State.OPEN));
+ assertThat(provider.getCluster(endpoint1.getHostAndPort()).getCircuitBreaker().getState(),
+ equalTo(CircuitBreaker.State.OPEN));
// Next command should be routed to the second endpoint
// Command 2
@@ -291,12 +296,13 @@ public void testCircuitBreakerCountsEachConnectionErrorSeparately() throws IOExc
@Test
public void testInflightCommandsAreRetriedAfterFailover() throws Exception {
- MultiClusterPooledConnectionProvider customProvider = createProvider();
+ MultiClusterPooledConnectionProvider customProvider = createProvider(builder -> builder.retryOnFailover(true));
// Create a custom client with retryOnFailover enabled for this specific test
- try (UnifiedJedis customClient = createClient(customProvider, builder -> builder.retryOnFailover(true))) {
+ try (UnifiedJedis customClient = new UnifiedJedis(customProvider)) {
assertThat(getNodeId(customClient.info("server")), equalTo(JEDIS1_ID));
+ Thread.sleep(1000);
// We will trigger failover while this command is in-flight
Future> blpop = executor.submit(() -> customClient.blpop(10000, "test-list-1"));
@@ -311,7 +317,7 @@ public void testInflightCommandsAreRetriedAfterFailover() throws Exception {
// immediately when CB state change to OPEN/FORCED_OPENs
assertThat(getNodeId(customClient.info("server")), equalTo(JEDIS2_ID));
// Check that the circuit breaker for Endpoint 1 is open
- assertThat(customProvider.getCluster(1).getCircuitBreaker().getState(),
+ assertThat(customProvider.getCluster(endpoint1.getHostAndPort()).getCircuitBreaker().getState(),
equalTo(CircuitBreaker.State.FORCED_OPEN));
// Disable redisProxy1 to enforce connection drop for the in-flight (blpop) command
@@ -329,9 +335,9 @@ public void testInflightCommandsAreRetriedAfterFailover() throws Exception {
@Test
public void testInflightCommandsAreNotRetriedAfterFailover() throws Exception {
// Create a custom provider and client with retry disabled for this specific test
- MultiClusterPooledConnectionProvider customProvider = createProvider();
+ MultiClusterPooledConnectionProvider customProvider = createProvider(builder -> builder.retryOnFailover(false));
- try (UnifiedJedis customClient = createClient(customProvider, builder -> builder.retryOnFailover(false))) {
+ try (UnifiedJedis customClient = new UnifiedJedis(customProvider)) {
assertThat(getNodeId(customClient.info("server")), equalTo(JEDIS1_ID));
Future> blpop = executor.submit(() -> customClient.blpop(500, "test-list-2"));
@@ -342,7 +348,8 @@ public void testInflightCommandsAreNotRetriedAfterFailover() throws Exception {
assertThrows(JedisConnectionException.class, () -> customClient.set("test-key", generateTestValue(150)));
// Check that the circuit breaker for Endpoint 1 is open
- assertThat(customProvider.getCluster(1).getCircuitBreaker().getState(), equalTo(CircuitBreaker.State.OPEN));
+ assertThat(customProvider.getCluster(endpoint1.getHostAndPort()).getCircuitBreaker().getState(),
+ equalTo(CircuitBreaker.State.OPEN));
// Disable redisProxy1 to enforce the current blpop command failure
redisProxy1.disable();
@@ -398,18 +405,24 @@ private MultiClusterPooledConnectionProvider createProvider() {
}
/**
- * Creates a UnifiedJedis client with customizable failover options
- * @param provider The connection provider to use
- * @param optionsCustomizer A function that customizes the failover options (can be null for defaults)
- * @return A configured failover client
+ * Creates a MultiClusterPooledConnectionProvider with standard configuration
+ * @return A configured provider
*/
- private UnifiedJedis createClient(MultiClusterPooledConnectionProvider provider,
- Function optionsCustomizer) {
- FailoverOptions.Builder builder = FailoverOptions.builder();
- if (optionsCustomizer != null) {
- builder = optionsCustomizer.apply(builder);
+ private MultiClusterPooledConnectionProvider createProvider(
+ Function configCustomizer) {
+ JedisClientConfig clientConfig = DefaultJedisClientConfig.builder()
+ .socketTimeoutMillis(RecommendedSettings.DEFAULT_TIMEOUT_MS)
+ .connectionTimeoutMillis(RecommendedSettings.DEFAULT_TIMEOUT_MS).build();
+
+ MultiClusterClientConfig.Builder builder = new MultiClusterClientConfig.Builder(
+ getClusterConfigs(clientConfig, endpoint1, endpoint2)).retryMaxAttempts(1).retryWaitDuration(1)
+ .circuitBreakerSlidingWindowType(COUNT_BASED).circuitBreakerSlidingWindowSize(1)
+ .circuitBreakerFailureRateThreshold(100).circuitBreakerSlidingWindowMinCalls(1);
+
+ if (configCustomizer != null) {
+ builder = configCustomizer.apply(builder);
}
- return new UnifiedJedis(provider, builder.build());
+ return new MultiClusterPooledConnectionProvider(builder.build());
}
}
diff --git a/src/test/java/redis/clients/jedis/mcf/ActiveActiveLocalFailoverTest.java b/src/test/java/redis/clients/jedis/mcf/ActiveActiveLocalFailoverTest.java
new file mode 100644
index 0000000000..3c0dd77ea3
--- /dev/null
+++ b/src/test/java/redis/clients/jedis/mcf/ActiveActiveLocalFailoverTest.java
@@ -0,0 +1,301 @@
+package redis.clients.jedis.mcf;
+
+import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig;
+
+import org.hamcrest.Matchers;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Tags;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import eu.rekawek.toxiproxy.Proxy;
+import eu.rekawek.toxiproxy.ToxiproxyClient;
+import eu.rekawek.toxiproxy.model.Toxic;
+import redis.clients.jedis.*;
+import redis.clients.jedis.MultiClusterClientConfig.ClusterConfig;
+import redis.clients.jedis.providers.MultiClusterPooledConnectionProvider;
+import redis.clients.jedis.scenario.ActiveActiveFailoverTest;
+import redis.clients.jedis.scenario.MultiThreadedFakeApp;
+import redis.clients.jedis.scenario.RecommendedSettings;
+import redis.clients.jedis.scenario.FaultInjectionClient.TriggerActionResponse;
+import redis.clients.jedis.exceptions.JedisConnectionException;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+@Tags({ @Tag("failover"), @Tag("scenario") })
+public class ActiveActiveLocalFailoverTest {
+ private static final Logger log = LoggerFactory.getLogger(ActiveActiveFailoverTest.class);
+
+ private static final EndpointConfig endpoint1 = HostAndPorts.getRedisEndpoint("redis-failover-1");
+ private static final EndpointConfig endpoint2 = HostAndPorts.getRedisEndpoint("redis-failover-2");
+ private static final ToxiproxyClient tp = new ToxiproxyClient("localhost", 8474);
+ private static Proxy redisProxy1;
+ private static Proxy redisProxy2;
+
+ @BeforeAll
+ public static void setupAdminClients() throws IOException {
+ if (tp.getProxyOrNull("redis-1") != null) {
+ tp.getProxy("redis-1").delete();
+ }
+ if (tp.getProxyOrNull("redis-2") != null) {
+ tp.getProxy("redis-2").delete();
+ }
+
+ redisProxy1 = tp.createProxy("redis-1", "0.0.0.0:29379", "redis-failover-1:9379");
+ redisProxy2 = tp.createProxy("redis-2", "0.0.0.0:29380", "redis-failover-2:9380");
+ }
+
+ @AfterAll
+ public static void cleanupAdminClients() throws IOException {
+ if (redisProxy1 != null) redisProxy1.delete();
+ if (redisProxy2 != null) redisProxy2.delete();
+ }
+
+ @BeforeEach
+ public void setup() throws IOException {
+ tp.getProxies().forEach(proxy -> {
+ try {
+ proxy.enable();
+ for (Toxic toxic : proxy.toxics().getAll()) {
+ toxic.remove();
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ @ParameterizedTest
+ @CsvSource({ "true, 0, 2, 4", "true, 0, 2, 6", "true, 0, 2, 7", "true, 0, 2, 8", "true, 0, 2, 9", "true, 0, 2, 16", })
+ public void testFailover(boolean fastFailover, long minFailoverCompletionDuration, long maxFailoverCompletionDuration,
+ int numberOfThreads) {
+
+ log.info(
+ "TESTING WITH PARAMETERS: fastFailover: {} numberOfThreads: {} minFailoverCompletionDuration: {} maxFailoverCompletionDuration: {] ",
+ fastFailover, numberOfThreads, minFailoverCompletionDuration, maxFailoverCompletionDuration);
+
+ MultiClusterClientConfig.ClusterConfig[] clusterConfig = new MultiClusterClientConfig.ClusterConfig[2];
+
+ JedisClientConfig config = endpoint1.getClientConfigBuilder()
+ .socketTimeoutMillis(RecommendedSettings.DEFAULT_TIMEOUT_MS)
+ .connectionTimeoutMillis(RecommendedSettings.DEFAULT_TIMEOUT_MS).build();
+
+ clusterConfig[0] = ClusterConfig.builder(endpoint1.getHostAndPort(), config)
+ .connectionPoolConfig(RecommendedSettings.poolConfig).weight(1.0f).build();
+ clusterConfig[1] = ClusterConfig.builder(endpoint2.getHostAndPort(), config)
+ .connectionPoolConfig(RecommendedSettings.poolConfig).weight(0.5f).build();
+
+ MultiClusterClientConfig.Builder builder = new MultiClusterClientConfig.Builder(clusterConfig);
+
+ builder.circuitBreakerSlidingWindowType(CircuitBreakerConfig.SlidingWindowType.TIME_BASED);
+ builder.circuitBreakerSlidingWindowSize(1); // SLIDING WINDOW SIZE IN SECONDS
+ builder.circuitBreakerSlidingWindowMinCalls(1);
+ builder.circuitBreakerFailureRateThreshold(10.0f); // percentage of failures to trigger circuit breaker
+
+ builder.failbackSupported(false);
+ // builder.failbackCheckInterval(1000);
+ builder.gracePeriod(10000);
+
+ builder.retryWaitDuration(10);
+ builder.retryMaxAttempts(1);
+ builder.retryWaitDurationExponentialBackoffMultiplier(1);
+
+ // Use the parameterized fastFailover setting
+ builder.fastFailover(fastFailover);
+
+ class FailoverReporter implements Consumer {
+
+ String currentClusterName = "not set";
+
+ boolean failoverHappened = false;
+
+ Instant failoverAt = null;
+
+ boolean failbackHappened = false;
+
+ Instant failbackAt = null;
+
+ public String getCurrentClusterName() {
+ return currentClusterName;
+ }
+
+ @Override
+ public void accept(ClusterSwitchEventArgs e) {
+ this.currentClusterName = e.getClusterName();
+ log.info("\n\n===={}=== \nJedis switching to cluster: {}\n====End of log===\n", e.getReason(),
+ e.getClusterName());
+ if ((e.getReason() == SwitchReason.CIRCUIT_BREAKER || e.getReason() == SwitchReason.HEALTH_CHECK)) {
+ failoverHappened = true;
+ failoverAt = Instant.now();
+ }
+ if (e.getReason() == SwitchReason.FAILBACK) {
+ failbackHappened = true;
+ failbackAt = Instant.now();
+ }
+ }
+ }
+
+ MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(builder.build());
+ FailoverReporter reporter = new FailoverReporter();
+ provider.setClusterSwitchListener(reporter);
+ provider.setActiveCluster(endpoint1.getHostAndPort());
+
+ UnifiedJedis client = new UnifiedJedis(provider);
+
+ AtomicLong retryingThreadsCounter = new AtomicLong(0);
+ AtomicLong failedCommandsAfterFailover = new AtomicLong(0);
+ AtomicReference lastFailedCommandAt = new AtomicReference<>();
+ AtomicReference lastFailedBeforeFailover = new AtomicReference<>();
+ AtomicBoolean errorOccuredAfterFailover = new AtomicBoolean(false);
+ AtomicBoolean unexpectedErrors = new AtomicBoolean(false);
+ AtomicReference lastException = new AtomicReference();
+ AtomicLong stopRunningAt = new AtomicLong();
+ String cluster2Id = provider.getCluster(endpoint2.getHostAndPort()).getCircuitBreaker().getName();
+
+ // Start thread that imitates an application that uses the client
+ MultiThreadedFakeApp fakeApp = new MultiThreadedFakeApp(client, (UnifiedJedis c) -> {
+
+ long threadId = Thread.currentThread().getId();
+
+ int attempt = 0;
+ int maxTries = 500;
+ int retryingDelay = 5;
+ String currentClusterId = null;
+ while (true) {
+ try {
+ if (System.currentTimeMillis() > stopRunningAt.get()) break;
+ currentClusterId = provider.getCluster().getCircuitBreaker().getName();
+ Map executionInfo = new HashMap() {
+ {
+ put("threadId", String.valueOf(threadId));
+ put("cluster", reporter.getCurrentClusterName());
+ }
+ };
+
+ client.xadd("execution_log", StreamEntryID.NEW_ENTRY, executionInfo);
+
+ if (attempt > 0) {
+ log.info("Thread {} recovered after {} ms. Threads still not recovered: {}", threadId,
+ attempt * retryingDelay, retryingThreadsCounter.decrementAndGet());
+ }
+
+ break;
+ } catch (JedisConnectionException e) {
+ if (cluster2Id.equals(currentClusterId)) {
+ break;
+ }
+ lastException.set(e);
+ lastFailedBeforeFailover.set(Instant.now());
+
+ if (reporter.failoverHappened) {
+ errorOccuredAfterFailover.set(true);
+
+ long failedCommands = failedCommandsAfterFailover.incrementAndGet();
+ lastFailedCommandAt.set(Instant.now());
+ log.warn("Thread {} failed to execute command after failover. Failed commands after failover: {}", threadId,
+ failedCommands);
+ }
+
+ if (attempt == 0) {
+ long failedThreads = retryingThreadsCounter.incrementAndGet();
+ log.warn("Thread {} failed to execute command. Failed threads: {}", threadId, failedThreads);
+ }
+ try {
+ Thread.sleep(retryingDelay);
+ } catch (InterruptedException ie) {
+ throw new RuntimeException(ie);
+ }
+ if (++attempt == maxTries) throw e;
+ } catch (Exception e) {
+ if (cluster2Id.equals(currentClusterId)) {
+ break;
+ }
+ lastException.set(e);
+ unexpectedErrors.set(true);
+ lastFailedBeforeFailover.set(Instant.now());
+ log.error("UNEXPECTED exception", e);
+ if (reporter.failoverHappened) {
+ errorOccuredAfterFailover.set(true);
+ lastFailedCommandAt.set(Instant.now());
+ }
+ }
+ }
+ return true;
+ }, numberOfThreads);
+ fakeApp.setKeepExecutingForSeconds(30);
+ Thread t = new Thread(fakeApp);
+ t.start();
+
+ stopRunningAt.set(System.currentTimeMillis() + 30000);
+
+ log.info("Triggering issue on endpoint1");
+ try (Jedis jedis = new Jedis(endpoint1.getHostAndPort(), endpoint1.getClientConfigBuilder().build())) {
+ jedis.clientPause(20000);
+ }
+
+ fakeApp.setAction(new TriggerActionResponse(null) {
+ private long completeAt = System.currentTimeMillis() + 10000;
+
+ @Override
+ public boolean isCompleted(Duration checkInterval, Duration delayAfter, Duration timeout) {
+ return System.currentTimeMillis() > completeAt;
+ }
+ });
+
+ log.info("Waiting for fake app to complete");
+ try {
+ t.join();
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ log.info("Fake app completed");
+
+ ConnectionPool pool = provider.getCluster(endpoint1.getHostAndPort()).getConnectionPool();
+
+ log.info("First connection pool state: active: {}, idle: {}", pool.getNumActive(), pool.getNumIdle());
+ log.info("Failover happened at: {}", reporter.failoverAt);
+ log.info("Failback happened at: {}", reporter.failbackAt);
+
+ assertEquals(0, pool.getNumActive());
+ assertTrue(fakeApp.capturedExceptions().isEmpty());
+ assertTrue(reporter.failoverHappened);
+ if (errorOccuredAfterFailover.get()) {
+ log.info("Last failed command at: {}", lastFailedCommandAt.get());
+ Duration fullFailoverTime = Duration.between(reporter.failoverAt, lastFailedCommandAt.get());
+ log.info("Full failover time: {} s", fullFailoverTime.getSeconds());
+ log.info("Last failed command exception: {}", lastException.get());
+
+ // assertTrue(reporter.failbackHappened);
+ assertThat(fullFailoverTime.getSeconds(), Matchers.greaterThanOrEqualTo(minFailoverCompletionDuration));
+ assertThat(fullFailoverTime.getSeconds(), Matchers.lessThanOrEqualTo(maxFailoverCompletionDuration));
+ } else {
+ log.info("No failed commands after failover!");
+ }
+
+ if (lastFailedBeforeFailover.get() != null) {
+ log.info("Last failed command before failover at: {}", lastFailedBeforeFailover.get());
+ }
+ assertFalse(unexpectedErrors.get());
+
+ client.close();
+ }
+
+}
\ No newline at end of file
diff --git a/src/test/java/redis/clients/jedis/mcf/FailbackMechanismIntegrationTest.java b/src/test/java/redis/clients/jedis/mcf/FailbackMechanismIntegrationTest.java
new file mode 100644
index 0000000000..56e23b54f4
--- /dev/null
+++ b/src/test/java/redis/clients/jedis/mcf/FailbackMechanismIntegrationTest.java
@@ -0,0 +1,362 @@
+package redis.clients.jedis.mcf;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.*;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.MockedConstruction;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import redis.clients.jedis.Connection;
+import redis.clients.jedis.ConnectionPool;
+import redis.clients.jedis.DefaultJedisClientConfig;
+import redis.clients.jedis.HostAndPort;
+import redis.clients.jedis.JedisClientConfig;
+import redis.clients.jedis.MultiClusterClientConfig;
+import redis.clients.jedis.providers.MultiClusterPooledConnectionProvider;
+import redis.clients.jedis.providers.MultiClusterPooledConnectionProviderHelper;
+
+@ExtendWith(MockitoExtension.class)
+class FailbackMechanismIntegrationTest {
+
+ private HostAndPort endpoint1;
+ private HostAndPort endpoint2;
+ private HostAndPort endpoint3;
+ private JedisClientConfig clientConfig;
+
+ @BeforeEach
+ void setUp() {
+ endpoint1 = new HostAndPort("localhost", 6379);
+ endpoint2 = new HostAndPort("localhost", 6380);
+ endpoint3 = new HostAndPort("localhost", 6381);
+ clientConfig = DefaultJedisClientConfig.builder().build();
+ }
+
+ private MockedConstruction mockPool() {
+ Connection mockConnection = mock(Connection.class);
+ lenient().when(mockConnection.ping()).thenReturn(true);
+ return mockConstruction(TrackingConnectionPool.class, (mock, context) -> {
+ when(mock.getResource()).thenReturn(mockConnection);
+ doNothing().when(mock).close();
+ });
+ }
+
+ @Test
+ void testFailbackDisabledDoesNotPerformFailback() throws InterruptedException {
+ try (MockedConstruction mockedPool = mockPool()) {
+ // Create clusters with different weights
+ MultiClusterClientConfig.ClusterConfig cluster1 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig).weight(1.0f).healthCheckEnabled(false).build(); // Lower weight
+
+ MultiClusterClientConfig.ClusterConfig cluster2 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint2, clientConfig).weight(2.0f) // Higher weight
+ .healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig config = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { cluster1, cluster2 }).failbackSupported(false) // Disabled
+ .failbackCheckInterval(100) // Short interval for testing
+ .build();
+
+ try (MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(config)) {
+ // Initially, cluster2 should be active (highest weight)
+ assertEquals(provider.getCluster(endpoint2), provider.getCluster());
+
+ // Make cluster2 unhealthy to force failover to cluster1
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint2, HealthStatus.HEALTHY, HealthStatus.UNHEALTHY);
+
+ // Should now be on cluster1 (only healthy option)
+ assertEquals(provider.getCluster(endpoint1), provider.getCluster());
+
+ // Make cluster2 healthy again (higher weight - would normally trigger failback)
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint2, HealthStatus.UNHEALTHY, HealthStatus.HEALTHY);
+
+ // Wait longer than failback interval
+ Thread.sleep(200);
+
+ // Should still be on cluster1 since failback is disabled
+ assertEquals(provider.getCluster(endpoint1), provider.getCluster());
+ }
+ }
+ }
+
+ @Test
+ void testFailbackToHigherWeightCluster() throws InterruptedException {
+ try (MockedConstruction mockedPool = mockPool()) {
+ // Create clusters with different weights
+ MultiClusterClientConfig.ClusterConfig cluster1 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig).weight(2.0f) // Higher weight
+ .healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig.ClusterConfig cluster2 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint2, clientConfig).weight(1.0f) // Lower weight
+ .healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig config = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { cluster1, cluster2 }).failbackSupported(true)
+ .failbackCheckInterval(100) // Short interval for testing
+ .gracePeriod(100)
+ .build();
+
+ try (MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(config)) {
+ // Initially, cluster1 should be active (highest weight)
+ assertEquals(provider.getCluster(endpoint1), provider.getCluster());
+
+ // Make cluster1 unhealthy to force failover to cluster2
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint1, HealthStatus.HEALTHY, HealthStatus.UNHEALTHY);
+
+ // Should now be on cluster2 (lower weight, but only healthy option)
+ assertEquals(provider.getCluster(endpoint2), provider.getCluster());
+
+ // Make cluster1 healthy again
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint1, HealthStatus.UNHEALTHY, HealthStatus.HEALTHY);
+
+ // Wait for failback check interval + some buffer
+ Thread.sleep(250);
+
+ // Should have failed back to cluster1 (higher weight)
+ assertEquals(provider.getCluster(endpoint1), provider.getCluster());
+ }
+ }
+ }
+
+ @Test
+ void testNoFailbackToLowerWeightCluster() throws InterruptedException {
+ try (MockedConstruction mockedPool = mockPool()) {
+ // Create three clusters with different weights to properly test no failback to lower weight
+ MultiClusterClientConfig.ClusterConfig cluster1 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig).weight(1.0f) // Lowest weight
+ .healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig.ClusterConfig cluster2 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint2, clientConfig).weight(2.0f) // Medium weight
+ .healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig.ClusterConfig cluster3 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint3, clientConfig).weight(3.0f) // Highest weight
+ .healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig config = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { cluster1, cluster2, cluster3 }).failbackSupported(true)
+ .failbackCheckInterval(100).build();
+
+ try (MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(config)) {
+ // Initially, cluster3 should be active (highest weight)
+ assertEquals(provider.getCluster(endpoint3), provider.getCluster());
+
+ // Make cluster3 unhealthy to force failover to cluster2 (medium weight)
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint3, HealthStatus.HEALTHY, HealthStatus.UNHEALTHY);
+
+ // Should now be on cluster2 (highest weight among healthy clusters)
+ assertEquals(provider.getCluster(endpoint2), provider.getCluster());
+
+ // Make cluster1 (lowest weight) healthy - this should NOT trigger failback
+ // since we don't failback to lower weight clusters
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint1, HealthStatus.UNHEALTHY, HealthStatus.HEALTHY);
+
+ // Wait for failback check interval
+ Thread.sleep(250);
+
+ // Should still be on cluster2 (no failback to lower weight cluster1)
+ assertEquals(provider.getCluster(endpoint2), provider.getCluster());
+ }
+ }
+ }
+
+ @Test
+ void testFailbackToHigherWeightClusterImmediately() throws InterruptedException {
+ try (MockedConstruction mockedPool = mockPool()) {
+ MultiClusterClientConfig.ClusterConfig cluster1 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig).weight(2.0f).healthCheckEnabled(false).build(); // Higher weight
+
+ MultiClusterClientConfig.ClusterConfig cluster2 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint2, clientConfig).weight(1.0f).healthCheckEnabled(false).build(); // Lower weight
+
+ MultiClusterClientConfig config = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { cluster1, cluster2 }).failbackSupported(true)
+ .failbackCheckInterval(100).gracePeriod(50).build();
+
+ try (MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(config)) {
+ // Initially, cluster1 should be active (highest weight)
+ assertEquals(provider.getCluster(endpoint1), provider.getCluster());
+
+ // Make cluster1 unhealthy to force failover to cluster2
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint1, HealthStatus.HEALTHY, HealthStatus.UNHEALTHY);
+
+ // Should now be on cluster2 (only healthy option)
+ assertEquals(provider.getCluster(endpoint2), provider.getCluster());
+
+ // Make cluster1 healthy again
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint1, HealthStatus.UNHEALTHY, HealthStatus.HEALTHY);
+
+ // Wait for failback check
+ Thread.sleep(150);
+
+ // Should have failed back to cluster1 immediately (higher weight, no stability period required)
+ assertEquals(provider.getCluster(endpoint1), provider.getCluster());
+ }
+ }
+ }
+
+ @Test
+ void testUnhealthyClusterCancelsFailback() throws InterruptedException {
+ try (MockedConstruction mockedPool = mockPool()) {
+ MultiClusterClientConfig.ClusterConfig cluster1 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig).weight(2.0f).healthCheckEnabled(false).build(); // Higher weight
+
+ MultiClusterClientConfig.ClusterConfig cluster2 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint2, clientConfig).weight(1.0f).healthCheckEnabled(false).build(); // Lower weight
+
+ MultiClusterClientConfig config = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { cluster1, cluster2 }).failbackSupported(true)
+ .failbackCheckInterval(200).build();
+
+ try (MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(config)) {
+ // Initially, cluster1 should be active (highest weight)
+ assertEquals(provider.getCluster(endpoint1), provider.getCluster());
+
+ // Make cluster1 unhealthy to force failover to cluster2
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint1, HealthStatus.HEALTHY, HealthStatus.UNHEALTHY);
+
+ // Should now be on cluster2 (only healthy option)
+ assertEquals(provider.getCluster(endpoint2), provider.getCluster());
+
+ // Make cluster1 healthy again (should trigger failback attempt)
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint1, HealthStatus.UNHEALTHY, HealthStatus.HEALTHY);
+
+ // Wait a bit
+ Thread.sleep(100);
+
+ // Make cluster1 unhealthy again before failback completes
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint1, HealthStatus.HEALTHY, HealthStatus.UNHEALTHY);
+
+ // Wait past the original failback interval
+ Thread.sleep(150);
+
+ // Should still be on cluster2 (failback was cancelled due to cluster1 becoming unhealthy)
+ assertEquals(provider.getCluster(endpoint2), provider.getCluster());
+ }
+ }
+ }
+
+ @Test
+ void testMultipleClusterFailbackPriority() throws InterruptedException {
+ try (MockedConstruction mockedPool = mockPool()) {
+ MultiClusterClientConfig.ClusterConfig cluster1 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig).weight(1.0f).healthCheckEnabled(false).build(); // Lowest weight
+
+ MultiClusterClientConfig.ClusterConfig cluster2 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint2, clientConfig).weight(2.0f).healthCheckEnabled(false).build(); // Medium weight
+
+ MultiClusterClientConfig.ClusterConfig cluster3 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint3, clientConfig).weight(3.0f) // Highest weight
+ .healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig config = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { cluster1, cluster2, cluster3 }).failbackSupported(true)
+ .failbackCheckInterval(100).gracePeriod(100).build();
+
+ try (MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(config)) {
+ // Initially, cluster3 should be active (highest weight)
+ assertEquals(provider.getCluster(endpoint3), provider.getCluster());
+
+ // Make cluster3 unhealthy to force failover to cluster2 (next highest weight)
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint3, HealthStatus.HEALTHY, HealthStatus.UNHEALTHY);
+
+ // Should now be on cluster2 (highest weight among healthy clusters)
+ assertEquals(provider.getCluster(endpoint2), provider.getCluster());
+
+ // Make cluster3 healthy again
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint3, HealthStatus.UNHEALTHY, HealthStatus.HEALTHY);
+
+ // Wait for failback
+ Thread.sleep(200);
+
+ // Should fail back to cluster3 (highest weight)
+ assertEquals(provider.getCluster(endpoint3), provider.getCluster());
+ }
+ }
+ }
+
+ @Test
+ void testGracePeriodDisablesClusterOnUnhealthy() throws InterruptedException {
+ try (MockedConstruction mockedPool = mockPool()) {
+ MultiClusterClientConfig.ClusterConfig cluster1 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig).weight(1.0f).healthCheckEnabled(false).build(); // Lower weight
+
+ MultiClusterClientConfig.ClusterConfig cluster2 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint2, clientConfig).weight(2.0f).healthCheckEnabled(false).build(); // Higher weight
+
+ MultiClusterClientConfig config = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { cluster1, cluster2 }).failbackSupported(true)
+ .failbackCheckInterval(100).gracePeriod(200) // 200ms grace period
+ .build();
+
+ try (MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(config)) {
+ // Initially, cluster2 should be active (highest weight)
+ assertEquals(provider.getCluster(endpoint2), provider.getCluster());
+
+ // Now make cluster2 unhealthy - it should be disabled for grace period
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint2, HealthStatus.HEALTHY, HealthStatus.UNHEALTHY);
+
+ // Should failover to cluster1
+ assertEquals(provider.getCluster(endpoint1), provider.getCluster());
+
+ // Cluster2 should be in grace period
+ assertTrue(provider.getCluster(endpoint2).isInGracePeriod());
+ }
+ }
+ }
+
+ @Test
+ void testGracePeriodReEnablesClusterAfterPeriod() throws InterruptedException {
+ try (MockedConstruction mockedPool = mockPool()) {
+ MultiClusterClientConfig.ClusterConfig cluster1 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig).weight(1.0f).healthCheckEnabled(false).build(); // Lower weight
+
+ MultiClusterClientConfig.ClusterConfig cluster2 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint2, clientConfig).weight(2.0f).healthCheckEnabled(false).build(); // Higher weight
+
+ MultiClusterClientConfig config = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { cluster1, cluster2 }).failbackSupported(true)
+ .failbackCheckInterval(50) // Short interval for testing
+ .gracePeriod(100) // Short grace period for testing
+ .build();
+
+ try (MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(config)) {
+ // Initially, cluster2 should be active (highest weight)
+ assertEquals(provider.getCluster(endpoint2), provider.getCluster());
+
+ // Make cluster2 unhealthy to start grace period and force failover
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint2, HealthStatus.HEALTHY, HealthStatus.UNHEALTHY);
+
+ // Should failover to cluster1
+ assertEquals(provider.getCluster(endpoint1), provider.getCluster());
+
+ // Cluster2 should be in grace period
+ assertTrue(provider.getCluster(endpoint2).isInGracePeriod());
+
+ // Make cluster2 healthy again while it's still in grace period
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint2, HealthStatus.UNHEALTHY, HealthStatus.HEALTHY);
+
+ // Should still be on cluster1 because cluster2 is in grace period
+ assertEquals(provider.getCluster(endpoint1), provider.getCluster());
+
+ // Wait for grace period to expire
+ Thread.sleep(150);
+
+ // Cluster2 should no longer be in grace period
+ assertFalse(provider.getCluster(endpoint2).isInGracePeriod());
+
+ // Wait for failback check to run
+ Thread.sleep(100);
+
+ // Should now failback to cluster2 (higher weight) since grace period has expired
+ assertEquals(provider.getCluster(endpoint2), provider.getCluster());
+ }
+ }
+ }
+
+
+}
diff --git a/src/test/java/redis/clients/jedis/mcf/FailbackMechanismUnitTest.java b/src/test/java/redis/clients/jedis/mcf/FailbackMechanismUnitTest.java
new file mode 100644
index 0000000000..535cdde2b9
--- /dev/null
+++ b/src/test/java/redis/clients/jedis/mcf/FailbackMechanismUnitTest.java
@@ -0,0 +1,184 @@
+package redis.clients.jedis.mcf;
+
+import static org.junit.jupiter.api.Assertions.*;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import redis.clients.jedis.DefaultJedisClientConfig;
+import redis.clients.jedis.HostAndPort;
+import redis.clients.jedis.JedisClientConfig;
+import redis.clients.jedis.MultiClusterClientConfig;
+
+@ExtendWith(MockitoExtension.class)
+class FailbackMechanismUnitTest {
+
+ private HostAndPort endpoint1;
+ private JedisClientConfig clientConfig;
+
+ @BeforeEach
+ void setUp() {
+ endpoint1 = new HostAndPort("localhost", 6379);
+ clientConfig = DefaultJedisClientConfig.builder().build();
+ }
+
+ @Test
+ void testFailbackCheckIntervalConfiguration() {
+ // Test default value
+ MultiClusterClientConfig.ClusterConfig clusterConfig = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig)
+ .healthCheckEnabled(false)
+ .build();
+
+ MultiClusterClientConfig defaultConfig = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[]{clusterConfig})
+ .build();
+
+ assertEquals(5000, defaultConfig.getFailbackCheckInterval());
+
+ // Test custom value
+ MultiClusterClientConfig customConfig = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[]{clusterConfig})
+ .failbackCheckInterval(3000)
+ .build();
+
+ assertEquals(3000, customConfig.getFailbackCheckInterval());
+ }
+
+ @Test
+ void testFailbackSupportedConfiguration() {
+ MultiClusterClientConfig.ClusterConfig clusterConfig = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig)
+ .healthCheckEnabled(false)
+ .build();
+
+ // Test default (should be true)
+ MultiClusterClientConfig defaultConfig = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[]{clusterConfig})
+ .build();
+
+ assertTrue(defaultConfig.isFailbackSupported());
+
+ // Test disabled
+ MultiClusterClientConfig disabledConfig = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[]{clusterConfig})
+ .failbackSupported(false)
+ .build();
+
+ assertFalse(disabledConfig.isFailbackSupported());
+ }
+
+ @Test
+ void testFailbackCheckIntervalValidation() {
+ MultiClusterClientConfig.ClusterConfig clusterConfig = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig)
+ .healthCheckEnabled(false)
+ .build();
+
+ // Test zero interval (should be allowed)
+ MultiClusterClientConfig zeroConfig = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[]{clusterConfig})
+ .failbackCheckInterval(0)
+ .build();
+
+ assertEquals(0, zeroConfig.getFailbackCheckInterval());
+
+ // Test negative interval (should be allowed - implementation decision)
+ MultiClusterClientConfig negativeConfig = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[]{clusterConfig})
+ .failbackCheckInterval(-1000)
+ .build();
+
+ assertEquals(-1000, negativeConfig.getFailbackCheckInterval());
+ }
+
+ @Test
+ void testBuilderChaining() {
+ MultiClusterClientConfig.ClusterConfig clusterConfig = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig)
+ .healthCheckEnabled(false)
+ .build();
+
+ // Test that builder methods can be chained
+ MultiClusterClientConfig config = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[]{clusterConfig})
+ .failbackSupported(true)
+ .failbackCheckInterval(2000)
+ .retryOnFailover(true)
+ .build();
+
+ assertTrue(config.isFailbackSupported());
+ assertEquals(2000, config.getFailbackCheckInterval());
+ assertTrue(config.isRetryOnFailover());
+ }
+
+ @Test
+ void testGracePeriodConfiguration() {
+ // Test default value
+ MultiClusterClientConfig.ClusterConfig clusterConfig = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig)
+ .healthCheckEnabled(false)
+ .build();
+
+ MultiClusterClientConfig defaultConfig = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[]{clusterConfig})
+ .build();
+
+ assertEquals(10000, defaultConfig.getGracePeriod()); // Default is 10 seconds
+
+ // Test custom value
+ MultiClusterClientConfig customConfig = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[]{clusterConfig})
+ .gracePeriod(5000)
+ .build();
+
+ assertEquals(5000, customConfig.getGracePeriod());
+ }
+
+ @Test
+ void testGracePeriodValidation() {
+ MultiClusterClientConfig.ClusterConfig clusterConfig = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig)
+ .healthCheckEnabled(false)
+ .build();
+
+ // Test zero grace period (should be allowed)
+ MultiClusterClientConfig zeroConfig = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[]{clusterConfig})
+ .gracePeriod(0)
+ .build();
+
+ assertEquals(0, zeroConfig.getGracePeriod());
+
+ // Test negative grace period (should be allowed - implementation decision)
+ MultiClusterClientConfig negativeConfig = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[]{clusterConfig})
+ .gracePeriod(-1000)
+ .build();
+
+ assertEquals(-1000, negativeConfig.getGracePeriod());
+ }
+
+ @Test
+ void testGracePeriodBuilderChaining() {
+ MultiClusterClientConfig.ClusterConfig clusterConfig = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig)
+ .healthCheckEnabled(false)
+ .build();
+
+ // Test that builder methods can be chained
+ MultiClusterClientConfig config = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[]{clusterConfig})
+ .failbackSupported(true)
+ .failbackCheckInterval(2000)
+ .gracePeriod(8000)
+ .retryOnFailover(true)
+ .build();
+
+ assertTrue(config.isFailbackSupported());
+ assertEquals(2000, config.getFailbackCheckInterval());
+ assertEquals(8000, config.getGracePeriod());
+ assertTrue(config.isRetryOnFailover());
+ }
+}
diff --git a/src/test/java/redis/clients/jedis/mcf/HealthCheckIntegrationTest.java b/src/test/java/redis/clients/jedis/mcf/HealthCheckIntegrationTest.java
new file mode 100644
index 0000000000..ed2cf8139e
--- /dev/null
+++ b/src/test/java/redis/clients/jedis/mcf/HealthCheckIntegrationTest.java
@@ -0,0 +1,110 @@
+package redis.clients.jedis.mcf;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import org.junit.jupiter.api.Test;
+
+import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig.SlidingWindowType;
+
+import redis.clients.jedis.EndpointConfig;
+import redis.clients.jedis.HostAndPorts;
+import redis.clients.jedis.JedisClientConfig;
+import redis.clients.jedis.MultiClusterClientConfig;
+import redis.clients.jedis.UnifiedJedis;
+import redis.clients.jedis.MultiClusterClientConfig.ClusterConfig;
+import redis.clients.jedis.providers.MultiClusterPooledConnectionProvider;
+import redis.clients.jedis.scenario.RecommendedSettings;
+
+public class HealthCheckIntegrationTest {
+
+ private final EndpointConfig endpoint1 = HostAndPorts.getRedisEndpoint("standalone0");
+ private final JedisClientConfig clientConfig = endpoint1.getClientConfigBuilder()
+ .socketTimeoutMillis(RecommendedSettings.DEFAULT_TIMEOUT_MS)
+ .connectionTimeoutMillis(RecommendedSettings.DEFAULT_TIMEOUT_MS).build();
+
+ @Test
+ public void testDisableHealthCheck() {
+ // No health check strategy supplier means health check is disabled
+ MultiClusterPooledConnectionProvider customProvider = getMCCF(null);
+ try (UnifiedJedis customClient = new UnifiedJedis(customProvider)) {
+ // Verify that the client can connect and execute commands
+ String result = customClient.ping();
+ assertEquals("PONG", result);
+ }
+ }
+
+ @Test
+ public void testDefaultStrategySupplier() {
+ // Create a default strategy supplier that creates EchoStrategy instances
+ MultiClusterClientConfig.StrategySupplier defaultSupplier = (hostAndPort, jedisClientConfig) -> {
+ return new EchoStrategy(hostAndPort, jedisClientConfig);
+ };
+ MultiClusterPooledConnectionProvider customProvider = getMCCF(defaultSupplier);
+ try (UnifiedJedis customClient = new UnifiedJedis(customProvider)) {
+ // Verify that the client can connect and execute commands
+ String result = customClient.ping();
+ assertEquals("PONG", result);
+ }
+ }
+
+ @Test
+ public void testCustomStrategySupplier() {
+ // Create a StrategySupplier that uses the JedisClientConfig when available
+ MultiClusterClientConfig.StrategySupplier strategySupplier = (hostAndPort, jedisClientConfig) -> {
+ return new HealthCheckStrategy() {
+
+ @Override
+ public int getInterval() {
+ return 500;
+ }
+
+ @Override
+ public int getTimeout() {
+ return 500;
+ }
+
+ @Override
+ public HealthStatus doHealthCheck(Endpoint endpoint) {
+ // Create connection per health check to avoid resource leak
+ try (UnifiedJedis pinger = new UnifiedJedis(hostAndPort, jedisClientConfig)) {
+ String result = pinger.ping();
+ return "PONG".equals(result) ? HealthStatus.HEALTHY : HealthStatus.UNHEALTHY;
+ } catch (Exception e) {
+ return HealthStatus.UNHEALTHY;
+ }
+ }
+
+ };
+ };
+
+ MultiClusterPooledConnectionProvider customProvider = getMCCF(strategySupplier);
+ try (UnifiedJedis customClient = new UnifiedJedis(customProvider)) {
+ // Verify that the client can connect and execute commands
+ String result = customClient.ping();
+ assertEquals("PONG", result);
+ }
+ }
+
+ private MultiClusterPooledConnectionProvider getMCCF(MultiClusterClientConfig.StrategySupplier strategySupplier) {
+ Function modifier = builder -> strategySupplier == null
+ ? builder.healthCheckEnabled(false)
+ : builder.healthCheckStrategySupplier(strategySupplier);
+
+ List clusterConfigs = Arrays
+ .stream(new EndpointConfig[] { endpoint1 }).map(e -> modifier
+ .apply(MultiClusterClientConfig.ClusterConfig.builder(e.getHostAndPort(), clientConfig)).build())
+ .collect(Collectors.toList());
+
+ MultiClusterClientConfig mccf = new MultiClusterClientConfig.Builder(clusterConfigs).retryMaxAttempts(1)
+ .retryWaitDuration(1).circuitBreakerSlidingWindowType(SlidingWindowType.COUNT_BASED)
+ .circuitBreakerSlidingWindowSize(1).circuitBreakerFailureRateThreshold(100)
+ .circuitBreakerSlidingWindowMinCalls(1).build();
+
+ return new MultiClusterPooledConnectionProvider(mccf);
+ }
+}
diff --git a/src/test/java/redis/clients/jedis/mcf/HealthCheckTest.java b/src/test/java/redis/clients/jedis/mcf/HealthCheckTest.java
new file mode 100644
index 0000000000..bfbfd452fe
--- /dev/null
+++ b/src/test/java/redis/clients/jedis/mcf/HealthCheckTest.java
@@ -0,0 +1,433 @@
+package redis.clients.jedis.mcf;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import redis.clients.jedis.DefaultJedisClientConfig;
+import redis.clients.jedis.HostAndPort;
+import redis.clients.jedis.JedisClientConfig;
+import redis.clients.jedis.MultiClusterClientConfig;
+import redis.clients.jedis.UnifiedJedis;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.Mockito.*;
+
+public class HealthCheckTest {
+
+ @Mock
+ private UnifiedJedis mockJedis;
+
+ @Mock
+ private HealthCheckStrategy mockStrategy;
+
+ private HealthCheckStrategy alwaysHealthyStrategy = new HealthCheckStrategy() {
+ @Override
+ public int getInterval() {
+ return 100;
+ }
+
+ @Override
+ public int getTimeout() {
+ return 50;
+ }
+
+ @Override
+ public HealthStatus doHealthCheck(Endpoint endpoint) {
+ return HealthStatus.HEALTHY;
+ }
+ };
+
+ @Mock
+ private Consumer mockCallback;
+
+ private HostAndPort testEndpoint;
+ private JedisClientConfig testConfig;
+
+ @BeforeEach
+ void setUp() {
+ MockitoAnnotations.openMocks(this);
+ testEndpoint = new HostAndPort("localhost", 6379);
+ testConfig = DefaultJedisClientConfig.builder().build();
+ }
+
+ // ========== HealthCheckCollection Tests ==========
+
+ @Test
+ void testHealthCheckCollectionAdd() {
+ HealthCheckCollection collection = new HealthCheckCollection();
+ HealthCheck healthCheck = new HealthCheck(testEndpoint, mockStrategy, mockCallback);
+
+ HealthCheck previous = collection.add(healthCheck);
+ assertNull(previous);
+
+ assertEquals(healthCheck, collection.get(testEndpoint));
+ }
+
+ @Test
+ void testHealthCheckCollectionRemoveByEndpoint() {
+ HealthCheckCollection collection = new HealthCheckCollection();
+ HealthCheck healthCheck = new HealthCheck(testEndpoint, mockStrategy, mockCallback);
+
+ collection.add(healthCheck);
+ HealthCheck removed = collection.remove(testEndpoint);
+
+ assertEquals(healthCheck, removed);
+ assertNull(collection.get(testEndpoint));
+ }
+
+ @Test
+ void testHealthCheckCollectionAddAll() {
+ HealthCheckCollection collection = new HealthCheckCollection();
+ HealthCheck[] healthChecks = { new HealthCheck(new HostAndPort("host1", 6379), mockStrategy, mockCallback),
+ new HealthCheck(new HostAndPort("host2", 6379), mockStrategy, mockCallback) };
+
+ HealthCheck[] previous = collection.addAll(healthChecks);
+
+ assertNotNull(previous);
+ assertEquals(2, previous.length);
+ assertNull(previous[0]); // No previous health check for host1
+ assertNull(previous[1]); // No previous health check for host2
+
+ assertEquals(healthChecks[0], collection.get(new HostAndPort("host1", 6379)));
+ assertEquals(healthChecks[1], collection.get(new HostAndPort("host2", 6379)));
+ }
+
+ @Test
+ void testHealthCheckCollectionReplacement() {
+ HealthCheckCollection collection = new HealthCheckCollection();
+ HealthCheck healthCheck1 = new HealthCheck(testEndpoint, mockStrategy, mockCallback);
+ HealthCheck healthCheck2 = new HealthCheck(testEndpoint, mockStrategy, mockCallback);
+
+ collection.add(healthCheck1);
+ HealthCheck previous = collection.add(healthCheck2);
+
+ assertEquals(healthCheck1, previous);
+ assertEquals(healthCheck2, collection.get(testEndpoint));
+ }
+
+ @Test
+ void testHealthCheckCollectionRemoveByHealthCheck() {
+ HealthCheckCollection collection = new HealthCheckCollection();
+ HealthCheck healthCheck = new HealthCheck(testEndpoint, mockStrategy, mockCallback);
+
+ collection.add(healthCheck);
+ HealthCheck removed = collection.remove(healthCheck);
+
+ assertEquals(healthCheck, removed);
+ assertNull(collection.get(testEndpoint));
+ }
+
+ // ========== HealthCheck Tests ==========
+
+ @Test
+ void testHealthCheckStatusUpdate() throws InterruptedException {
+ when(mockStrategy.getInterval()).thenReturn(100);
+ when(mockStrategy.getTimeout()).thenReturn(50);
+ when(mockStrategy.doHealthCheck(any(Endpoint.class))).thenReturn(HealthStatus.UNHEALTHY);
+
+ CountDownLatch latch = new CountDownLatch(1);
+ Consumer callback = event -> {
+ assertEquals(HealthStatus.UNKNOWN, event.getOldStatus());
+ assertEquals(HealthStatus.UNHEALTHY, event.getNewStatus());
+ latch.countDown();
+ };
+
+ HealthCheck healthCheck = new HealthCheck(testEndpoint, mockStrategy, callback);
+ healthCheck.start();
+
+ assertTrue(latch.await(2, TimeUnit.SECONDS));
+ healthCheck.stop();
+ }
+
+ @Test
+ void testHealthCheckStop() {
+ when(mockStrategy.getInterval()).thenReturn(1000);
+ when(mockStrategy.getTimeout()).thenReturn(500);
+
+ HealthCheck healthCheck = new HealthCheck(testEndpoint, mockStrategy, mockCallback);
+ healthCheck.start();
+
+ assertDoesNotThrow(() -> healthCheck.stop());
+ }
+
+ // ========== HealthStatusManager Tests ==========
+
+ @Test
+ void testHealthStatusManagerRegisterListener() {
+ HealthStatusManager manager = new HealthStatusManager();
+ HealthStatusListener listener = mock(HealthStatusListener.class);
+
+ manager.registerListener(listener);
+
+ // Verify listener is registered by triggering an event
+ manager.add(testEndpoint, alwaysHealthyStrategy);
+ // Give some time for health check to run
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ }
+
+ verify(listener, atLeastOnce()).onStatusChange(any(HealthStatusChangeEvent.class));
+ }
+
+ @Test
+ void testHealthStatusManagerUnregisterListener() {
+ HealthStatusManager manager = new HealthStatusManager();
+ HealthStatusListener listener = mock(HealthStatusListener.class);
+
+ manager.registerListener(listener);
+ manager.unregisterListener(listener);
+
+ manager.add(testEndpoint, alwaysHealthyStrategy);
+
+ // Give some time for potential health check
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ }
+
+ verify(listener, never()).onStatusChange(any(HealthStatusChangeEvent.class));
+ }
+
+ @Test
+ void testHealthStatusManagerEndpointSpecificListener() {
+ HealthStatusManager manager = new HealthStatusManager();
+ HealthStatusListener listener = mock(HealthStatusListener.class);
+ HostAndPort otherEndpoint = new HostAndPort("other", 6379);
+
+ manager.registerListener(testEndpoint, listener);
+ manager.add(testEndpoint, alwaysHealthyStrategy);
+ manager.add(otherEndpoint, alwaysHealthyStrategy);
+
+ // Give some time for health checks
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ }
+
+ // Listener should only receive events for testEndpoint
+ verify(listener, atLeastOnce()).onStatusChange(argThat(event -> event.getEndpoint().equals(testEndpoint)));
+ }
+
+ @Test
+ void testHealthStatusManagerLifecycle() throws InterruptedException {
+ HealthStatusManager manager = new HealthStatusManager();
+
+ // Before adding health check
+ assertEquals(HealthStatus.UNKNOWN, manager.getHealthStatus(testEndpoint));
+
+ // Set up event listener to wait for initial health check completion
+ CountDownLatch healthCheckCompleteLatch = new CountDownLatch(1);
+ HealthStatusListener listener = event -> healthCheckCompleteLatch.countDown();
+
+ // Register listener before adding health check to capture the initial event
+ manager.registerListener(testEndpoint, listener);
+
+ // Add health check - this will start async health checking
+ manager.add(testEndpoint, alwaysHealthyStrategy);
+
+ // Initially should still be UNKNOWN until first check completes
+ assertEquals(HealthStatus.UNKNOWN, manager.getHealthStatus(testEndpoint));
+
+ // Wait for initial health check to complete
+ assertTrue(healthCheckCompleteLatch.await(2, TimeUnit.SECONDS),
+ "Initial health check should complete within timeout");
+
+ // Now should be HEALTHY after initial check
+ assertEquals(HealthStatus.HEALTHY, manager.getHealthStatus(testEndpoint));
+
+ // Clean up and verify removal
+ manager.remove(testEndpoint);
+ assertEquals(HealthStatus.UNKNOWN, manager.getHealthStatus(testEndpoint));
+ }
+
+ // ========== EchoStrategy Tests ==========
+
+ @Test
+ void testEchoStrategyCustomIntervalTimeout() {
+ EchoStrategy strategy = new EchoStrategy(testEndpoint, testConfig, 2000, 1500);
+
+ assertEquals(2000, strategy.getInterval());
+ assertEquals(1500, strategy.getTimeout());
+ }
+
+ @Test
+ void testEchoStrategyDefaultSupplier() {
+ MultiClusterClientConfig.StrategySupplier supplier = EchoStrategy.DEFAULT;
+ HealthCheckStrategy strategy = supplier.get(testEndpoint, testConfig);
+
+ assertInstanceOf(EchoStrategy.class, strategy);
+ }
+
+ // ========== Failover configuration Tests ==========
+
+ @Test
+ void testNewFieldLocations() {
+ // Test new field locations in ClusterConfig and MultiClusterClientConfig
+ MultiClusterClientConfig.ClusterConfig clusterConfig = MultiClusterClientConfig.ClusterConfig
+ .builder(testEndpoint, testConfig).weight(2.5f).build();
+
+ MultiClusterClientConfig multiConfig = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { clusterConfig }).retryOnFailover(true)
+ .failbackSupported(false).build();
+
+ assertEquals(2.5f, clusterConfig.getWeight());
+ assertTrue(multiConfig.isRetryOnFailover());
+ assertFalse(multiConfig.isFailbackSupported());
+ }
+
+ @Test
+ void testDefaultValues() {
+ // Test default values in ClusterConfig
+ MultiClusterClientConfig.ClusterConfig clusterConfig = MultiClusterClientConfig.ClusterConfig
+ .builder(testEndpoint, testConfig).build();
+
+ assertEquals(1.0f, clusterConfig.getWeight()); // Default weight
+ assertEquals(EchoStrategy.DEFAULT, clusterConfig.getHealthCheckStrategySupplier()); // Default is null (no
+ // health check)
+
+ // Test default values in MultiClusterClientConfig
+ MultiClusterClientConfig multiConfig = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { clusterConfig }).build();
+
+ assertFalse(multiConfig.isRetryOnFailover()); // Default is false
+ assertTrue(multiConfig.isFailbackSupported()); // Default is true
+ }
+
+ @Test
+ void testClusterConfigWithHealthCheckStrategy() {
+ HealthCheckStrategy customStrategy = mock(HealthCheckStrategy.class);
+
+ MultiClusterClientConfig.StrategySupplier supplier = (hostAndPort, jedisClientConfig) -> customStrategy;
+
+ MultiClusterClientConfig.ClusterConfig clusterConfig = MultiClusterClientConfig.ClusterConfig
+ .builder(testEndpoint, testConfig).healthCheckStrategySupplier(supplier).build();
+
+ assertNotNull(clusterConfig.getHealthCheckStrategySupplier());
+ HealthCheckStrategy result = clusterConfig.getHealthCheckStrategySupplier().get(testEndpoint, testConfig);
+ assertEquals(customStrategy, result);
+ }
+
+ @Test
+ void testClusterConfigWithStrategySupplier() {
+ MultiClusterClientConfig.StrategySupplier customSupplier = (hostAndPort, jedisClientConfig) -> {
+ return mock(HealthCheckStrategy.class);
+ };
+
+ MultiClusterClientConfig.ClusterConfig clusterConfig = MultiClusterClientConfig.ClusterConfig
+ .builder(testEndpoint, testConfig).healthCheckStrategySupplier(customSupplier).build();
+
+ assertEquals(customSupplier, clusterConfig.getHealthCheckStrategySupplier());
+ }
+
+ @Test
+ void testClusterConfigWithEchoStrategy() {
+ MultiClusterClientConfig.StrategySupplier echoSupplier = (hostAndPort, jedisClientConfig) -> {
+ return new EchoStrategy(hostAndPort, jedisClientConfig);
+ };
+
+ MultiClusterClientConfig.ClusterConfig clusterConfig = MultiClusterClientConfig.ClusterConfig
+ .builder(testEndpoint, testConfig).healthCheckStrategySupplier(echoSupplier).build();
+
+ MultiClusterClientConfig.StrategySupplier supplier = clusterConfig.getHealthCheckStrategySupplier();
+ assertNotNull(supplier);
+ assertInstanceOf(EchoStrategy.class, supplier.get(testEndpoint, testConfig));
+ }
+
+ @Test
+ void testClusterConfigWithDefaultHealthCheck() {
+ MultiClusterClientConfig.ClusterConfig clusterConfig = MultiClusterClientConfig.ClusterConfig
+ .builder(testEndpoint, testConfig).build(); // Should use default EchoStrategy
+
+ assertNotNull(clusterConfig.getHealthCheckStrategySupplier());
+ assertEquals(EchoStrategy.DEFAULT, clusterConfig.getHealthCheckStrategySupplier());
+ }
+
+ @Test
+ void testClusterConfigWithDisabledHealthCheck() {
+ MultiClusterClientConfig.ClusterConfig clusterConfig = MultiClusterClientConfig.ClusterConfig
+ .builder(testEndpoint, testConfig).healthCheckEnabled(false).build();
+
+ assertNull(clusterConfig.getHealthCheckStrategySupplier());
+ }
+
+ @Test
+ void testClusterConfigHealthCheckEnabledExplicitly() {
+ MultiClusterClientConfig.ClusterConfig clusterConfig = MultiClusterClientConfig.ClusterConfig
+ .builder(testEndpoint, testConfig).healthCheckEnabled(true).build();
+
+ assertNotNull(clusterConfig.getHealthCheckStrategySupplier());
+ assertEquals(EchoStrategy.DEFAULT, clusterConfig.getHealthCheckStrategySupplier());
+ }
+
+ // ========== Integration Tests ==========
+
+ @Test
+ @Timeout(5)
+ void testHealthCheckIntegration() throws InterruptedException {
+ // Create a mock strategy that alternates between healthy and unhealthy
+ AtomicReference statusToReturn = new AtomicReference<>(HealthStatus.HEALTHY);
+ HealthCheckStrategy alternatingStrategy = new HealthCheckStrategy() {
+ @Override
+ public int getInterval() {
+ return 100;
+ }
+
+ @Override
+ public int getTimeout() {
+ return 50;
+ }
+
+ @Override
+ public HealthStatus doHealthCheck(Endpoint endpoint) {
+ HealthStatus current = statusToReturn.get();
+ statusToReturn.set(current == HealthStatus.HEALTHY ? HealthStatus.UNHEALTHY : HealthStatus.HEALTHY);
+ return current;
+ }
+ };
+
+ CountDownLatch statusChangeLatch = new CountDownLatch(2); // Wait for 2 status changes
+ HealthStatusListener listener = event -> statusChangeLatch.countDown();
+
+ HealthStatusManager manager = new HealthStatusManager();
+ manager.registerListener(listener);
+ manager.add(testEndpoint, alternatingStrategy);
+
+ assertTrue(statusChangeLatch.await(3, TimeUnit.SECONDS));
+
+ manager.remove(testEndpoint);
+ }
+
+ @Test
+ void testStrategySupplierPolymorphism() {
+ // Test that the polymorphic design works correctly
+ MultiClusterClientConfig.StrategySupplier supplier = (hostAndPort, jedisClientConfig) -> {
+ if (jedisClientConfig != null) {
+ return new EchoStrategy(hostAndPort, jedisClientConfig, 500, 250);
+ } else {
+ return new EchoStrategy(hostAndPort, DefaultJedisClientConfig.builder().build());
+ }
+ };
+
+ // Test with config
+ HealthCheckStrategy strategyWithConfig = supplier.get(testEndpoint, testConfig);
+ assertNotNull(strategyWithConfig);
+ assertEquals(500, strategyWithConfig.getInterval());
+ assertEquals(250, strategyWithConfig.getTimeout());
+
+ // Test without config
+ HealthCheckStrategy strategyWithoutConfig = supplier.get(testEndpoint, null);
+ assertNotNull(strategyWithoutConfig);
+ assertEquals(1000, strategyWithoutConfig.getInterval()); // Default values
+ assertEquals(1000, strategyWithoutConfig.getTimeout());
+ }
+}
diff --git a/src/test/java/redis/clients/jedis/mcf/MultiClusterDynamicEndpointUnitTest.java b/src/test/java/redis/clients/jedis/mcf/MultiClusterDynamicEndpointUnitTest.java
new file mode 100644
index 0000000000..acc3446c52
--- /dev/null
+++ b/src/test/java/redis/clients/jedis/mcf/MultiClusterDynamicEndpointUnitTest.java
@@ -0,0 +1,203 @@
+package redis.clients.jedis.mcf;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.MockedConstruction;
+
+import redis.clients.jedis.Connection;
+import redis.clients.jedis.ConnectionPool;
+import redis.clients.jedis.DefaultJedisClientConfig;
+import redis.clients.jedis.EndpointConfig;
+import redis.clients.jedis.HostAndPort;
+import redis.clients.jedis.HostAndPorts;
+import redis.clients.jedis.JedisClientConfig;
+import redis.clients.jedis.MultiClusterClientConfig;
+import redis.clients.jedis.MultiClusterClientConfig.ClusterConfig;
+import redis.clients.jedis.exceptions.JedisValidationException;
+import redis.clients.jedis.providers.MultiClusterPooledConnectionProvider;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockConstruction;
+import static org.mockito.Mockito.when;
+
+public class MultiClusterDynamicEndpointUnitTest {
+
+ private MultiClusterPooledConnectionProvider provider;
+ private JedisClientConfig clientConfig;
+ private final EndpointConfig endpoint1 = HostAndPorts.getRedisEndpoint("standalone0");
+ private final EndpointConfig endpoint2 = HostAndPorts.getRedisEndpoint("standalone1");
+
+ @BeforeEach
+ void setUp() {
+ clientConfig = DefaultJedisClientConfig.builder().build();
+
+ // Create initial provider with endpoint1
+ ClusterConfig initialConfig = createClusterConfig(endpoint1.getHostAndPort(), 1.0f);
+
+ MultiClusterClientConfig multiConfig = new MultiClusterClientConfig.Builder(
+ new ClusterConfig[] { initialConfig }).build();
+
+ provider = new MultiClusterPooledConnectionProvider(multiConfig);
+ }
+
+ // Helper method to create cluster configurations
+ private ClusterConfig createClusterConfig(HostAndPort hostAndPort, float weight) {
+ // Disable health check for unit tests to avoid real connections
+ return ClusterConfig.builder(hostAndPort, clientConfig).weight(weight).healthCheckEnabled(false).build();
+ }
+
+ @Test
+ void testAddNewCluster() {
+ ClusterConfig newConfig = createClusterConfig(endpoint2.getHostAndPort(), 2.0f);
+
+ // Should not throw exception
+ assertDoesNotThrow(() -> provider.add(newConfig));
+
+ // Verify the cluster was added by checking it can be retrieved
+ assertNotNull(provider.getCluster(endpoint2.getHostAndPort()));
+ }
+
+ @Test
+ void testAddDuplicateCluster() {
+ ClusterConfig duplicateConfig = createClusterConfig(endpoint1.getHostAndPort(), 2.0f);
+
+ // Should throw validation exception for duplicate endpoint
+ assertThrows(JedisValidationException.class, () -> provider.add(duplicateConfig));
+ }
+
+ @Test
+ void testAddNullClusterConfig() {
+ // Should throw validation exception for null config
+ assertThrows(JedisValidationException.class, () -> provider.add(null));
+ }
+
+ @Test
+ void testRemoveExistingCluster() {
+ Connection mockConnection = mock(Connection.class);
+ when(mockConnection.ping()).thenReturn(true);
+
+ try (MockedConstruction mockedPool = mockPool(mockConnection)) {
+ // Create initial provider with endpoint1
+ ClusterConfig clusterConfig1 = createClusterConfig(endpoint1.getHostAndPort(), 1.0f);
+
+ MultiClusterClientConfig multiConfig = MultiClusterClientConfig
+ .builder(new ClusterConfig[] { clusterConfig1 }).build();
+
+ try (MultiClusterPooledConnectionProvider providerWithMockedPool = new MultiClusterPooledConnectionProvider(
+ multiConfig)) {
+
+ // Add endpoint2 as second cluster
+ ClusterConfig newConfig = createClusterConfig(endpoint2.getHostAndPort(), 2.0f);
+ providerWithMockedPool.add(newConfig);
+
+ // Now remove endpoint1 (original cluster)
+ assertDoesNotThrow(() -> providerWithMockedPool.remove(endpoint1.getHostAndPort()));
+
+ // Verify endpoint1 was removed
+ assertNull(providerWithMockedPool.getCluster(endpoint1.getHostAndPort()));
+ // Verify endpoint2 still exists
+ assertNotNull(providerWithMockedPool.getCluster(endpoint2.getHostAndPort()));
+ }
+ }
+ }
+
+ private MockedConstruction mockPool(Connection mockConnection) {
+ return mockConstruction(TrackingConnectionPool.class, (mock, context) -> {
+ when(mock.getResource()).thenReturn(mockConnection);
+ doNothing().when(mock).close();
+ });
+ }
+
+ @Test
+ void testRemoveNonExistentCluster() {
+ HostAndPort nonExistentEndpoint = new HostAndPort("localhost", 9999);
+
+ // Should throw validation exception for non-existent endpoint
+ assertThrows(JedisValidationException.class, () -> provider.remove(nonExistentEndpoint));
+ }
+
+ @Test
+ void testRemoveLastRemainingCluster() {
+ // Should throw validation exception when trying to remove the last cluster
+ assertThrows(JedisValidationException.class, () -> provider.remove(endpoint1.getHostAndPort()));
+ }
+
+ @Test
+ void testRemoveNullEndpoint() {
+ // Should throw validation exception for null endpoint
+ assertThrows(JedisValidationException.class, () -> provider.remove(null));
+ }
+
+ @Test
+ void testAddAndRemoveMultipleClusters() {
+ // Add endpoint2 as second cluster
+ ClusterConfig config2 = createClusterConfig(endpoint2.getHostAndPort(), 2.0f);
+
+ // Create a third endpoint for this test
+ HostAndPort endpoint3 = new HostAndPort("localhost", 6381);
+ ClusterConfig config3 = createClusterConfig(endpoint3, 3.0f);
+
+ provider.add(config2);
+ provider.add(config3);
+
+ // Verify all clusters exist
+ assertNotNull(provider.getCluster(endpoint1.getHostAndPort()));
+ assertNotNull(provider.getCluster(endpoint2.getHostAndPort()));
+ assertNotNull(provider.getCluster(endpoint3));
+
+ // Remove endpoint2
+ provider.remove(endpoint2.getHostAndPort());
+
+ // Verify correct cluster was removed
+ assertNull(provider.getCluster(endpoint2.getHostAndPort()));
+ assertNotNull(provider.getCluster(endpoint1.getHostAndPort()));
+ assertNotNull(provider.getCluster(endpoint3));
+ }
+
+ @Test
+ void testActiveClusterHandlingOnAdd() {
+ // The initial cluster should be active
+ assertNotNull(provider.getCluster());
+
+ // Add endpoint2 with higher weight
+ ClusterConfig newConfig = createClusterConfig(endpoint2.getHostAndPort(), 5.0f);
+ provider.add(newConfig);
+
+ // Active cluster should still be valid (implementation may or may not switch)
+ assertNotNull(provider.getCluster());
+ }
+
+ @Test
+ void testActiveClusterHandlingOnRemove() {
+ Connection mockConnection = mock(Connection.class);
+ when(mockConnection.ping()).thenReturn(true);
+
+ try (MockedConstruction mockedPool = mockPool(mockConnection)) {
+ // Create initial provider with endpoint1
+ ClusterConfig clusterConfig1 = createClusterConfig(endpoint1.getHostAndPort(), 1.0f);
+
+ MultiClusterClientConfig multiConfig = MultiClusterClientConfig
+ .builder(new ClusterConfig[] { clusterConfig1 }).build();
+
+ try (MultiClusterPooledConnectionProvider providerWithMockedPool = new MultiClusterPooledConnectionProvider(
+ multiConfig)) {
+
+ // Add endpoint2 as second cluster
+ ClusterConfig newConfig = createClusterConfig(endpoint2.getHostAndPort(), 2.0f);
+ providerWithMockedPool.add(newConfig);
+
+ // Get current active cluster
+ Object initialActiveCluster = providerWithMockedPool.getCluster();
+ assertNotNull(initialActiveCluster);
+
+ // Remove endpoint1 (original cluster, might be active)
+ providerWithMockedPool.remove(endpoint1.getHostAndPort());
+
+ // Should still have an active cluster
+ assertNotNull(providerWithMockedPool.getCluster());
+ }
+ }
+ }
+}
diff --git a/src/test/java/redis/clients/jedis/mcf/MultiClusterInitializationTest.java b/src/test/java/redis/clients/jedis/mcf/MultiClusterInitializationTest.java
new file mode 100644
index 0000000000..3c42aefea6
--- /dev/null
+++ b/src/test/java/redis/clients/jedis/mcf/MultiClusterInitializationTest.java
@@ -0,0 +1,175 @@
+package redis.clients.jedis.mcf;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.*;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.MockedConstruction;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import redis.clients.jedis.Connection;
+import redis.clients.jedis.ConnectionPool;
+import redis.clients.jedis.DefaultJedisClientConfig;
+import redis.clients.jedis.HostAndPort;
+import redis.clients.jedis.JedisClientConfig;
+import redis.clients.jedis.MultiClusterClientConfig;
+import redis.clients.jedis.exceptions.JedisValidationException;
+import redis.clients.jedis.providers.MultiClusterPooledConnectionProvider;
+
+/**
+ * Tests for MultiClusterPooledConnectionProvider initialization edge cases
+ */
+@ExtendWith(MockitoExtension.class)
+public class MultiClusterInitializationTest {
+
+ private HostAndPort endpoint1;
+ private HostAndPort endpoint2;
+ private HostAndPort endpoint3;
+ private JedisClientConfig clientConfig;
+
+ @BeforeEach
+ void setUp() {
+ endpoint1 = new HostAndPort("localhost", 6379);
+ endpoint2 = new HostAndPort("localhost", 6380);
+ endpoint3 = new HostAndPort("localhost", 6381);
+ clientConfig = DefaultJedisClientConfig.builder().build();
+ }
+
+ private MockedConstruction mockPool() {
+ Connection mockConnection = mock(Connection.class);
+ lenient().when(mockConnection.ping()).thenReturn(true);
+ return mockConstruction(ConnectionPool.class, (mock, context) -> {
+ when(mock.getResource()).thenReturn(mockConnection);
+ doNothing().when(mock).close();
+ });
+ }
+
+ @Test
+ void testInitializationWithMixedHealthCheckConfiguration() {
+ try (MockedConstruction mockedPool = mockPool()) {
+ // Create clusters with mixed health check configuration
+ MultiClusterClientConfig.ClusterConfig cluster1 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig)
+ .weight(1.0f)
+ .healthCheckEnabled(false) // No health check
+ .build();
+
+ MultiClusterClientConfig.ClusterConfig cluster2 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint2, clientConfig)
+ .weight(2.0f)
+ .healthCheckStrategySupplier(EchoStrategy.DEFAULT) // With health check
+ .build();
+
+ MultiClusterClientConfig config = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { cluster1, cluster2 })
+ .build();
+
+ try (MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(config)) {
+ // Should initialize successfully
+ assertNotNull(provider.getCluster());
+
+ // Should select cluster1 (no health check, assumed healthy) or cluster2 based on weight
+ // Since cluster2 has higher weight and health checks, it should be selected if healthy
+ assertTrue(provider.getCluster() == provider.getCluster(endpoint1) ||
+ provider.getCluster() == provider.getCluster(endpoint2));
+ }
+ }
+ }
+
+ @Test
+ void testInitializationWithAllHealthChecksDisabled() {
+ try (MockedConstruction mockedPool = mockPool()) {
+ // Create clusters with no health checks
+ MultiClusterClientConfig.ClusterConfig cluster1 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig)
+ .weight(1.0f)
+ .healthCheckEnabled(false)
+ .build();
+
+ MultiClusterClientConfig.ClusterConfig cluster2 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint2, clientConfig)
+ .weight(3.0f) // Higher weight
+ .healthCheckEnabled(false)
+ .build();
+
+ MultiClusterClientConfig config = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { cluster1, cluster2 })
+ .build();
+
+ try (MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(config)) {
+ // Should select cluster2 (highest weight, no health checks)
+ assertEquals(provider.getCluster(endpoint2), provider.getCluster());
+ }
+ }
+ }
+
+ @Test
+ void testInitializationWithSingleCluster() {
+ try (MockedConstruction mockedPool = mockPool()) {
+ MultiClusterClientConfig.ClusterConfig cluster = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig)
+ .weight(1.0f)
+ .healthCheckEnabled(false)
+ .build();
+
+ MultiClusterClientConfig config = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { cluster })
+ .build();
+
+ try (MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(config)) {
+ // Should select the only available cluster
+ assertEquals(provider.getCluster(endpoint1), provider.getCluster());
+ }
+ }
+ }
+
+ @Test
+ void testErrorHandlingWithNullConfiguration() {
+ assertThrows(JedisValidationException.class, () -> {
+ new MultiClusterPooledConnectionProvider(null);
+ });
+ }
+
+ @Test
+ void testErrorHandlingWithEmptyClusterArray() {
+ assertThrows(JedisValidationException.class, () -> {
+ new MultiClusterClientConfig.Builder(new MultiClusterClientConfig.ClusterConfig[0]).build();
+ });
+ }
+
+ @Test
+ void testErrorHandlingWithNullClusterConfig() {
+ assertThrows(IllegalArgumentException.class, () -> {
+ new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { null }).build();
+ });
+ }
+
+ @Test
+ void testInitializationWithZeroWeights() {
+ try (MockedConstruction mockedPool = mockPool()) {
+ MultiClusterClientConfig.ClusterConfig cluster1 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig)
+ .weight(0.0f) // Zero weight
+ .healthCheckEnabled(false)
+ .build();
+
+ MultiClusterClientConfig.ClusterConfig cluster2 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint2, clientConfig)
+ .weight(0.0f) // Zero weight
+ .healthCheckEnabled(false)
+ .build();
+
+ MultiClusterClientConfig config = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { cluster1, cluster2 })
+ .build();
+
+ try (MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(config)) {
+ // Should still initialize and select one of the clusters
+ assertNotNull(provider.getCluster());
+ }
+ }
+ }
+}
diff --git a/src/test/java/redis/clients/jedis/mcf/PeriodicFailbackTest.java b/src/test/java/redis/clients/jedis/mcf/PeriodicFailbackTest.java
new file mode 100644
index 0000000000..beede62798
--- /dev/null
+++ b/src/test/java/redis/clients/jedis/mcf/PeriodicFailbackTest.java
@@ -0,0 +1,211 @@
+package redis.clients.jedis.mcf;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.*;
+import static redis.clients.jedis.providers.MultiClusterPooledConnectionProviderHelper.onHealthStatusChange;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.MockedConstruction;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import redis.clients.jedis.Connection;
+import redis.clients.jedis.ConnectionPool;
+import redis.clients.jedis.DefaultJedisClientConfig;
+import redis.clients.jedis.HostAndPort;
+import redis.clients.jedis.JedisClientConfig;
+import redis.clients.jedis.MultiClusterClientConfig;
+import redis.clients.jedis.providers.MultiClusterPooledConnectionProvider;
+import redis.clients.jedis.providers.MultiClusterPooledConnectionProviderHelper;
+
+@ExtendWith(MockitoExtension.class)
+class PeriodicFailbackTest {
+
+ private HostAndPort endpoint1;
+ private HostAndPort endpoint2;
+ private JedisClientConfig clientConfig;
+
+ @BeforeEach
+ void setUp() {
+ endpoint1 = new HostAndPort("localhost", 6379);
+ endpoint2 = new HostAndPort("localhost", 6380);
+ clientConfig = DefaultJedisClientConfig.builder().build();
+ }
+
+ private MockedConstruction mockPool() {
+ Connection mockConnection = mock(Connection.class);
+ lenient().when(mockConnection.ping()).thenReturn(true);
+ return mockConstruction(TrackingConnectionPool.class, (mock, context) -> {
+ when(mock.getResource()).thenReturn(mockConnection);
+ doNothing().when(mock).close();
+ });
+ }
+
+ @Test
+ void testPeriodicFailbackCheckWithDisabledCluster() throws InterruptedException {
+ try (MockedConstruction mockedPool = mockPool()) {
+ MultiClusterClientConfig.ClusterConfig cluster1 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig).weight(1.0f).healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig.ClusterConfig cluster2 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint2, clientConfig).weight(2.0f).healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig config = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { cluster1, cluster2 }).failbackSupported(true)
+ .failbackCheckInterval(100).build();
+
+ try (MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(config)) {
+ // Initially, cluster2 should be active (highest weight: 2.0f vs 1.0f)
+ assertEquals(provider.getCluster(endpoint2), provider.getCluster());
+
+ // Start grace period for cluster2 manually
+ provider.getCluster(endpoint2).setGracePeriod();
+ provider.getCluster(endpoint2).setDisabled(true);
+
+ // Force failover to cluster1 since cluster2 is disabled
+ provider.iterateActiveCluster(SwitchReason.FORCED);
+
+ // Manually trigger periodic check
+ MultiClusterPooledConnectionProviderHelper.periodicFailbackCheck(provider);
+
+ // Should still be on cluster1 (cluster2 is in grace period)
+ assertEquals(provider.getCluster(endpoint1), provider.getCluster());
+ }
+ }
+ }
+
+ @Test
+ void testPeriodicFailbackCheckWithHealthyCluster() throws InterruptedException {
+ try (MockedConstruction mockedPool = mockPool()) {
+ MultiClusterClientConfig.ClusterConfig cluster1 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig).weight(1.0f).healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig.ClusterConfig cluster2 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint2, clientConfig).weight(2.0f).healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig config = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { cluster1, cluster2 }).failbackSupported(true)
+ .failbackCheckInterval(50).gracePeriod(100).build(); // Add grace period
+
+ try (MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(config)) {
+ // Initially, cluster2 should be active (highest weight: 2.0f vs 1.0f)
+ assertEquals(provider.getCluster(endpoint2), provider.getCluster());
+
+ // Make cluster2 unhealthy to force failover to cluster1
+ onHealthStatusChange(provider, endpoint2, HealthStatus.HEALTHY, HealthStatus.UNHEALTHY);
+
+ // Should now be on cluster1 (cluster2 is in grace period)
+ assertEquals(provider.getCluster(endpoint1), provider.getCluster());
+
+ // Verify cluster2 is in grace period
+ assertTrue(provider.getCluster(endpoint2).isInGracePeriod());
+
+ // Make cluster2 healthy again (but it's still in grace period)
+ onHealthStatusChange(provider, endpoint2, HealthStatus.UNHEALTHY, HealthStatus.HEALTHY);
+
+ // Trigger periodic check immediately - should still be on cluster1
+ MultiClusterPooledConnectionProviderHelper.periodicFailbackCheck(provider);
+ assertEquals(provider.getCluster(endpoint1), provider.getCluster());
+
+ // Wait for grace period to expire
+ Thread.sleep(150);
+
+ // Trigger periodic check after grace period expires
+ MultiClusterPooledConnectionProviderHelper.periodicFailbackCheck(provider);
+
+ // Should have failed back to cluster2 (higher weight, grace period expired)
+ assertEquals(provider.getCluster(endpoint2), provider.getCluster());
+ }
+ }
+ }
+
+ @Test
+ void testPeriodicFailbackCheckWithFailbackDisabled() throws InterruptedException {
+ try (MockedConstruction mockedPool = mockPool()) {
+ MultiClusterClientConfig.ClusterConfig cluster1 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig).weight(1.0f).healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig.ClusterConfig cluster2 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint2, clientConfig).weight(2.0f).healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig config = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { cluster1, cluster2 }).failbackSupported(false) // Disabled
+ .failbackCheckInterval(50).build();
+
+ try (MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(config)) {
+ // Initially, cluster2 should be active (highest weight: 2.0f vs 1.0f)
+ assertEquals(provider.getCluster(endpoint2), provider.getCluster());
+
+ // Make cluster2 unhealthy to force failover to cluster1
+ onHealthStatusChange(provider, endpoint2, HealthStatus.HEALTHY, HealthStatus.UNHEALTHY);
+
+ // Should now be on cluster1
+ assertEquals(provider.getCluster(endpoint1), provider.getCluster());
+
+ // Make cluster2 healthy again
+ onHealthStatusChange(provider, endpoint2, HealthStatus.UNHEALTHY, HealthStatus.HEALTHY);
+
+ // Wait for stability period
+ Thread.sleep(100);
+
+ // Trigger periodic check
+ MultiClusterPooledConnectionProviderHelper.periodicFailbackCheck(provider);
+
+ // Should still be on cluster1 (failback disabled)
+ assertEquals(provider.getCluster(endpoint1), provider.getCluster());
+ }
+ }
+ }
+
+ @Test
+ void testPeriodicFailbackCheckSelectsHighestWeightCluster() throws InterruptedException {
+ try (MockedConstruction mockedPool = mockPool()) {
+ HostAndPort endpoint3 = new HostAndPort("localhost", 6381);
+
+ MultiClusterClientConfig.ClusterConfig cluster1 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig).weight(1.0f).healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig.ClusterConfig cluster2 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint2, clientConfig).weight(2.0f).healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig.ClusterConfig cluster3 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint3, clientConfig).weight(3.0f) // Highest weight
+ .healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig config = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { cluster1, cluster2, cluster3 }).failbackSupported(true)
+ .failbackCheckInterval(50).gracePeriod(100).build(); // Add grace period
+
+ try (MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(config)) {
+ // Initially, cluster3 should be active (highest weight: 3.0f vs 2.0f vs 1.0f)
+ assertEquals(provider.getCluster(endpoint3), provider.getCluster());
+
+ // Make cluster3 unhealthy to force failover to cluster2 (next highest weight)
+ onHealthStatusChange(provider, endpoint3, HealthStatus.HEALTHY, HealthStatus.UNHEALTHY);
+
+ // Should now be on cluster2 (weight 2.0f, higher than cluster1's 1.0f)
+ assertEquals(provider.getCluster(endpoint2), provider.getCluster());
+
+ // Make cluster2 unhealthy to force failover to cluster1
+ onHealthStatusChange(provider, endpoint2, HealthStatus.HEALTHY, HealthStatus.UNHEALTHY);
+
+ // Should now be on cluster1 (only healthy cluster left)
+ assertEquals(provider.getCluster(endpoint1), provider.getCluster());
+
+ // Make cluster2 and cluster3 healthy again
+ onHealthStatusChange(provider, endpoint2, HealthStatus.UNHEALTHY, HealthStatus.HEALTHY);
+ onHealthStatusChange(provider, endpoint3, HealthStatus.UNHEALTHY, HealthStatus.HEALTHY);
+
+ // Wait for grace period to expire
+ Thread.sleep(150);
+
+ // Trigger periodic check
+ MultiClusterPooledConnectionProviderHelper.periodicFailbackCheck(provider);
+
+ // Should have failed back to cluster3 (highest weight, grace period expired)
+ assertEquals(provider.getCluster(endpoint3), provider.getCluster());
+ }
+ }
+ }
+}
diff --git a/src/test/java/redis/clients/jedis/mcf/StatusTrackerTest.java b/src/test/java/redis/clients/jedis/mcf/StatusTrackerTest.java
new file mode 100644
index 0000000000..3a97267c96
--- /dev/null
+++ b/src/test/java/redis/clients/jedis/mcf/StatusTrackerTest.java
@@ -0,0 +1,243 @@
+package redis.clients.jedis.mcf;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.*;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+import redis.clients.jedis.HostAndPort;
+
+public class StatusTrackerTest {
+
+ @Mock
+ private HealthStatusManager mockHealthStatusManager;
+
+ private StatusTracker statusTracker;
+ private HostAndPort testEndpoint;
+
+ @BeforeEach
+ void setUp() {
+ MockitoAnnotations.openMocks(this);
+ statusTracker = new StatusTracker(mockHealthStatusManager);
+ testEndpoint = new HostAndPort("localhost", 6379);
+ }
+
+ @Test
+ void testWaitForHealthStatus_AlreadyDetermined() {
+ // Given: Health status is already HEALTHY
+ when(mockHealthStatusManager.getHealthStatus(testEndpoint)).thenReturn(HealthStatus.HEALTHY);
+
+ // When: Waiting for health status
+ HealthStatus result = statusTracker.waitForHealthStatus(testEndpoint);
+
+ // Then: Should return immediately without waiting
+ assertEquals(HealthStatus.HEALTHY, result);
+ verify(mockHealthStatusManager, never()).registerListener(eq(testEndpoint), any(HealthStatusListener.class));
+ }
+
+ @Test
+ void testWaitForHealthStatus_EventDriven() throws InterruptedException {
+ // Given: Health status is initially UNKNOWN
+ when(mockHealthStatusManager.getHealthStatus(testEndpoint))
+ .thenReturn(HealthStatus.UNKNOWN) // First call
+ .thenReturn(HealthStatus.UNKNOWN); // Second call after registering listener
+
+ // Capture the registered listener
+ final HealthStatusListener[] capturedListener = new HealthStatusListener[1];
+ doAnswer(invocation -> {
+ capturedListener[0] = invocation.getArgument(1);
+ return null;
+ }).when(mockHealthStatusManager).registerListener(eq(testEndpoint), any(HealthStatusListener.class));
+
+ // When: Start waiting in a separate thread
+ CountDownLatch testLatch = new CountDownLatch(1);
+ final HealthStatus[] result = new HealthStatus[1];
+
+ Thread waitingThread = new Thread(() -> {
+ result[0] = statusTracker.waitForHealthStatus(testEndpoint);
+ testLatch.countDown();
+ });
+ waitingThread.start();
+
+ // Give some time for the listener to be registered
+ Thread.sleep(50);
+
+ // Simulate health status change event
+ assertNotNull(capturedListener[0], "Listener should have been registered");
+ HealthStatusChangeEvent event = new HealthStatusChangeEvent(testEndpoint, HealthStatus.UNKNOWN, HealthStatus.HEALTHY);
+ capturedListener[0].onStatusChange(event);
+
+ // Then: Should complete and return the new status
+ assertTrue(testLatch.await(1, TimeUnit.SECONDS), "Should complete within timeout");
+ assertEquals(HealthStatus.HEALTHY, result[0]);
+
+ // Verify cleanup
+ verify(mockHealthStatusManager).unregisterListener(eq(testEndpoint), eq(capturedListener[0]));
+ }
+
+ @Test
+ void testWaitForHealthStatus_IgnoresUnknownStatus() throws InterruptedException {
+ // Given: Health status is initially UNKNOWN
+ when(mockHealthStatusManager.getHealthStatus(testEndpoint)).thenReturn(HealthStatus.UNKNOWN);
+
+ // Capture the registered listener
+ final HealthStatusListener[] capturedListener = new HealthStatusListener[1];
+ doAnswer(invocation -> {
+ capturedListener[0] = invocation.getArgument(1);
+ return null;
+ }).when(mockHealthStatusManager).registerListener(eq(testEndpoint), any(HealthStatusListener.class));
+
+ // When: Start waiting in a separate thread
+ CountDownLatch testLatch = new CountDownLatch(1);
+ final HealthStatus[] result = new HealthStatus[1];
+
+ Thread waitingThread = new Thread(() -> {
+ result[0] = statusTracker.waitForHealthStatus(testEndpoint);
+ testLatch.countDown();
+ });
+ waitingThread.start();
+
+ // Give some time for the listener to be registered
+ Thread.sleep(50);
+
+ // Simulate UNKNOWN status change (should be ignored)
+ assertNotNull(capturedListener[0], "Listener should have been registered");
+ HealthStatusChangeEvent unknownEvent = new HealthStatusChangeEvent(testEndpoint, HealthStatus.UNKNOWN, HealthStatus.UNKNOWN);
+ capturedListener[0].onStatusChange(unknownEvent);
+
+ // Should not complete yet
+ assertFalse(testLatch.await(100, TimeUnit.MILLISECONDS), "Should not complete with UNKNOWN status");
+
+ // Now send a real status change
+ HealthStatusChangeEvent realEvent = new HealthStatusChangeEvent(testEndpoint, HealthStatus.UNKNOWN, HealthStatus.UNHEALTHY);
+ capturedListener[0].onStatusChange(realEvent);
+
+ // Then: Should complete now
+ assertTrue(testLatch.await(1, TimeUnit.SECONDS), "Should complete with real status");
+ assertEquals(HealthStatus.UNHEALTHY, result[0]);
+ }
+
+ @Test
+ void testWaitForHealthStatus_IgnoresOtherEndpoints() throws InterruptedException {
+ // Given: Health status is initially UNKNOWN
+ when(mockHealthStatusManager.getHealthStatus(testEndpoint)).thenReturn(HealthStatus.UNKNOWN);
+ HostAndPort otherEndpoint = new HostAndPort("other", 6379);
+
+ // Capture the registered listener
+ final HealthStatusListener[] capturedListener = new HealthStatusListener[1];
+ doAnswer(invocation -> {
+ capturedListener[0] = invocation.getArgument(1);
+ return null;
+ }).when(mockHealthStatusManager).registerListener(eq(testEndpoint), any(HealthStatusListener.class));
+
+ // When: Start waiting in a separate thread
+ CountDownLatch testLatch = new CountDownLatch(1);
+ final HealthStatus[] result = new HealthStatus[1];
+
+ Thread waitingThread = new Thread(() -> {
+ result[0] = statusTracker.waitForHealthStatus(testEndpoint);
+ testLatch.countDown();
+ });
+ waitingThread.start();
+
+ // Give some time for the listener to be registered
+ Thread.sleep(50);
+
+ // Simulate status change for different endpoint (should be ignored)
+ assertNotNull(capturedListener[0], "Listener should have been registered");
+ HealthStatusChangeEvent otherEvent = new HealthStatusChangeEvent(otherEndpoint, HealthStatus.UNKNOWN, HealthStatus.HEALTHY);
+ capturedListener[0].onStatusChange(otherEvent);
+
+ // Should not complete yet
+ assertFalse(testLatch.await(100, TimeUnit.MILLISECONDS), "Should not complete with other endpoint");
+
+ // Now send event for correct endpoint
+ HealthStatusChangeEvent correctEvent = new HealthStatusChangeEvent(testEndpoint, HealthStatus.UNKNOWN, HealthStatus.HEALTHY);
+ capturedListener[0].onStatusChange(correctEvent);
+
+ // Then: Should complete now
+ assertTrue(testLatch.await(1, TimeUnit.SECONDS), "Should complete with correct endpoint");
+ assertEquals(HealthStatus.HEALTHY, result[0]);
+ }
+
+ @Test
+ void testWaitForHealthStatus_InterruptHandling() {
+ // Given: Health status is initially UNKNOWN and will stay that way
+ when(mockHealthStatusManager.getHealthStatus(testEndpoint)).thenReturn(HealthStatus.UNKNOWN);
+
+ // When: Interrupt the waiting thread
+ Thread testThread = new Thread(() -> {
+ try {
+ statusTracker.waitForHealthStatus(testEndpoint);
+ fail("Should have thrown JedisConnectionException due to interrupt");
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("Interrupted while waiting"));
+ assertTrue(Thread.currentThread().isInterrupted());
+ }
+ });
+
+ testThread.start();
+
+ // Give thread time to start waiting
+ try {
+ Thread.sleep(50);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+
+ // Interrupt the waiting thread
+ testThread.interrupt();
+
+ try {
+ testThread.join(1000);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+
+ assertFalse(testThread.isAlive(), "Test thread should have completed");
+ }
+
+ @Test
+ void testWaitForHealthStatus_RaceConditionProtection() {
+ // Given: Health status changes between first check and listener registration
+ when(mockHealthStatusManager.getHealthStatus(testEndpoint))
+ .thenReturn(HealthStatus.UNKNOWN) // First call
+ .thenReturn(HealthStatus.HEALTHY); // Second call after registering listener
+
+ // When: Waiting for health status
+ HealthStatus result = statusTracker.waitForHealthStatus(testEndpoint);
+
+ // Then: Should return the status from the second check without waiting
+ assertEquals(HealthStatus.HEALTHY, result);
+
+ // Verify listener was registered and unregistered
+ verify(mockHealthStatusManager).registerListener(eq(testEndpoint), any(HealthStatusListener.class));
+ verify(mockHealthStatusManager).unregisterListener(eq(testEndpoint), any(HealthStatusListener.class));
+ }
+
+ @Test
+ void testWaitForHealthStatus_ListenerCleanupOnException() {
+ // Given: Health status is initially UNKNOWN
+ when(mockHealthStatusManager.getHealthStatus(testEndpoint)).thenReturn(HealthStatus.UNKNOWN);
+
+ // Mock registerListener to throw an exception
+ doThrow(new RuntimeException("Registration failed"))
+ .when(mockHealthStatusManager).registerListener(eq(testEndpoint), any(HealthStatusListener.class));
+
+ // When: Waiting for health status
+ assertThrows(RuntimeException.class, () -> {
+ statusTracker.waitForHealthStatus(testEndpoint);
+ });
+
+ // Then: Should still attempt to unregister (cleanup in finally block)
+ verify(mockHealthStatusManager).registerListener(eq(testEndpoint), any(HealthStatusListener.class));
+ // Note: unregisterListener might not be called if registerListener fails,
+ // but the finally block should handle this gracefully
+ }
+}
diff --git a/src/test/java/redis/clients/jedis/misc/AutomaticFailoverTest.java b/src/test/java/redis/clients/jedis/misc/AutomaticFailoverTest.java
index c6ccbc6636..4bb3fa8bd1 100644
--- a/src/test/java/redis/clients/jedis/misc/AutomaticFailoverTest.java
+++ b/src/test/java/redis/clients/jedis/misc/AutomaticFailoverTest.java
@@ -16,6 +16,8 @@
import redis.clients.jedis.*;
import redis.clients.jedis.exceptions.JedisAccessControlException;
import redis.clients.jedis.exceptions.JedisConnectionException;
+import redis.clients.jedis.mcf.ClusterSwitchEventArgs;
+import redis.clients.jedis.mcf.SwitchReason;
import redis.clients.jedis.providers.MultiClusterPooledConnectionProvider;
import redis.clients.jedis.util.IOUtils;
@@ -65,7 +67,7 @@ public void pipelineWithSwitch() {
AbstractPipeline pipe = client.pipelined();
pipe.set("pstr", "foobar");
pipe.hset("phash", "foo", "bar");
- provider.incrementActiveMultiClusterIndex();
+ provider.iterateActiveCluster(SwitchReason.HEALTH_CHECK);
pipe.sync();
}
@@ -83,7 +85,7 @@ public void transactionWithSwitch() {
AbstractTransaction tx = client.multi();
tx.set("tstr", "foobar");
tx.hset("thash", "foo", "bar");
- provider.incrementActiveMultiClusterIndex();
+ provider.iterateActiveCluster(SwitchReason.HEALTH_CHECK);
assertEquals(Arrays.asList("OK", 1L), tx.exec());
}
@@ -106,7 +108,7 @@ public void commandFailover() {
RedisFailoverReporter failoverReporter = new RedisFailoverReporter();
MultiClusterPooledConnectionProvider connectionProvider = new MultiClusterPooledConnectionProvider(
builder.build());
- connectionProvider.setClusterFailoverPostProcessor(failoverReporter);
+ connectionProvider.setClusterSwitchListener(failoverReporter);
UnifiedJedis jedis = new UnifiedJedis(connectionProvider);
@@ -145,7 +147,7 @@ public void pipelineFailover() {
RedisFailoverReporter failoverReporter = new RedisFailoverReporter();
MultiClusterPooledConnectionProvider cacheProvider = new MultiClusterPooledConnectionProvider(builder.build());
- cacheProvider.setClusterFailoverPostProcessor(failoverReporter);
+ cacheProvider.setClusterSwitchListener(failoverReporter);
UnifiedJedis jedis = new UnifiedJedis(cacheProvider);
@@ -178,7 +180,7 @@ public void failoverFromAuthError() {
RedisFailoverReporter failoverReporter = new RedisFailoverReporter();
MultiClusterPooledConnectionProvider cacheProvider = new MultiClusterPooledConnectionProvider(builder.build());
- cacheProvider.setClusterFailoverPostProcessor(failoverReporter);
+ cacheProvider.setClusterSwitchListener(failoverReporter);
UnifiedJedis jedis = new UnifiedJedis(cacheProvider);
@@ -194,13 +196,13 @@ public void failoverFromAuthError() {
jedis.close();
}
- static class RedisFailoverReporter implements Consumer {
+ static class RedisFailoverReporter implements Consumer {
boolean failedOver = false;
@Override
- public void accept(String clusterName) {
- log.info("Jedis fail over to cluster: " + clusterName);
+ public void accept(ClusterSwitchEventArgs e) {
+ log.info("Jedis fail over to cluster: " + e.getClusterName());
failedOver = true;
}
}
diff --git a/src/test/java/redis/clients/jedis/providers/MultiClusterPooledConnectionProviderHelper.java b/src/test/java/redis/clients/jedis/providers/MultiClusterPooledConnectionProviderHelper.java
new file mode 100644
index 0000000000..138d524ec0
--- /dev/null
+++ b/src/test/java/redis/clients/jedis/providers/MultiClusterPooledConnectionProviderHelper.java
@@ -0,0 +1,17 @@
+package redis.clients.jedis.providers;
+
+import redis.clients.jedis.mcf.Endpoint;
+import redis.clients.jedis.mcf.HealthStatus;
+import redis.clients.jedis.mcf.HealthStatusChangeEvent;
+
+public class MultiClusterPooledConnectionProviderHelper {
+
+ public static void onHealthStatusChange(MultiClusterPooledConnectionProvider provider, Endpoint endpoint,
+ HealthStatus oldStatus, HealthStatus newStatus) {
+ provider.onHealthStatusChange(new HealthStatusChangeEvent(endpoint, oldStatus, newStatus));
+ }
+
+ public static void periodicFailbackCheck(MultiClusterPooledConnectionProvider provider) {
+ provider.periodicFailbackCheck();
+ }
+}
diff --git a/src/test/java/redis/clients/jedis/providers/MultiClusterPooledConnectionProviderTest.java b/src/test/java/redis/clients/jedis/providers/MultiClusterPooledConnectionProviderTest.java
index 8996f0e285..27762ce097 100644
--- a/src/test/java/redis/clients/jedis/providers/MultiClusterPooledConnectionProviderTest.java
+++ b/src/test/java/redis/clients/jedis/providers/MultiClusterPooledConnectionProviderTest.java
@@ -1,13 +1,21 @@
package redis.clients.jedis.providers;
import io.github.resilience4j.circuitbreaker.CircuitBreaker;
+
+import org.awaitility.Awaitility;
+import org.awaitility.Durations;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import redis.clients.jedis.*;
import redis.clients.jedis.MultiClusterClientConfig.ClusterConfig;
import redis.clients.jedis.exceptions.JedisConnectionException;
import redis.clients.jedis.exceptions.JedisValidationException;
+import redis.clients.jedis.mcf.Endpoint;
+import redis.clients.jedis.mcf.SwitchReason;
+import redis.clients.jedis.providers.MultiClusterPooledConnectionProvider.Cluster;
+import java.util.Arrays;
import java.util.concurrent.atomic.AtomicBoolean;
import static org.junit.jupiter.api.Assertions.*;
@@ -26,16 +34,27 @@ public class MultiClusterPooledConnectionProviderTest {
public void setUp() {
ClusterConfig[] clusterConfigs = new ClusterConfig[2];
- clusterConfigs[0] = new ClusterConfig(endpointStandalone0.getHostAndPort(), endpointStandalone0.getClientConfigBuilder().build());
- clusterConfigs[1] = new ClusterConfig(endpointStandalone1.getHostAndPort(), endpointStandalone0.getClientConfigBuilder().build());
+ clusterConfigs[0] = ClusterConfig
+ .builder(endpointStandalone0.getHostAndPort(), endpointStandalone0.getClientConfigBuilder().build())
+ .weight(0.5f).build();
+ clusterConfigs[1] = ClusterConfig
+ .builder(endpointStandalone1.getHostAndPort(), endpointStandalone1.getClientConfigBuilder().build())
+ .weight(0.3f).build();
+
+ provider = new MultiClusterPooledConnectionProvider(
+ new MultiClusterClientConfig.Builder(clusterConfigs).build());
+ }
- provider = new MultiClusterPooledConnectionProvider(new MultiClusterClientConfig.Builder(clusterConfigs).build());
+ @AfterEach
+ public void destroy() {
+ provider.close();
+ provider = null;
}
@Test
public void testCircuitBreakerForcedTransitions() {
- CircuitBreaker circuitBreaker = provider.getClusterCircuitBreaker(1);
+ CircuitBreaker circuitBreaker = provider.getClusterCircuitBreaker();
circuitBreaker.getState();
if (CircuitBreaker.State.FORCED_OPEN.equals(circuitBreaker.getState()))
@@ -49,45 +68,56 @@ public void testCircuitBreakerForcedTransitions() {
}
@Test
- public void testIncrementActiveMultiClusterIndex() {
- int index = provider.incrementActiveMultiClusterIndex();
- assertEquals(2, index);
+ public void testIterateActiveCluster() throws InterruptedException {
+ waitForClustersToGetHealthy(provider.getCluster(endpointStandalone0.getHostAndPort()),
+ provider.getCluster(endpointStandalone1.getHostAndPort()));
+
+ Endpoint e2 = provider.iterateActiveCluster(SwitchReason.HEALTH_CHECK);
+ assertEquals(endpointStandalone1.getHostAndPort(), e2);
}
@Test
- public void testIncrementActiveMultiClusterIndexOutOfRange() {
- provider.setActiveMultiClusterIndex(1);
+ public void testIterateActiveClusterOutOfRange() {
+ waitForClustersToGetHealthy(provider.getCluster(endpointStandalone0.getHostAndPort()),
+ provider.getCluster(endpointStandalone1.getHostAndPort()));
- int index = provider.incrementActiveMultiClusterIndex();
- assertEquals(2, index);
+ provider.setActiveCluster(endpointStandalone0.getHostAndPort());
+ provider.getCluster().setDisabled(true);
- assertThrows(JedisConnectionException.class,
- () -> provider.incrementActiveMultiClusterIndex()); // Should throw an exception
+ Endpoint e2 = provider.iterateActiveCluster(SwitchReason.HEALTH_CHECK);
+ provider.getCluster().setDisabled(true);
+
+ assertEquals(endpointStandalone1.getHostAndPort(), e2);
+ // Should throw an exception
+ assertThrows(JedisConnectionException.class, () -> provider.iterateActiveCluster(SwitchReason.HEALTH_CHECK));
}
@Test
- public void testIsLastClusterCircuitBreakerForcedOpen() {
- provider.setActiveMultiClusterIndex(1);
-
- try {
- provider.incrementActiveMultiClusterIndex();
- } catch (Exception e) {}
-
- // This should set the isLastClusterCircuitBreakerForcedOpen to true
- try {
- provider.incrementActiveMultiClusterIndex();
- } catch (Exception e) {}
+ public void testCanIterateOnceMore() {
+ waitForClustersToGetHealthy(provider.getCluster(endpointStandalone0.getHostAndPort()),
+ provider.getCluster(endpointStandalone1.getHostAndPort()));
+
+ provider.setActiveCluster(endpointStandalone0.getHostAndPort());
+ provider.getCluster().setDisabled(true);
+ provider.iterateActiveCluster(SwitchReason.HEALTH_CHECK);
+
+ assertFalse(provider.canIterateOnceMore());
+ }
- assertTrue(provider.isLastClusterCircuitBreakerForcedOpen());
+ private void waitForClustersToGetHealthy(Cluster... clusters) {
+ Awaitility.await().pollInterval(Durations.ONE_HUNDRED_MILLISECONDS).atMost(Durations.TWO_SECONDS)
+ .until(() -> Arrays.stream(clusters).allMatch(Cluster::isHealthy));
}
@Test
public void testRunClusterFailoverPostProcessor() {
ClusterConfig[] clusterConfigs = new ClusterConfig[2];
- clusterConfigs[0] = new ClusterConfig(new HostAndPort("purposefully-incorrect", 0000),
- DefaultJedisClientConfig.builder().build());
- clusterConfigs[1] = new ClusterConfig(new HostAndPort("purposefully-incorrect", 0001),
- DefaultJedisClientConfig.builder().build());
+ clusterConfigs[0] = ClusterConfig
+ .builder(new HostAndPort("purposefully-incorrect", 0000), DefaultJedisClientConfig.builder().build())
+ .weight(0.5f).healthCheckEnabled(false).build();
+ clusterConfigs[1] = ClusterConfig
+ .builder(new HostAndPort("purposefully-incorrect", 0001), DefaultJedisClientConfig.builder().build())
+ .weight(0.4f).healthCheckEnabled(false).build();
MultiClusterClientConfig.Builder builder = new MultiClusterClientConfig.Builder(clusterConfigs);
@@ -98,41 +128,48 @@ public void testRunClusterFailoverPostProcessor() {
AtomicBoolean isValidTest = new AtomicBoolean(false);
MultiClusterPooledConnectionProvider localProvider = new MultiClusterPooledConnectionProvider(builder.build());
- localProvider.setClusterFailoverPostProcessor(a -> { isValidTest.set(true); });
+ localProvider.setClusterSwitchListener(a -> {
+ isValidTest.set(true);
+ });
try (UnifiedJedis jedis = new UnifiedJedis(localProvider)) {
- // This should fail after 3 retries and meet the requirements to open the circuit on the next iteration
- try {
- jedis.get("foo");
- } catch (Exception e) {}
-
- // This should fail after 3 retries and open the circuit which will trigger the post processor
+ // This will fail due to unable to connect and open the circuit which will trigger the post processor
try {
jedis.get("foo");
- } catch (Exception e) {}
+ } catch (Exception e) {
+ }
}
- assertTrue(isValidTest.get());
+ assertTrue(isValidTest.get());
}
@Test
public void testSetActiveMultiClusterIndexEqualsZero() {
- assertThrows(JedisValidationException.class,
- () -> provider.setActiveMultiClusterIndex(0)); // Should throw an exception
+ assertThrows(JedisValidationException.class, () -> provider.setActiveCluster(null)); // Should throw an
+ // exception
}
@Test
public void testSetActiveMultiClusterIndexLessThanZero() {
- assertThrows(JedisValidationException.class,
- () -> provider.setActiveMultiClusterIndex(-1)); // Should throw an exception
+ assertThrows(JedisValidationException.class, () -> provider.setActiveCluster(null)); // Should throw an
+ // exception
}
@Test
public void testSetActiveMultiClusterIndexOutOfRange() {
- assertThrows(JedisValidationException.class,
- () -> provider.setActiveMultiClusterIndex(3)); // Should throw an exception
+ assertThrows(JedisValidationException.class, () -> provider.setActiveCluster(new Endpoint() {
+ @Override
+ public String getHost() {
+ return "purposefully-incorrect";
+ }
+
+ @Override
+ public int getPort() {
+ return 0000;
+ }
+ })); // Should throw an exception
}
@Test
@@ -142,10 +179,12 @@ public void testConnectionPoolConfigApplied() {
poolConfig.setMaxIdle(4);
poolConfig.setMinIdle(1);
ClusterConfig[] clusterConfigs = new ClusterConfig[2];
- clusterConfigs[0] = new ClusterConfig(endpointStandalone0.getHostAndPort(), endpointStandalone0.getClientConfigBuilder().build(), poolConfig);
- clusterConfigs[1] = new ClusterConfig(endpointStandalone1.getHostAndPort(), endpointStandalone0.getClientConfigBuilder().build(), poolConfig);
+ clusterConfigs[0] = new ClusterConfig(endpointStandalone0.getHostAndPort(),
+ endpointStandalone0.getClientConfigBuilder().build(), poolConfig);
+ clusterConfigs[1] = new ClusterConfig(endpointStandalone1.getHostAndPort(),
+ endpointStandalone0.getClientConfigBuilder().build(), poolConfig);
try (MultiClusterPooledConnectionProvider customProvider = new MultiClusterPooledConnectionProvider(
- new MultiClusterClientConfig.Builder(clusterConfigs).build())) {
+ new MultiClusterClientConfig.Builder(clusterConfigs).build())) {
MultiClusterPooledConnectionProvider.Cluster activeCluster = customProvider.getCluster();
ConnectionPool connectionPool = activeCluster.getConnectionPool();
assertEquals(8, connectionPool.getMaxTotal());
diff --git a/src/test/java/redis/clients/jedis/providers/MultiClusterProviderHealthStatusChangeEventTest.java b/src/test/java/redis/clients/jedis/providers/MultiClusterProviderHealthStatusChangeEventTest.java
new file mode 100644
index 0000000000..cb9993c8c7
--- /dev/null
+++ b/src/test/java/redis/clients/jedis/providers/MultiClusterProviderHealthStatusChangeEventTest.java
@@ -0,0 +1,346 @@
+package redis.clients.jedis.providers;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.*;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.MockedConstruction;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import redis.clients.jedis.Connection;
+import redis.clients.jedis.ConnectionPool;
+import redis.clients.jedis.DefaultJedisClientConfig;
+import redis.clients.jedis.HostAndPort;
+import redis.clients.jedis.JedisClientConfig;
+import redis.clients.jedis.MultiClusterClientConfig;
+import redis.clients.jedis.mcf.Endpoint;
+import redis.clients.jedis.mcf.HealthCheckStrategy;
+import redis.clients.jedis.mcf.HealthStatus;
+import redis.clients.jedis.mcf.HealthStatusListener;
+import redis.clients.jedis.mcf.HealthStatusManager;
+
+/**
+ * Tests for MultiClusterPooledConnectionProvider event handling behavior during initialization and throughout its
+ * lifecycle with HealthStatusChangeEvents.
+ */
+@ExtendWith(MockitoExtension.class)
+public class MultiClusterProviderHealthStatusChangeEventTest {
+
+ private HostAndPort endpoint1;
+ private HostAndPort endpoint2;
+ private HostAndPort endpoint3;
+ private JedisClientConfig clientConfig;
+
+ @BeforeEach
+ void setUp() {
+ endpoint1 = new HostAndPort("localhost", 6879);
+ endpoint2 = new HostAndPort("localhost", 6880);
+ endpoint3 = new HostAndPort("localhost", 6881);
+ clientConfig = DefaultJedisClientConfig.builder().build();
+ }
+
+ private MockedConstruction mockConnectionPool() {
+ Connection mockConnection = mock(Connection.class);
+ lenient().when(mockConnection.ping()).thenReturn(true);
+ return mockConstruction(ConnectionPool.class, (mock, context) -> {
+ when(mock.getResource()).thenReturn(mockConnection);
+ doNothing().when(mock).close();
+ });
+ }
+
+ @Test
+ void testEventsProcessedAfterInitialization() throws Exception {
+ try (MockedConstruction mockedPool = mockConnectionPool()) {
+ // Create clusters without health checks
+ MultiClusterClientConfig.ClusterConfig cluster1 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig).weight(1.0f).healthCheckEnabled(false).build();
+ MultiClusterClientConfig.ClusterConfig cluster2 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint2, clientConfig).weight(0.5f).healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig config = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { cluster1, cluster2 }).build();
+
+ try (MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(config)) {
+ // This should process immediately since initialization is complete
+ assertDoesNotThrow(() -> {
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint1,
+ HealthStatus.HEALTHY, HealthStatus.UNHEALTHY);
+ }, "Post-initialization events should be processed immediately");
+
+ // Verify the cluster status was updated
+ assertEquals(HealthStatus.UNHEALTHY, provider.getCluster(endpoint1).getHealthStatus(),
+ "Cluster health status should be updated after post-init event");
+ }
+ }
+ }
+
+ @Test
+ void testMultipleEventsProcessedSequentially() throws Exception {
+ try (MockedConstruction mockedPool = mockConnectionPool()) {
+ MultiClusterClientConfig.ClusterConfig cluster1 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig).weight(1.0f).healthCheckEnabled(false).build();
+ MultiClusterClientConfig.ClusterConfig cluster2 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint2, clientConfig).weight(0.5f).healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig config = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { cluster1, cluster2 }).build();
+
+ try (MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(config)) {
+ // Verify initial state
+ assertEquals(HealthStatus.HEALTHY, provider.getCluster(endpoint1).getHealthStatus(),
+ "Should start as HEALTHY");
+
+ // Simulate multiple rapid events for the same endpoint
+ // Process events sequentially (post-init behavior)
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint1,
+ HealthStatus.HEALTHY, HealthStatus.UNHEALTHY);
+ assertEquals(HealthStatus.UNHEALTHY, provider.getCluster(endpoint1).getHealthStatus(),
+ "Should be UNHEALTHY after first event");
+
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint1,
+ HealthStatus.UNHEALTHY, HealthStatus.HEALTHY);
+ assertEquals(HealthStatus.HEALTHY, provider.getCluster(endpoint1).getHealthStatus(),
+ "Should be HEALTHY after second event");
+
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint1,
+ HealthStatus.HEALTHY, HealthStatus.UNHEALTHY);
+ assertEquals(HealthStatus.UNHEALTHY, provider.getCluster(endpoint1).getHealthStatus(),
+ "Should be UNHEALTHY after third event");
+ }
+ }
+ }
+
+ @Test
+ void testEventsForMultipleEndpointsPreserveOrder() throws Exception {
+ try (MockedConstruction mockedPool = mockConnectionPool()) {
+ MultiClusterClientConfig.ClusterConfig cluster1 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig).weight(1.0f).healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig.ClusterConfig cluster2 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint2, clientConfig).weight(2.0f).healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig config = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { cluster1, cluster2 }).build();
+
+ try (MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(config)) {
+ // This test verifies that multiple endpoints are properly initialized
+
+ // Verify both clusters are initialized properly
+ assertNotNull(provider.getCluster(endpoint1), "Cluster 1 should be available");
+ assertNotNull(provider.getCluster(endpoint2), "Cluster 2 should be available");
+
+ // Both should be healthy (no health checks = assumed healthy)
+ assertTrue(provider.getCluster(endpoint1).isHealthy(), "Cluster 1 should be healthy");
+ assertTrue(provider.getCluster(endpoint2).isHealthy(), "Cluster 2 should be healthy");
+ }
+ }
+ }
+
+ @Test
+ void testEventProcessingWithMixedHealthCheckConfiguration() throws Exception {
+ try (MockedConstruction mockedPool = mockConnectionPool()) {
+ // One cluster with health checks disabled, one with enabled (but mocked)
+ MultiClusterClientConfig.ClusterConfig cluster1 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig).weight(1.0f).healthCheckEnabled(false) // No health checks
+ .build();
+
+ MultiClusterClientConfig.ClusterConfig cluster2 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint2, clientConfig).weight(2.0f).healthCheckEnabled(false) // No health checks for
+ // simplicity
+ .build();
+
+ MultiClusterClientConfig config = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { cluster1, cluster2 }).build();
+
+ try (MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(config)) {
+ // Both clusters should be available and healthy
+ assertNotNull(provider.getCluster(endpoint1), "Cluster 1 should be available");
+ assertNotNull(provider.getCluster(endpoint2), "Cluster 2 should be available");
+
+ assertTrue(provider.getCluster(endpoint1).isHealthy(), "Cluster 1 should be healthy");
+ assertTrue(provider.getCluster(endpoint2).isHealthy(), "Cluster 2 should be healthy");
+
+ // Provider should select the higher weight cluster (endpoint2)
+ assertEquals(provider.getCluster(endpoint2), provider.getCluster(),
+ "Should select higher weight cluster as active");
+ }
+ }
+ }
+
+ @Test
+ void testNoEventsLostDuringInitialization() throws Exception {
+ try (MockedConstruction mockedPool = mockConnectionPool()) {
+ MultiClusterClientConfig.ClusterConfig cluster1 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig).weight(1.0f).healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig config = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { cluster1 }).build();
+
+ // This test verifies that the provider initializes correctly and doesn't lose events
+ // In practice, with health checks disabled, no events should be generated during init
+ try (MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(config)) {
+ // Verify successful initialization
+ assertNotNull(provider.getCluster(), "Provider should have initialized successfully");
+ assertEquals(provider.getCluster(endpoint1), provider.getCluster(),
+ "Should have selected the configured cluster");
+ assertTrue(provider.getCluster().isHealthy(),
+ "Cluster should be healthy (assumed healthy with no health checks)");
+ }
+ }
+ }
+
+ // ========== POST-INITIALIZATION EVENT ORDERING TESTS ==========
+
+ @Test
+ void testPostInitEventOrderingWithMultipleEndpoints() throws Exception {
+ try (MockedConstruction mockedPool = mockConnectionPool()) {
+ MultiClusterClientConfig.ClusterConfig cluster1 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig).weight(1.0f).healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig.ClusterConfig cluster2 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint2, clientConfig).weight(0.5f).healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig.ClusterConfig cluster3 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint3, clientConfig).weight(0.2f).healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig config = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { cluster1, cluster2, cluster3 }).build();
+
+ try (MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(config)) {
+ // Process events immediately (post-init behavior)
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint1,
+ HealthStatus.HEALTHY, HealthStatus.UNHEALTHY);
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint2,
+ HealthStatus.HEALTHY, HealthStatus.UNHEALTHY);
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint1,
+ HealthStatus.UNHEALTHY, HealthStatus.HEALTHY);
+
+ // Verify events were processed and cluster states updated
+ assertEquals(HealthStatus.HEALTHY, provider.getCluster(endpoint1).getHealthStatus(),
+ "Endpoint1 should have final HEALTHY status");
+ assertEquals(HealthStatus.UNHEALTHY, provider.getCluster(endpoint2).getHealthStatus(),
+ "Endpoint2 should have UNHEALTHY status");
+ }
+ }
+ }
+
+ @Test
+ void testPostInitRapidEventsOptimization() throws Exception {
+ try (MockedConstruction mockedPool = mockConnectionPool()) {
+ MultiClusterClientConfig.ClusterConfig cluster1 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint1, clientConfig).weight(1.0f).healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig.ClusterConfig cluster2 = MultiClusterClientConfig.ClusterConfig
+ .builder(endpoint2, clientConfig).weight(0.5f).healthCheckEnabled(false).build();
+
+ MultiClusterClientConfig config = new MultiClusterClientConfig.Builder(
+ new MultiClusterClientConfig.ClusterConfig[] { cluster1, cluster2 }).build();
+
+ try (MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(config)) {
+ // Verify initial state
+ assertEquals(HealthStatus.HEALTHY, provider.getCluster(endpoint1).getHealthStatus(),
+ "Should start as HEALTHY");
+
+ // Send rapid sequence of events (should all be processed since init is complete)
+ // Process events immediately (post-init behavior)
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint1,
+ HealthStatus.HEALTHY, HealthStatus.UNHEALTHY);
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint1,
+ HealthStatus.UNHEALTHY, HealthStatus.HEALTHY);
+ MultiClusterPooledConnectionProviderHelper.onHealthStatusChange(provider, endpoint1,
+ HealthStatus.HEALTHY, HealthStatus.UNHEALTHY);
+
+ // Final state should reflect the last event
+ assertEquals(HealthStatus.UNHEALTHY, provider.getCluster(endpoint1).getHealthStatus(),
+ "Should have final UNHEALTHY status from last event");
+ }
+ }
+ }
+
+ @Test
+ void testHealthStatusManagerEventOrdering() throws InterruptedException {
+ HealthStatusManager manager = new HealthStatusManager();
+
+ // Counter to track events received
+ AtomicInteger eventCount = new AtomicInteger(0);
+ CountDownLatch eventLatch = new CountDownLatch(1);
+
+ // Create a listener that counts events
+ HealthStatusListener listener = event -> {
+ eventCount.incrementAndGet();
+ eventLatch.countDown();
+ };
+
+ // Register listener BEFORE adding endpoint (correct order to prevent missing events)
+ manager.registerListener(endpoint1, listener);
+
+ // Create a strategy that immediately returns HEALTHY
+ HealthCheckStrategy immediateStrategy = new HealthCheckStrategy() {
+ @Override
+ public int getInterval() {
+ return 100;
+ }
+
+ @Override
+ public int getTimeout() {
+ return 50;
+ }
+
+ @Override
+ public HealthStatus doHealthCheck(Endpoint endpoint) {
+ return HealthStatus.HEALTHY;
+ }
+ };
+
+ // Add endpoint - this should trigger health check and event
+ manager.add(endpoint1, immediateStrategy);
+
+ // Wait for event to be processed
+ assertTrue(eventLatch.await(2, TimeUnit.SECONDS), "Should receive health status event");
+
+ // Should have received at least one event (UNKNOWN -> HEALTHY)
+ assertTrue(eventCount.get() >= 1, "Should have received at least one health status event");
+
+ manager.remove(endpoint1);
+ }
+
+ @Test
+ void testHealthStatusManagerHasHealthCheck() {
+ HealthStatusManager manager = new HealthStatusManager();
+
+ // Initially no health check
+ assertFalse(manager.hasHealthCheck(endpoint1), "Should not have health check initially");
+
+ // Create a simple strategy
+ HealthCheckStrategy strategy = new HealthCheckStrategy() {
+ @Override
+ public int getInterval() {
+ return 100;
+ }
+
+ @Override
+ public int getTimeout() {
+ return 50;
+ }
+
+ @Override
+ public HealthStatus doHealthCheck(Endpoint endpoint) {
+ return HealthStatus.HEALTHY;
+ }
+ };
+
+ // Add health check
+ manager.add(endpoint1, strategy);
+ assertTrue(manager.hasHealthCheck(endpoint1), "Should have health check after adding");
+
+ // Remove health check
+ manager.remove(endpoint1);
+ assertFalse(manager.hasHealthCheck(endpoint1), "Should not have health check after removing");
+ }
+}
diff --git a/src/test/java/redis/clients/jedis/scenario/ActiveActiveFailoverTest.java b/src/test/java/redis/clients/jedis/scenario/ActiveActiveFailoverTest.java
index 7ec4edb14e..e27debb188 100644
--- a/src/test/java/redis/clients/jedis/scenario/ActiveActiveFailoverTest.java
+++ b/src/test/java/redis/clients/jedis/scenario/ActiveActiveFailoverTest.java
@@ -1,6 +1,8 @@
package redis.clients.jedis.scenario;
import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig;
+
+import org.hamcrest.Matchers;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Tags;
@@ -8,8 +10,10 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.*;
+import redis.clients.jedis.MultiClusterClientConfig.ClusterConfig;
import redis.clients.jedis.providers.MultiClusterPooledConnectionProvider;
import redis.clients.jedis.exceptions.JedisConnectionException;
+import redis.clients.jedis.mcf.ClusterSwitchEventArgs;
import java.io.IOException;
import java.time.Duration;
@@ -24,11 +28,9 @@
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import static org.junit.jupiter.api.Assumptions.assumeTrue;
+import static org.hamcrest.MatcherAssert.assertThat;
-@Tags({
- @Tag("failover"),
- @Tag("scenario")
-})
+@Tags({ @Tag("failover"), @Tag("scenario") })
public class ActiveActiveFailoverTest {
private static final Logger log = LoggerFactory.getLogger(ActiveActiveFailoverTest.class);
@@ -52,13 +54,13 @@ public void testFailover() {
MultiClusterClientConfig.ClusterConfig[] clusterConfig = new MultiClusterClientConfig.ClusterConfig[2];
JedisClientConfig config = endpoint.getClientConfigBuilder()
- .socketTimeoutMillis(RecommendedSettings.DEFAULT_TIMEOUT_MS)
- .connectionTimeoutMillis(RecommendedSettings.DEFAULT_TIMEOUT_MS).build();
+ .socketTimeoutMillis(RecommendedSettings.DEFAULT_TIMEOUT_MS)
+ .connectionTimeoutMillis(RecommendedSettings.DEFAULT_TIMEOUT_MS).build();
- clusterConfig[0] = new MultiClusterClientConfig.ClusterConfig(endpoint.getHostAndPort(0),
- config, RecommendedSettings.poolConfig);
- clusterConfig[1] = new MultiClusterClientConfig.ClusterConfig(endpoint.getHostAndPort(1),
- config, RecommendedSettings.poolConfig);
+ clusterConfig[0] = ClusterConfig.builder(endpoint.getHostAndPort(0), config)
+ .connectionPoolConfig(RecommendedSettings.poolConfig).weight(1.0f).build();
+ clusterConfig[1] = ClusterConfig.builder(endpoint.getHostAndPort(1), config)
+ .connectionPoolConfig(RecommendedSettings.poolConfig).weight(0.5f).build();
MultiClusterClientConfig.Builder builder = new MultiClusterClientConfig.Builder(clusterConfig);
@@ -67,11 +69,15 @@ public void testFailover() {
builder.circuitBreakerSlidingWindowMinCalls(1);
builder.circuitBreakerFailureRateThreshold(10.0f); // percentage of failures to trigger circuit breaker
+ builder.failbackSupported(true);
+ builder.failbackCheckInterval(1000);
+ builder.gracePeriod(10000);
+
builder.retryWaitDuration(10);
builder.retryMaxAttempts(1);
builder.retryWaitDurationExponentialBackoffMultiplier(1);
- class FailoverReporter implements Consumer {
+ class FailoverReporter implements Consumer {
String currentClusterName = "not set";
@@ -79,27 +85,34 @@ class FailoverReporter implements Consumer {
Instant failoverAt = null;
+ boolean failbackHappened = false;
+
+ Instant failbackAt = null;
+
public String getCurrentClusterName() {
return currentClusterName;
}
@Override
- public void accept(String clusterName) {
- this.currentClusterName = clusterName;
- log.info(
- "\n\n====FailoverEvent=== \nJedis failover to cluster: {}\n====FailoverEvent===\n\n",
- clusterName);
-
- failoverHappened = true;
- failoverAt = Instant.now();
+ public void accept(ClusterSwitchEventArgs e) {
+ this.currentClusterName = e.getClusterName();
+ log.info("\n\n====FailoverEvent=== \nJedis failover to cluster: {}\n====FailoverEvent===\n\n",
+ e.getClusterName());
+
+ if (failoverHappened) {
+ failbackHappened = true;
+ failbackAt = Instant.now();
+ } else {
+ failoverHappened = true;
+ failoverAt = Instant.now();
+ }
}
}
- MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(
- builder.build());
+ MultiClusterPooledConnectionProvider provider = new MultiClusterPooledConnectionProvider(builder.build());
FailoverReporter reporter = new FailoverReporter();
- provider.setClusterFailoverPostProcessor(reporter);
- provider.setActiveMultiClusterIndex(1);
+ provider.setClusterSwitchListener(reporter);
+ provider.setActiveCluster(endpoint.getHostAndPort(0));
UnifiedJedis client = new UnifiedJedis(provider);
@@ -117,15 +130,17 @@ public void accept(String clusterName) {
int retryingDelay = 5;
while (true) {
try {
- Map executionInfo = new HashMap() {{
- put("threadId", String.valueOf(threadId));
- put("cluster", reporter.getCurrentClusterName());
- }};
+ Map executionInfo = new HashMap() {
+ {
+ put("threadId", String.valueOf(threadId));
+ put("cluster", reporter.getCurrentClusterName());
+ }
+ };
client.xadd("execution_log", StreamEntryID.NEW_ENTRY, executionInfo);
if (attempt > 0) {
log.info("Thread {} recovered after {} ms. Threads still not recovered: {}", threadId,
- attempt * retryingDelay, retryingThreadsCounter.decrementAndGet());
+ attempt * retryingDelay, retryingThreadsCounter.decrementAndGet());
}
break;
@@ -134,15 +149,13 @@ public void accept(String clusterName) {
if (reporter.failoverHappened) {
long failedCommands = failedCommandsAfterFailover.incrementAndGet();
lastFailedCommandAt.set(Instant.now());
- log.warn(
- "Thread {} failed to execute command after failover. Failed commands after failover: {}",
- threadId, failedCommands);
+ log.warn("Thread {} failed to execute command after failover. Failed commands after failover: {}", threadId,
+ failedCommands);
}
if (attempt == 0) {
long failedThreads = retryingThreadsCounter.incrementAndGet();
- log.warn("Thread {} failed to execute command. Failed threads: {}", threadId,
- failedThreads);
+ log.warn("Thread {} failed to execute command. Failed threads: {}", threadId, failedThreads);
}
try {
Thread.sleep(retryingDelay);
@@ -153,20 +166,21 @@ public void accept(String clusterName) {
}
}
return true;
- }, 18);
+ }, 4);
fakeApp.setKeepExecutingForSeconds(30);
Thread t = new Thread(fakeApp);
t.start();
HashMap params = new HashMap<>();
params.put("bdb_id", endpoint.getBdbId());
- params.put("rlutil_command", "pause_bdb");
+ params.put("actions",
+ "[{\"type\":\"execute_rlutil_command\",\"params\":{\"rlutil_command\":\"pause_bdb\"}},{\"type\":\"wait\",\"params\":{\"wait_time\":\"15\"}},{\"type\":\"execute_rlutil_command\",\"params\":{\"rlutil_command\":\"resume_bdb\"}}]");
FaultInjectionClient.TriggerActionResponse actionResponse = null;
try {
- log.info("Triggering bdb_pause");
- actionResponse = faultClient.triggerAction("execute_rlutil_command", params);
+ log.info("Triggering bdb_pause + wait 15 seconds + bdb_resume");
+ actionResponse = faultClient.triggerAction("sequence_of_actions", params);
} catch (IOException e) {
fail("Fault Injection Server error:" + e.getMessage());
}
@@ -180,17 +194,22 @@ public void accept(String clusterName) {
throw new RuntimeException(e);
}
- ConnectionPool pool = provider.getCluster(1).getConnectionPool();
+ ConnectionPool pool = provider.getCluster(endpoint.getHostAndPort(0)).getConnectionPool();
- log.info("First connection pool state: active: {}, idle: {}", pool.getNumActive(),
- pool.getNumIdle());
- log.info("Full failover time: {} s",
- Duration.between(reporter.failoverAt, lastFailedCommandAt.get()).getSeconds());
+ log.info("First connection pool state: active: {}, idle: {}", pool.getNumActive(), pool.getNumIdle());
+ log.info("Failover happened at: {}", reporter.failoverAt);
+ log.info("Failback happened at: {}", reporter.failbackAt);
+ log.info("Last failed command at: {}", lastFailedCommandAt.get());
+ Duration fullFailoverTime = Duration.between(reporter.failoverAt, lastFailedCommandAt.get());
+ log.info("Full failover time: {} s", fullFailoverTime.getSeconds());
assertEquals(0, pool.getNumActive());
assertTrue(fakeApp.capturedExceptions().isEmpty());
+ assertTrue(reporter.failoverHappened);
+ assertTrue(reporter.failbackHappened);
+ assertThat(fullFailoverTime.getSeconds(), Matchers.greaterThanOrEqualTo(30L));
client.close();
}
-}
+}
\ No newline at end of file
diff --git a/src/test/java/redis/clients/jedis/scenario/FaultInjectionClient.java b/src/test/java/redis/clients/jedis/scenario/FaultInjectionClient.java
index c4e1c5717b..17170251cd 100644
--- a/src/test/java/redis/clients/jedis/scenario/FaultInjectionClient.java
+++ b/src/test/java/redis/clients/jedis/scenario/FaultInjectionClient.java
@@ -89,8 +89,8 @@ public boolean isCompleted(Duration checkInterval, Duration delayAfter, Duration
private static CloseableHttpClient getHttpClient() {
RequestConfig requestConfig = RequestConfig.custom()
- .setConnectionRequestTimeout(5000, TimeUnit.MILLISECONDS)
- .setResponseTimeout(5000, TimeUnit.MILLISECONDS).build();
+ .setConnectionRequestTimeout(10000, TimeUnit.MILLISECONDS)
+ .setResponseTimeout(10000, TimeUnit.MILLISECONDS).build();
return HttpClientBuilder.create()
.setDefaultRequestConfig(requestConfig).build();
diff --git a/src/test/resources/logback-test.xml b/src/test/resources/logback-test.xml
index 29aed8ea57..0f4dc685b9 100644
--- a/src/test/resources/logback-test.xml
+++ b/src/test/resources/logback-test.xml
@@ -23,6 +23,7 @@
+