diff --git a/docs/modules/ROOT/pages/leader-election.adoc b/docs/modules/ROOT/pages/leader-election.adoc index 58f3bf3b47..3e497775e2 100644 --- a/docs/modules/ROOT/pages/leader-election.adoc +++ b/docs/modules/ROOT/pages/leader-election.adoc @@ -53,7 +53,7 @@ management.info.leader.enabled=false ''' -There is another way you can configure leader election, and it comes with native support in the fabric8 library (k8s native client support is not yet implemented). In the long run, this will be the default way to configure leader election, while the previous one will be dropped. You can treat this one much like the JDK's "preview" features. +There is another way you can configure leader election, and it comes with native support in both fabric8 and kubernetes client. In the long run, this will be the default way to configure leader election, while the previous one will be dropped. You can treat this one much like the JDK's "preview" features. To be able to use it, you need to set the property: diff --git a/pom.xml b/pom.xml index 78bd9172ba..6663e06272 100644 --- a/pom.xml +++ b/pom.xml @@ -131,6 +131,7 @@ spring-cloud-starter-kubernetes-fabric8-loadbalancer spring-cloud-kubernetes-discovery spring-cloud-starter-kubernetes-discoveryclient + spring-cloud-kubernetes-client-leader diff --git a/spring-cloud-kubernetes-client-leader/pom.xml b/spring-cloud-kubernetes-client-leader/pom.xml new file mode 100644 index 0000000000..ac7ee0c289 --- /dev/null +++ b/spring-cloud-kubernetes-client-leader/pom.xml @@ -0,0 +1,61 @@ + + + 4.0.0 + + org.springframework.cloud + spring-cloud-kubernetes + 5.0.1-SNAPSHOT + + + spring-cloud-kubernetes-client-leader + K8s Client Spring Cloud Kubernetes :: Leader + + + + org.springframework.cloud + spring-cloud-kubernetes-client-autoconfig + + + org.springframework.boot + spring-boot-starter-actuator + true + + + org.springframework.boot + spring-boot-configuration-processor + true + + + org.springframework.boot + spring-boot-starter-web + test + + + org.springframework.boot + spring-boot-starter-webflux + test + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.cloud + spring-cloud-kubernetes-test-support + test + + + org.springframework.boot + spring-boot-webtestclient + test + + + org.wiremock + wiremock-standalone + test + + + diff --git a/spring-cloud-kubernetes-client-leader/src/main/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionAutoConfiguration.java b/spring-cloud-kubernetes-client-leader/src/main/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionAutoConfiguration.java new file mode 100644 index 0000000000..57377c3766 --- /dev/null +++ b/spring-cloud-kubernetes-client-leader/src/main/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionAutoConfiguration.java @@ -0,0 +1,165 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.leader.election; + +import java.util.List; +import java.util.Objects; +import java.util.function.BooleanSupplier; + +import io.kubernetes.client.extended.leaderelection.LeaderElectionConfig; +import io.kubernetes.client.extended.leaderelection.Lock; +import io.kubernetes.client.extended.leaderelection.resourcelock.ConfigMapLock; +import io.kubernetes.client.extended.leaderelection.resourcelock.LeaseLock; +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.apis.CustomObjectsApi; +import io.kubernetes.client.openapi.models.V1APIResource; +import io.kubernetes.client.openapi.models.V1Pod; +import io.kubernetes.client.openapi.models.V1PodCondition; + +import org.springframework.boot.actuate.autoconfigure.info.ConditionalOnEnabledInfoContributor; +import org.springframework.boot.actuate.info.InfoContributor; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnCloudPlatform; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.cloud.CloudPlatform; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.kubernetes.commons.leader.election.ConditionalOnLeaderElectionEnabled; +import org.springframework.cloud.kubernetes.commons.leader.election.LeaderElectionProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.log.LogAccessor; + +import static org.springframework.cloud.kubernetes.commons.leader.LeaderUtils.COORDINATION_GROUP; +import static org.springframework.cloud.kubernetes.commons.leader.LeaderUtils.COORDINATION_VERSION; + +/** + * @author wind57 + */ +@Configuration(proxyBeanMethods = false) +@EnableConfigurationProperties(LeaderElectionProperties.class) +@ConditionalOnBean(ApiClient.class) +@ConditionalOnLeaderElectionEnabled +@ConditionalOnCloudPlatform(CloudPlatform.KUBERNETES) +@AutoConfigureAfter(KubernetesClientLeaderElectionCallbacksAutoConfiguration.class) +class KubernetesClientLeaderElectionAutoConfiguration { + + private static final LogAccessor LOG = new LogAccessor(KubernetesClientLeaderElectionAutoConfiguration.class); + + @Bean + @ConditionalOnClass(InfoContributor.class) + @ConditionalOnEnabledInfoContributor("leader.election") + KubernetesClientLeaderElectionInfoContributor kubernetesClientLeaderElectionInfoContributor( + String candidateIdentity, LeaderElectionConfig leaderElectionConfig) { + return new KubernetesClientLeaderElectionInfoContributor(candidateIdentity, leaderElectionConfig); + } + + @Bean + @ConditionalOnMissingBean + KubernetesClientLeaderElectionInitiator kubernetesClientLeaderElectionInitiator(String candidateIdentity, + String podNamespace, LeaderElectionConfig leaderElectionConfig, + LeaderElectionProperties leaderElectionProperties, BooleanSupplier podReadySupplier, + KubernetesClientLeaderElectionCallbacks callbacks) { + return new KubernetesClientLeaderElectionInitiator(candidateIdentity, podNamespace, leaderElectionConfig, + leaderElectionProperties, podReadySupplier, callbacks); + } + + @Bean + @ConditionalOnMissingBean + BooleanSupplier kubernetesClientPodReadySupplier(CoreV1Api coreV1Api, String candidateIdentity, + String podNamespace) { + return () -> { + try { + V1Pod pod = coreV1Api.readNamespacedPod(candidateIdentity, podNamespace).execute(); + return isPodReady(pod); + } + catch (ApiException e) { + throw new RuntimeException(e); + } + }; + } + + @Bean + @ConditionalOnMissingBean + LeaderElectionConfig kubernetesClientLeaderElectionConfig(LeaderElectionProperties properties, Lock lock) { + return new LeaderElectionConfig(lock, properties.leaseDuration(), properties.renewDeadline(), + properties.retryPeriod()); + } + + @Bean + @ConditionalOnMissingBean + Lock kubernetesClientLeaderElectionLock(ApiClient apiClient, LeaderElectionProperties properties, + String candidateIdentity) { + + CustomObjectsApi customObjectsApi = new CustomObjectsApi(apiClient); + boolean leaseSupported; + try { + List resources = customObjectsApi.getAPIResources(COORDINATION_GROUP, COORDINATION_VERSION) + .execute() + .getResources(); + + leaseSupported = resources.stream().map(V1APIResource::getKind).anyMatch("Lease"::equals); + } + catch (ApiException e) { + throw new RuntimeException(e); + } + + if (leaseSupported) { + if (properties.useConfigMapAsLock()) { + LOG.info(() -> "leases are supported on the cluster, but config map will be used " + + "(because 'spring.cloud.kubernetes.leader.election.use-config-map-as-lock=true')"); + return new ConfigMapLock(properties.lockNamespace(), properties.lockName(), candidateIdentity); + } + else { + LOG.info(() -> "will use lease as the lock for leader election"); + return new LeaseLock(properties.lockNamespace(), properties.lockName(), candidateIdentity, apiClient); + } + } + else { + LOG.info(() -> "will use configmap as the lock for leader election"); + return new ConfigMapLock(properties.lockNamespace(), properties.lockName(), candidateIdentity, apiClient); + } + } + + // following two methods are a verbatim copy of the fabric8 implementation + private static boolean isPodReady(V1Pod pod) { + Objects.requireNonNull(pod, "Pod can't be null."); + V1PodCondition condition = getPodReadyCondition(pod); + + if (condition == null) { + return false; + } + return condition.getStatus().equalsIgnoreCase("True"); + } + + private static V1PodCondition getPodReadyCondition(V1Pod pod) { + if (pod.getStatus() == null || pod.getStatus().getConditions() == null) { + return null; + } + + for (V1PodCondition condition : pod.getStatus().getConditions()) { + if ("Ready".equals(condition.getType())) { + return condition; + } + } + return null; + } + +} diff --git a/spring-cloud-kubernetes-client-leader/src/main/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionCallbacks.java b/spring-cloud-kubernetes-client-leader/src/main/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionCallbacks.java new file mode 100644 index 0000000000..780cc862e2 --- /dev/null +++ b/spring-cloud-kubernetes-client-leader/src/main/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionCallbacks.java @@ -0,0 +1,26 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.leader.election; + +import java.util.function.Consumer; + +/** + * @author wind57 + */ +record KubernetesClientLeaderElectionCallbacks(Runnable onStartLeadingCallback, Runnable onStopLeadingCallback, + Consumer onNewLeaderCallback) { +} diff --git a/spring-cloud-kubernetes-client-leader/src/main/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionCallbacksAutoConfiguration.java b/spring-cloud-kubernetes-client-leader/src/main/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionCallbacksAutoConfiguration.java new file mode 100644 index 0000000000..02af5386e7 --- /dev/null +++ b/spring-cloud-kubernetes-client-leader/src/main/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionCallbacksAutoConfiguration.java @@ -0,0 +1,55 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.leader.election; + +import java.util.function.Consumer; + +import io.kubernetes.client.openapi.ApiClient; + +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnCloudPlatform; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.cloud.CloudPlatform; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.kubernetes.client.KubernetesClientAutoConfiguration; +import org.springframework.cloud.kubernetes.commons.leader.election.ConditionalOnLeaderElectionEnabled; +import org.springframework.cloud.kubernetes.commons.leader.election.LeaderElectionCallbacks; +import org.springframework.cloud.kubernetes.commons.leader.election.LeaderElectionProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * @author wind57 + */ +@Configuration(proxyBeanMethods = false) +@EnableConfigurationProperties(LeaderElectionProperties.class) +@ConditionalOnBean(ApiClient.class) +@ConditionalOnCloudPlatform(CloudPlatform.KUBERNETES) +@ConditionalOnLeaderElectionEnabled +@AutoConfigureAfter({ KubernetesClientAutoConfiguration.class }) +public class KubernetesClientLeaderElectionCallbacksAutoConfiguration extends LeaderElectionCallbacks { + + @Bean + @ConditionalOnMissingBean + KubernetesClientLeaderElectionCallbacks kubernetesClientLeaderElectionCallbacks(Runnable onStartLeadingCallback, + Runnable onStopLeadingCallback, Consumer onNewLeaderCallback) { + return new KubernetesClientLeaderElectionCallbacks(onStartLeadingCallback, onStopLeadingCallback, + onNewLeaderCallback); + } + +} diff --git a/spring-cloud-kubernetes-client-leader/src/main/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionInfoContributor.java b/spring-cloud-kubernetes-client-leader/src/main/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionInfoContributor.java new file mode 100644 index 0000000000..728b71f93e --- /dev/null +++ b/spring-cloud-kubernetes-client-leader/src/main/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionInfoContributor.java @@ -0,0 +1,63 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.leader.election; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import io.kubernetes.client.extended.leaderelection.LeaderElectionConfig; +import io.kubernetes.client.openapi.ApiException; + +import org.springframework.boot.actuate.info.Info; +import org.springframework.boot.actuate.info.InfoContributor; +import org.springframework.core.log.LogAccessor; + +/** + * @author wind57 + */ +final class KubernetesClientLeaderElectionInfoContributor implements InfoContributor { + + private static final LogAccessor LOG = new LogAccessor(KubernetesClientLeaderElectionInfoContributor.class); + + private final String candidateIdentity; + + private final LeaderElectionConfig leaderElectionConfig; + + KubernetesClientLeaderElectionInfoContributor(String candidateIdentity, LeaderElectionConfig leaderElectionConfig) { + this.candidateIdentity = candidateIdentity; + this.leaderElectionConfig = leaderElectionConfig; + } + + @Override + public void contribute(Info.Builder builder) { + Map details = new HashMap<>(); + try { + Optional.ofNullable(leaderElectionConfig.getLock().get()).ifPresentOrElse(leaderRecord -> { + boolean isLeader = candidateIdentity.equals(leaderRecord.getHolderIdentity()); + details.put("leaderId", candidateIdentity); + details.put("isLeader", isLeader); + }, () -> details.put("leaderId", "Unknown")); + } + catch (ApiException e) { + LOG.error(e, "error in leader election info contributor"); + } + + builder.withDetail("leaderElection", details); + } + +} diff --git a/spring-cloud-kubernetes-client-leader/src/main/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionInitiator.java b/spring-cloud-kubernetes-client-leader/src/main/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionInitiator.java new file mode 100644 index 0000000000..9abd7f86a8 --- /dev/null +++ b/spring-cloud-kubernetes-client-leader/src/main/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionInitiator.java @@ -0,0 +1,193 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.leader.election; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.function.BooleanSupplier; + +import io.kubernetes.client.extended.leaderelection.LeaderElectionConfig; +import io.kubernetes.client.extended.leaderelection.LeaderElector; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; + +import org.springframework.cloud.kubernetes.commons.leader.election.LeaderElectionProperties; +import org.springframework.cloud.kubernetes.commons.leader.election.PodReadyRunner; +import org.springframework.core.log.LogAccessor; + +import static java.util.concurrent.Executors.newSingleThreadExecutor; +import static org.springframework.cloud.kubernetes.commons.leader.election.LeaderElectionInitiatorUtil.attachReadinessLoggerPipeline; +import static org.springframework.cloud.kubernetes.commons.leader.election.LeaderElectionInitiatorUtil.blockReadinessCheck; +import static org.springframework.cloud.kubernetes.commons.leader.election.LeaderElectionInitiatorUtil.shutDownExecutor; +import static org.springframework.cloud.kubernetes.commons.leader.election.LeaderElectionInitiatorUtil.sleep; + +/** + * @author wind57 + */ +final class KubernetesClientLeaderElectionInitiator { + + private static final LogAccessor LOG = new LogAccessor(KubernetesClientLeaderElectionInitiator.class); + + private final PodReadyRunner podReadyRunner; + + private final String candidateIdentity; + + private final LeaderElectionConfig leaderElectionConfig; + + private final LeaderElectionProperties leaderElectionProperties; + + private final boolean waitForPodReady; + + private final ExecutorService podReadyWaitingExecutor; + + private final BooleanSupplier podReadySupplier; + + private final KubernetesClientLeaderElectionCallbacks callbacks; + + private volatile LeaderElector leaderElector; + + private volatile CompletableFuture podReadyFuture; + + private volatile boolean shutDownCalled = false; + + KubernetesClientLeaderElectionInitiator(String candidateIdentity, String candidateNamespace, + LeaderElectionConfig leaderElectionConfig, LeaderElectionProperties leaderElectionProperties, + BooleanSupplier podReadySupplier, KubernetesClientLeaderElectionCallbacks callbacks) { + this.candidateIdentity = candidateIdentity; + this.leaderElectionConfig = leaderElectionConfig; + this.leaderElectionProperties = leaderElectionProperties; + this.waitForPodReady = leaderElectionProperties.waitForPodReady(); + this.podReadySupplier = podReadySupplier; + this.callbacks = callbacks; + + this.podReadyWaitingExecutor = newSingleThreadExecutor( + runnable -> new Thread(runnable, "KubernetesClientLeaderElectionInitiator-" + candidateIdentity)); + + this.podReadyRunner = new PodReadyRunner(candidateIdentity, candidateNamespace); + } + + // visible for testing only + LeaderElector leaderElector() { + return leaderElector; + } + + /** + * 
+	 * 	We first try to see if we need to wait for the pod to be ready
+	 * 	before starting the leader election process.
+	 *
+ * + */ + @PostConstruct + void postConstruct() { + + LOG.info(() -> "starting leader initiator : " + candidateIdentity); + + // wait until the pod is ready + if (waitForPodReady) { + LOG.info(() -> "will wait until pod " + candidateIdentity + " is ready"); + podReadyFuture = podReadyRunner.podReady(podReadySupplier); + } + else { + podReadyFuture = CompletableFuture.completedFuture(null); + } + + // wait in a different thread until the pod is ready + // and don't block the main application from starting + podReadyWaitingExecutor.execute(() -> { + try { + if (waitForPodReady) { + CompletableFuture ready = attachReadinessLoggerPipeline(podReadyFuture, candidateIdentity); + blockReadinessCheck(ready); + startLeaderElection(); + } + else { + startLeaderElection(); + } + } + catch (Exception e) { + LOG.error(e, () -> "failure : " + e.getMessage()); + } + }); + + } + + @PreDestroy + void preDestroy() { + LOG.info(() -> "preDestroy called on the leader initiator : " + candidateIdentity); + shutDownCalled = true; + + if (podReadyFuture != null && !podReadyFuture.isDone()) { + // if the task is not running, this has no effect. + // if the task is running, calling this will also make sure + // that the caching executor will shut down too. + LOG.debug(() -> "podReadyFuture will be canceled for : " + candidateIdentity); + podReadyFuture.cancel(true); + } + + if (!podReadyWaitingExecutor.isShutdown()) { + shutDownExecutor(podReadyWaitingExecutor, candidateIdentity); + } + + if (leaderElector != null) { + leaderElector.close(); + } + } + + private void startLeaderElection() { + + LOG.info(() -> "starting leader election : " + candidateIdentity); + + boolean failedDuringStartup = false; + leaderElector = new LeaderElector(leaderElectionConfig); + try { + // this runs in a while(true) loop and every throwable is just logged, + // it does not spill over to our code. It means that 'failedDuringStartup' + // can only be true before we enter the while(true) loop. This can be some + // basic validations + // like not sufficient RBAC, for example. + leaderElector.run(callbacks.onStartLeadingCallback(), callbacks.onStopLeadingCallback(), + callbacks.onNewLeaderCallback()); + } + catch (Exception e) { + // this is only possible when we can't start leader election, not during + // its inner workings + LOG.error(e, () -> "failure starting leader election: " + e.getMessage()); + failedDuringStartup = true; + } + finally { + leaderElector.close(); + } + + if (shutDownCalled) { + LOG.debug(() -> "leadership terminated for : " + candidateIdentity); + return; + } + + if (failedDuringStartup) { + LOG.error(() -> "leadership failed during startup for : " + candidateIdentity); + return; + } + + // as soon as leader election is over, re-start it + LOG.debug(() -> "will re-start leader election for : " + candidateIdentity); + sleep(leaderElectionProperties); + podReadyWaitingExecutor.execute(this::startLeaderElection); + + } + +} diff --git a/spring-cloud-kubernetes-client-leader/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/spring-cloud-kubernetes-client-leader/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports new file mode 100644 index 0000000000..3e40fe917a --- /dev/null +++ b/spring-cloud-kubernetes-client-leader/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -0,0 +1,2 @@ +org.springframework.cloud.kubernetes.client.leader.election.KubernetesClientLeaderElectionCallbacksAutoConfiguration +org.springframework.cloud.kubernetes.client.leader.election.KubernetesClientLeaderElectionAutoConfiguration diff --git a/spring-cloud-kubernetes-client-leader/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionAutoConfigurationTests.java b/spring-cloud-kubernetes-client-leader/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionAutoConfigurationTests.java new file mode 100644 index 0000000000..06b59e942c --- /dev/null +++ b/spring-cloud-kubernetes-client-leader/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionAutoConfigurationTests.java @@ -0,0 +1,206 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.leader.election; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.client.WireMock; +import io.kubernetes.client.openapi.ApiClient; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import org.springframework.boot.autoconfigure.AutoConfigurations; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.cloud.kubernetes.client.KubernetesClientAutoConfiguration; +import org.springframework.cloud.kubernetes.commons.KubernetesCommonsAutoConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; + +import static org.springframework.cloud.kubernetes.client.leader.election.KubernetesClientLeaderElectionUtil.apiClientWithLeaseSupport; +import static org.springframework.cloud.kubernetes.client.leader.election.KubernetesClientLeaderElectionUtil.wireMockServer; + +/** + * @author wind57 + */ +class KubernetesClientLeaderElectionAutoConfigurationTests { + + private static WireMockServer wireMockServer; + + @BeforeAll + static void beforeAll() { + wireMockServer = wireMockServer(); + } + + @AfterAll + static void after() { + wireMockServer.stop(); + } + + @AfterEach + void afterEach() { + WireMock.reset(); + } + + /** + * 
+	 *     - spring.cloud.kubernetes.leader.election is not present
+	 *
+	 *     As such:
+	 *
+	 *     - KubernetesClientLeaderElectionAutoConfiguration          is not present
+	 *     - KubernetesClientLeaderElectionCallbacksAutoConfiguration is not present
+	 *
+ */ + @Test + void leaderElectionAnnotationMissing() { + new ApplicationContextRunner() + .withConfiguration(AutoConfigurations.of(KubernetesCommonsAutoConfiguration.class, + KubernetesClientAutoConfiguration.class, KubernetesClientLeaderElectionAutoConfiguration.class, + KubernetesClientLeaderElectionCallbacksAutoConfiguration.class)) + .withAllowBeanDefinitionOverriding(true) + .withUserConfiguration(ApiClientConfiguration.class) + .run(context -> { + Assertions.assertThat(context).doesNotHaveBean(KubernetesClientLeaderElectionAutoConfiguration.class); + Assertions.assertThat(context) + .doesNotHaveBean(KubernetesClientLeaderElectionCallbacksAutoConfiguration.class); + }); + } + + /** + * 
+	 *     - spring.cloud.kubernetes.leader.election = false
+	 *
+	 *     As such:
+	 *
+	 *     - KubernetesClientLeaderElectionAutoConfiguration          is not present
+	 *     - KubernetesClientLeaderElectionCallbacksAutoConfiguration is not present
+	 *
+ */ + @Test + void leaderElectionAnnotationPresentEqualToFalse() { + new ApplicationContextRunner() + .withConfiguration(AutoConfigurations.of(KubernetesCommonsAutoConfiguration.class, + KubernetesClientAutoConfiguration.class, KubernetesClientLeaderElectionAutoConfiguration.class, + KubernetesClientLeaderElectionCallbacksAutoConfiguration.class)) + .withAllowBeanDefinitionOverriding(true) + .withUserConfiguration(ApiClientConfiguration.class) + .withPropertyValues("spring.cloud.kubernetes.leader.election.enabled=false") + .run(context -> { + Assertions.assertThat(context).doesNotHaveBean(KubernetesClientLeaderElectionAutoConfiguration.class); + Assertions.assertThat(context) + .doesNotHaveBean(KubernetesClientLeaderElectionCallbacksAutoConfiguration.class); + }); + } + + /** + * 
+	 *     - spring.cloud.kubernetes.leader.election = true
+	 *
+	 *     As such:
+	 *
+	 *     - KubernetesClientLeaderElectionAutoConfiguration          is present
+	 *     - KubernetesClientLeaderElectionCallbacksAutoConfiguration is present
+	 *
+ */ + @Test + void leaderElectionAnnotationPresentEqualToTrue() { + new ApplicationContextRunner().withAllowBeanDefinitionOverriding(true) + .withUserConfiguration(ApiClientConfiguration.class) + .withConfiguration(AutoConfigurations.of(KubernetesCommonsAutoConfiguration.class, + KubernetesClientAutoConfiguration.class, KubernetesClientLeaderElectionAutoConfiguration.class, + KubernetesClientLeaderElectionCallbacksAutoConfiguration.class)) + .withPropertyValues("spring.cloud.kubernetes.leader.election.enabled=true", + "spring.main.cloud-platform=kubernetes") + .run(context -> { + Assertions.assertThat(context).hasSingleBean(KubernetesClientLeaderElectionAutoConfiguration.class); + Assertions.assertThat(context) + .hasSingleBean(KubernetesClientLeaderElectionCallbacksAutoConfiguration.class); + }); + } + + /** + * 
+	 *     - spring.cloud.kubernetes.leader.election = true
+	 *     - management.info.leader.election.enabled = true
+	 *
+	 *     As such:
+	 *
+	 *     - KubernetesClientLeaderElectionAutoConfiguration          is present
+	 *     - KubernetesClientLeaderElectionCallbacksAutoConfiguration is present
+	 *     - KubernetesClientLeaderElectionInfoContributor            is present
+	 *
+ */ + @Test + void leaderInfoContributorPresent() { + new ApplicationContextRunner().withUserConfiguration(ApiClientConfiguration.class) + .withConfiguration(AutoConfigurations.of(KubernetesCommonsAutoConfiguration.class, + KubernetesClientAutoConfiguration.class, KubernetesClientLeaderElectionAutoConfiguration.class, + KubernetesClientLeaderElectionCallbacksAutoConfiguration.class)) + .withPropertyValues("spring.main.cloud-platform=kubernetes", "management.info.leader.election.enabled=true", + "spring.cloud.kubernetes.leader.election.enabled=true") + .run(context -> { + Assertions.assertThat(context).hasSingleBean(KubernetesClientLeaderElectionAutoConfiguration.class); + Assertions.assertThat(context) + .hasSingleBean(KubernetesClientLeaderElectionCallbacksAutoConfiguration.class); + Assertions.assertThat(context).hasSingleBean(KubernetesClientLeaderElectionInfoContributor.class); + }); + } + + /** + * 
+	 *     - spring.cloud.kubernetes.leader.election = true
+	 *     - management.info.leader.election.enabled = false
+	 *
+	 *     As such:
+	 *
+	 *     - KubernetesClientLeaderElectionAutoConfiguration          is present
+	 *     - KubernetesClientLeaderElectionCallbacksAutoConfiguration is present
+	 *     - KubernetesClientLeaderElectionInfoContributor            is not present
+	 *
+ */ + @Test + void leaderInfoContributorMissing() { + new ApplicationContextRunner().withUserConfiguration(ApiClientConfiguration.class) + .withConfiguration(AutoConfigurations.of(KubernetesCommonsAutoConfiguration.class, + KubernetesClientAutoConfiguration.class, KubernetesClientLeaderElectionAutoConfiguration.class, + KubernetesClientLeaderElectionCallbacksAutoConfiguration.class)) + .withPropertyValues("spring.main.cloud-platform=kubernetes", + "management.info.leader.election.enabled=false", + "spring.cloud.kubernetes.leader.election.enabled=true") + .run(context -> { + Assertions.assertThat(context).hasSingleBean(KubernetesClientLeaderElectionAutoConfiguration.class); + Assertions.assertThat(context) + .hasSingleBean(KubernetesClientLeaderElectionCallbacksAutoConfiguration.class); + Assertions.assertThat(context).doesNotHaveBean(KubernetesClientLeaderElectionInfoContributor.class); + }); + } + + @Configuration + static class ApiClientConfiguration { + + @Bean + @Primary + ApiClient apiClient() { + return apiClientWithLeaseSupport(wireMockServer); + } + + } + +} diff --git a/spring-cloud-kubernetes-client-leader/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionInfoContributorIsLeaderTest.java b/spring-cloud-kubernetes-client-leader/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionInfoContributorIsLeaderTest.java new file mode 100644 index 0000000000..b0261df998 --- /dev/null +++ b/spring-cloud-kubernetes-client-leader/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionInfoContributorIsLeaderTest.java @@ -0,0 +1,97 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.leader.election; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.client.WireMock; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.server.LocalManagementPort; +import org.springframework.boot.webtestclient.autoconfigure.AutoConfigureWebTestClient; +import org.springframework.cloud.kubernetes.commons.leader.LeaderUtils; +import org.springframework.http.MediaType; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.web.reactive.server.WebTestClient; + +import static org.springframework.cloud.kubernetes.client.leader.election.KubernetesClientLeaderElectionUtil.HOLDER_IDENTITY; +import static org.springframework.cloud.kubernetes.client.leader.election.KubernetesClientLeaderElectionUtil.wireMockServer; + +/** + * @author wind57 + */ +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = { "spring.main.cloud-platform=KUBERNETES", "management.endpoints.web.exposure.include=info", + "management.endpoint.info.show-details=always", "spring.cloud.kubernetes.leader.election.enabled=true", + "spring.main.allow-bean-definition-overriding=true", + "spring.cloud.kubernetes.leader.election.wait-for-pod-ready=false" }, + classes = { KubernetesClientLeaderElectionTestApp.class, + KubernetesClientLeaderElectionUtil.ApiClientConfiguration.class }) +@AutoConfigureWebTestClient +@DirtiesContext +class KubernetesClientLeaderElectionInfoContributorIsLeaderTest { + + @LocalManagementPort + private int port; + + @Autowired + private WebTestClient webClient; + + private static MockedStatic leaderUtilsMockedStatic; + + private static WireMockServer wireMockServer; + + @BeforeAll + static void beforeAll() { + leaderUtilsMockedStatic = Mockito.mockStatic(LeaderUtils.class); + leaderUtilsMockedStatic.when(LeaderUtils::hostName).thenReturn(HOLDER_IDENTITY); + wireMockServer = wireMockServer(); + } + + @AfterAll + static void afterAll() { + leaderUtilsMockedStatic.close(); + wireMockServer.stop(); + } + + @AfterEach + void afterEach() { + WireMock.reset(); + } + + @Test + void infoEndpointIsLeaderTest() { + webClient.get() + .uri("http://localhost:{port}/actuator/info", port) + .accept(MediaType.APPLICATION_JSON) + .exchange() + .expectStatus() + .isOk() + .expectBody() + .jsonPath("leaderElection.isLeader") + .isEqualTo(true) + .jsonPath("leaderElection.leaderId") + .isEqualTo(HOLDER_IDENTITY); + } + +} diff --git a/spring-cloud-kubernetes-client-leader/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionInfoContributorIsNotLeaderTest.java b/spring-cloud-kubernetes-client-leader/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionInfoContributorIsNotLeaderTest.java new file mode 100644 index 0000000000..4035b3e536 --- /dev/null +++ b/spring-cloud-kubernetes-client-leader/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionInfoContributorIsNotLeaderTest.java @@ -0,0 +1,97 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.leader.election; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.client.WireMock; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.server.LocalManagementPort; +import org.springframework.boot.webtestclient.autoconfigure.AutoConfigureWebTestClient; +import org.springframework.cloud.kubernetes.commons.leader.LeaderUtils; +import org.springframework.http.MediaType; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.web.reactive.server.WebTestClient; + +import static org.springframework.cloud.kubernetes.client.leader.election.KubernetesClientLeaderElectionUtil.HOLDER_IDENTITY; +import static org.springframework.cloud.kubernetes.client.leader.election.KubernetesClientLeaderElectionUtil.wireMockServer; + +/** + * @author wind57 + */ +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = { "spring.main.cloud-platform=KUBERNETES", "management.endpoints.web.exposure.include=info", + "management.endpoint.info.show-details=always", "spring.cloud.kubernetes.leader.election.enabled=true", + "spring.main.allow-bean-definition-overriding=true", + "spring.cloud.kubernetes.leader.election.wait-for-pod-ready=false" }, + classes = { KubernetesClientLeaderElectionTestApp.class, + KubernetesClientLeaderElectionUtil.ApiClientConfiguration.class }) +@AutoConfigureWebTestClient +@DirtiesContext +class KubernetesClientLeaderElectionInfoContributorIsNotLeaderTest { + + @LocalManagementPort + private int port; + + @Autowired + private WebTestClient webClient; + + private static MockedStatic leaderUtilsMockedStatic; + + private static WireMockServer wireMockServer; + + @BeforeAll + static void beforeAll() { + leaderUtilsMockedStatic = Mockito.mockStatic(LeaderUtils.class); + leaderUtilsMockedStatic.when(LeaderUtils::hostName).thenReturn("non-" + HOLDER_IDENTITY); + wireMockServer = wireMockServer(); + } + + @AfterAll + static void afterAll() { + leaderUtilsMockedStatic.close(); + wireMockServer.stop(); + } + + @AfterEach + void afterEach() { + WireMock.reset(); + } + + @Test + void infoEndpointIsNotLeaderTest() { + webClient.get() + .uri("http://localhost:{port}/actuator/info", port) + .accept(MediaType.APPLICATION_JSON) + .exchange() + .expectStatus() + .isOk() + .expectBody() + .jsonPath("leaderElection.isLeader") + .isEqualTo(false) + .jsonPath("leaderElection.leaderId") + .isEqualTo("non-" + HOLDER_IDENTITY); + } + +} diff --git a/spring-cloud-kubernetes-client-leader/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionTestApp.java b/spring-cloud-kubernetes-client-leader/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionTestApp.java new file mode 100644 index 0000000000..7a6307b9b2 --- /dev/null +++ b/spring-cloud-kubernetes-client-leader/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionTestApp.java @@ -0,0 +1,32 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.leader.election; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +/** + * @author wind57 + */ +@SpringBootApplication +class KubernetesClientLeaderElectionTestApp { + + public static void main(String[] args) { + SpringApplication.run(KubernetesClientLeaderElectionTestApp.class, args); + } + +} diff --git a/spring-cloud-kubernetes-client-leader/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionUtil.java b/spring-cloud-kubernetes-client-leader/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionUtil.java new file mode 100644 index 0000000000..ed3bfec3c5 --- /dev/null +++ b/spring-cloud-kubernetes-client-leader/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/KubernetesClientLeaderElectionUtil.java @@ -0,0 +1,98 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.leader.election; + +import java.time.OffsetDateTime; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.JSON; +import io.kubernetes.client.openapi.models.V1APIResource; +import io.kubernetes.client.openapi.models.V1APIResourceList; +import io.kubernetes.client.openapi.models.V1APIResourceListBuilder; +import io.kubernetes.client.openapi.models.V1Lease; +import io.kubernetes.client.openapi.models.V1LeaseBuilder; +import io.kubernetes.client.openapi.models.V1LeaseSpecBuilder; +import io.kubernetes.client.util.ClientBuilder; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; + +/** + * @author wind57 + */ +final class KubernetesClientLeaderElectionUtil { + + static final String HOLDER_IDENTITY = "leader"; + + private KubernetesClientLeaderElectionUtil() { + + } + + static WireMockServer wireMockServer() { + WireMockServer wireMockServer = new WireMockServer(WireMockConfiguration.options().dynamicPort()); + wireMockServer.start(); + WireMock.configureFor(wireMockServer.port()); + return wireMockServer; + } + + static ApiClient apiClientWithLeaseSupport(WireMockServer wireMockServer) { + + // lease lock is supported + V1APIResourceList leaseList = new V1APIResourceListBuilder() + .addToResources(new V1APIResource().kind("Lease").name("my-lease").namespaced(false).singularName("Lease")) + .withApiVersion("v1") + .withGroupVersion("v1") + .withKind("Foo") + .build(); + stubFor(get("/apis/coordination.k8s.io/v1") + .willReturn(aResponse().withStatus(200).withBody(JSON.serialize(leaseList)))); + + // lease that is requested + V1Lease lease = new V1LeaseBuilder().withKind("Lease") + .withSpec(new V1LeaseSpecBuilder().withLeaseTransitions(1) + .withAcquireTime(OffsetDateTime.now()) + .withLeaseDurationSeconds(2) + .withRenewTime(OffsetDateTime.now()) + .withHolderIdentity(HOLDER_IDENTITY) + .build()) + .build(); + stubFor(get("/apis/coordination.k8s.io/v1/namespaces/default/leases/spring-k8s-leader-election-lock") + .willReturn(aResponse().withStatus(200).withBody(JSON.serialize(lease)))); + + return new ClientBuilder().setBasePath(wireMockServer.baseUrl()).build(); + } + + @Configuration + static class ApiClientConfiguration { + + @Bean + @Primary + ApiClient apiClient() { + return apiClientWithLeaseSupport(wireMockServer()); + } + + } + +} diff --git a/spring-cloud-kubernetes-dependencies/pom.xml b/spring-cloud-kubernetes-dependencies/pom.xml index 6ea725bf6f..e657da8086 100644 --- a/spring-cloud-kubernetes-dependencies/pom.xml +++ b/spring-cloud-kubernetes-dependencies/pom.xml @@ -105,6 +105,12 @@ ${project.version} + + org.springframework.cloud + spring-cloud-kubernetes-client-leader + ${project.version} + + org.springframework.cloud spring-cloud-kubernetes-commons diff --git a/spring-cloud-kubernetes-fabric8-leader/pom.xml b/spring-cloud-kubernetes-fabric8-leader/pom.xml index 648e3082ea..6c1da24144 100644 --- a/spring-cloud-kubernetes-fabric8-leader/pom.xml +++ b/spring-cloud-kubernetes-fabric8-leader/pom.xml @@ -10,7 +10,7 @@ spring-cloud-kubernetes-fabric8-leader - Spring Cloud Kubernetes :: Leader + Fabric8 Spring Cloud Kubernetes :: Leader diff --git a/spring-cloud-kubernetes-fabric8-leader/src/main/java/org/springframework/cloud/kubernetes/fabric8/leader/election/Fabric8LeaderElectionCallbacksAutoConfiguration.java b/spring-cloud-kubernetes-fabric8-leader/src/main/java/org/springframework/cloud/kubernetes/fabric8/leader/election/Fabric8LeaderElectionCallbacksAutoConfiguration.java index 14f4255985..1d0c157cc2 100644 --- a/spring-cloud-kubernetes-fabric8-leader/src/main/java/org/springframework/cloud/kubernetes/fabric8/leader/election/Fabric8LeaderElectionCallbacksAutoConfiguration.java +++ b/spring-cloud-kubernetes-fabric8-leader/src/main/java/org/springframework/cloud/kubernetes/fabric8/leader/election/Fabric8LeaderElectionCallbacksAutoConfiguration.java @@ -40,7 +40,7 @@ @ConditionalOnCloudPlatform(CloudPlatform.KUBERNETES) @ConditionalOnLeaderElectionEnabled @AutoConfigureAfter({ Fabric8AutoConfiguration.class, KubernetesCommonsAutoConfiguration.class }) -final class Fabric8LeaderElectionCallbacksAutoConfiguration extends LeaderElectionCallbacks { +public class Fabric8LeaderElectionCallbacksAutoConfiguration extends LeaderElectionCallbacks { @Bean @ConditionalOnMissingBean diff --git a/spring-cloud-kubernetes-fabric8-leader/src/test/java/org/springframework/cloud/kubernetes/fabric8/leader/election/Fabric8LeaderAutoConfigurationTests.java b/spring-cloud-kubernetes-fabric8-leader/src/test/java/org/springframework/cloud/kubernetes/fabric8/leader/election/Fabric8LeaderElectionAutoConfigurationTests.java similarity index 99% rename from spring-cloud-kubernetes-fabric8-leader/src/test/java/org/springframework/cloud/kubernetes/fabric8/leader/election/Fabric8LeaderAutoConfigurationTests.java rename to spring-cloud-kubernetes-fabric8-leader/src/test/java/org/springframework/cloud/kubernetes/fabric8/leader/election/Fabric8LeaderElectionAutoConfigurationTests.java index 05918d2bbf..6afd5dade4 100644 --- a/spring-cloud-kubernetes-fabric8-leader/src/test/java/org/springframework/cloud/kubernetes/fabric8/leader/election/Fabric8LeaderAutoConfigurationTests.java +++ b/spring-cloud-kubernetes-fabric8-leader/src/test/java/org/springframework/cloud/kubernetes/fabric8/leader/election/Fabric8LeaderElectionAutoConfigurationTests.java @@ -29,7 +29,7 @@ * * @author wind57 */ -class Fabric8LeaderAutoConfigurationTests { +class Fabric8LeaderElectionAutoConfigurationTests { /** * 
diff --git a/spring-cloud-kubernetes-integration-tests/pom.xml b/spring-cloud-kubernetes-integration-tests/pom.xml
index 1908c2ead4..a1a9a6b108 100644
--- a/spring-cloud-kubernetes-integration-tests/pom.xml
+++ b/spring-cloud-kubernetes-integration-tests/pom.xml
@@ -108,13 +108,14 @@
 		spring-cloud-kubernetes-k8s-client-configuration-watcher
 
 		
-        spring-cloud-kubernetes-k8s-client-kafka-configmap-reload
+		spring-cloud-kubernetes-k8s-client-kafka-configmap-reload
 
 		
-        spring-cloud-kubernetes-k8s-client-rabbitmq-secret-reload
+		spring-cloud-kubernetes-k8s-client-rabbitmq-secret-reload
 
 		
-        spring-cloud-kubernetes-fabric8-leader-election
+		spring-cloud-kubernetes-fabric8-leader-election
+		spring-cloud-kubernetes-k8s-client-leader-election
 
-    
+	
 
diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-leader-election/src/test/java/org/springframework/cloud/kubernetes/fabric8/leader/election/Fabric8LeaderElectionCanceledAndNotRestartedIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-leader-election/src/test/java/org/springframework/cloud/kubernetes/fabric8/leader/election/Fabric8LeaderElectionCanceledAndNotRestartedIT.java
index dee050c416..d18d0e82c3 100644
--- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-leader-election/src/test/java/org/springframework/cloud/kubernetes/fabric8/leader/election/Fabric8LeaderElectionCanceledAndNotRestartedIT.java
+++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-leader-election/src/test/java/org/springframework/cloud/kubernetes/fabric8/leader/election/Fabric8LeaderElectionCanceledAndNotRestartedIT.java
@@ -66,6 +66,8 @@ void test(CapturedOutput output) {
 		// lease is going to reset
 		awaitUntil(10, 100, () -> getLease().getSpec().getHolderIdentity().isEmpty());
 
+		awaitUntil(10, 100, () -> output.getOut().contains("terminating leadership for : " + NAME));
+
 	}
 
 }
diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/pom.xml b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/pom.xml
new file mode 100644
index 0000000000..68865a336c
--- /dev/null
+++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/pom.xml
@@ -0,0 +1,39 @@
+
+
+	4.0.0
+	
+		org.springframework.cloud
+		spring-cloud-kubernetes-integration-tests
+		5.0.1-SNAPSHOT
+	
+
+	
+		true
+		true
+	
+
+	spring-cloud-kubernetes-k8s-client-leader-election
+
+	
+		
+			org.springframework.boot
+			spring-boot-starter-test
+			test
+		
+		
+			org.springframework.cloud
+			spring-cloud-kubernetes-test-support
+			test
+		
+		
+			org.springframework.cloud
+			spring-cloud-kubernetes-client-leader
+		
+		
+			org.springframework.boot
+			spring-boot-starter-actuator
+		
+	
+
diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/AbstractLeaderElection.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/AbstractLeaderElection.java
new file mode 100644
index 0000000000..d23e23a057
--- /dev/null
+++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/AbstractLeaderElection.java
@@ -0,0 +1,180 @@
+/*
+ * Copyright 2013-present the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.kubernetes.client.leader.election;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BooleanSupplier;
+
+import io.kubernetes.client.openapi.ApiClient;
+import io.kubernetes.client.openapi.ApiException;
+import io.kubernetes.client.openapi.apis.CoordinationV1Api;
+import io.kubernetes.client.openapi.models.V1Lease;
+import io.kubernetes.client.util.Config;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+import org.testcontainers.k3s.K3sContainer;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.context.TestConfiguration;
+import org.springframework.boot.test.system.OutputCaptureExtension;
+import org.springframework.cloud.kubernetes.commons.leader.LeaderUtils;
+import org.springframework.cloud.kubernetes.integration.tests.commons.Commons;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Primary;
+import org.springframework.test.annotation.DirtiesContext;
+
+/**
+ * @author wind57
+ */
+@ExtendWith(OutputCaptureExtension.class)
+@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT,
+		properties = { "spring.main.cloud-platform=KUBERNETES", "spring.cloud.kubernetes.leader.election.enabled=true",
+				"spring.cloud.kubernetes.leader.election.lease-duration=6s",
+				"spring.cloud.kubernetes.leader.election.renew-deadline=5s",
+				"logging.level.org.springframework.cloud.kubernetes.commons.leader.election=debug",
+				"logging.level.org.springframework.cloud.kubernetes.client.leader.election=debug",
+				"logging.level.io.kubernetes.client.extended.leaderelection=debug" },
+		classes = { App.class, AbstractLeaderElection.TestConfig.class,
+				AbstractLeaderElection.PodReadyTestConfiguration.class })
+@DirtiesContext
+abstract class AbstractLeaderElection {
+
+	@Autowired
+	private ApiClient apiClient;
+
+	private static K3sContainer container;
+
+	private static MockedStatic LEADER_UTILS_MOCKED_STATIC;
+
+	static void beforeAll(String candidateIdentity) {
+		container = Commons.container();
+		container.start();
+
+		LEADER_UTILS_MOCKED_STATIC = Mockito.mockStatic(LeaderUtils.class);
+		LEADER_UTILS_MOCKED_STATIC.when(LeaderUtils::hostName).thenReturn(candidateIdentity);
+	}
+
+	@AfterAll
+	static void afterAll() {
+		LEADER_UTILS_MOCKED_STATIC.close();
+	}
+
+	void stopLeaderAndDeleteLease(KubernetesClientLeaderElectionInitiator initiator, boolean deleteLease) {
+		initiator.preDestroy();
+
+		if (deleteLease) {
+			CoordinationV1Api api = new CoordinationV1Api(apiClient);
+
+			try {
+				api.deleteNamespacedLease("spring-k8s-leader-election-lock", "default").execute();
+			}
+			catch (ApiException e) {
+				throw new RuntimeException(e);
+			}
+		}
+	}
+
+	V1Lease getLease() {
+		CoordinationV1Api api = new CoordinationV1Api(apiClient);
+		try {
+			return api.readNamespacedLease("spring-k8s-leader-election-lock", "default").execute();
+		}
+		catch (ApiException e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+	V1Lease updateLease(V1Lease lease) {
+		CoordinationV1Api api = new CoordinationV1Api(apiClient);
+		try {
+			return api.replaceNamespacedLease("spring-k8s-leader-election-lock", "default", lease).execute();
+		}
+		catch (ApiException e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+	@TestConfiguration
+	static class TestConfig {
+
+		@Bean
+		@Primary
+		ApiClient client() {
+			String kubeConfigYaml = container.getKubeConfigYaml();
+
+			ApiClient client;
+			try {
+				client = Config.fromConfig(new StringReader(kubeConfigYaml));
+			}
+			catch (IOException e) {
+				throw new RuntimeException(e);
+			}
+			return client;
+		}
+
+	}
+
+	@TestConfiguration
+	static class PodReadyTestConfiguration {
+
+		// readiness passes after 2 retries
+		@Bean
+		@Primary
+		@ConditionalOnProperty(value = "readiness.passes", havingValue = "true", matchIfMissing = false)
+		BooleanSupplier readinessSupplierPasses() {
+			AtomicInteger counter = new AtomicInteger(0);
+			return () -> {
+				if (counter.get() != 2) {
+					counter.incrementAndGet();
+					return false;
+				}
+				return true;
+			};
+		}
+
+		// readiness fails after 2 retries
+		@Bean
+		@Primary
+		@ConditionalOnProperty(value = "readiness.fails", havingValue = "true", matchIfMissing = false)
+		BooleanSupplier readinessSupplierFails() {
+			AtomicInteger counter = new AtomicInteger(0);
+			return () -> {
+				if (counter.get() != 2) {
+					counter.incrementAndGet();
+					return false;
+				}
+				throw new RuntimeException("readiness fails");
+			};
+		}
+
+		// readiness always fails
+		@Bean
+		@Primary
+		@ConditionalOnProperty(value = "readiness.never.finishes", havingValue = "true", matchIfMissing = false)
+		BooleanSupplier readinessNeverFinishes() {
+			return () -> false;
+		}
+
+	}
+
+}
diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/App.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/App.java
new file mode 100644
index 0000000000..7b5bcc26f1
--- /dev/null
+++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/App.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2013-present the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.kubernetes.client.leader.election;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class App {
+
+	public static void main(String[] args) {
+		SpringApplication.run(App.class, args);
+	}
+
+}
diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/Assertions.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/Assertions.java
new file mode 100644
index 0000000000..1d0f752dee
--- /dev/null
+++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/Assertions.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2013-present the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.kubernetes.client.leader.election;
+
+import java.time.OffsetDateTime;
+import java.util.function.Supplier;
+
+import io.kubernetes.client.openapi.models.V1Lease;
+
+import org.springframework.boot.test.system.CapturedOutput;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.springframework.cloud.kubernetes.integration.tests.commons.Awaitilities.awaitUntil;
+
+/**
+ * @author wind57
+ */
+final class Assertions {
+
+	private Assertions() {
+
+	}
+
+	/**
+	 * lease was acquired and we renewed it, at least once.
+	 */
+	static void assertAcquireAndRenew(CapturedOutput output, Supplier leaseSupplier,
+			String candidateIdentity) {
+		// we have become the leader
+		awaitUntil(60, 100, () -> output.getOut().contains(candidateIdentity + " is the new leader"));
+
+		// let's unwind some logs to see that the process is how we expect it to be
+
+		// 1. lease is used as the lock (comes from our code)
+		awaitUntil(5, 100, () -> output.getOut().contains("will use lease as the lock for leader election"));
+
+		// 2. we start leader initiator for our hostname (comes from our code)
+		awaitUntil(5, 100, () -> output.getOut().contains("starting leader initiator : " + candidateIdentity));
+
+		// 3. start leader election with the configured lock
+		awaitUntil(10, 100, () -> output.getOut()
+			.contains("Start leader election with lock default/spring-k8s-leader-election-lock"));
+
+		// 4. we try to acquire the lease
+		awaitUntil(5, 100, () -> output.getOut().contains("Attempting to acquire leader lease"));
+
+		// 5. lease has been acquired
+		awaitUntil(5, 100,
+				() -> output.getOut().contains("LeaderElection lock is currently held by " + candidateIdentity));
+
+		// 6. we are the leader
+		awaitUntil(10, 100, () -> output.getOut().contains("Successfully acquired lease, became leader"));
+
+		// 7. wait until a renewal happens
+		// this one means that we have extended our leadership
+		awaitUntil(10, 100, () -> output.getOut().contains("Successfully renewed lease"));
+
+		V1Lease lease = leaseSupplier.get();
+
+		OffsetDateTime currentAcquiredTime = lease.getSpec().getAcquireTime();
+		assertThat(currentAcquiredTime).isNotNull();
+		assertThat(lease.getSpec().getLeaseDurationSeconds()).isEqualTo(6);
+		assertThat(lease.getSpec().getLeaseTransitions()).isEqualTo(0);
+
+		OffsetDateTime currentRenewalTime = lease.getSpec().getRenewTime();
+		assertThat(currentRenewalTime).isNotNull();
+
+		// 8. renewal happens
+		awaitUntil(4, 500, () -> {
+			OffsetDateTime newRenewalTime = leaseSupplier.get().getSpec().getRenewTime();
+			return newRenewalTime.isAfter(currentRenewalTime);
+		});
+	}
+
+}
diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/K8sClientLeaderElectionCanceledAndNotRestartedIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/K8sClientLeaderElectionCanceledAndNotRestartedIT.java
new file mode 100644
index 0000000000..9e040693e4
--- /dev/null
+++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/K8sClientLeaderElectionCanceledAndNotRestartedIT.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2013-present the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.kubernetes.client.leader.election;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.system.CapturedOutput;
+import org.springframework.test.context.TestPropertySource;
+
+import static org.springframework.cloud.kubernetes.client.leader.election.Assertions.assertAcquireAndRenew;
+import static org.springframework.cloud.kubernetes.integration.tests.commons.Awaitilities.awaitUntil;
+
+/**
+ * 
+ *     - we acquire the leadership
+ *     - leadership feature fails
+ * 

+ *
+ * @author wind57
+ */
+@TestPropertySource(properties = { "spring.cloud.kubernetes.leader.election.wait-for-pod-ready=true",
+		"spring.cloud.kubernetes.leader.election.restart-on-failure=true", "readiness.passes=true" })
+class K8sClientLeaderElectionCanceledAndNotRestartedIT extends AbstractLeaderElection {
+
+	private static final String NAME = "leader-acquired-then-canceled-it";
+
+	@Autowired
+	private KubernetesClientLeaderElectionInitiator initiator;
+
+	@BeforeAll
+	static void beforeAll() {
+		AbstractLeaderElection.beforeAll(NAME);
+	}
+
+	@AfterEach
+	void afterEach() {
+		stopLeaderAndDeleteLease(initiator, true);
+	}
+
+	@Test
+	void test(CapturedOutput output) {
+
+		assertAcquireAndRenew(output, this::getLease, NAME);
+
+		// this will kill leadership and it will not be re-started
+		initiator.preDestroy();
+
+		awaitUntil(10, 100, () -> output.getOut().contains("leadership terminated for : " + NAME));
+
+		awaitUntil(10, 100, () -> output.getOut().contains("Giving up the lock"));
+
+	}
+
+}
diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/K8sClientLeaderElectionCompletedExceptionallyAndRestartedIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/K8sClientLeaderElectionCompletedExceptionallyAndRestartedIT.java
new file mode 100644
index 0000000000..c098671180
--- /dev/null
+++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/K8sClientLeaderElectionCompletedExceptionallyAndRestartedIT.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright 2013-present the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.kubernetes.client.leader.election;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.system.CapturedOutput;
+import org.springframework.test.context.TestPropertySource;
+
+import static org.springframework.cloud.kubernetes.client.leader.election.Assertions.assertAcquireAndRenew;
+import static org.springframework.cloud.kubernetes.integration.tests.commons.Awaitilities.awaitUntil;
+
+/**
+ * 
+ *     - we acquire the leadership
+ *     - leadership feature fails
+ *     - we retry and acquire it again
+ * 

+ *
+ * @author wind57
+ */
+@TestPropertySource(properties = { "spring.cloud.kubernetes.leader.election.wait-for-pod-ready=true",
+		"spring.cloud.kubernetes.leader.election.restart-on-failure=true", "readiness.passes=true" })
+class K8sClientLeaderElectionCompletedExceptionallyAndRestartedIT extends AbstractLeaderElection {
+
+	private static final String NAME = "leader-completed-and-restarted-it";
+
+	@Autowired
+	private KubernetesClientLeaderElectionInitiator initiator;
+
+	@BeforeAll
+	static void beforeAll() {
+		AbstractLeaderElection.beforeAll(NAME);
+	}
+
+	@AfterEach
+	void afterEach() {
+		stopLeaderAndDeleteLease(initiator, true);
+	}
+
+	@Test
+	void test(CapturedOutput output) {
+
+		assertAcquireAndRenew(output, this::getLease, NAME);
+
+		// simulate that the lock is released
+		initiator.leaderElector().close();
+
+		// from the callback
+		awaitUntil(5, 50, () -> output.getOut().contains("id : " + NAME + " stopped being a leader"));
+
+		awaitUntil(5, 50, () -> output.getOut().contains("will re-start leader election for : " + NAME));
+
+		int afterLeaderFailure = output.getOut().indexOf("will re-start leader election for : " + NAME);
+
+		afterLeaderFailure(afterLeaderFailure, output);
+
+	}
+
+	private void afterLeaderFailure(int afterLeaderFailure, CapturedOutput output) {
+		awaitUntil(60, 100, () -> output.getOut().substring(afterLeaderFailure).contains(NAME + " is the new leader"));
+		awaitUntil(5, 100, () -> output.getOut().contains("Update lock to renew lease"));
+		awaitUntil(5, 100, () -> output.getOut().contains("TryAcquireOrRenew return success"));
+		awaitUntil(5, 100, () -> output.getOut().contains("Successfully renewed lease"));
+		awaitUntil(5, 100, () -> output.getOut().contains("Update lock to renew lease"));
+	}
+
+}
diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/K8sClientLeaderElectionConcurrentIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/K8sClientLeaderElectionConcurrentIT.java
new file mode 100644
index 0000000000..0ad66c7136
--- /dev/null
+++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/K8sClientLeaderElectionConcurrentIT.java
@@ -0,0 +1,230 @@
+/*
+ * Copyright 2013-present the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.kubernetes.client.leader.election;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.time.Duration;
+import java.util.function.Consumer;
+
+import io.kubernetes.client.extended.leaderelection.LeaderElectionConfig;
+import io.kubernetes.client.extended.leaderelection.Lock;
+import io.kubernetes.client.extended.leaderelection.resourcelock.LeaseLock;
+import io.kubernetes.client.openapi.ApiClient;
+import io.kubernetes.client.openapi.ApiException;
+import io.kubernetes.client.openapi.apis.CoordinationV1Api;
+import io.kubernetes.client.util.Config;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.testcontainers.k3s.K3sContainer;
+
+import org.springframework.boot.test.system.CapturedOutput;
+import org.springframework.boot.test.system.OutputCaptureExtension;
+import org.springframework.cloud.kubernetes.commons.leader.election.LeaderElectionProperties;
+import org.springframework.cloud.kubernetes.integration.tests.commons.Commons;
+
+import static org.springframework.cloud.kubernetes.integration.tests.commons.Awaitilities.awaitUntil;
+
+@ExtendWith(OutputCaptureExtension.class)
+class K8sClientLeaderElectionConcurrentIT {
+
+	private static final String LEASE_NAME = "lease-lock";
+
+	private static final LeaderElectionProperties PROPERTIES = new LeaderElectionProperties(false, false,
+			Duration.ofSeconds(15), "default", LEASE_NAME, Duration.ofSeconds(5), Duration.ofSeconds(2),
+			Duration.ofSeconds(5), false, true);
+
+	private static final String CANDIDATE_IDENTITY_ONE = "one";
+
+	private static final String CANDIDATE_IDENTITY_TWO = "two";
+
+	private KubernetesClientLeaderElectionInitiator one;
+
+	private KubernetesClientLeaderElectionInitiator two;
+
+	private static ApiClient apiClient;
+
+	@BeforeAll
+	static void beforeAll() {
+
+		K3sContainer container = Commons.container();
+		container.start();
+
+		String kubeConfigYaml = container.getKubeConfigYaml();
+
+		try {
+			apiClient = Config.fromConfig(new StringReader(kubeConfigYaml));
+		}
+		catch (IOException e) {
+			throw new RuntimeException(e);
+		}
+
+	}
+
+	@AfterAll
+	static void afterAll() {
+
+		CoordinationV1Api api = new CoordinationV1Api(apiClient);
+
+		try {
+			api.deleteNamespacedLease(LEASE_NAME, "default").execute();
+		}
+		catch (ApiException e) {
+			throw new RuntimeException(e);
+		}
+
+	}
+
+	@AfterEach
+	void afterEach() {
+		one.preDestroy();
+		two.preDestroy();
+	}
+
+	@Test
+	void test(CapturedOutput output) {
+
+		LeaderElectionConfig leaderElectionConfigOne = leaderElectionConfig(CANDIDATE_IDENTITY_ONE);
+		KubernetesClientLeaderElectionCallbacks callbacksOne = callbacks(CANDIDATE_IDENTITY_ONE);
+		one = new KubernetesClientLeaderElectionInitiator(CANDIDATE_IDENTITY_ONE, "default", leaderElectionConfigOne,
+				PROPERTIES, () -> true, callbacksOne);
+
+		LeaderElectionConfig leaderElectionConfigTwo = leaderElectionConfig(CANDIDATE_IDENTITY_TWO);
+		KubernetesClientLeaderElectionCallbacks callbacksTwo = callbacks(CANDIDATE_IDENTITY_TWO);
+		two = new KubernetesClientLeaderElectionInitiator(CANDIDATE_IDENTITY_TWO, "default", leaderElectionConfigTwo,
+				PROPERTIES, () -> true, callbacksTwo);
+
+		one.postConstruct();
+		two.postConstruct();
+
+		// both try to acquire the lock
+		awaitUntil(5, 100, () -> output.getOut().contains("starting leader initiator : one"));
+		awaitUntil(5, 100, () -> output.getOut().contains("starting leader initiator : two"));
+
+		// someone has become the leader
+		awaitUntil(5, 100, () -> output.getOut().contains("LeaderElection lock is currently held by"));
+
+		LeaderAndFollower leaderAndFollower = leaderAndFollower(leaderElectionConfigOne);
+		String leader = leaderAndFollower.leader();
+		String follower = leaderAndFollower.follower();
+
+		// someone has become the leader
+		awaitUntil(5, 100, () -> output.getOut().contains("LeaderElection lock is currently held by " + leader));
+		awaitUntil(3, 100, () -> output.getOut().contains("id : " + leader + " is the new leader"));
+
+		// the other elector says it can't acquire the lock
+		awaitUntil(10, 100, () -> output.getOut().contains("Lock is held by " + leader + " and has not yet expired"));
+		awaitUntil(10, 100, () -> output.getOut().contains("The tryAcquireOrRenew result is false"));
+
+		int beforeRelease = output.getOut().length();
+		failLeaderRenewal(leader, one, two);
+
+		/*
+		 * we simulated above that renewal failed and leader future was canceled. In this
+		 * case, the 'notLeader' picks up the leadership, the 'leader' is now a
+		 * "follower", it re-tries to take leadership.
+		 */
+		awaitUntil(10, 100,
+				() -> output.getOut().substring(beforeRelease).contains("id : " + follower + " is the new leader"));
+		awaitUntil(10, 100,
+				() -> output.getOut().substring(beforeRelease).contains("Failed to renew lease, lose leadership"));
+
+		// the other candidate still tries to become leader
+		awaitUntil(10, 100,
+				() -> output.getOut()
+					.substring(beforeRelease)
+					.contains("Lock is held by " + follower + " and has not yet expired"));
+
+		/*
+		 * we simulate the renewal failure one more time. we know that leader = 'follower'
+		 */
+		int failAgain = output.getOut().length();
+		failLeaderRenewal(follower, one, two);
+
+		awaitUntil(10, 100,
+				() -> output.getOut().substring(failAgain).contains("id : " + leader + " is the new leader"));
+		// the other candidate still tries to become leader
+		awaitUntil(10, 100,
+				() -> output.getOut()
+					.substring(beforeRelease)
+					.contains("Lock is held by " + leader + " and has not yet expired"));
+	}
+
+	private LeaderElectionConfig leaderElectionConfig(String holderIdentity) {
+
+		Lock lock = leaseLock(holderIdentity);
+
+		LeaderElectionConfig leaderElectionConfig = new LeaderElectionConfig();
+		leaderElectionConfig.setLock(lock);
+		leaderElectionConfig.setLeaseDuration(PROPERTIES.leaseDuration());
+		leaderElectionConfig.setRenewDeadline(PROPERTIES.renewDeadline());
+		leaderElectionConfig.setRetryPeriod(PROPERTIES.retryPeriod());
+
+		return leaderElectionConfig;
+	}
+
+	private LeaseLock leaseLock(String holderIdentity) {
+		return new LeaseLock("default", LEASE_NAME, holderIdentity, apiClient);
+	}
+
+	private KubernetesClientLeaderElectionCallbacks callbacks(String holderIdentity) {
+		KubernetesClientLeaderElectionCallbacksAutoConfiguration configuration = new KubernetesClientLeaderElectionCallbacksAutoConfiguration();
+
+		Runnable onStartLeadingCallback = configuration.onStartLeadingCallback(null, holderIdentity, PROPERTIES);
+		Runnable onStopLeadingCallback = configuration.onStopLeadingCallback(null, holderIdentity, PROPERTIES);
+		Consumer onNewLeaderCallback = configuration.onNewLeaderCallback(null, PROPERTIES);
+
+		return new KubernetesClientLeaderElectionCallbacks(onStartLeadingCallback, onStopLeadingCallback,
+				onNewLeaderCallback);
+	}
+
+	private LeaderAndFollower leaderAndFollower(LeaderElectionConfig leaderElectionConfig) {
+
+		boolean oneIsLeader;
+		try {
+			oneIsLeader = leaderElectionConfig.getLock().get().getHolderIdentity().equals(CANDIDATE_IDENTITY_ONE);
+		}
+		catch (ApiException e) {
+			throw new RuntimeException(e);
+		}
+
+		if (oneIsLeader) {
+			return new LeaderAndFollower("one", "two");
+		}
+		else {
+			return new LeaderAndFollower("two", "one");
+		}
+	}
+
+	private void failLeaderRenewal(String currentLeader, KubernetesClientLeaderElectionInitiator one,
+			KubernetesClientLeaderElectionInitiator two) {
+		if (currentLeader.equals("one")) {
+			one.leaderElector().close();
+		}
+		else {
+			two.leaderElector().close();
+		}
+	}
+
+	private record LeaderAndFollower(String leader, String follower) {
+
+	}
+
+}
diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/K8sClientLeaderElectionIsLostAndRestartedIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/K8sClientLeaderElectionIsLostAndRestartedIT.java
new file mode 100644
index 0000000000..c3c6e8a2f2
--- /dev/null
+++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/K8sClientLeaderElectionIsLostAndRestartedIT.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright 2013-present the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.kubernetes.client.leader.election;
+
+import io.kubernetes.client.openapi.models.V1Lease;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.system.CapturedOutput;
+import org.springframework.test.context.TestPropertySource;
+
+import static org.springframework.cloud.kubernetes.client.leader.election.Assertions.assertAcquireAndRenew;
+import static org.springframework.cloud.kubernetes.integration.tests.commons.Awaitilities.awaitUntil;
+
+/**
+ * We acquire leadership, then lose it, then acquire it back. This tests the "leaderFuture
+ * finished normally, will re-start it for" branch
+ *
+ * @author wind57
+ */
+@TestPropertySource(
+		properties = { "spring.cloud.kubernetes.leader.election.wait-for-pod-ready=true", "readiness.passes=true" })
+class K8sClientLeaderElectionIsLostAndRestartedIT extends AbstractLeaderElection {
+
+	private static final String NAME = "leader-lost-then-recovers-it";
+
+	@Autowired
+	private KubernetesClientLeaderElectionInitiator initiator;
+
+	@BeforeAll
+	static void beforeAll() {
+		AbstractLeaderElection.beforeAll(NAME);
+	}
+
+	@AfterEach
+	void afterEach() {
+		stopLeaderAndDeleteLease(initiator, true);
+	}
+
+	@Test
+	void test(CapturedOutput output) {
+
+		assertAcquireAndRenew(output, this::getLease, NAME);
+
+		// 8. simulate that leadership has changed
+		V1Lease lease = getLease();
+		lease.getSpec().setHolderIdentity("leader-lost-then-recovers-it-is-not-the-leader-anymore");
+		updateLease(lease);
+
+		// 9. we lost leadership
+		awaitUntil(10, 100, () -> output.getOut().contains("Failed to renew lease, lose leadership"));
+
+		// 10. callback confirms we lost leadership
+		awaitUntil(10, 100, () -> output.getOut().contains("id : " + NAME + " stopped being a leader"));
+
+		// 11. leader has changed
+		awaitUntil(10, 20, () -> output.getOut()
+			.contains(
+					"LeaderElection lock is currently held by leader-lost-then-recovers-it-is-not-the-leader-anymore"));
+
+		// 12. from our callback
+		awaitUntil(10, 100, () -> output.getOut()
+			.contains("id : leader-lost-then-recovers-it-is-not-the-leader-anymore is the new leader"));
+
+		// 13. leadership is restarted for us
+		awaitUntil(10, 100, () -> output.getOut().contains("will re-start leader election for : " + NAME));
+
+		int leadershipFinished = output.getOut().indexOf("will re-start leader election for : " + NAME);
+		afterLeadershipRestart(output, leadershipFinished);
+
+	}
+
+	private void afterLeadershipRestart(CapturedOutput output, int leadershipFinished) {
+
+		// 14. since the new leader is artificial, renew is not going to happen for it
+		awaitUntil(60, 100, () -> output.getOut().substring(leadershipFinished).contains(NAME + " is the new leader"));
+
+		// 15. callback is again triggered
+		awaitUntil(10, 100, () -> output.getOut().substring(leadershipFinished).contains(NAME + " is now a leader"));
+
+	}
+
+}
diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/K8sClientLeaderElectionReadinessCanceledIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/K8sClientLeaderElectionReadinessCanceledIT.java
new file mode 100644
index 0000000000..177dcec022
--- /dev/null
+++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/K8sClientLeaderElectionReadinessCanceledIT.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2013-present the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.kubernetes.client.leader.election;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.system.CapturedOutput;
+import org.springframework.test.context.TestPropertySource;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.springframework.cloud.kubernetes.integration.tests.commons.Awaitilities.awaitUntil;
+
+/**
+ * Readiness is canceled. This is the case when pod is shut down gracefully
+ *
+ * @author wind57
+ */
+@TestPropertySource(properties = { "readiness.never.finishes=true",
+		"spring.cloud.kubernetes.leader.election.wait-for-pod-ready=true" })
+class K8sClientLeaderElectionReadinessCanceledIT extends AbstractLeaderElection {
+
+	private static final String NAME = "readiness-canceled-it";
+
+	@Autowired
+	private KubernetesClientLeaderElectionInitiator initiator;
+
+	@BeforeAll
+	static void beforeAll() {
+		AbstractLeaderElection.beforeAll(NAME);
+	}
+
+	@AfterEach
+	void afterEach() {
+		stopLeaderAndDeleteLease(initiator, false);
+	}
+
+	@Test
+	void test(CapturedOutput output) {
+
+		// we are trying readiness at least once
+		awaitUntil(60, 500, () -> output.getOut()
+			.contains("Pod : " + NAME + " in namespace : " + "default is not ready, will retry in one second"));
+
+		initiator.preDestroy();
+
+		// 1. preDestroy method logs what it will do
+		assertThat(output.getOut()).contains("podReadyFuture will be canceled for : " + NAME);
+
+		// 2. readiness failed
+		assertThat(output.getOut()).contains("readiness failed for : " + NAME + ", leader election will not start");
+
+		// 3. will cancel the future that is supposed to do the readiness
+		assertThat(output.getOut()).contains("canceling scheduled future because completable future was cancelled");
+
+		// 4. podReadyWaitingExecutor is shut down also
+		assertThat(output.getOut()).contains("podReadyWaitingExecutor will be shutdown for : " + NAME);
+
+		// 5. the scheduled executor where pod readiness is checked is shut down also
+		awaitUntil(2, 100, () -> output.getOut().contains("Shutting down executor : podReadyExecutor"));
+
+		// we need to call preDestroy again, to make sure that leaderFuture was not
+		// started
+		initiator.preDestroy();
+
+		// 6. leader election is not started, since readiness does not finish
+		assertThat(output.getOut()).doesNotContain("starting leader election : " + NAME);
+
+	}
+
+}
diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/K8sClientLeaderElectionReadinessFailsIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/K8sClientLeaderElectionReadinessFailsIT.java
new file mode 100644
index 0000000000..82dc925551
--- /dev/null
+++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/K8sClientLeaderElectionReadinessFailsIT.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright 2013-present the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.kubernetes.client.leader.election;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.system.CapturedOutput;
+import org.springframework.test.context.TestPropertySource;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.springframework.cloud.kubernetes.integration.tests.commons.Awaitilities.awaitUntil;
+
+/**
+ * Readiness fails with an Exception, and we don't establish leadership
+ *
+ * @author wind57
+ */
+@TestPropertySource(
+		properties = { "readiness.fails=true", "spring.cloud.kubernetes.leader.election.wait-for-pod-ready=true" })
+class K8sClientLeaderElectionReadinessFailsIT extends AbstractLeaderElection {
+
+	private static final String NAME = "readiness-fails-it";
+
+	@Autowired
+	private KubernetesClientLeaderElectionInitiator initiator;
+
+	@BeforeAll
+	static void beforeAll() {
+		AbstractLeaderElection.beforeAll(NAME);
+	}
+
+	@AfterEach
+	void afterEach() {
+		stopLeaderAndDeleteLease(initiator, false);
+	}
+
+	/**
+	 * 
+	 *   - readiness fails after 2 seconds - leader election process is not started at all
+	 * 

+	 */
+	@Test
+	void test(CapturedOutput output) {
+
+		// we do not start leader election at all
+		awaitUntil(60, 1000,
+				() -> output.getOut().contains("readiness failed for : " + NAME + ", leader election will not start"));
+
+		// 1. lease is used as the lock (comes from our code)
+		assertThat(output.getOut()).contains("will use lease as the lock for leader election");
+
+		// 2. leader initiator is started
+		assertThat(output.getOut()).contains("starting leader initiator : " + NAME);
+
+		// 3. wait for when pod is ready (we mock this one)
+		assertThat(output.getOut()).contains("will wait until pod " + NAME + " is ready");
+
+		// 4. we run readiness check in podReadyExecutor
+		assertThat(output.getOut()).contains("Scheduling command to run in : podReadyExecutor");
+
+		// 5. pod fails on the first two attempts
+		assertThat(output.getOut())
+			.contains("Pod : " + NAME + " in namespace : default is not ready, will retry in one second");
+
+		// 6. readiness fails
+		assertThat(output.getOut()).contains("exception waiting for pod : " + NAME);
+
+		// 7. we shut down the executor
+		assertThat(output.getOut()).contains("canceling scheduled future because readiness failed");
+
+		// 8. leader election did not even start properly
+		assertThat(output.getOut()).contains("pod readiness for : " + NAME + " failed with : readiness fails");
+
+		// 9. executor is shutdown, even when readiness failed
+		awaitUntil(60, 100,
+				() -> output.getOut().contains("readiness failed for : " + NAME + ", leader election will not start"));
+
+	}
+
+}
diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/K8sClientLeaderElectionReadinessPassesIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/K8sClientLeaderElectionReadinessPassesIT.java
new file mode 100644
index 0000000000..a5fab8d35d
--- /dev/null
+++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/java/org/springframework/cloud/kubernetes/client/leader/election/K8sClientLeaderElectionReadinessPassesIT.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2013-present the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.kubernetes.client.leader.election;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.system.CapturedOutput;
+import org.springframework.test.context.TestPropertySource;
+
+import static org.springframework.cloud.kubernetes.client.leader.election.Assertions.assertAcquireAndRenew;
+import static org.springframework.cloud.kubernetes.integration.tests.commons.Awaitilities.awaitUntil;
+
+/**
+ * A simple test where we are the sole participant in the leader election and everything
+ * goes fine from start to end. It's a happy path scenario test.
+ *
+ * @author wind57
+ */
+
+@TestPropertySource(
+		properties = { "spring.cloud.kubernetes.leader.election.wait-for-pod-ready=true", "readiness.passes=true" })
+class K8sClientLeaderElectionReadinessPassesIT extends AbstractLeaderElection {
+
+	@Autowired
+	private KubernetesClientLeaderElectionInitiator initiator;
+
+	private static final String NAME = "readiness-passes-it";
+
+	@BeforeAll
+	static void beforeAll() {
+		AbstractLeaderElection.beforeAll(NAME);
+	}
+
+	@AfterEach
+	void afterEach() {
+		stopLeaderAndDeleteLease(initiator, true);
+	}
+
+	/**
+	 * 
+	 *     - readiness is checked
+	 *     - leader election process happens after that
+	 *     - we establish leadership and renew it
+	 * 

+	 */
+	@Test
+	void test(CapturedOutput output) {
+
+		awaitUntil(10, 100, () -> output.getOut()
+			.contains("Pod : " + NAME + " in namespace : default is not ready, will retry in one second"));
+		awaitUntil(10, 100, () -> output.getOut().contains("Pod : " + NAME + " in namespace : default is ready"));
+		awaitUntil(10, 100, () -> output.getOut().contains(NAME + " is ready"));
+		awaitUntil(10, 100, () -> output.getOut().contains("canceling scheduled future because readiness succeeded"));
+
+		assertAcquireAndRenew(output, this::getLease, NAME);
+
+		// comes from the callback, where we post the spring lifecycle event
+		awaitUntil(5, 100, () -> output.getOut().contains(NAME + " is the new leader"));
+
+		awaitUntil(60, 100, () -> output.getOut().contains("Shutting down executor : podReadyExecutor"));
+	}
+
+}
diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/resources/logback-test.xml b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/resources/logback-test.xml
new file mode 100644
index 0000000000..33cd90c3b9
--- /dev/null
+++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-leader-election/src/test/resources/logback-test.xml
@@ -0,0 +1,17 @@
+
+	
+		
+			%d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n
+		
+	
+
+	
+		
+	
+
+	
+	
+
+	
+
+